diff --git a/.github/actions/get-msys2/action.yml b/.github/actions/get-msys2/action.yml index 3e6c3417a311d..27726245c15dc 100644 --- a/.github/actions/get-msys2/action.yml +++ b/.github/actions/get-msys2/action.yml @@ -30,7 +30,8 @@ runs: using: composite steps: - name: 'Install MSYS2' - uses: msys2/setup-msys2@v2 + # use a specific release of msys2/setup-msys2 to prevent jtreg build failures on newer release + uses: msys2/setup-msys2@7efe20baefed56359985e327d329042cde2434ff with: install: 'autoconf tar unzip zip make' path-type: minimal diff --git a/.github/workflows/build-cross-compile.yml b/.github/workflows/build-cross-compile.yml index 168c5924d8610..d4a31714d73c2 100644 --- a/.github/workflows/build-cross-compile.yml +++ b/.github/workflows/build-cross-compile.yml @@ -35,7 +35,7 @@ on: apt-gcc-version: required: false type: string - default: '10.3.0-1ubuntu1~20.04' + default: '10.4.0-4ubuntu1~22.04' apt-gcc-cross-suffix: required: false type: string @@ -44,7 +44,7 @@ on: jobs: build-cross-compile: name: build - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false @@ -54,20 +54,34 @@ jobs: - arm - s390x - ppc64le + - riscv64 include: - target-cpu: aarch64 - debian-arch: arm64 gnu-arch: aarch64 + debian-arch: arm64 + debian-repository: https://httpredir.debian.org/debian/ + debian-version: buster - target-cpu: arm - debian-arch: armhf gnu-arch: arm + debian-arch: armhf + debian-repository: https://httpredir.debian.org/debian/ + debian-version: buster gnu-abi: eabihf - target-cpu: s390x - debian-arch: s390x gnu-arch: s390x + debian-arch: s390x + debian-repository: https://httpredir.debian.org/debian/ + debian-version: buster - target-cpu: ppc64le - debian-arch: ppc64el gnu-arch: powerpc64le + debian-arch: ppc64el + debian-repository: https://httpredir.debian.org/debian/ + debian-version: buster + - target-cpu: riscv64 + gnu-arch: riscv64 + debian-arch: riscv64 + debian-repository: https://deb.debian.org/debian-ports + debian-version: sid steps: - name: 'Checkout the JDK source' @@ -118,9 +132,9 @@ jobs: --verbose --include=fakeroot,symlinks,build-essential,libx11-dev,libxext-dev,libxrender-dev,libxrandr-dev,libxtst-dev,libxt-dev,libcups2-dev,libfontconfig1-dev,libasound2-dev,libfreetype6-dev,libpng-dev --resolve-deps - buster + ${{ matrix.debian-version }} sysroot - https://httpredir.debian.org/debian/ + ${{ matrix.debian-repository }} if: steps.get-cached-sysroot.outputs.cache-hit != 'true' - name: 'Prepare sysroot' diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 86fdba8670fe8..d8ff96712596c 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -92,12 +92,26 @@ jobs: id: gtest uses: ./.github/actions/get-gtest + - name: 'Check toolchain installed' + id: toolchain-check + run: | + set +e + '/c/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/vc/auxiliary/build/vcvars64.bat' -vcvars_ver=${{ inputs.msvc-toolset-version }} + if [ $? -eq 0 ]; then + echo "Toolchain is already installed" + echo "toolchain-installed=true" >> $GITHUB_OUTPUT + else + echo "Toolchain is not yet installed" + echo "toolchain-installed=false" >> $GITHUB_OUTPUT + fi + - name: 'Install toolchain and dependencies' run: | # Run Visual Studio Installer '/c/Program Files (x86)/Microsoft Visual Studio/Installer/vs_installer.exe' \ - modify --quiet --installPath 'C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise' \ + modify --quiet --installPath 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise' \ --add Microsoft.VisualStudio.Component.VC.${{ inputs.msvc-toolset-version }}.${{ inputs.msvc-toolset-architecture }} + if: steps.toolchain-check.outputs.toolchain-installed != 'true' - name: 'Configure' run: > diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4577be3248b08..ed159046d0b0d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -71,19 +71,17 @@ jobs: # 'false' otherwise. # arg $1: platform name or names to look for function check_platform() { - if [[ '${{ !secrets.JDK_SUBMIT_FILTER || startsWith(github.ref, 'refs/heads/submit/') }}' == 'false' ]]; then - # If JDK_SUBMIT_FILTER is set, and this is not a "submit/" branch, don't run anything - echo 'false' - return - fi - if [[ $GITHUB_EVENT_NAME == workflow_dispatch ]]; then input='${{ github.event.inputs.platforms }}' elif [[ $GITHUB_EVENT_NAME == push ]]; then - input='${{ secrets.JDK_SUBMIT_PLATFORMS }}' - else - echo 'Internal error in GHA' - exit 1 + if [[ '${{ !secrets.JDK_SUBMIT_FILTER || startsWith(github.ref, 'refs/heads/submit/') }}' == 'false' ]]; then + # If JDK_SUBMIT_FILTER is set, and this is not a "submit/" branch, don't run anything + >&2 echo 'JDK_SUBMIT_FILTER is set and not a "submit/" branch' + echo 'false' + return + else + input='${{ secrets.JDK_SUBMIT_PLATFORMS }}' + fi fi normalized_input="$(echo ,$input, | tr -d ' ')" @@ -204,7 +202,7 @@ jobs: uses: ./.github/workflows/build-macos.yml with: platform: macos-x64 - xcode-toolset-version: '11.7' + xcode-toolset-version: '12.5.1' if: needs.select.outputs.macos-x64 == 'true' build-macos-aarch64: @@ -213,7 +211,7 @@ jobs: uses: ./.github/workflows/build-macos.yml with: platform: macos-aarch64 - xcode-toolset-version: '12.4' + xcode-toolset-version: '12.5.1' extra-conf-options: '--openjdk-target=aarch64-apple-darwin' if: needs.select.outputs.macos-aarch64 == 'true' @@ -223,7 +221,7 @@ jobs: uses: ./.github/workflows/build-windows.yml with: platform: windows-x64 - msvc-toolset-version: '14.25' + msvc-toolset-version: '14.29' msvc-toolset-architecture: 'x86.x64' if: needs.select.outputs.windows-x64 == 'true' diff --git a/.gitignore b/.gitignore index 6787b23253522..d0736707b804a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ NashornProfile.txt /src/utils/LogCompilation/target/ /.project/ /.settings/ +/compile_commands.json +/.cache diff --git a/.jcheck/conf b/.jcheck/conf index 495417d197213..ab251de5d4f13 100644 --- a/.jcheck/conf +++ b/.jcheck/conf @@ -1,7 +1,7 @@ [general] project=jdk-updates jbs=JDK -version=17.0.7 +version=17.0.9 [checks] error=author,committer,reviewers,merge,issues,executable,symlink,message,hg-tag,whitespace,problemlists diff --git a/bin/jib.sh b/bin/jib.sh index aab198990ccd0..9f34cf9ab7b6d 100644 --- a/bin/jib.sh +++ b/bin/jib.sh @@ -128,6 +128,15 @@ install_jib() { exit 1 fi fi + # Want to check the filetype using file, to see if we got served a HTML error page. + # This is sensitive to the filename containing a specific string, but good enough. + file "${installed_jib_script}.gz" | grep "gzip compressed data" > /dev/null + if [ $? -ne 0 ]; then + echo "Warning: ${installed_jib_script}.gz is not a gzip file." + echo "If you are behind a proxy you may need to configure exceptions using no_proxy." + echo "The download URL was: ${jib_url}" + exit 1 + fi echo "Extracting JIB bootstrap script" rm -f "${installed_jib_script}" gunzip "${installed_jib_script}.gz" diff --git a/doc/building.html b/doc/building.html index 49035c0d625ef..f92b3680009f3 100644 --- a/doc/building.html +++ b/doc/building.html @@ -273,7 +273,7 @@

Native Compiler (Toolchain) Requ Linux -gcc 10.2.0 +gcc 11.2.0 macOS @@ -288,7 +288,7 @@

Native Compiler (Toolchain) Requ

All compilers are expected to be able to compile to the C99 language standard, as some C99 features are used in the source code. Microsoft Visual Studio doesn't fully support C99 so in practice shared code is limited to using C99 features that it does support.

gcc

The minimum accepted version of gcc is 5.0. Older versions will generate a warning by configure and are unlikely to work.

-

The JDK is currently known to be able to compile with at least version 10.2 of gcc.

+

The JDK is currently known to be able to compile with at least version 11.2 of gcc.

In general, any version between these two should be usable.

clang

The minimum accepted version of clang is 3.5. Older versions will not be accepted by configure.

diff --git a/doc/building.md b/doc/building.md index 30ec48b78e2b8..a11ed8846104e 100644 --- a/doc/building.md +++ b/doc/building.md @@ -321,7 +321,7 @@ issues. | Operating system | Toolchain version | | ------------------ | ------------------------------------------ | -| Linux | gcc 10.2.0 | +| Linux | gcc 11.2.0 | | macOS | Apple Xcode 10.1 (using clang 10.0.0) | | Windows | Microsoft Visual Studio 2022 update 17.1.0 | @@ -335,7 +335,7 @@ features that it does support. The minimum accepted version of gcc is 5.0. Older versions will generate a warning by `configure` and are unlikely to work. -The JDK is currently known to be able to compile with at least version 10.2 of +The JDK is currently known to be able to compile with at least version 11.2 of gcc. In general, any version between these two should be usable. diff --git a/doc/testing.html b/doc/testing.html index effc6f0c446a9..e9ff786349743 100644 --- a/doc/testing.html +++ b/doc/testing.html @@ -242,14 +242,39 @@

PKCS11 Tests

JTREG="JAVA_OPTIONS=-Dtest.nss.lib.paths=/path/to/your/latest/NSS-libs"

For more notes about the PKCS11 tests, please refer to test/jdk/sun/security/pkcs11/README.

Client UI Tests

+

System key shortcuts

Some Client UI tests use key sequences which may be reserved by the operating system. Usually that causes the test failure. So it is highly recommended to disable system key shortcuts prior testing. The steps to access and disable system key shortcuts for various platforms are provided below.

-

MacOS

+
MacOS

Choose Apple menu; System Preferences, click Keyboard, then click Shortcuts; select or deselect desired shortcut.

For example, test/jdk/javax/swing/TooltipManager/JMenuItemToolTipKeyBindingsTest/JMenuItemToolTipKeyBindingsTest.java fails on MacOS because it uses CTRL + F1 key sequence to show or hide tooltip message but the key combination is reserved by the operating system. To run the test correctly the default global key shortcut should be disabled using the steps described above, and then deselect "Turn keyboard access on or off" option which is responsible for CTRL + F1 combination.

-

Linux

+
Linux

Open the Activities overview and start typing Settings; Choose Settings, click Devices, then click Keyboard; set or override desired shortcut.

-

Windows

+
Windows

Type gpedit in the Search and then click Edit group policy; navigate to User Configuration -> Administrative Templates -> Windows Components -> File Explorer; in the right-side pane look for "Turn off Windows key hotkeys" and double click on it; enable or disable hotkeys.

Note: restart is required to make the settings take effect.

+

Robot API

+

Most automated Client UI tests use Robot API to control +the UI. Usually, the default operating system settings need to be +adjusted for Robot to work correctly. The detailed steps how to access +and update these settings for different platforms are provided +below.

+
macOS
+

Robot is not permitted to control your Mac by default +since macOS 10.15. To allow it, choose Apple menu -> System Settings, +click Privacy & Security; then click Accessibility and ensure the +following apps are allowed to control your computer: Java and +Terminal. If the tests are run from an IDE, the IDE should be +granted this permission too.

+
Windows
+

On Windows if Cygwin terminal is used to run the tests, there is a +delay in focus transfer. Usually it causes automated UI test failure. To +disable the delay, type regedit in the Search and then +select Registry Editor; navigate to the following key: +HKEY_CURRENT_USER\Control Panel\Desktop; make sure the +ForegroundLockTimeout value is set to 0.

+

Additional information about Client UI tests configuration for +various operating systems can be obtained at Automated +client GUI testing system set up requirements

diff --git a/doc/testing.md b/doc/testing.md index 241bc08b40b9e..7f11fb2b51195 100644 --- a/doc/testing.md +++ b/doc/testing.md @@ -546,12 +546,14 @@ test/jdk/sun/security/pkcs11/README. ### Client UI Tests +#### System key shortcuts + Some Client UI tests use key sequences which may be reserved by the operating system. Usually that causes the test failure. So it is highly recommended to disable system key shortcuts prior testing. The steps to access and disable system key shortcuts for various platforms are provided below. -#### MacOS +##### macOS Choose Apple menu; System Preferences, click Keyboard, then click Shortcuts; select or deselect desired shortcut. @@ -564,12 +566,12 @@ test correctly the default global key shortcut should be disabled using the steps described above, and then deselect "Turn keyboard access on or off" option which is responsible for `CTRL + F1` combination. -#### Linux +##### Linux Open the Activities overview and start typing Settings; Choose Settings, click Devices, then click Keyboard; set or override desired shortcut. -#### Windows +##### Windows Type `gpedit` in the Search and then click Edit group policy; navigate to User Configuration -> Administrative Templates -> Windows Components -> File @@ -578,6 +580,33 @@ double click on it; enable or disable hotkeys. Note: restart is required to make the settings take effect. +#### Robot API + +Most automated Client UI tests use `Robot` API to control the UI. Usually, +the default operating system settings need to be adjusted for Robot +to work correctly. The detailed steps how to access and update these settings +for different platforms are provided below. + +##### macOS + +`Robot` is not permitted to control your Mac by default since +macOS 10.15. To allow it, choose Apple menu -> System Settings, click +Privacy & Security; then click Accessibility and ensure the following apps are +allowed to control your computer: *Java* and *Terminal*. If the tests are run +from an IDE, the IDE should be granted this permission too. + +##### Windows + +On Windows if Cygwin terminal is used to run the tests, there is a delay in +focus transfer. Usually it causes automated UI test failure. To disable the +delay, type `regedit` in the Search and then select Registry Editor; navigate +to the following key: `HKEY_CURRENT_USER\Control Panel\Desktop`; make sure +the `ForegroundLockTimeout` value is set to 0. + +Additional information about Client UI tests configuration for various operating +systems can be obtained at [Automated client GUI testing system set up +requirements](https://wiki.openjdk.org/display/ClientLibs/Automated+client+GUI+testing+system+set+up+requirements) + --- # Override some definitions in the global css file that are not optimal for # this document. diff --git a/make/Init.gmk b/make/Init.gmk index 4d297ee85e9e5..3e48409354f96 100644 --- a/make/Init.gmk +++ b/make/Init.gmk @@ -1,5 +1,5 @@ # -# Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -138,7 +138,10 @@ ifeq ($(HAS_SPEC),) # The spec files depend on the autoconf source code. This check makes sure # the configuration is up to date after changes to configure. $(SPECS): $(wildcard $(topdir)/make/autoconf/*) \ - $(if $(CUSTOM_CONFIG_DIR), $(wildcard $(CUSTOM_CONFIG_DIR)/*)) + $(if $(CUSTOM_CONFIG_DIR), $(wildcard $(CUSTOM_CONFIG_DIR)/*)) \ + $(addprefix $(topdir)/make/conf/, version-numbers.conf branding.conf) \ + $(if $(CUSTOM_CONF_DIR), $(wildcard $(addprefix $(CUSTOM_CONF_DIR)/, \ + version-numbers.conf branding.conf))) ifeq ($(CONF_CHECK), fail) @echo Error: The configuration is not up to date for \ "'$(lastword $(subst /, , $(dir $@)))'." diff --git a/make/ReleaseFile.gmk b/make/ReleaseFile.gmk index 0424e2fb62321..5e8d123fac9c8 100644 --- a/make/ReleaseFile.gmk +++ b/make/ReleaseFile.gmk @@ -51,6 +51,7 @@ define create-info-file $(if $(VENDOR_VERSION_STRING), \ $(call info-file-item, "IMPLEMENTOR_VERSION", "$(VENDOR_VERSION_STRING)")) $(call info-file-item, "JAVA_VERSION_DATE", "$(VERSION_DATE)") + $(call info-file-item, "JAVA_RUNTIME_VERSION", "$(VERSION_STRING)") $(call info-file-item, "OS_NAME", "$(RELEASE_FILE_OS_NAME)") $(call info-file-item, "OS_ARCH", "$(RELEASE_FILE_OS_ARCH)") $(call info-file-item, "LIBC", "$(RELEASE_FILE_LIBC)") diff --git a/make/autoconf/basic.m4 b/make/autoconf/basic.m4 index c16aba0881e54..8461815d7718a 100644 --- a/make/autoconf/basic.m4 +++ b/make/autoconf/basic.m4 @@ -55,6 +55,7 @@ AC_DEFUN([BASIC_CHECK_LEFTOVER_OVERRIDDEN], ############################################################################### # Setup basic configuration paths, and platform-specific stuff related to PATHs. +# Make sure to only use tools set up in BASIC_SETUP_FUNDAMENTAL_TOOLS. AC_DEFUN_ONCE([BASIC_SETUP_PATHS], [ # Save the current directory this script was started from diff --git a/make/autoconf/basic_tools.m4 b/make/autoconf/basic_tools.m4 index ac24bbcb831f1..8c518d05d41d1 100644 --- a/make/autoconf/basic_tools.m4 +++ b/make/autoconf/basic_tools.m4 @@ -24,8 +24,8 @@ # ############################################################################### -# Setup the most fundamental tools that relies on not much else to set up, -# but is used by much of the early bootstrap code. +# Setup the most fundamental tools, used for setting up build platform and +# path handling. AC_DEFUN_ONCE([BASIC_SETUP_FUNDAMENTAL_TOOLS], [ # Bootstrapping: These tools are needed by UTIL_LOOKUP_PROGS @@ -37,7 +37,28 @@ AC_DEFUN_ONCE([BASIC_SETUP_FUNDAMENTAL_TOOLS], UTIL_CHECK_NONEMPTY(FILE) AC_PATH_PROGS(LDD, ldd) - # First are all the fundamental required tools. + # Required tools + UTIL_REQUIRE_PROGS(ECHO, echo) + UTIL_REQUIRE_PROGS(TR, tr) + UTIL_REQUIRE_PROGS(UNAME, uname) + UTIL_REQUIRE_PROGS(WC, wc) + + # Required tools with some special treatment + UTIL_REQUIRE_SPECIAL(GREP, [AC_PROG_GREP]) + UTIL_REQUIRE_SPECIAL(EGREP, [AC_PROG_EGREP]) + UTIL_REQUIRE_SPECIAL(SED, [AC_PROG_SED]) + + # Tools only needed on some platforms + UTIL_LOOKUP_PROGS(PATHTOOL, cygpath wslpath) + UTIL_LOOKUP_PROGS(CMD, cmd.exe, $PATH:/cygdrive/c/windows/system32:/mnt/c/windows/system32:/c/windows/system32) +]) + +############################################################################### +# Setup further tools that should be resolved early but after setting up +# build platform and path handling. +AC_DEFUN_ONCE([BASIC_SETUP_TOOLS], +[ + # Required tools UTIL_REQUIRE_PROGS(BASH, bash) UTIL_REQUIRE_PROGS(CAT, cat) UTIL_REQUIRE_PROGS(CHMOD, chmod) @@ -45,7 +66,6 @@ AC_DEFUN_ONCE([BASIC_SETUP_FUNDAMENTAL_TOOLS], UTIL_REQUIRE_PROGS(CUT, cut) UTIL_REQUIRE_PROGS(DATE, date) UTIL_REQUIRE_PROGS(DIFF, gdiff diff) - UTIL_REQUIRE_PROGS(ECHO, echo) UTIL_REQUIRE_PROGS(EXPR, expr) UTIL_REQUIRE_PROGS(FIND, find) UTIL_REQUIRE_PROGS(GUNZIP, gunzip) @@ -67,26 +87,18 @@ AC_DEFUN_ONCE([BASIC_SETUP_FUNDAMENTAL_TOOLS], UTIL_REQUIRE_PROGS(TAR, gtar tar) UTIL_REQUIRE_PROGS(TEE, tee) UTIL_REQUIRE_PROGS(TOUCH, touch) - UTIL_REQUIRE_PROGS(TR, tr) - UTIL_REQUIRE_PROGS(UNAME, uname) - UTIL_REQUIRE_PROGS(WC, wc) UTIL_REQUIRE_PROGS(XARGS, xargs) - # Then required tools that require some special treatment. - UTIL_REQUIRE_SPECIAL(GREP, [AC_PROG_GREP]) - UTIL_REQUIRE_SPECIAL(EGREP, [AC_PROG_EGREP]) + # Required tools with some special treatment UTIL_REQUIRE_SPECIAL(FGREP, [AC_PROG_FGREP]) - UTIL_REQUIRE_SPECIAL(SED, [AC_PROG_SED]) # Optional tools, we can do without them UTIL_LOOKUP_PROGS(DF, df) UTIL_LOOKUP_PROGS(NICE, nice) UTIL_LOOKUP_PROGS(READLINK, greadlink readlink) - # These are only needed on some platforms - UTIL_LOOKUP_PROGS(PATHTOOL, cygpath wslpath) + # Tools only needed on some platforms UTIL_LOOKUP_PROGS(LSB_RELEASE, lsb_release) - UTIL_LOOKUP_PROGS(CMD, cmd.exe, $PATH:/cygdrive/c/windows/system32:/mnt/c/windows/system32:/c/windows/system32) # For compare.sh only UTIL_LOOKUP_PROGS(CMP, cmp) diff --git a/make/autoconf/build-aux/config.sub b/make/autoconf/build-aux/config.sub index 8f5a5cf524492..f347a4623bd22 100644 --- a/make/autoconf/build-aux/config.sub +++ b/make/autoconf/build-aux/config.sub @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (c) 2014, 2020, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2014, 2022, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -46,6 +46,13 @@ if echo $* | grep pc-msys >/dev/null ; then exit fi +# Canonicalize for riscv which autoconf-config.sub doesn't handle +if echo $* | grep '^riscv\(32\|64\)-linux' >/dev/null ; then + result=`echo $@ | sed 's/linux/unknown-linux/'` + echo $result + exit +fi + # Filter out everything that doesn't begin with "aarch64-" if ! echo $* | grep '^aarch64-' >/dev/null ; then . $DIR/autoconf-config.sub "$@" @@ -78,4 +85,3 @@ result=`echo $result | sed "s/^arm-/aarch64-/"` echo $result exit $exitcode - diff --git a/make/autoconf/configure.ac b/make/autoconf/configure.ac index c7046ae41ea7b..ebf43c363eab0 100644 --- a/make/autoconf/configure.ac +++ b/make/autoconf/configure.ac @@ -86,6 +86,7 @@ PLATFORM_SETUP_OPENJDK_BUILD_AND_TARGET # Continue setting up basic stuff. Most remaining code require fundamental tools. BASIC_SETUP_PATHS +BASIC_SETUP_TOOLS # Check if it's a pure open build or if custom sources are to be used. JDKOPT_SETUP_OPEN_OR_CUSTOM diff --git a/make/autoconf/flags-cflags.m4 b/make/autoconf/flags-cflags.m4 index 96d68b587f1c3..4f86ed5a4bc0d 100644 --- a/make/autoconf/flags-cflags.m4 +++ b/make/autoconf/flags-cflags.m4 @@ -1,5 +1,5 @@ # -# Copyright (c) 2011, 2021, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -188,6 +188,10 @@ AC_DEFUN([FLAGS_SETUP_WARNINGS], WARNINGS_ENABLE_ALL_CXXFLAGS="$WARNINGS_ENABLE_ALL_CFLAGS $WARNINGS_ENABLE_ADDITIONAL_CXX" DISABLED_WARNINGS="unused-parameter unused" + # gcc10/11 on ppc generate lots of abi warnings about layout of aggregates containing vectors + if test "x$OPENJDK_TARGET_CPU_ARCH" = "xppc"; then + DISABLED_WARNINGS="$DISABLED_WARNINGS psabi" + fi ;; clang) diff --git a/make/autoconf/jvm-features.m4 b/make/autoconf/jvm-features.m4 index 906a285787721..aa99b037b2b2f 100644 --- a/make/autoconf/jvm-features.m4 +++ b/make/autoconf/jvm-features.m4 @@ -1,5 +1,5 @@ # -# Copyright (c) 2011, 2021, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -248,8 +248,11 @@ AC_DEFUN_ONCE([JVM_FEATURES_CHECK_CDS], AC_DEFUN_ONCE([JVM_FEATURES_CHECK_DTRACE], [ JVM_FEATURES_CHECK_AVAILABILITY(dtrace, [ - AC_MSG_CHECKING([for dtrace tool]) - if test "x$DTRACE" != "x" && test -x "$DTRACE"; then + AC_MSG_CHECKING([for dtrace tool and platform support]) + if test "x$OPENJDK_TARGET_CPU_ARCH" = "xppc"; then + AC_MSG_RESULT([no, $OPENJDK_TARGET_CPU_ARCH]) + AVAILABLE=false + elif test "x$DTRACE" != "x" && test -x "$DTRACE"; then AC_MSG_RESULT([$DTRACE]) else AC_MSG_RESULT([no]) @@ -308,7 +311,8 @@ AC_DEFUN_ONCE([JVM_FEATURES_CHECK_SHENANDOAHGC], AC_MSG_CHECKING([if platform is supported by Shenandoah]) if test "x$OPENJDK_TARGET_CPU_ARCH" = "xx86" || \ test "x$OPENJDK_TARGET_CPU" = "xaarch64" || \ - test "x$OPENJDK_TARGET_CPU" = "xppc64le"; then + test "x$OPENJDK_TARGET_CPU" = "xppc64le" || \ + test "x$OPENJDK_TARGET_CPU" = "xriscv64"; then AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no, $OPENJDK_TARGET_CPU]) @@ -358,7 +362,8 @@ AC_DEFUN_ONCE([JVM_FEATURES_CHECK_ZGC], AC_MSG_RESULT([no, $OPENJDK_TARGET_OS-$OPENJDK_TARGET_CPU]) AVAILABLE=false fi - elif test "x$OPENJDK_TARGET_CPU" = "xppc64le"; then + elif test "x$OPENJDK_TARGET_CPU" = "xppc64le" || \ + test "x$OPENJDK_TARGET_CPU" = "xriscv64"; then if test "x$OPENJDK_TARGET_OS" = "xlinux"; then AC_MSG_RESULT([yes]) else diff --git a/make/autoconf/lib-bundled.m4 b/make/autoconf/lib-bundled.m4 index f2b103e09df01..d30ad70b7be93 100644 --- a/make/autoconf/lib-bundled.m4 +++ b/make/autoconf/lib-bundled.m4 @@ -166,7 +166,9 @@ AC_DEFUN_ONCE([LIB_SETUP_ZLIB], DEFAULT_ZLIB=system if test "x$OPENJDK_TARGET_OS" = xwindows -o "x$OPENJDK_TARGET_OS" = xaix; then - # On windows and aix default is bundled, on others default is system + # On windows and aix default is bundled + DEFAULT_ZLIB=bundled + elif test "x$OPENJDK_TARGET_OS" = xmacosx -a "x$OPENJDK_TARGET_CPU" = xaarch64; then DEFAULT_ZLIB=bundled fi diff --git a/make/autoconf/libraries.m4 b/make/autoconf/libraries.m4 index a8f054c139743..1f8d782f419be 100644 --- a/make/autoconf/libraries.m4 +++ b/make/autoconf/libraries.m4 @@ -1,5 +1,5 @@ # -# Copyright (c) 2011, 2021, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -148,6 +148,12 @@ AC_DEFUN_ONCE([LIB_SETUP_LIBRARIES], fi fi + # Because RISC-V only has word-sized atomics, it requries libatomic where + # other common architectures do not. So link libatomic by default. + if test "x$OPENJDK_TARGET_OS" = xlinux && test "x$OPENJDK_TARGET_CPU" = xriscv64; then + BASIC_JVM_LIBS="$BASIC_JVM_LIBS -latomic" + fi + # perfstat lib if test "x$OPENJDK_TARGET_OS" = xaix; then BASIC_JVM_LIBS="$BASIC_JVM_LIBS -lperfstat" @@ -155,7 +161,7 @@ AC_DEFUN_ONCE([LIB_SETUP_LIBRARIES], if test "x$OPENJDK_TARGET_OS" = xwindows; then BASIC_JVM_LIBS="$BASIC_JVM_LIBS kernel32.lib user32.lib gdi32.lib winspool.lib \ - comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib \ + comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib powrprof.lib uuid.lib \ wsock32.lib winmm.lib version.lib psapi.lib" fi diff --git a/make/autoconf/platform.m4 b/make/autoconf/platform.m4 index 9e9e9454f0e09..5752d3bd1a6ad 100644 --- a/make/autoconf/platform.m4 +++ b/make/autoconf/platform.m4 @@ -1,5 +1,5 @@ # -# Copyright (c) 2011, 2021, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -561,6 +561,8 @@ AC_DEFUN([PLATFORM_SETUP_LEGACY_VARS_HELPER], HOTSPOT_$1_CPU_DEFINE=PPC64 elif test "x$OPENJDK_$1_CPU" = xppc64le; then HOTSPOT_$1_CPU_DEFINE=PPC64 + elif test "x$OPENJDK_$1_CPU" = xriscv64; then + HOTSPOT_$1_CPU_DEFINE=RISCV64 # The cpu defines below are for zero, we don't support them directly. elif test "x$OPENJDK_$1_CPU" = xsparc; then @@ -571,8 +573,6 @@ AC_DEFUN([PLATFORM_SETUP_LEGACY_VARS_HELPER], HOTSPOT_$1_CPU_DEFINE=S390 elif test "x$OPENJDK_$1_CPU" = xs390x; then HOTSPOT_$1_CPU_DEFINE=S390 - elif test "x$OPENJDK_$1_CPU" = xriscv64; then - HOTSPOT_$1_CPU_DEFINE=RISCV elif test "x$OPENJDK_$1_CPU" != x; then HOTSPOT_$1_CPU_DEFINE=$(echo $OPENJDK_$1_CPU | tr a-z A-Z) fi @@ -632,6 +632,7 @@ AC_DEFUN([PLATFORM_SET_MODULE_TARGET_OS_VALUES], ]) #%%% Build and target systems %%% +# Make sure to only use tools set up in BASIC_SETUP_FUNDAMENTAL_TOOLS. AC_DEFUN_ONCE([PLATFORM_SETUP_OPENJDK_BUILD_AND_TARGET], [ # Figure out the build and target systems. # Note that in autoconf terminology, "build" is obvious, but "target" @@ -718,7 +719,7 @@ AC_DEFUN_ONCE([PLATFORM_SETUP_OPENJDK_TARGET_ENDIANNESS], [ ############################################################################### # - # Is the target little of big endian? + # Is the target little or big endian? # AC_C_BIGENDIAN([ENDIAN="big"],[ENDIAN="little"],[ENDIAN="unknown"],[ENDIAN="universal_endianness"]) diff --git a/make/autoconf/toolchain.m4 b/make/autoconf/toolchain.m4 index 99c780532ee87..72710c7a7150f 100644 --- a/make/autoconf/toolchain.m4 +++ b/make/autoconf/toolchain.m4 @@ -58,6 +58,9 @@ TOOLCHAIN_MINIMUM_VERSION_xlc="" # Minimum supported linker versions, empty means unspecified TOOLCHAIN_MINIMUM_LD_VERSION_gcc="2.18" +# Minimum supported version +JTREG_MINIMUM_VERSION=6.1 + # Prepare the system so that TOOLCHAIN_CHECK_COMPILER_VERSION can be called. # Must have CC_VERSION_NUMBER and CXX_VERSION_NUMBER. # $1 - optional variable prefix for compiler and version variables (BUILD_) @@ -1082,6 +1085,23 @@ AC_DEFUN_ONCE([TOOLCHAIN_SETUP_JTREG], UTIL_FIXUP_PATH(JT_HOME) AC_SUBST(JT_HOME) + + # Verify jtreg version + if test "x$JT_HOME" != x; then + AC_MSG_CHECKING([jtreg version number]) + # jtreg -version looks like this: "jtreg 6.1+1-19" + # Extract actual version part ("6.1" in this case) + jtreg_version_full=`$JAVA -jar $JT_HOME/lib/jtreg.jar -version | $HEAD -n 1 | $CUT -d ' ' -f 2` + jtreg_version=${jtreg_version_full/%+*} + AC_MSG_RESULT([$jtreg_version]) + + # This is a simplified version of TOOLCHAIN_CHECK_COMPILER_VERSION + comparable_actual_version=`$AWK -F. '{ printf("%05d%05d%05d%05d\n", [$]1, [$]2, [$]3, [$]4) }' <<< "$jtreg_version"` + comparable_minimum_version=`$AWK -F. '{ printf("%05d%05d%05d%05d\n", [$]1, [$]2, [$]3, [$]4) }' <<< "$JTREG_MINIMUM_VERSION"` + if test $comparable_actual_version -lt $comparable_minimum_version ; then + AC_MSG_ERROR([jtreg version is too old, at least version $JTREG_MINIMUM_VERSION is required]) + fi + fi ]) # Setup the JIB dependency resolver diff --git a/make/common/NativeCompilation.gmk b/make/common/NativeCompilation.gmk index 0f315019d0e6d..215d90d17e96f 100644 --- a/make/common/NativeCompilation.gmk +++ b/make/common/NativeCompilation.gmk @@ -343,10 +343,15 @@ define SetupCompileNativeFileBody endif endif + ifneq ($(DISABLE_WARNING_PREFIX), ) + $1_WARNINGS_FLAGS := $$(addprefix $(DISABLE_WARNING_PREFIX), \ + $$($$($1_BASE)_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)_$$($1_FILENAME))) + endif + $1_BASE_CFLAGS := $$($$($1_BASE)_CFLAGS) $$($$($1_BASE)_EXTRA_CFLAGS) \ - $$($$($1_BASE)_SYSROOT_CFLAGS) + $$($$($1_BASE)_SYSROOT_CFLAGS) $$($1_WARNINGS_FLAGS) $1_BASE_CXXFLAGS := $$($$($1_BASE)_CXXFLAGS) $$($$($1_BASE)_EXTRA_CXXFLAGS) \ - $$($$($1_BASE)_SYSROOT_CFLAGS) $$($1_EXTRA_CXXFLAGS) + $$($$($1_BASE)_SYSROOT_CFLAGS) $$($1_EXTRA_CXXFLAGS) $$($1_WARNINGS_FLAGS) $1_BASE_ASFLAGS := $$($$($1_BASE)_ASFLAGS) $$($$($1_BASE)_EXTRA_ASFLAGS) ifneq ($$(filter %.c, $$($1_FILENAME)), ) diff --git a/make/conf/jib-profiles.js b/make/conf/jib-profiles.js index 72373a0837d0e..ef3aca5bbe38c 100644 --- a/make/conf/jib-profiles.js +++ b/make/conf/jib-profiles.js @@ -457,7 +457,7 @@ var getJibProfilesProfiles = function (input, common, data) { target_os: "macosx", target_cpu: "aarch64", dependencies: ["devkit", "gtest"], - configure_args: concat(common.configure_args_64bit, "--with-zlib=system", + configure_args: concat(common.configure_args_64bit, "--with-macosx-version-max=11.00.00"), }, @@ -1048,10 +1048,10 @@ var getJibProfilesProfiles = function (input, common, data) { var getJibProfilesDependencies = function (input, common) { var devkit_platform_revisions = { - linux_x64: "gcc10.3.0-OL6.4+1.0", + linux_x64: "gcc11.2.0-OL6.4+1.0", macosx: "Xcode12.4+1.0", windows_x64: "VS2022-17.1.0+1.0", - linux_aarch64: "gcc10.3.0-OL7.6+1.0", + linux_aarch64: "gcc11.2.0-OL7.6+1.0", linux_arm: "gcc8.2.0-Fedora27+1.0", linux_ppc64le: "gcc8.2.0-Fedora27+1.0", linux_s390x: "gcc8.2.0-Fedora27+1.0" @@ -1477,7 +1477,7 @@ var getVersionNumbers = function () { var isWsl = function (input) { return ( input.build_osenv == "wsl" || (input.build_os == "linux" - && java.lang.System.getProperty("os.version").contains("Microsoft"))); + && java.lang.System.getProperty("os.version").toLowerCase().contains("microsoft"))); } var error = function (s) { diff --git a/make/conf/version-numbers.conf b/make/conf/version-numbers.conf index 2b871e10237fe..c188a694618ec 100644 --- a/make/conf/version-numbers.conf +++ b/make/conf/version-numbers.conf @@ -1,5 +1,5 @@ # -# Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -28,12 +28,12 @@ DEFAULT_VERSION_FEATURE=17 DEFAULT_VERSION_INTERIM=0 -DEFAULT_VERSION_UPDATE=7 +DEFAULT_VERSION_UPDATE=9 DEFAULT_VERSION_PATCH=0 DEFAULT_VERSION_EXTRA1=0 DEFAULT_VERSION_EXTRA2=0 DEFAULT_VERSION_EXTRA3=0 -DEFAULT_VERSION_DATE=2023-04-18 +DEFAULT_VERSION_DATE=2023-10-17 DEFAULT_VERSION_CLASSFILE_MAJOR=61 # "`$EXPR $DEFAULT_VERSION_FEATURE + 44`" DEFAULT_VERSION_CLASSFILE_MINOR=0 DEFAULT_VERSION_DOCS_API_SINCE=11 diff --git a/make/data/cacerts/gtsrootcar1 b/make/data/cacerts/gtsrootcar1 new file mode 100644 index 0000000000000..ac879b232b26e --- /dev/null +++ b/make/data/cacerts/gtsrootcar1 @@ -0,0 +1,38 @@ +Owner: CN=GTS Root R1, O=Google Trust Services LLC, C=US +Issuer: CN=GTS Root R1, O=Google Trust Services LLC, C=US +Serial number: 203e5936f31b01349886ba217 +Valid from: Wed Jun 22 00:00:00 GMT 2016 until: Sun Jun 22 00:00:00 GMT 2036 +Signature algorithm name: SHA384withRSA +Subject Public Key Algorithm: 4096-bit RSA key +Version: 3 +-----BEGIN CERTIFICATE----- +MIIFVzCCAz+gAwIBAgINAgPlk28xsBNJiGuiFzANBgkqhkiG9w0BAQwFADBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjEwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwggIiMA0GCSqGSIb3DQEBAQUA +A4ICDwAwggIKAoICAQC2EQKLHuOhd5s73L+UPreVp0A8of2C+X0yBoJx9vaMf/vo +27xqLpeXo4xL+Sv2sfnOhB2x+cWX3u+58qPpvBKJXqeqUqv4IyfLpLGcY9vXmX7w +Cl7raKb0xlpHDU0QM+NOsROjyBhsS+z8CZDfnWQpJSMHobTSPS5g4M/SCYe7zUjw +TcLCeoiKu7rPWRnWr4+wB7CeMfGCwcDfLqZtbBkOtdh+JhpFAz2weaSUKK0Pfybl +qAj+lug8aJRT7oM6iCsVlgmy4HqMLnXWnOunVmSPlk9orj2XwoSPwLxAwAtcvfaH +szVsrBhQf4TgTM2S0yDpM7xSma8ytSmzJSq0SPly4cpk9+aCEI3oncKKiPo4Zor8 +Y/kB+Xj9e1x3+naH+uzfsQ55lVe0vSbv1gHR6xYKu44LtcXFilWr06zqkUspzBmk +MiVOKvFlRNACzqrOSbTqn3yDsEB750Orp2yjj32JgfpMpf/VjsPOS+C12LOORc92 +wO1AK/1TD7Cn1TsNsYqiA94xrcx36m97PtbfkSIS5r762DL8EGMUUXLeXdYWk70p +aDPvOmbsB4om3xPXV2V4J95eSRQAogB/mqghtqmxlbCluQ0WEdrHbEg8QOB+DVrN +VjzRlwW5y0vtOUucxD/SVRNuJLDWcfr0wbrM7Rv1/oFB2ACYPTrIrnqYNxgFlQID +AQABo0IwQDAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQU5K8rJnEaK0gnhS9SZizv8IkTcT4wDQYJKoZIhvcNAQEMBQADggIBAJ+qQibb +C5u+/x6Wki4+omVKapi6Ist9wTrYggoGxval3sBOh2Z5ofmmWJyq+bXmYOfg6LEe +QkEzCzc9zolwFcq1JKjPa7XSQCGYzyI0zzvFIoTgxQ6KfF2I5DUkzps+GlQebtuy +h6f88/qBVRRiClmpIgUxPoLW7ttXNLwzldMXG+gnoot7TiYaelpkttGsN/H9oPM4 +7HLwEXWdyzRSjeZ2axfG34arJ45JK3VmgRAhpuo+9K4l/3wV3s6MJT/KYnAK9y8J +ZgfIPxz88NtFMN9iiMG1D53Dn0reWVlHxYciNuaCp+0KueIHoI17eko8cdLiA6Ef +MgfdG+RCzgwARWGAtQsgWSl4vflVy2PFPEz0tv/bal8xa5meLMFrUKTX5hgUvYU/ +Z6tGn6D/Qqc6f1zLXbBwHSs09dR2CQzreExZBfMzQsNhFRAbd03OIozUhfJFfbdT +6u9AWpQKXCBfTkBdYiJ23//OYb2MI3jSNwLgjt7RETeJ9r/tSQdirpLsQBqvFAnZ +0E6yove+7u7Y/9waLd64NnHi/Hm3lCXRSHNboTXns5lndcEZOitHTtNCjv0xyBZm +2tIMPNuzjsmhDYAPexZ3FL//2wmUspO8IFgV6dtxQ/PeEMMA3KgqlbbC1j+Qa3bb +bP6MvPJwNQzcmRk13NfIRmPVNnGuV/u3gm3c +-----END CERTIFICATE----- \ No newline at end of file diff --git a/make/data/cacerts/gtsrootcar2 b/make/data/cacerts/gtsrootcar2 new file mode 100644 index 0000000000000..a71e7c5e9513a --- /dev/null +++ b/make/data/cacerts/gtsrootcar2 @@ -0,0 +1,38 @@ +Owner: CN=GTS Root R2, O=Google Trust Services LLC, C=US +Issuer: CN=GTS Root R2, O=Google Trust Services LLC, C=US +Serial number: 203e5aec58d04251aab1125aa +Valid from: Wed Jun 22 00:00:00 GMT 2016 until: Sun Jun 22 00:00:00 GMT 2036 +Signature algorithm name: SHA384withRSA +Subject Public Key Algorithm: 4096-bit RSA key +Version: 3 +-----BEGIN CERTIFICATE----- +MIIFVzCCAz+gAwIBAgINAgPlrsWNBCUaqxElqjANBgkqhkiG9w0BAQwFADBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjIwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwggIiMA0GCSqGSIb3DQEBAQUA +A4ICDwAwggIKAoICAQDO3v2m++zsFDQ8BwZabFn3GTXd98GdVarTzTukk3LvCvpt +nfbwhYBboUhSnznFt+4orO/LdmgUud+tAWyZH8QiHZ/+cnfgLFuv5AS/T3KgGjSY +6Dlo7JUle3ah5mm5hRm9iYz+re026nO8/4Piy33B0s5Ks40FnotJk9/BW9BuXvAu +MC6C/Pq8tBcKSOWIm8Wba96wyrQD8Nr0kLhlZPdcTK3ofmZemde4wj7I0BOdre7k +RXuJVfeKH2JShBKzwkCX44ofR5GmdFrS+LFjKBC4swm4VndAoiaYecb+3yXuPuWg +f9RhD1FLPD+M2uFwdNjCaKH5wQzpoeJ/u1U8dgbuak7MkogwTZq9TwtImoS1mKPV ++3PBV2HdKFZ1E66HjucMUQkQdYhMvI35ezzUIkgfKtzra7tEscszcTJGr61K8Yzo +dDqs5xoic4DSMPclQsciOzsSrZYuxsN2B6ogtzVJV+mSSeh2FnIxZyuWfoqjx5RW +Ir9qS34BIbIjMt/kmkRtWVtd9QCgHJvGeJeNkP+byKq0rxFROV7Z+2et1VsRnTKa +G73VululycslaVNVJ1zgyjbLiGH7HrfQy+4W+9OmTN6SpdTi3/UGVN4unUu0kzCq +gc7dGtxRcw1PcOnlthYhGXmy5okLdWTK1au8CcEYof/UVKGFPP0UJAOyh9OktwID +AQABo0IwQDAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQUu//KjiOfT5nK2+JopqUVJxce2Q4wDQYJKoZIhvcNAQEMBQADggIBAB/Kzt3H +vqGf2SdMC9wXmBFqiN495nFWcrKeGk6c1SuYJF2ba3uwM4IJvd8lRuqYnrYb/oM8 +0mJhwQTtzuDFycgTE1XnqGOtjHsB/ncw4c5omwX4Eu55MaBBRTUoCnGkJE+M3DyC +B19m3H0Q/gxhswWV7uGugQ+o+MePTagjAiZrHYNSVc61LwDKgEDg4XSsYPWHgJ2u +NmSRXbBoGOqKYcl3qJfEycel/FVL8/B/uWU9J2jQzGv6U53hkRrJXRqWbTKH7QMg +yALOWr7Z6v2yTcQvG99fevX4i8buMTolUVVnjWQye+mew4K6Ki3pHrTgSAai/Gev +HyICc/sgCq+dVEuhzf9gR7A/Xe8bVr2XIZYtCtFenTgCR2y59PYjJbigapordwj6 +xLEokCZYCDzifqrXPW+6MYgKBesntaFJ7qBFVHvmJ2WZICGoo7z7GJa7Um8M7YNR +TOlZ4iBgxcJlkoKM8xAfDoqXvneCbT+PHV28SSe9zE8P4c52hgQjxcCMElv924Sg +JPFI/2R80L5cFtHvma3AH/vLrrw4IgYmZNralw4/KBVEqE8AyvCazM90arQ+POuV +7LXTWtiBmelDGDfrs7vRWGJB82bSj6p4lVQgw1oudCvV0b4YacCs1aTPObpRhANl +6WLAYv7YTVWW4tAR+kg0Eeye7QUd5MjWHYbL +-----END CERTIFICATE----- \ No newline at end of file diff --git a/make/data/cacerts/gtsrootecccar3 b/make/data/cacerts/gtsrootecccar3 new file mode 100644 index 0000000000000..38f7533ca58ef --- /dev/null +++ b/make/data/cacerts/gtsrootecccar3 @@ -0,0 +1,20 @@ +Owner: CN=GTS Root R3, O=Google Trust Services LLC, C=US +Issuer: CN=GTS Root R3, O=Google Trust Services LLC, C=US +Serial number: 203e5b882eb20f825276d3d66 +Valid from: Wed Jun 22 00:00:00 GMT 2016 until: Sun Jun 22 00:00:00 GMT 2036 +Signature algorithm name: SHA384withECDSA +Subject Public Key Algorithm: 384-bit EC (secp384r1) key +Version: 3 +-----BEGIN CERTIFICATE----- +MIICCTCCAY6gAwIBAgINAgPluILrIPglJ209ZjAKBggqhkjOPQQDAzBHMQswCQYD +VQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEUMBIG +A1UEAxMLR1RTIFJvb3QgUjMwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAwMDAw +WjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2Vz +IExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjMwdjAQBgcqhkjOPQIBBgUrgQQAIgNi +AAQfTzOHMymKoYTey8chWEGJ6ladK0uFxh1MJ7x/JlFyb+Kf1qPKzEUURout736G +jOyxfi//qXGdGIRFBEFVbivqJn+7kAHjSxm65FSWRQmx1WyRRK2EE46ajA2ADDL2 +4CejQjBAMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW +BBTB8Sa6oC2uhYHP0/EqEr24Cmf9vDAKBggqhkjOPQQDAwNpADBmAjEA9uEglRR7 +VKOQFhG/hMjqb2sXnh5GmCCbn9MN2azTL818+FsuVbu/3ZL3pAzcMeGiAjEA/Jdm +ZuVDFhOD3cffL74UOO0BzrEXGhF16b0DjyZ+hOXJYKaV11RZt+cRLInUue4X +-----END CERTIFICATE----- \ No newline at end of file diff --git a/make/data/cacerts/gtsrootecccar4 b/make/data/cacerts/gtsrootecccar4 new file mode 100644 index 0000000000000..e812a16c24bd1 --- /dev/null +++ b/make/data/cacerts/gtsrootecccar4 @@ -0,0 +1,20 @@ +Owner: CN=GTS Root R4, O=Google Trust Services LLC, C=US +Issuer: CN=GTS Root R4, O=Google Trust Services LLC, C=US +Serial number: 203e5c068ef631a9c72905052 +Valid from: Wed Jun 22 00:00:00 GMT 2016 until: Sun Jun 22 00:00:00 GMT 2036 +Signature algorithm name: SHA384withECDSA +Subject Public Key Algorithm: 384-bit EC (secp384r1) key +Version: 3 +-----BEGIN CERTIFICATE----- +MIICCTCCAY6gAwIBAgINAgPlwGjvYxqccpBQUjAKBggqhkjOPQQDAzBHMQswCQYD +VQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEUMBIG +A1UEAxMLR1RTIFJvb3QgUjQwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAwMDAw +WjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2Vz +IExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjQwdjAQBgcqhkjOPQIBBgUrgQQAIgNi +AATzdHOnaItgrkO4NcWBMHtLSZ37wWHO5t5GvWvVYRg1rkDdc/eJkTBa6zzuhXyi +QHY7qca4R9gq55KRanPpsXI5nymfopjTX15YhmUPoYRlBtHci8nHc8iMai/lxKvR +HYqjQjBAMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW +BBSATNbrdP9JNqPV2Py1PsVq8JQdjDAKBggqhkjOPQQDAwNpADBmAjEA6ED/g94D +9J+uHXqnLrmvT/aDHQ4thQEd0dlq7A/Cr8deVl5c1RxYIigL9zC2L7F8AjEA8GE8 +p/SgguMh1YQdc4acLa/KNJvxn7kjNuK8YAOdgLOaVsjh4rsUecrNIdSUtUlD +-----END CERTIFICATE----- \ No newline at end of file diff --git a/make/data/cacerts/microsoftecc2017 b/make/data/cacerts/microsoftecc2017 new file mode 100644 index 0000000000000..8fc6733204553 --- /dev/null +++ b/make/data/cacerts/microsoftecc2017 @@ -0,0 +1,22 @@ +Owner: CN=Microsoft ECC Root Certificate Authority 2017, O=Microsoft Corporation, C=US +Issuer: CN=Microsoft ECC Root Certificate Authority 2017, O=Microsoft Corporation, C=US +Serial number: 66f23daf87de8bb14aea0c573101c2ec +Valid from: Wed Dec 18 23:06:45 GMT 2019 until: Fri Jul 18 23:16:04 GMT 2042 +Signature algorithm name: SHA384withECDSA +Subject Public Key Algorithm: 384-bit EC (secp384r1) key +Version: 3 +-----BEGIN CERTIFICATE----- +MIICWTCCAd+gAwIBAgIQZvI9r4fei7FK6gxXMQHC7DAKBggqhkjOPQQDAzBlMQsw +CQYDVQQGEwJVUzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYwNAYD +VQQDEy1NaWNyb3NvZnQgRUNDIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIw +MTcwHhcNMTkxMjE4MjMwNjQ1WhcNNDIwNzE4MjMxNjA0WjBlMQswCQYDVQQGEwJV +UzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYwNAYDVQQDEy1NaWNy +b3NvZnQgRUNDIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMTcwdjAQBgcq +hkjOPQIBBgUrgQQAIgNiAATUvD0CQnVBEyPNgASGAlEvaqiBYgtlzPbKnR5vSmZR +ogPZnZH6thaxjG7efM3beaYvzrvOcS/lpaso7GMEZpn4+vKTEAXhgShC48Zo9OYb +hGBKia/teQ87zvH2RPUBeMCjVDBSMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8E +BTADAQH/MB0GA1UdDgQWBBTIy5lycFIM+Oa+sgRXKSrPQhDtNTAQBgkrBgEEAYI3 +FQEEAwIBADAKBggqhkjOPQQDAwNoADBlAjBY8k3qDPlfXu5gKcs68tvWMoQZP3zV +L8KxzJOuULsJMsbG7X7JNpQS5GiFBqIb0C8CMQCZ6Ra0DvpWSNSkMBaReNtUjGUB +iudQZsIxtzm6uBoiB078a1QWIP8rtedMDE2mT3M= +-----END CERTIFICATE----- diff --git a/make/data/cacerts/microsoftrsa2017 b/make/data/cacerts/microsoftrsa2017 new file mode 100644 index 0000000000000..2d30c9688b7ab --- /dev/null +++ b/make/data/cacerts/microsoftrsa2017 @@ -0,0 +1,40 @@ +Owner: CN=Microsoft RSA Root Certificate Authority 2017, O=Microsoft Corporation, C=US +Issuer: CN=Microsoft RSA Root Certificate Authority 2017, O=Microsoft Corporation, C=US +Serial number: 1ed397095fd8b4b347701eaabe7f45b3 +Valid from: Wed Dec 18 22:51:22 GMT 2019 until: Fri Jul 18 23:00:23 GMT 2042 +Signature algorithm name: SHA384withRSA +Subject Public Key Algorithm: 4096-bit RSA key +Version: 3 +-----BEGIN CERTIFICATE----- +MIIFqDCCA5CgAwIBAgIQHtOXCV/YtLNHcB6qvn9FszANBgkqhkiG9w0BAQwFADBl +MQswCQYDVQQGEwJVUzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYw +NAYDVQQDEy1NaWNyb3NvZnQgUlNBIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5 +IDIwMTcwHhcNMTkxMjE4MjI1MTIyWhcNNDIwNzE4MjMwMDIzWjBlMQswCQYDVQQG +EwJVUzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYwNAYDVQQDEy1N +aWNyb3NvZnQgUlNBIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMTcwggIi +MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDKW76UM4wplZEWCpW9R2LBifOZ +Nt9GkMml7Xhqb0eRaPgnZ1AzHaGm++DlQ6OEAlcBXZxIQIJTELy/xztokLaCLeX0 +ZdDMbRnMlfl7rEqUrQ7eS0MdhweSE5CAg2Q1OQT85elss7YfUJQ4ZVBcF0a5toW1 +HLUX6NZFndiyJrDKxHBKrmCk3bPZ7Pw71VdyvD/IybLeS2v4I2wDwAW9lcfNcztm +gGTjGqwu+UcF8ga2m3P1eDNbx6H7JyqhtJqRjJHTOoI+dkC0zVJhUXAoP8XFWvLJ +jEm7FFtNyP9nTUwSlq31/niol4fX/V4ggNyhSyL71Imtus5Hl0dVe49FyGcohJUc +aDDv70ngNXtk55iwlNpNhTs+VcQor1fznhPbRiefHqJeRIOkpcrVE7NLP8TjwuaG +YaRSMLl6IE9vDzhTyzMMEyuP1pq9KsgtsRx9S1HKR9FIJ3Jdh+vVReZIZZ2vUpC6 +W6IYZVcSn2i51BVrlMRpIpj0M+Dt+VGOQVDJNE92kKz8OMHY4Xu54+OU4UZpyw4K +UGsTuqwPN1q3ErWQgR5WrlcihtnJ0tHXUeOrO8ZV/R4O03QK0dqq6mm4lyiPSMQH ++FJDOvTKVTUssKZqwJz58oHhEmrARdlns87/I6KJClTUFLkqqNfs+avNJVgyeY+Q +W5g5xAgGwax/Dj0ApQIDAQABo1QwUjAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/ +BAUwAwEB/zAdBgNVHQ4EFgQUCctZf4aycI8awznjwNnpv7tNsiMwEAYJKwYBBAGC +NxUBBAMCAQAwDQYJKoZIhvcNAQEMBQADggIBAKyvPl3CEZaJjqPnktaXFbgToqZC +LgLNFgVZJ8og6Lq46BrsTaiXVq5lQ7GPAJtSzVXNUzltYkyLDVt8LkS/gxCP81OC +gMNPOsduET/m4xaRhPtthH80dK2Jp86519efhGSSvpWhrQlTM93uCupKUY5vVau6 +tZRGrox/2KJQJWVggEbbMwSubLWYdFQl3JPk+ONVFT24bcMKpBLBaYVu32TxU5nh +SnUgnZUP5NbcA/FZGOhHibJXWpS2qdgXKxdJ5XbLwVaZOjex/2kskZGT4d9Mozd2 +TaGf+G0eHdP67Pv0RR0Tbc/3WeUiJ3IrhvNXuzDtJE3cfVa7o7P4NHmJweDyAmH3 +pvwPuxwXC65B2Xy9J6P9LjrRk5Sxcx0ki69bIImtt2dmefU6xqaWM/5TkshGsRGR +xpl/j8nWZjEgQRCHLQzWwa80mMpkg/sTV9HB8Dx6jKXB/ZUhoHHBk2dxEuqPiApp +GWSZI1b7rCoucL5mxAyE7+WL85MB+GqQk2dLsmijtWKP6T+MejteD+eMuMZ87zf9 +dOLITzNy4ZQ5bb0Sr74MTnB8G2+NszKTc0QWbej09+CVgI+WXTik9KveCjCHk9hN +AHFiRSdLOkKEW39lt2c0Ui2cFmuqqNh7o0JMcccMyj6D5KbvtwEwXlGjefVwaaZB +RA+GsCyRxj3qrg+E +-----END CERTIFICATE----- diff --git a/make/data/cacerts/secomscrootca1 b/make/data/cacerts/secomscrootca1 deleted file mode 100644 index d300e31195d34..0000000000000 --- a/make/data/cacerts/secomscrootca1 +++ /dev/null @@ -1,27 +0,0 @@ -Owner: OU=Security Communication RootCA1, O=SECOM Trust.net, C=JP -Issuer: OU=Security Communication RootCA1, O=SECOM Trust.net, C=JP -Serial number: 0 -Valid from: Tue Sep 30 04:20:49 GMT 2003 until: Sat Sep 30 04:20:49 GMT 2023 -Signature algorithm name: SHA1withRSA -Subject Public Key Algorithm: 2048-bit RSA key -Version: 3 ------BEGIN CERTIFICATE----- -MIIDWjCCAkKgAwIBAgIBADANBgkqhkiG9w0BAQUFADBQMQswCQYDVQQGEwJKUDEY -MBYGA1UEChMPU0VDT00gVHJ1c3QubmV0MScwJQYDVQQLEx5TZWN1cml0eSBDb21t -dW5pY2F0aW9uIFJvb3RDQTEwHhcNMDMwOTMwMDQyMDQ5WhcNMjMwOTMwMDQyMDQ5 -WjBQMQswCQYDVQQGEwJKUDEYMBYGA1UEChMPU0VDT00gVHJ1c3QubmV0MScwJQYD -VQQLEx5TZWN1cml0eSBDb21tdW5pY2F0aW9uIFJvb3RDQTEwggEiMA0GCSqGSIb3 -DQEBAQUAA4IBDwAwggEKAoIBAQCzs/5/022x7xZ8V6UMbXaKL0u/ZPtM7orw8yl8 -9f/uKuDp6bpbZCKamm8sOiZpUQWZJtzVHGpxxpp9Hp3dfGzGjGdnSj74cbAZJ6kJ -DKaVv0uMDPpVmDvY6CKhS3E4eayXkmmziX7qIWgGmBSWh9JhNrxtJ1aeV+7AwFb9 -Ms+k2Y7CI9eNqPPYJayX5HA49LY6tJ07lyZDo6G8SVlyTCMwhwFY9k6+HGhWZq/N -QV3Is00qVUarH9oe4kA92819uZKAnDfdDJZkndwi92SL32HeFZRSFaB9UslLqCHJ -xrHty8OVYNEP8Ktw+N/LTX7s1vqr2b1/VPKl6Xn62dZ2JChzAgMBAAGjPzA9MB0G -A1UdDgQWBBSgc0mZaNyFW2XjmygvV5+9M7wHSDALBgNVHQ8EBAMCAQYwDwYDVR0T -AQH/BAUwAwEB/zANBgkqhkiG9w0BAQUFAAOCAQEAaECpqLvkT115swW1F7NgE+vG -kl3g0dNq/vu+m22/xwVtWSDEHPC32oRYAmP6SBbvT6UL90qY8j+eG61Ha2POCEfr -Uj94nK9NrvjVT8+amCoQQTlSxN3Zmw7vkwGusi7KaEIkQmywszo+zenaSMQVy+n5 -Bw+SUEmK3TGXX8npN6o7WWWXlDLJs58+OmJYxUmtYg5xpTKqL8aJdkNAExNnPaJU -JRDL8Try2frbSVa7pv6nQTXD4IhhyYjH3zYQIphZ6rBK+1YWc26sTfcioU+tHXot -RSflMMFe8toTyyVCUZVHA4xsIcx0Qu1T/zOLjw9XARYvz6buyXAiFL39vmwLAw== ------END CERTIFICATE----- diff --git a/make/data/cacerts/twcaglobalrootca b/make/data/cacerts/twcaglobalrootca new file mode 100644 index 0000000000000..f17112ad9a546 --- /dev/null +++ b/make/data/cacerts/twcaglobalrootca @@ -0,0 +1,38 @@ +Owner: CN=TWCA Global Root CA, OU=Root CA, O=TAIWAN-CA, C=TW +Issuer: CN=TWCA Global Root CA, OU=Root CA, O=TAIWAN-CA, C=TW +Serial number: cbe +Valid from: Wed Jun 27 06:28:33 GMT 2012 until: Tue Dec 31 15:59:59 GMT 2030 +Signature algorithm name: SHA256withRSA +Subject Public Key Algorithm: 4096-bit RSA key +Version: 3 +-----BEGIN CERTIFICATE----- +MIIFQTCCAymgAwIBAgICDL4wDQYJKoZIhvcNAQELBQAwUTELMAkGA1UEBhMCVFcx +EjAQBgNVBAoTCVRBSVdBTi1DQTEQMA4GA1UECxMHUm9vdCBDQTEcMBoGA1UEAxMT +VFdDQSBHbG9iYWwgUm9vdCBDQTAeFw0xMjA2MjcwNjI4MzNaFw0zMDEyMzExNTU5 +NTlaMFExCzAJBgNVBAYTAlRXMRIwEAYDVQQKEwlUQUlXQU4tQ0ExEDAOBgNVBAsT +B1Jvb3QgQ0ExHDAaBgNVBAMTE1RXQ0EgR2xvYmFsIFJvb3QgQ0EwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQCwBdvI64zEbooh745NnHEKH1Jw7W2CnJfF +10xORUnLQEK1EjRsGcJ0pDFfhQKX7EMzClPSnIyOt7h52yvVavKOZsTuKwEHktSz +0ALfUPZVr2YOy+BHYC8rMjk1Ujoog/h7FsYYuGLWRyWRzvAZEk2tY/XTP3VfKfCh +MBwqoJimFb3u/Rk28OKRQ4/6ytYQJ0lM793B8YVwm8rqqFpD/G2Gb3PpN0Wp8DbH +zIh1HrtsBv+baz4X7GGqcXzGHaL3SekVtTzWoWH1EfcFbx39Eb7QMAfCKbAJTibc +46KokWofwpFFiFzlmLhxpRUZyXx1EcxwdE8tmx2RRP1WKKD+u4ZqyPpcC1jcxkt2 +yKsi2XMPpfRaAok/T54igu6idFMqPVMnaR1sjjIsZAAmY2E2TqNGtz99sy2sbZCi +laLOz9qC5wc0GZbpuCGqKX6mOL6OKUohZnkfs8O1CWfe1tQHRvMq2uYiN2DLgbYP +oA/pyJV/v1WRBXrPPRXAb94JlAGD1zQbzECl8LibZ9WYkTunhHiVJqRaCPgrdLQA +BDzfuBSO6N+pjWxnkjMdwLfS7JLIvgm/LCkFbwJrnu+8vyq8W8BQj0FwcYeyTbcE +qYSjMq+u7msXi7Kx/mzhkIyIqJdIzshNy/MGz19qCkKxHh53L46g5pIOBvwFItIm +4TFRfTLcDwIDAQABoyMwITAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB +/zANBgkqhkiG9w0BAQsFAAOCAgEAXzSBdu+WHdXltdkCY4QWwa6gcFGn90xHNcgL +1yg9iXHZqjNB6hQbbCEAwGxCGX6faVsgQt+i0trEfJdLjbDorMjupWkEmQqSpqsn +LhpNgb+E1HAerUf+/UqdM+DyucRFCCEK2mlpc3INvjT+lIutwx4116KD7+U4x6WF +H6vPNOw/KP4M8VeGTslV9xzU2KV9Bnpv1d8Q34FOIWWxtuEXeZVFBs5fzNxGiWNo +RI2T9GRwoD2dKAXDOXC4Ynsg/eTb6QihuJ49CcdP+yz4k3ZB3lLg4VfSnQO8d57+ +nile98FRYB/e2guyLXW3Q0iT5/Z5xoRdgFlglPx4mI88k1HtQJAH32RjJMtOcQWh +15QaiDLxInQirqWm2BJpTGCjAu4r7NRjkgtevi92a6O2JryPA9gK8kxkRr05YuWW +6zRjESjMlfGt7+/cgFhI6Uu46mWs6fyAtbXIRfmswZ/ZuepiiI7E8UuDEq3mi4TW +nsLrgxifarsbJGAzcMzs9zLzXNl5fe+epP7JI8Mk7hWSsT2RTyaGvWZzJBPqpK5j +wa19hAM8EHiGG3njxPPyBJUgriOCxLM6AGK/5jYk4Ve6xx6QddVfP5VhK8E7zeWz +aGHQRiapIVJpLesux+t3zqY6tQMzT3bR51xUAV3LePTJDL/PEo4XLSNolOer/qmy +KwbQBM0= +-----END CERTIFICATE----- diff --git a/make/data/charsetmapping/charsets b/make/data/charsetmapping/charsets index a06e4038d697f..950347a6202b7 100644 --- a/make/data/charsetmapping/charsets +++ b/make/data/charsetmapping/charsets @@ -1,5 +1,5 @@ # -# Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -477,6 +477,11 @@ charset x-IBM874 IBM874 alias ibm-874 alias 874 +# alias for GB18030 is generated at runtime +charset GB18030 GB18030 + package sun.nio.cs + type source + ######################################################## # # charsets provided by ExtendedCharsets provider. @@ -564,11 +569,6 @@ charset GBK GBK # Simplified Chinese alias windows-936 alias CP936 -charset GB18030 GB18030 - package sun.nio.cs.ext - type template - alias gb18030-2000 - charset GB2312 EUC_CN package sun.nio.cs.ext type dbcs diff --git a/make/data/charsetmapping/stdcs-aix b/make/data/charsetmapping/stdcs-aix index f17468bbdbcf3..750308b3bfb19 100644 --- a/make/data/charsetmapping/stdcs-aix +++ b/make/data/charsetmapping/stdcs-aix @@ -7,7 +7,6 @@ Big5_HKSCS EUC_CN EUC_KR GBK -GB18030 IBM856 IBM921 IBM922 diff --git a/make/data/charsetmapping/stdcs-linux b/make/data/charsetmapping/stdcs-linux index d7b3b4d9672db..b9e80ca64555f 100644 --- a/make/data/charsetmapping/stdcs-linux +++ b/make/data/charsetmapping/stdcs-linux @@ -11,7 +11,6 @@ EUC_JP_LINUX EUC_JP_Open EUC_TW GBK -GB18030 ISO_8859_11 ISO_8859_3 ISO_8859_6 diff --git a/make/data/charsetmapping/stdcs-solaris b/make/data/charsetmapping/stdcs-solaris deleted file mode 100644 index cf3c01f12f853..0000000000000 --- a/make/data/charsetmapping/stdcs-solaris +++ /dev/null @@ -1,26 +0,0 @@ -# -# generate these charsets into sun.nio.cs -# -Big5 -Big5_Solaris -Big5_HKSCS # always together with Big5 -EUC_CN -EUC_KR -EUC_JP -EUC_JP_LINUX -EUC_JP_Open -EUC_TW -GBK -GB18030 -ISO_8859_11 -ISO_8859_3 -ISO_8859_6 -ISO_8859_8 -Johab -PCK -TIS_620 -JIS_X_0201 -JIS_X_0208 -JIS_X_0212 -JIS_X_0208_Solaris -JIS_X_0212_Solaris diff --git a/make/data/charsetmapping/stdcs-windows b/make/data/charsetmapping/stdcs-windows index 3185e9a966dde..482b699bcd1e7 100644 --- a/make/data/charsetmapping/stdcs-windows +++ b/make/data/charsetmapping/stdcs-windows @@ -2,7 +2,6 @@ # generate these charsets into sun.nio.cs # GBK -GB18030 Johab MS1255 MS1256 diff --git a/make/data/currency/CurrencyData.properties b/make/data/currency/CurrencyData.properties index d234c96c47628..12c0c69801efc 100644 --- a/make/data/currency/CurrencyData.properties +++ b/make/data/currency/CurrencyData.properties @@ -1,5 +1,5 @@ # -# Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -32,7 +32,7 @@ formatVersion=3 # Version of the currency code information in this class. # It is a serial number that accompanies with each amendment. -dataVersion=174 +dataVersion=175 # List of all valid ISO 4217 currency codes. # To ensure compatibility, do not remove codes. diff --git a/make/data/hotspot-symbols/symbols-unix b/make/data/hotspot-symbols/symbols-unix index a34732739d35b..b14c5338d6e55 100644 --- a/make/data/hotspot-symbols/symbols-unix +++ b/make/data/hotspot-symbols/symbols-unix @@ -154,7 +154,6 @@ JVM_IsRecord JVM_IsSameClassPackage JVM_IsSharingEnabled JVM_IsSupportedJNIVersion -JVM_IsThreadAlive JVM_IsVMGeneratedMethodIx JVM_LatestUserDefinedLoader JVM_LoadLibrary diff --git a/make/data/tzdata/VERSION b/make/data/tzdata/VERSION index 0f328a4a7ffa7..66bd061e8bcf9 100644 --- a/make/data/tzdata/VERSION +++ b/make/data/tzdata/VERSION @@ -21,4 +21,4 @@ # or visit www.oracle.com if you need additional information or have any # questions. # -tzdata2022g +tzdata2023c diff --git a/make/data/tzdata/africa b/make/data/tzdata/africa index 830d7d10b7e7c..a73405fdb01f2 100644 --- a/make/data/tzdata/africa +++ b/make/data/tzdata/africa @@ -344,6 +344,14 @@ Rule Egypt 2007 only - Sep Thu>=1 24:00 0 - # From Mina Samuel (2016-07-04): # Egyptian government took the decision to cancel the DST, +# From Ahmad ElDardiry (2023-03-01): +# Egypt officially announced today that daylight savings will be +# applied from last Friday of April to last Thursday of October. +# From Paul Eggert (2023-03-01): +# Assume transitions are at 00:00 and 24:00 respectively. +# From Amir Adib (2023-03-07): +# https://www.facebook.com/EgyptianCabinet/posts/638829614954129/ + Rule Egypt 2008 only - Aug lastThu 24:00 0 - Rule Egypt 2009 only - Aug 20 24:00 0 - Rule Egypt 2010 only - Aug 10 24:00 0 - @@ -353,6 +361,8 @@ Rule Egypt 2014 only - May 15 24:00 1:00 S Rule Egypt 2014 only - Jun 26 24:00 0 - Rule Egypt 2014 only - Jul 31 24:00 1:00 S Rule Egypt 2014 only - Sep lastThu 24:00 0 - +Rule Egypt 2023 max - Apr lastFri 0:00 1:00 S +Rule Egypt 2023 max - Oct lastThu 24:00 0 - # Zone NAME STDOFF RULES FORMAT [UNTIL] #STDOFF 2:05:08.9 @@ -452,7 +462,7 @@ Zone Africa/Nairobi 2:27:16 - LMT 1908 May # President William R. Tolbert, Jr., July 23, 1971-July 31, 1972. # Monrovia: Executive Mansion. # -# Use the abbreviation "MMT" before 1972, as the more-accurate numeric +# Use the abbreviation "MMT" before 1972, as the more accurate numeric # abbreviation "-004430" would be one byte over the POSIX limit. # # Zone NAME STDOFF RULES FORMAT [UNTIL] @@ -589,8 +599,8 @@ Zone Africa/Tripoli 0:52:44 - LMT 1920 # DST the coming summer... # # Some sources, in French: -# http://www.defimedia.info/news/946/Rashid-Beebeejaun-:-%C2%AB-L%E2%80%99heure-d%E2%80%99%C3%A9t%C3%A9-ne-sera-pas-appliqu%C3%A9e-cette-ann%C3%A9e-%C2%BB -# http://lexpress.mu/Story/3398~Beebeejaun---Les-objectifs-d-%C3%A9conomie-d-%C3%A9nergie-de-l-heure-d-%C3%A9t%C3%A9-ont-%C3%A9t%C3%A9-atteints- +# http://www.defimedia.info/news/946/Rashid-Beebeejaun-:-«-L%E2%80%99heure-d%E2%80%99été-ne-sera-pas-appliquée-cette-année-» +# http://lexpress.mu/Story/3398~Beebeejaun---Les-objectifs-d-économie-d-énergie-de-l-heure-d-été-ont-été-atteints- # # Our wrap-up: # https://www.timeanddate.com/news/time/mauritius-dst-will-not-repeat.html @@ -721,7 +731,7 @@ Zone Indian/Mauritius 3:50:00 - LMT 1907 # Port Louis # More articles in the press # https://www.yabiladi.com/articles/details/5058/secret-l-heure-d-ete-maroc-leve.html # http://www.lematin.ma/Actualite/Express/Article.asp?id=148923 -# http://www.lavieeco.com/actualite/Le-Maroc-passe-sur-GMT%2B1-a-partir-de-dim +# http://www.lavieeco.com/actualite/Le-Maroc-passe-sur-GMT+1-a-partir-de-dim # From Petr Machata (2011-03-30): # They have it written in English here: @@ -736,7 +746,7 @@ Zone Indian/Mauritius 3:50:00 - LMT 1907 # Port Louis # According to Infomédiaire web site from Morocco (infomediaire.ma), # on March 9, 2012, (in French) Heure légale: # Le Maroc adopte officiellement l'heure d'été -# http://www.infomediaire.ma/news/maroc/heure-l%C3%A9gale-le-maroc-adopte-officiellement-lheure-d%C3%A9t%C3%A9 +# http://www.infomediaire.ma/news/maroc/heure-légale-le-maroc-adopte-officiellement-lheure-dété # Governing Council adopted draft decree, that Morocco DST starts on # the last Sunday of March (March 25, 2012) and ends on # last Sunday of September (September 30, 2012) @@ -860,19 +870,28 @@ Zone Indian/Mauritius 3:50:00 - LMT 1907 # Port Louis # Friday or Saturday (and so the 2 days off are on a weekend), the next time # shift will be the next weekend. # -# From Paul Eggert (2020-05-31): +# From Milamber (2021-03-31, 2022-03-10): +# https://www.mmsp.gov.ma/fr/actualites.aspx?id=2076 +# https://www.ecoactu.ma/horaires-administration-ramadan-gmtheure-gmt-a-partir-de-dimanche-27-mars/ +# +# From Milamber (2023-03-14, 2023-03-15): +# The return to legal GMT time will take place this Sunday, March 19 at 3 a.m. +# ... the return to GMT+1 will be made on Sunday April 23, 2023 at 2 a.m. +# https://www.mmsp.gov.ma/fr/actualites/passage-à-l%E2%80%99heure-gmt-à-partir-du-dimanche-19-mars-2023 +# +# From Paul Eggert (2023-03-14): # For now, guess that in the future Morocco will fall back at 03:00 # the last Sunday before Ramadan, and spring forward at 02:00 the -# first Sunday after two days after Ramadan. To implement this, +# first Sunday after one day after Ramadan. To implement this, # transition dates and times for 2019 through 2087 were determined by -# running the following program under GNU Emacs 26.3. (This algorithm +# running the following program under GNU Emacs 28.2. (This algorithm # also produces the correct transition dates for 2016 through 2018, # though the times differ due to Morocco's time zone change in 2018.) # (let ((islamic-year 1440)) # (require 'cal-islam) # (while (< islamic-year 1511) # (let ((a (calendar-islamic-to-absolute (list 9 1 islamic-year))) -# (b (+ 2 (calendar-islamic-to-absolute (list 10 1 islamic-year)))) +# (b (+ 1 (calendar-islamic-to-absolute (list 10 1 islamic-year)))) # (sunday 0)) # (while (/= sunday (mod (setq a (1- a)) 7))) # (while (/= sunday (mod b 7)) @@ -886,10 +905,6 @@ Zone Indian/Mauritius 3:50:00 - LMT 1907 # Port Louis # (car (cdr (cdr a))) (calendar-month-name (car a) t) (car (cdr a)) # (car (cdr (cdr b))) (calendar-month-name (car b) t) (car (cdr b))))) # (setq islamic-year (+ 1 islamic-year)))) -# -# From Milamber (2021-03-31, 2022-03-10), confirming these predictions: -# https://www.mmsp.gov.ma/fr/actualites.aspx?id=2076 -# https://www.ecoactu.ma/horaires-administration-ramadan-gmtheure-gmt-a-partir-de-dimanche-27-mars/ # Rule NAME FROM TO - IN ON AT SAVE LETTER/S Rule Morocco 1939 only - Sep 12 0:00 1:00 - @@ -942,7 +957,7 @@ Rule Morocco 2021 only - May 16 2:00 0 - Rule Morocco 2022 only - Mar 27 3:00 -1:00 - Rule Morocco 2022 only - May 8 2:00 0 - Rule Morocco 2023 only - Mar 19 3:00 -1:00 - -Rule Morocco 2023 only - Apr 30 2:00 0 - +Rule Morocco 2023 only - Apr 23 2:00 0 - Rule Morocco 2024 only - Mar 10 3:00 -1:00 - Rule Morocco 2024 only - Apr 14 2:00 0 - Rule Morocco 2025 only - Feb 23 3:00 -1:00 - @@ -958,7 +973,7 @@ Rule Morocco 2029 only - Feb 18 2:00 0 - Rule Morocco 2029 only - Dec 30 3:00 -1:00 - Rule Morocco 2030 only - Feb 10 2:00 0 - Rule Morocco 2030 only - Dec 22 3:00 -1:00 - -Rule Morocco 2031 only - Feb 2 2:00 0 - +Rule Morocco 2031 only - Jan 26 2:00 0 - Rule Morocco 2031 only - Dec 14 3:00 -1:00 - Rule Morocco 2032 only - Jan 18 2:00 0 - Rule Morocco 2032 only - Nov 28 3:00 -1:00 - @@ -974,7 +989,7 @@ Rule Morocco 2036 only - Nov 23 2:00 0 - Rule Morocco 2037 only - Oct 4 3:00 -1:00 - Rule Morocco 2037 only - Nov 15 2:00 0 - Rule Morocco 2038 only - Sep 26 3:00 -1:00 - -Rule Morocco 2038 only - Nov 7 2:00 0 - +Rule Morocco 2038 only - Oct 31 2:00 0 - Rule Morocco 2039 only - Sep 18 3:00 -1:00 - Rule Morocco 2039 only - Oct 23 2:00 0 - Rule Morocco 2040 only - Sep 2 3:00 -1:00 - @@ -990,7 +1005,7 @@ Rule Morocco 2044 only - Aug 28 2:00 0 - Rule Morocco 2045 only - Jul 9 3:00 -1:00 - Rule Morocco 2045 only - Aug 20 2:00 0 - Rule Morocco 2046 only - Jul 1 3:00 -1:00 - -Rule Morocco 2046 only - Aug 12 2:00 0 - +Rule Morocco 2046 only - Aug 5 2:00 0 - Rule Morocco 2047 only - Jun 23 3:00 -1:00 - Rule Morocco 2047 only - Jul 28 2:00 0 - Rule Morocco 2048 only - Jun 7 3:00 -1:00 - @@ -1006,7 +1021,7 @@ Rule Morocco 2052 only - Jun 2 2:00 0 - Rule Morocco 2053 only - Apr 13 3:00 -1:00 - Rule Morocco 2053 only - May 25 2:00 0 - Rule Morocco 2054 only - Apr 5 3:00 -1:00 - -Rule Morocco 2054 only - May 17 2:00 0 - +Rule Morocco 2054 only - May 10 2:00 0 - Rule Morocco 2055 only - Mar 28 3:00 -1:00 - Rule Morocco 2055 only - May 2 2:00 0 - Rule Morocco 2056 only - Mar 12 3:00 -1:00 - @@ -1022,7 +1037,7 @@ Rule Morocco 2060 only - Mar 7 2:00 0 - Rule Morocco 2061 only - Jan 16 3:00 -1:00 - Rule Morocco 2061 only - Feb 27 2:00 0 - Rule Morocco 2062 only - Jan 8 3:00 -1:00 - -Rule Morocco 2062 only - Feb 19 2:00 0 - +Rule Morocco 2062 only - Feb 12 2:00 0 - Rule Morocco 2062 only - Dec 31 3:00 -1:00 - Rule Morocco 2063 only - Feb 4 2:00 0 - Rule Morocco 2063 only - Dec 16 3:00 -1:00 - @@ -1038,7 +1053,7 @@ Rule Morocco 2067 only - Dec 11 2:00 0 - Rule Morocco 2068 only - Oct 21 3:00 -1:00 - Rule Morocco 2068 only - Dec 2 2:00 0 - Rule Morocco 2069 only - Oct 13 3:00 -1:00 - -Rule Morocco 2069 only - Nov 24 2:00 0 - +Rule Morocco 2069 only - Nov 17 2:00 0 - Rule Morocco 2070 only - Oct 5 3:00 -1:00 - Rule Morocco 2070 only - Nov 9 2:00 0 - Rule Morocco 2071 only - Sep 20 3:00 -1:00 - @@ -1054,7 +1069,7 @@ Rule Morocco 2075 only - Sep 15 2:00 0 - Rule Morocco 2076 only - Jul 26 3:00 -1:00 - Rule Morocco 2076 only - Sep 6 2:00 0 - Rule Morocco 2077 only - Jul 18 3:00 -1:00 - -Rule Morocco 2077 only - Aug 29 2:00 0 - +Rule Morocco 2077 only - Aug 22 2:00 0 - Rule Morocco 2078 only - Jul 10 3:00 -1:00 - Rule Morocco 2078 only - Aug 14 2:00 0 - Rule Morocco 2079 only - Jun 25 3:00 -1:00 - @@ -1064,13 +1079,13 @@ Rule Morocco 2080 only - Jul 21 2:00 0 - Rule Morocco 2081 only - Jun 1 3:00 -1:00 - Rule Morocco 2081 only - Jul 13 2:00 0 - Rule Morocco 2082 only - May 24 3:00 -1:00 - -Rule Morocco 2082 only - Jul 5 2:00 0 - +Rule Morocco 2082 only - Jun 28 2:00 0 - Rule Morocco 2083 only - May 16 3:00 -1:00 - Rule Morocco 2083 only - Jun 20 2:00 0 - Rule Morocco 2084 only - Apr 30 3:00 -1:00 - Rule Morocco 2084 only - Jun 11 2:00 0 - Rule Morocco 2085 only - Apr 22 3:00 -1:00 - -Rule Morocco 2085 only - Jun 3 2:00 0 - +Rule Morocco 2085 only - May 27 2:00 0 - Rule Morocco 2086 only - Apr 14 3:00 -1:00 - Rule Morocco 2086 only - May 19 2:00 0 - Rule Morocco 2087 only - Mar 30 3:00 -1:00 - @@ -1213,15 +1228,15 @@ Zone Africa/Windhoek 1:08:24 - LMT 1892 Feb 8 # From P Chan (2020-12-03): # GMT was adopted as the standard time of Lagos on 1905-07-01. # Lagos Weekly Record, 1905-06-24, p 3 -# http://ddsnext.crl.edu/titles/31558#?c=0&m=668&s=0&cv=2&r=0&xywh=1446%2C5221%2C1931%2C1235 +# http://ddsnext.crl.edu/titles/31558#?c=0&m=668&s=0&cv=2&r=0&xywh=1446,5221,1931,1235 # says "It is officially notified that on and after the 1st of July 1905 -# Greenwich Mean Solar Time will be adopted thought the Colony and +# Greenwich Mean Solar Time will be adopted throughout the Colony and # Protectorate, and that it will be necessary to put all clocks 13 minutes and # 35 seconds back, recording local mean time." # # It seemed that Lagos returned to LMT on 1908-07-01. # [The Lagos Standard], 1908-07-01, p 5 -# http://ddsnext.crl.edu/titles/31556#?c=0&m=78&s=0&cv=4&r=0&xywh=-92%2C3590%2C3944%2C2523 +# http://ddsnext.crl.edu/titles/31556#?c=0&m=78&s=0&cv=4&r=0&xywh=-92,3590,3944,2523 # says "Scarcely have the people become accustomed to this new time, when # another official notice has now appeared announcing that from and after the # 1st July next, return will be made to local mean time." @@ -1233,7 +1248,7 @@ Zone Africa/Windhoek 1:08:24 - LMT 1892 Feb 8 # https://libsysdigi.library.illinois.edu/ilharvest/Africana/Books2011-05/3064634/3064634_1914/3064634_1914_opt.pdf#page=27 # "On January 1st [1914], a universal standard time for Nigeria was adopted, # viz., half an hour fast on Greenwich mean time, corresponding to the meridian -# 7 [degrees] 30' E. long." +# 7° 30' E. long." # Lloyd's Register of Shipping (1915) says "Hitherto the time observed in Lagos # was the local mean time. On 1st January, 1914, standard time for the whole of # Nigeria was introduced ... Lagos time has been advanced about 16 minutes @@ -1251,7 +1266,7 @@ Zone Africa/Windhoek 1:08:24 - LMT 1892 Feb 8 # The Lagos Weekly Record, 1919-09-20, p 3 details discussion on the first # reading of this Bill by the Legislative Council of the Colony of Nigeria on # Thursday 1919-08-28: -# http://ddsnext.crl.edu/titles/31558?terms&item_id=303484#?m=1118&c=1&s=0&cv=2&r=0&xywh=1261%2C3408%2C2994%2C1915 +# http://ddsnext.crl.edu/titles/31558?terms&item_id=303484#?m=1118&c=1&s=0&cv=2&r=0&xywh=1261,3408,2994,1915 # "The proposal is that the Globe should be divided into twelve zones East and # West of Greenwich, of one hour each, Nigeria falling into the zone with a # standard of one hour fast on Greenwich Mean Time. Nigeria standard time is diff --git a/make/data/tzdata/antarctica b/make/data/tzdata/antarctica index 792542b92246b..3de5e726eb4d7 100644 --- a/make/data/tzdata/antarctica +++ b/make/data/tzdata/antarctica @@ -315,7 +315,7 @@ Zone Antarctica/Rothera 0 - -00 1976 Dec 1 # but that he found it more convenient to keep GMT+12 # as supplies for the station were coming from McMurdo Sound, # which was on GMT+12 because New Zealand was on GMT+12 all year -# at that time (1957). (Source: Siple's book 90 Degrees South.) +# at that time (1957). (Source: Siple's book 90° South.) # # From Susan Smith # http://www.cybertours.com/whs/pole10.html diff --git a/make/data/tzdata/asia b/make/data/tzdata/asia index ff81978bc47eb..6a048c3ad283e 100644 --- a/make/data/tzdata/asia +++ b/make/data/tzdata/asia @@ -2714,6 +2714,40 @@ Zone Asia/Pyongyang 8:23:00 - LMT 1908 Apr 1 # Lebanon +# +# From Saadallah Itani (2023-03-23): +# Lebanon ... announced today delay of Spring forward from March 25 to April 20. +# +# From Paul Eggert (2023-03-27): +# This announcement was by the Lebanese caretaker prime minister Najib Mikati. +# https://www.mtv.com.lb/en/News/Local/1352516/lebanon-postpones-daylight-saving-time-adoption +# A video was later leaked to the media of parliament speaker Nabih Berri +# asking Mikati to postpone DST to aid observance of Ramadan, Mikati objecting +# that this would cause problems such as scheduling airline flights, to which +# Berri interjected, "What flights?" +# +# The change was controversial and led to a partly-sectarian divide. +# Many Lebanese institutions, including the education ministry, the Maronite +# church, and two news channels LCBI and MTV, ignored the announcement and +# went ahead with the long-scheduled spring-forward on March 25/26, some +# arguing that the prime minister had not followed the law because the change +# had not been approved by the cabinet. Google went with the announcement; +# Apple ignored it. At least one bank followed the announcement for its doors, +# but ignored the announcement in internal computer systems. +# Beirut international airport listed two times for each departure. +# Dan Azzi wrote "My view is that this whole thing is a Dumb and Dumber movie." +# Eventually the prime minister backed down, said the cabinet had decided to +# stick with its 1998 decision, and that DST would begin midnight March 29/30. +# https://www.nna-leb.gov.lb/en/miscellaneous/604093/lebanon-has-two-times-of-day-amid-daylight-savings +# https://www.cnbc.com/2023/03/27/lebanon-in-two-different-time-zones-as-government-disagrees-on-daylight-savings.html +# +# Although we could model the chaos with two Zones, that would likely cause +# more trouble than it would cure. Since so many manual clocks and +# computer-based timestamps ignored the announcement, stick with official +# cabinet resolutions in the data while recording the prime minister's +# announcement as a comment. This is how we treated a similar situation in +# Rio de Janeiro in spring 1993. +# # Rule NAME FROM TO - IN ON AT SAVE LETTER/S Rule Lebanon 1920 only - Mar 28 0:00 1:00 S Rule Lebanon 1920 only - Oct 25 0:00 0 - @@ -2739,6 +2773,10 @@ Rule Lebanon 1992 only - Oct 4 0:00 0 - Rule Lebanon 1993 max - Mar lastSun 0:00 1:00 S Rule Lebanon 1993 1998 - Sep lastSun 0:00 0 - Rule Lebanon 1999 max - Oct lastSun 0:00 0 - +# This one-time rule, announced by the prime minister first for April 21 +# then for March 30, is commented out for reasons described above. +#Rule Lebanon 2023 only - Mar 30 0:00 1:00 S + # Zone NAME STDOFF RULES FORMAT [UNTIL] Zone Asia/Beirut 2:22:00 - LMT 1880 2:00 Lebanon EE%sT @@ -2977,7 +3015,7 @@ Zone Asia/Kathmandu 5:41:16 - LMT 1920 # 9pm and moving clocks forward by one hour for the next three months. ...." # # http://www.worldtimezone.com/dst_news/dst_news_pakistan01.html -# http://www.dailytimes.com.pk/default.asp?page=2008%5C05%5C15%5Cstory_15-5-2008_pg1_4 +# http://www.dailytimes.com.pk/default.asp?page=2008\05\15\story_15-5-2008_pg1_4 # From Arthur David Olson (2008-05-19): # XXX--midnight transitions is a guess; 2008 only is a guess. @@ -3300,7 +3338,7 @@ Zone Asia/Karachi 4:28:12 - LMT 1907 # Some of many sources in Arabic: # http://www.samanews.com/index.php?act=Show&id=122638 # -# http://safa.ps/details/news/74352/%D8%A8%D8%AF%D8%A1-%D8%A7%D9%84%D8%AA%D9%88%D9%82%D9%8A%D8%AA-%D8%A7%D9%84%D8%B5%D9%8A%D9%81%D9%8A-%D8%A8%D8%A7%D9%84%D8%B6%D9%81%D8%A9-%D9%88%D8%BA%D8%B2%D8%A9-%D9%84%D9%8A%D9%84%D8%A9-%D8%A7%D9%84%D8%AC%D9%85%D8%B9%D8%A9.html +# http://safa.ps/details/news/74352/بدء-التوقيت-الصيفي-بالضفة-وغزة-ليلة-الجمعة.html # # Our brief summary: # https://www.timeanddate.com/news/time/gaza-west-bank-dst-2012.html @@ -3310,7 +3348,7 @@ Zone Asia/Karachi 4:28:12 - LMT 1907 # time from midnight on Friday, March 29, 2013" (translated). # [These are in Arabic and are for Gaza and for Ramallah, respectively.] # http://www.samanews.com/index.php?act=Show&id=154120 -# http://safa.ps/details/news/99844/%D8%B1%D8%A7%D9%85-%D8%A7%D9%84%D9%84%D9%87-%D8%A8%D8%AF%D8%A1-%D8%A7%D9%84%D8%AA%D9%88%D9%82%D9%8A%D8%AA-%D8%A7%D9%84%D8%B5%D9%8A%D9%81%D9%8A-29-%D8%A7%D9%84%D8%AC%D8%A7%D8%B1%D9%8A.html +# http://safa.ps/details/news/99844/رام-الله-بدء-التوقيت-الصيفي-29-الجاري.html # From Steffen Thorsen (2013-09-24): # The Gaza and West Bank are ending DST Thursday at midnight @@ -3408,9 +3446,41 @@ Zone Asia/Karachi 4:28:12 - LMT 1907 # (2022-08-31): ... the Saturday before the last Sunday in March and October # at 2:00 AM ,for the years from 2023 to 2026. # (2022-09-05): https://mtit.pna.ps/Site/New/1453 -# -# From Paul Eggert (2022-08-31): -# For now, assume that this rule will also be used after 2026. + +# From Heba Hamad (2023-03-22): +# ... summer time will begin in Palestine from Saturday 04-29-2023, +# 02:00 AM by 60 minutes forward. +# +# From Paul Eggert (2023-03-22): +# For now, guess that spring and fall transitions will normally +# continue to use 2022's rules, that during DST Palestine will switch +# to standard time at 02:00 the last Saturday before Ramadan and back +# to DST at 02:00 the first Saturday after Ramadan, and that +# if the normal spring-forward or fall-back transition occurs during +# Ramadan the former is delayed and the latter advanced. +# To implement this, I predicted Ramadan-oriented transition dates for +# 2023 through 2086 by running the following program under GNU Emacs 28.2, +# with the results integrated by hand into the table below. +# Predictions after 2086 are approximated without Ramadan. +# +# (let ((islamic-year 1444)) +# (require 'cal-islam) +# (while (< islamic-year 1510) +# (let ((a (calendar-islamic-to-absolute (list 9 1 islamic-year))) +# (b (+ 1 (calendar-islamic-to-absolute (list 10 1 islamic-year)))) +# (saturday 6)) +# (while (/= saturday (mod (setq a (1- a)) 7))) +# (while (/= saturday (mod b 7)) +# (setq b (1+ b))) +# (setq a (calendar-gregorian-from-absolute a)) +# (setq b (calendar-gregorian-from-absolute b)) +# (insert +# (format +# (concat "Rule Palestine\t%d\tonly\t-\t%s\t%2d\t2:00\t0\t-\n" +# "Rule Palestine\t%d\tonly\t-\t%s\t%2d\t2:00\t1:00\tS\n") +# (car (cdr (cdr a))) (calendar-month-name (car a) t) (car (cdr a)) +# (car (cdr (cdr b))) (calendar-month-name (car b) t) (car (cdr b))))) +# (setq islamic-year (+ 1 islamic-year)))) # Rule NAME FROM TO - IN ON AT SAVE LETTER/S Rule EgyptAsia 1957 only - May 10 0:00 1:00 S @@ -3450,8 +3520,86 @@ Rule Palestine 2020 2021 - Mar Sat<=30 0:00 1:00 S Rule Palestine 2020 only - Oct 24 1:00 0 - Rule Palestine 2021 only - Oct 29 1:00 0 - Rule Palestine 2022 only - Mar 27 0:00 1:00 S -Rule Palestine 2022 max - Oct Sat<=30 2:00 0 - -Rule Palestine 2023 max - Mar Sat<=30 2:00 1:00 S +Rule Palestine 2022 2035 - Oct Sat<=30 2:00 0 - +Rule Palestine 2023 only - Apr 29 2:00 1:00 S +Rule Palestine 2024 only - Apr 13 2:00 1:00 S +Rule Palestine 2025 only - Apr 5 2:00 1:00 S +Rule Palestine 2026 2054 - Mar Sat<=30 2:00 1:00 S +Rule Palestine 2036 only - Oct 18 2:00 0 - +Rule Palestine 2037 only - Oct 10 2:00 0 - +Rule Palestine 2038 only - Sep 25 2:00 0 - +Rule Palestine 2039 only - Sep 17 2:00 0 - +Rule Palestine 2039 only - Oct 22 2:00 1:00 S +Rule Palestine 2039 2067 - Oct Sat<=30 2:00 0 - +Rule Palestine 2040 only - Sep 1 2:00 0 - +Rule Palestine 2040 only - Oct 13 2:00 1:00 S +Rule Palestine 2041 only - Aug 24 2:00 0 - +Rule Palestine 2041 only - Sep 28 2:00 1:00 S +Rule Palestine 2042 only - Aug 16 2:00 0 - +Rule Palestine 2042 only - Sep 20 2:00 1:00 S +Rule Palestine 2043 only - Aug 1 2:00 0 - +Rule Palestine 2043 only - Sep 12 2:00 1:00 S +Rule Palestine 2044 only - Jul 23 2:00 0 - +Rule Palestine 2044 only - Aug 27 2:00 1:00 S +Rule Palestine 2045 only - Jul 15 2:00 0 - +Rule Palestine 2045 only - Aug 19 2:00 1:00 S +Rule Palestine 2046 only - Jun 30 2:00 0 - +Rule Palestine 2046 only - Aug 11 2:00 1:00 S +Rule Palestine 2047 only - Jun 22 2:00 0 - +Rule Palestine 2047 only - Jul 27 2:00 1:00 S +Rule Palestine 2048 only - Jun 6 2:00 0 - +Rule Palestine 2048 only - Jul 18 2:00 1:00 S +Rule Palestine 2049 only - May 29 2:00 0 - +Rule Palestine 2049 only - Jul 3 2:00 1:00 S +Rule Palestine 2050 only - May 21 2:00 0 - +Rule Palestine 2050 only - Jun 25 2:00 1:00 S +Rule Palestine 2051 only - May 6 2:00 0 - +Rule Palestine 2051 only - Jun 17 2:00 1:00 S +Rule Palestine 2052 only - Apr 27 2:00 0 - +Rule Palestine 2052 only - Jun 1 2:00 1:00 S +Rule Palestine 2053 only - Apr 12 2:00 0 - +Rule Palestine 2053 only - May 24 2:00 1:00 S +Rule Palestine 2054 only - Apr 4 2:00 0 - +Rule Palestine 2054 only - May 16 2:00 1:00 S +Rule Palestine 2055 only - May 1 2:00 1:00 S +Rule Palestine 2056 only - Apr 22 2:00 1:00 S +Rule Palestine 2057 only - Apr 7 2:00 1:00 S +Rule Palestine 2058 max - Mar Sat<=30 2:00 1:00 S +Rule Palestine 2068 only - Oct 20 2:00 0 - +Rule Palestine 2069 only - Oct 12 2:00 0 - +Rule Palestine 2070 only - Oct 4 2:00 0 - +Rule Palestine 2071 only - Sep 19 2:00 0 - +Rule Palestine 2072 only - Sep 10 2:00 0 - +Rule Palestine 2072 only - Oct 15 2:00 1:00 S +Rule Palestine 2073 only - Sep 2 2:00 0 - +Rule Palestine 2073 only - Oct 7 2:00 1:00 S +Rule Palestine 2074 only - Aug 18 2:00 0 - +Rule Palestine 2074 only - Sep 29 2:00 1:00 S +Rule Palestine 2075 only - Aug 10 2:00 0 - +Rule Palestine 2075 only - Sep 14 2:00 1:00 S +Rule Palestine 2075 max - Oct Sat<=30 2:00 0 - +Rule Palestine 2076 only - Jul 25 2:00 0 - +Rule Palestine 2076 only - Sep 5 2:00 1:00 S +Rule Palestine 2077 only - Jul 17 2:00 0 - +Rule Palestine 2077 only - Aug 28 2:00 1:00 S +Rule Palestine 2078 only - Jul 9 2:00 0 - +Rule Palestine 2078 only - Aug 13 2:00 1:00 S +Rule Palestine 2079 only - Jun 24 2:00 0 - +Rule Palestine 2079 only - Aug 5 2:00 1:00 S +Rule Palestine 2080 only - Jun 15 2:00 0 - +Rule Palestine 2080 only - Jul 20 2:00 1:00 S +Rule Palestine 2081 only - Jun 7 2:00 0 - +Rule Palestine 2081 only - Jul 12 2:00 1:00 S +Rule Palestine 2082 only - May 23 2:00 0 - +Rule Palestine 2082 only - Jul 4 2:00 1:00 S +Rule Palestine 2083 only - May 15 2:00 0 - +Rule Palestine 2083 only - Jun 19 2:00 1:00 S +Rule Palestine 2084 only - Apr 29 2:00 0 - +Rule Palestine 2084 only - Jun 10 2:00 1:00 S +Rule Palestine 2085 only - Apr 21 2:00 0 - +Rule Palestine 2085 only - Jun 2 2:00 1:00 S +Rule Palestine 2086 only - Apr 13 2:00 0 - +Rule Palestine 2086 only - May 18 2:00 1:00 S # Zone NAME STDOFF RULES FORMAT [UNTIL] Zone Asia/Gaza 2:17:52 - LMT 1900 Oct @@ -3655,7 +3803,7 @@ Zone Asia/Singapore 6:55:25 - LMT 1901 Jan 1 # standard time is SLST. # # From Paul Eggert (2016-10-18): -# "SLST" seems to be reasonably recent and rarely-used outside time +# "SLST" seems to be reasonably recent and rarely used outside time # zone nerd sources. I searched Google News and found three uses of # it in the International Business Times of India in February and # March of this year when discussing cricket match times, but nothing diff --git a/make/data/tzdata/australasia b/make/data/tzdata/australasia index fbe3b8a6d72c0..893d7055eaba4 100644 --- a/make/data/tzdata/australasia +++ b/make/data/tzdata/australasia @@ -346,7 +346,7 @@ Zone Antarctica/Macquarie 0 - -00 1899 Nov # From Steffen Thorsen (2013-01-10): # Fiji will end DST on 2014-01-19 02:00: -# http://www.fiji.gov.fj/Media-Center/Press-Releases/DAYLIGHT-SAVINGS-TO-END-THIS-MONTH-%281%29.aspx +# http://www.fiji.gov.fj/Media-Center/Press-Releases/DAYLIGHT-SAVINGS-TO-END-THIS-MONTH-(1).aspx # From Ken Rylander (2014-10-20): # DST will start Nov. 2 this year. @@ -746,7 +746,7 @@ Zone Pacific/Pago_Pago 12:37:12 - LMT 1892 Jul 5 # # Samoa's Daylight Saving Time Act 2009 is available here, but does not # contain any dates: -# http://www.parliament.gov.ws/documents/acts/Daylight%20Saving%20Act%20%202009%20%28English%29%20-%20Final%207-7-091.pdf +# http://www.parliament.gov.ws/documents/acts/Daylight%20Saving%20Act%20%202009%20(English)%20-%20Final%207-7-091.pdf # From Laupue Raymond Hughes (2010-10-07): # Please see @@ -1831,7 +1831,7 @@ Zone Pacific/Efate 11:13:16 - LMT 1912 Jan 13 # Vila # period. It would probably be reasonable to assume Guam use GMT+9 during # that period of time like the surrounding area. -# From Paul Eggert (2018-11-18): +# From Paul Eggert (2023-01-23): # Howse writes (p 153) "The Spaniards, on the other hand, reached the # Philippines and the Ladrones from America," and implies that the Ladrones # (now called the Marianas) kept American date for quite some time. @@ -1844,7 +1844,7 @@ Zone Pacific/Efate 11:13:16 - LMT 1912 Jan 13 # Vila # they did as that avoids the need for a separate zone due to our 1970 cutoff. # # US Public Law 106-564 (2000-12-23) made UT +10 the official standard time, -# under the name "Chamorro Standard Time". There is no official abbreviation, +# under the name "Chamorro standard time". There is no official abbreviation, # but Congressman Robert A. Underwood, author of the bill that became law, # wrote in a press release (2000-12-27) that he will seek the use of "ChST". @@ -2222,24 +2222,18 @@ Zone Pacific/Efate 11:13:16 - LMT 1912 Jan 13 # Vila # an international standard, there are some places on the high seas where the # correct date is ambiguous. -# From Wikipedia (2005-08-31): -# Before 1920, all ships kept local apparent time on the high seas by setting -# their clocks at night or at the morning sight so that, given the ship's -# speed and direction, it would be 12 o'clock when the Sun crossed the ship's -# meridian (12 o'clock = local apparent noon). During 1917, at the -# Anglo-French Conference on Time-keeping at Sea, it was recommended that all -# ships, both military and civilian, should adopt hourly standard time zones -# on the high seas. Whenever a ship was within the territorial waters of any -# nation it would use that nation's standard time. The captain was permitted -# to change his ship's clocks at a time of his choice following his ship's -# entry into another zone time - he often chose midnight. These zones were -# adopted by all major fleets between 1920 and 1925 but not by many -# independent merchant ships until World War II. - -# From Paul Eggert, using references suggested by Oscar van Vlijmen -# (2005-03-20): -# -# The American Practical Navigator (2002) -# http://pollux.nss.nima.mil/pubs/pubs_j_apn_sections.html?rid=187 -# talks only about the 180-degree meridian with respect to ships in -# international waters; it ignores the international date line. +# From Wikipedia (2023-01-23): +# The nautical time zone system is analogous to the terrestrial time zone +# system for use on high seas. Under the system time changes are required for +# changes of longitude in one-hour steps. The one-hour step corresponds to a +# time zone width of 15° longitude. The 15° gore that is offset from GMT or +# UT1 (not UTC) by twelve hours is bisected by the nautical date line into two +# 7°30' gores that differ from GMT by ±12 hours. A nautical date line is +# implied but not explicitly drawn on time zone maps. It follows the 180th +# meridian except where it is interrupted by territorial waters adjacent to +# land, forming gaps: it is a pole-to-pole dashed line. + +# From Paul Eggert (2023-01-23): +# The American Practical Navigator , +# 2019 edition, merely says that the International Date Line +# "coincides with the 180th meridian over most of its length." diff --git a/make/data/tzdata/backward b/make/data/tzdata/backward index fa44f65500917..c0746d6dd1bc8 100644 --- a/make/data/tzdata/backward +++ b/make/data/tzdata/backward @@ -297,6 +297,7 @@ Link America/Argentina/Cordoba America/Rosario Link America/Tijuana America/Santa_Isabel Link America/Denver America/Shiprock Link America/Toronto America/Thunder_Bay +Link America/Edmonton America/Yellowknife Link Pacific/Auckland Antarctica/South_Pole Link Asia/Shanghai Asia/Chongqing Link Asia/Shanghai Asia/Harbin diff --git a/make/data/tzdata/europe b/make/data/tzdata/europe index acc5da3ec7954..446d2e1e658fa 100644 --- a/make/data/tzdata/europe +++ b/make/data/tzdata/europe @@ -540,9 +540,7 @@ Zone Europe/London -0:01:15 - LMT 1847 Dec 1 # other form with a traditional approximation for Irish timestamps # after 1971-10-31 02:00 UTC; although this approximation has tm_isdst # flags that are reversed, its UTC offsets are correct and this often -# suffices. This source file currently uses only nonnegative SAVE -# values, but this is intended to change and downstream code should -# not rely on it. +# suffices.... # # The following is like GB-Eire and EU, except with standard time in # summer and negative daylight saving time in winter. It is for when @@ -1136,19 +1134,18 @@ Zone Atlantic/Faroe -0:27:04 - LMT 1908 Jan 11 # Tórshavn # # From Jürgen Appel (2022-11-25): # https://ina.gl/samlinger/oversigt-over-samlinger/samling/dagsordener/dagsorden.aspx?lang=da&day=24-11-2022 -# If I understand this correctly, from the next planned switch to -# summer time, Greenland will permanently stay at that time, i.e. no -# switch back to winter time in 2023 will occur. -# -# From Paul Eggert (2022-11-28): -# The official document in Danish -# https://naalakkersuisut.gl/-/media/naalakkersuisut/filer/kundgoerelser/2022/11/2511/31_da_inatsisartutlov-om-tidens-bestemmelse.pdf?la=da&hash=A33597D8A38CC7038465241119EF34F3 -# says standard time for Greenland is -02, that Naalakkersuisut can lay down -# rules for DST and can require some areas to use a different time zone, -# and that this all takes effect 2023-03-25 22:00. The abovementioned -# "bekymringer" URL says the intent is no transition March 25, that -# Greenland will not go back to winter time in fall 2023, and that -# only America/Nuuk is affected (though further changes may occur). +# +# From Thomas M. Steenholdt (2022-12-02): +# - The bill to move America/Nuuk from UTC-03 to UTC-02 passed. +# - The bill to stop observing DST did not (Greenland will stop observing DST +# when EU does). +# Details on the implementation are here (section 6): +# https://ina.gl/dvd/EM%202022/pdf/media/2553529/pkt17_em2022_tidens_bestemmelse_bem_da.pdf +# This is how the change will be implemented: +# 1. The shift *to* DST in 2023 happens as normal. +# 2. The shift *from* DST in 2023 happens as normal, but coincides with the +# shift to UTC-02 normaltime (people will not change their clocks here). +# 3. After this, DST is still observed, but as -02/-01 instead of -03/-02. # Rule NAME FROM TO - IN ON AT SAVE LETTER/S Rule Thule 1991 1992 - Mar lastSun 2:00 1:00 D @@ -1172,8 +1169,8 @@ Zone America/Scoresbysund -1:27:52 - LMT 1916 Jul 28 # Ittoqqortoormiit -1:00 EU -01/+00 Zone America/Nuuk -3:26:56 - LMT 1916 Jul 28 # Godthåb -3:00 - -03 1980 Apr 6 2:00 - -3:00 EU -03/-02 2023 Mar 25 22:00 - -2:00 - -02 + -3:00 EU -03/-02 2023 Oct 29 1:00u + -2:00 EU -02/-01 Zone America/Thule -4:35:08 - LMT 1916 Jul 28 # Pituffik -4:00 Thule A%sT @@ -1509,9 +1506,9 @@ Zone Europe/Paris 0:09:21 - LMT 1891 Mar 16 Rule Germany 1946 only - Apr 14 2:00s 1:00 S Rule Germany 1946 only - Oct 7 2:00s 0 - Rule Germany 1947 1949 - Oct Sun>=1 2:00s 0 - -# http://www.ptb.de/de/org/4/44/441/salt.htm says the following transition -# occurred at 3:00 MEZ, not the 2:00 MEZ given in Shanks & Pottenger. -# Go with the PTB. +# https://www.ptb.de/cms/en/ptb/fachabteilungen/abt4/fb-44/ag-441/realisation-of-legal-time-in-germany/dst-and-midsummer-dst-in-germany-until-1979.html +# says the following transition occurred at 3:00 MEZ, not the 2:00 MEZ +# given in Shanks & Pottenger. Go with the PTB. Rule Germany 1947 only - Apr 6 3:00s 1:00 S Rule Germany 1947 only - May 11 2:00s 2:00 M Rule Germany 1947 only - Jun 29 3:00 1:00 S @@ -2272,7 +2269,7 @@ Zone Europe/Bucharest 1:44:24 - LMT 1891 Oct # the State Duma has approved ... the draft bill on returning to # winter time standard and return Russia 11 time zones. The new # regulations will come into effect on October 26, 2014 at 02:00 ... -# http://asozd2.duma.gov.ru/main.nsf/%28Spravka%29?OpenAgent&RN=431985-6&02 +# http://asozd2.duma.gov.ru/main.nsf/(Spravka)?OpenAgent&RN=431985-6&02 # Here is a link where we put together table (based on approved Bill N # 431985-6) with proposed 11 Russian time zones and corresponding # areas/cities/administrative centers in the Russian Federation (in English): @@ -2682,13 +2679,13 @@ Zone Europe/Volgograd 2:57:40 - LMT 1920 Jan 3 3:00 - +03 1930 Jun 21 4:00 - +04 1961 Nov 11 4:00 Russia +04/+05 1988 Mar 27 2:00s - 3:00 Russia +03/+04 1991 Mar 31 2:00s + 3:00 Russia MSK/MSD 1991 Mar 31 2:00s 4:00 - +04 1992 Mar 29 2:00s - 3:00 Russia +03/+04 2011 Mar 27 2:00s - 4:00 - +04 2014 Oct 26 2:00s - 3:00 - +03 2018 Oct 28 2:00s + 3:00 Russia MSK/MSD 2011 Mar 27 2:00s + 4:00 - MSK 2014 Oct 26 2:00s + 3:00 - MSK 2018 Oct 28 2:00s 4:00 - +04 2020 Dec 27 2:00s - 3:00 - +03 + 3:00 - MSK # From Paul Eggert (2016-11-11): # Europe/Saratov covers: @@ -2719,11 +2716,11 @@ Zone Europe/Saratov 3:04:18 - LMT 1919 Jul 1 0:00u Zone Europe/Kirov 3:18:48 - LMT 1919 Jul 1 0:00u 3:00 - +03 1930 Jun 21 4:00 Russia +04/+05 1989 Mar 26 2:00s - 3:00 Russia +03/+04 1991 Mar 31 2:00s + 3:00 Russia MSK/MSD 1991 Mar 31 2:00s 4:00 - +04 1992 Mar 29 2:00s - 3:00 Russia +03/+04 2011 Mar 27 2:00s - 4:00 - +04 2014 Oct 26 2:00s - 3:00 - +03 + 3:00 Russia MSK/MSD 2011 Mar 27 2:00s + 4:00 - MSK 2014 Oct 26 2:00s + 3:00 - MSK # From Tim Parenti (2014-07-03), per Oscar van Vlijmen (2001-08-25): # Europe/Samara covers... diff --git a/make/data/tzdata/iso3166.tab b/make/data/tzdata/iso3166.tab index fbfb74bec45ce..cea17732dd138 100644 --- a/make/data/tzdata/iso3166.tab +++ b/make/data/tzdata/iso3166.tab @@ -261,7 +261,7 @@ SY Syria SZ Eswatini (Swaziland) TC Turks & Caicos Is TD Chad -TF French Southern Territories +TF French S. Terr. TG Togo TH Thailand TJ Tajikistan diff --git a/make/data/tzdata/leapseconds b/make/data/tzdata/leapseconds index d6fb840f512ce..89ce8b89cd28f 100644 --- a/make/data/tzdata/leapseconds +++ b/make/data/tzdata/leapseconds @@ -95,11 +95,11 @@ Leap 2016 Dec 31 23:59:60 + S # Any additional leap seconds will come after this. # This Expires line is commented out for now, # so that pre-2020a zic implementations do not reject this file. -#Expires 2023 Jun 28 00:00:00 +#Expires 2023 Dec 28 00:00:00 # POSIX timestamps for the data in this file: #updated 1467936000 (2016-07-08 00:00:00 UTC) -#expires 1687910400 (2023-06-28 00:00:00 UTC) +#expires 1703721600 (2023-12-28 00:00:00 UTC) -# Updated through IERS Bulletin C64 -# File expires on: 28 June 2023 +# Updated through IERS Bulletin C65 +# File expires on: 28 December 2023 diff --git a/make/data/tzdata/northamerica b/make/data/tzdata/northamerica index a5fd701f88ce4..e240cf35103c4 100644 --- a/make/data/tzdata/northamerica +++ b/make/data/tzdata/northamerica @@ -1,4 +1,3 @@ -# # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -299,9 +298,10 @@ Zone PST8PDT -8:00 US P%sT # -10 Standard Alaska Time (AST) Alaska-Hawaii standard time (AHST) # -11 (unofficial) Nome (NST) Bering standard time (BST) # -# From Paul Eggert (2000-01-08), following a heads-up from Rives McDow: -# Public law 106-564 (2000-12-23) introduced ... "Chamorro Standard Time" +# From Paul Eggert (2023-01-23), from a 2001-01-08 heads-up from Rives McDow: +# Public law 106-564 (2000-12-23) introduced "Chamorro standard time" # for time in Guam and the Northern Marianas. See the file "australasia". +# Also see 15 U.S.C. §263 . # # From Paul Eggert (2015-04-17): # HST and HDT are standardized abbreviations for Hawaii-Aleutian @@ -618,7 +618,7 @@ Zone America/Los_Angeles -7:52:58 - LMT 1883 Nov 18 20:00u # local times of other Alaskan locations so that they change simultaneously. # From Paul Eggert (2014-07-18): -# One opinion of the early-1980s turmoil in Alaska over time zones and +# One opinion of the early 1980s turmoil in Alaska over time zones and # daylight saving time appeared as graffiti on a Juneau airport wall: # "Welcome to Juneau. Please turn your watch back to the 19th century." # See: Turner W. Alaska's four time zones now two. NY Times 1983-11-01. @@ -690,6 +690,10 @@ Zone America/Los_Angeles -7:52:58 - LMT 1883 Nov 18 20:00u # So they won't be waiting for Alaska to join them on 2019-03-10, but will # rather change their clocks twice in seven weeks. +# From Paul Eggert (2023-01-23): +# America/Adak is for the Aleutian Islands that are part of Alaska +# and are west of 169.5° W. + # Zone NAME STDOFF RULES FORMAT [UNTIL] Zone America/Juneau 15:02:19 - LMT 1867 Oct 19 15:33:32 -8:57:41 - LMT 1900 Aug 20 12:00 @@ -2148,10 +2152,6 @@ Zone America/Fort_Nelson -8:10:47 - LMT 1884 # Nunavut ... moved ... to incorporate the whole territory into one time zone. # Nunavut moves to single time zone Oct. 31 # http://www.nunatsiaq.com/nunavut/nvt90903_13.html -# -# From Antoine Leca (1999-09-06): -# We then need to create a new timezone for the Kitikmeot region of Nunavut -# to differentiate it from the Yellowknife region. # From Paul Eggert (1999-09-20): # Basic Facts: The New Territory @@ -2345,9 +2345,6 @@ Zone America/Cambridge_Bay 0 - -00 1920 # trading post est.? -5:00 - EST 2000 Nov 5 0:00 -6:00 - CST 2001 Apr 1 3:00 -7:00 Canada M%sT -Zone America/Yellowknife 0 - -00 1935 # Yellowknife founded? - -7:00 NT_YK M%sT 1980 - -7:00 Canada M%sT Zone America/Inuvik 0 - -00 1953 # Inuvik founded -8:00 NT_YK P%sT 1979 Apr lastSun 2:00 -7:00 NT_YK M%sT 1980 @@ -2584,7 +2581,7 @@ Zone America/Dawson -9:17:40 - LMT 1900 Aug 20 # and in addition changes all of Chihuahua to -06 with no DST. # From Heitor David Pinto (2022-11-28): -# Now the northern municipalities want to have the same time zone as the +# Now the northern [municipios] want to have the same time zone as the # respective neighboring cities in the US, for example Juárez in UTC-7 with # DST, matching El Paso, and Ojinaga in UTC-6 with DST, matching Presidio.... # the president authorized the publication of the decree for November 29, @@ -2621,7 +2618,7 @@ Zone America/Merida -5:58:28 - LMT 1922 Jan 1 6:00u -5:00 - EST 1982 Dec 2 -6:00 Mexico C%sT # Coahuila, Nuevo León, Tamaulipas (near US border) -# This includes the following municipalities: +# This includes the following municipios: # in Coahuila: Acuña, Allende, Guerrero, Hidalgo, Jiménez, Morelos, Nava, # Ocampo, Piedras Negras, Villa Unión, Zaragoza # in Nuevo León: Anáhuac @@ -2647,8 +2644,8 @@ Zone America/Mexico_City -6:36:36 - LMT 1922 Jan 1 7:00u -6:00 - CST 2002 Feb 20 -6:00 Mexico C%sT # Chihuahua (near US border - western side) -# This includes the municipalities of Janos, Ascensión, Juárez, Guadalupe, -# and Práxedis G Guerrero. +# This includes the municipios of Janos, Ascensión, Juárez, Guadalupe, and +# Práxedis G Guerrero. # http://gaceta.diputados.gob.mx/PDF/65/2a022/nov/20221124-VII.pdf Zone America/Ciudad_Juarez -7:05:56 - LMT 1922 Jan 1 7:00u -7:00 - MST 1927 Jun 10 23:00 @@ -2662,7 +2659,8 @@ Zone America/Ciudad_Juarez -7:05:56 - LMT 1922 Jan 1 7:00u -6:00 - CST 2022 Nov 30 0:00 -7:00 US M%sT # Chihuahua (near US border - eastern side) -# The municipalities of Coyame del Sotol, Ojinaga, and Manuel Benavides. +# This includes the municipios of Coyame del Sotol, Ojinaga, and Manuel +# Benavides. # http://gaceta.diputados.gob.mx/PDF/65/2a022/nov/20221124-VII.pdf Zone America/Ojinaga -6:57:40 - LMT 1922 Jan 1 7:00u -7:00 - MST 1927 Jun 10 23:00 @@ -3083,7 +3081,7 @@ Zone America/Costa_Rica -5:36:13 - LMT 1890 # San José # # He supplied these references: # -# http://www.prensalatina.com.mx/article.asp?ID=%7B4CC32C1B-A9F7-42FB-8A07-8631AFC923AF%7D&language=ES +# http://www.prensalatina.com.mx/article.asp?ID={4CC32C1B-A9F7-42FB-8A07-8631AFC923AF}&language=ES # http://actualidad.terra.es/sociedad/articulo/cuba_llama_ahorrar_energia_cambio_1957044.htm # # From Alex Krivenyshev (2007-10-25): diff --git a/make/data/tzdata/southamerica b/make/data/tzdata/southamerica index 81fdd793df43a..4024e7180cdb9 100644 --- a/make/data/tzdata/southamerica +++ b/make/data/tzdata/southamerica @@ -231,7 +231,7 @@ Rule Arg 2008 only - Oct Sun>=15 0:00 1:00 - # Hora de verano para la República Argentina # http://buenasiembra.com.ar/esoterismo/astrologia/hora-de-verano-de-la-republica-argentina-27.html # says that standard time in Argentina from 1894-10-31 -# to 1920-05-01 was -4:16:48.25. Go with this more-precise value +# to 1920-05-01 was -4:16:48.25. Go with this more precise value # over Shanks & Pottenger. It is upward compatible with Milne, who # says Córdoba time was -4:16:48.2. diff --git a/make/data/tzdata/zone.tab b/make/data/tzdata/zone.tab index 939432d3456fb..3edb0d61c8099 100644 --- a/make/data/tzdata/zone.tab +++ b/make/data/tzdata/zone.tab @@ -144,9 +144,8 @@ CA +744144-0944945 America/Resolute Central - NU (Resolute) CA +624900-0920459 America/Rankin_Inlet Central - NU (central) CA +5024-10439 America/Regina CST - SK (most areas) CA +5017-10750 America/Swift_Current CST - SK (midwest) -CA +5333-11328 America/Edmonton Mountain - AB; BC (E); SK (W) +CA +5333-11328 America/Edmonton Mountain - AB; BC (E); NT (E); SK (W) CA +690650-1050310 America/Cambridge_Bay Mountain - NU (west) -CA +6227-11421 America/Yellowknife Mountain - NT (central) CA +682059-1334300 America/Inuvik Mountain - NT (west) CA +4906-11631 America/Creston MST - BC (Creston) CA +5546-12014 America/Dawson_Creek MST - BC (Dawson Cr, Ft St John) @@ -162,7 +161,7 @@ CG -0416+01517 Africa/Brazzaville CH +4723+00832 Europe/Zurich CI +0519-00402 Africa/Abidjan CK -2114-15946 Pacific/Rarotonga -CL -3327-07040 America/Santiago Chile (most areas) +CL -3327-07040 America/Santiago most of Chile CL -5309-07055 America/Punta_Arenas Region of Magallanes CL -2709-10926 Pacific/Easter Easter Island CM +0403+00942 Africa/Douala @@ -174,10 +173,10 @@ CU +2308-08222 America/Havana CV +1455-02331 Atlantic/Cape_Verde CW +1211-06900 America/Curacao CX -1025+10543 Indian/Christmas -CY +3510+03322 Asia/Nicosia Cyprus (most areas) +CY +3510+03322 Asia/Nicosia most of Cyprus CY +3507+03357 Asia/Famagusta Northern Cyprus CZ +5005+01426 Europe/Prague -DE +5230+01322 Europe/Berlin Germany (most areas) +DE +5230+01322 Europe/Berlin most of Germany DE +4742+00841 Europe/Busingen Busingen DJ +1136+04309 Africa/Djibouti DK +5540+01235 Europe/Copenhagen @@ -210,7 +209,7 @@ GF +0456-05220 America/Cayenne GG +492717-0023210 Europe/Guernsey GH +0533-00013 Africa/Accra GI +3608-00521 Europe/Gibraltar -GL +6411-05144 America/Nuuk Greenland (most areas) +GL +6411-05144 America/Nuuk most of Greenland GL +7646-01840 America/Danmarkshavn National Park (east coast) GL +7029-02158 America/Scoresbysund Scoresbysund/Ittoqqortoormiit GL +7634-06847 America/Thule Thule/Pituffik @@ -258,7 +257,7 @@ KP +3901+12545 Asia/Pyongyang KR +3733+12658 Asia/Seoul KW +2920+04759 Asia/Kuwait KY +1918-08123 America/Cayman -KZ +4315+07657 Asia/Almaty Kazakhstan (most areas) +KZ +4315+07657 Asia/Almaty most of Kazakhstan KZ +4448+06528 Asia/Qyzylorda Qyzylorda/Kyzylorda/Kzyl-Orda KZ +5312+06337 Asia/Qostanay Qostanay/Kostanay/Kustanay KZ +5017+05710 Asia/Aqtobe Aqtobe/Aktobe @@ -282,12 +281,12 @@ MD +4700+02850 Europe/Chisinau ME +4226+01916 Europe/Podgorica MF +1804-06305 America/Marigot MG -1855+04731 Indian/Antananarivo -MH +0709+17112 Pacific/Majuro Marshall Islands (most areas) +MH +0709+17112 Pacific/Majuro most of Marshall Islands MH +0905+16720 Pacific/Kwajalein Kwajalein MK +4159+02126 Europe/Skopje ML +1239-00800 Africa/Bamako MM +1647+09610 Asia/Yangon -MN +4755+10653 Asia/Ulaanbaatar Mongolia (most areas) +MN +4755+10653 Asia/Ulaanbaatar most of Mongolia MN +4801+09139 Asia/Hovd Bayan-Olgiy, Govi-Altai, Hovd, Uvs, Zavkhan MN +4804+11430 Asia/Choibalsan Dornod, Sukhbaatar MO +221150+1133230 Asia/Macau @@ -325,7 +324,7 @@ NO +5955+01045 Europe/Oslo NP +2743+08519 Asia/Kathmandu NR -0031+16655 Pacific/Nauru NU -1901-16955 Pacific/Niue -NZ -3652+17446 Pacific/Auckland New Zealand (most areas) +NZ -3652+17446 Pacific/Auckland most of New Zealand NZ -4357-17633 Pacific/Chatham Chatham Islands OM +2336+05835 Asia/Muscat PA +0858-07932 America/Panama @@ -333,7 +332,7 @@ PE -1203-07703 America/Lima PF -1732-14934 Pacific/Tahiti Society Islands PF -0900-13930 Pacific/Marquesas Marquesas Islands PF -2308-13457 Pacific/Gambier Gambier Islands -PG -0930+14710 Pacific/Port_Moresby Papua New Guinea (most areas) +PG -0930+14710 Pacific/Port_Moresby most of Papua New Guinea PG -0613+15534 Pacific/Bougainville Bougainville PH +1435+12100 Asia/Manila PK +2452+06703 Asia/Karachi @@ -379,7 +378,7 @@ RU +4310+13156 Asia/Vladivostok MSK+07 - Amur River RU +643337+1431336 Asia/Ust-Nera MSK+07 - Oymyakonsky RU +5934+15048 Asia/Magadan MSK+08 - Magadan RU +4658+14242 Asia/Sakhalin MSK+08 - Sakhalin Island -RU +6728+15343 Asia/Srednekolymsk MSK+08 - Sakha (E); North Kuril Is +RU +6728+15343 Asia/Srednekolymsk MSK+08 - Sakha (E); N Kuril Is RU +5301+15839 Asia/Kamchatka MSK+09 - Kamchatka RU +6445+17729 Asia/Anadyr MSK+09 - Bering Sea RW -0157+03004 Africa/Kigali @@ -420,7 +419,7 @@ TT +1039-06131 America/Port_of_Spain TV -0831+17913 Pacific/Funafuti TW +2503+12130 Asia/Taipei TZ -0648+03917 Africa/Dar_es_Salaam -UA +5026+03031 Europe/Kyiv Ukraine (most areas) +UA +5026+03031 Europe/Kyiv most of Ukraine UG +0019+03225 Africa/Kampala UM +2813-17722 Pacific/Midway Midway Islands UM +1917+16637 Pacific/Wake Wake Island @@ -443,7 +442,7 @@ US +465042-1012439 America/North_Dakota/New_Salem Central - ND (Morton rural) US +471551-1014640 America/North_Dakota/Beulah Central - ND (Mercer) US +394421-1045903 America/Denver Mountain (most areas) US +433649-1161209 America/Boise Mountain - ID (south); OR (east) -US +332654-1120424 America/Phoenix MST - Arizona (except Navajo) +US +332654-1120424 America/Phoenix MST - AZ (except Navajo) US +340308-1181434 America/Los_Angeles Pacific US +611305-1495401 America/Anchorage Alaska (most areas) US +581807-1342511 America/Juneau Alaska - Juneau area @@ -451,7 +450,7 @@ US +571035-1351807 America/Sitka Alaska - Sitka area US +550737-1313435 America/Metlakatla Alaska - Annette Island US +593249-1394338 America/Yakutat Alaska - Yakutat US +643004-1652423 America/Nome Alaska (west) -US +515248-1763929 America/Adak Aleutian Islands +US +515248-1763929 America/Adak Alaska - western Aleutians US +211825-1575130 Pacific/Honolulu Hawaii UY -345433-0561245 America/Montevideo UZ +3940+06648 Asia/Samarkand Uzbekistan (west) diff --git a/make/devkit/Tools.gmk b/make/devkit/Tools.gmk index 19eccf89be2ac..e94a74d0063e1 100644 --- a/make/devkit/Tools.gmk +++ b/make/devkit/Tools.gmk @@ -87,8 +87,17 @@ endif # Define external dependencies # Latest that could be made to work. -GCC_VER := 10.3.0 -ifeq ($(GCC_VER), 10.3.0) +GCC_VER := 11.2.0 +ifeq ($(GCC_VER), 11.2.0) + gcc_ver := gcc-11.2.0 + binutils_ver := binutils-2.37 + ccache_ver := ccache-3.7.12 + mpfr_ver := mpfr-4.1.0 + gmp_ver := gmp-6.2.1 + mpc_ver := mpc-1.2.1 + gdb_ver := gdb-11.1 + REQUIRED_MIN_MAKE_MAJOR_VERSION := 4 +else ifeq ($(GCC_VER), 10.3.0) gcc_ver := gcc-10.3.0 binutils_ver := binutils-2.36.1 ccache_ver := ccache-3.7.11 diff --git a/make/hotspot/gensrc/GensrcAdlc.gmk b/make/hotspot/gensrc/GensrcAdlc.gmk index 362184b7c820c..a3969fc6988a9 100644 --- a/make/hotspot/gensrc/GensrcAdlc.gmk +++ b/make/hotspot/gensrc/GensrcAdlc.gmk @@ -1,5 +1,5 @@ # -# Copyright (c) 2013, 2021, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2013, 2022, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -149,6 +149,13 @@ ifeq ($(call check-jvm-feature, compiler2), true) ))) endif + ifeq ($(HOTSPOT_TARGET_CPU_ARCH), riscv) + AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \ + $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/$(HOTSPOT_TARGET_CPU_ARCH)_v.ad \ + $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/$(HOTSPOT_TARGET_CPU_ARCH)_b.ad \ + ))) + endif + ifeq ($(call check-jvm-feature, shenandoahgc), true) AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \ $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/shenandoah/shenandoah_$(HOTSPOT_TARGET_CPU).ad \ diff --git a/make/hotspot/lib/CompileGtest.gmk b/make/hotspot/lib/CompileGtest.gmk index cb2bbccc1686a..f16b9a747bcc4 100644 --- a/make/hotspot/lib/CompileGtest.gmk +++ b/make/hotspot/lib/CompileGtest.gmk @@ -1,5 +1,5 @@ # -# Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -49,7 +49,7 @@ $(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBGTEST, \ $(GTEST_FRAMEWORK_SRC)/googletest/src \ $(GTEST_FRAMEWORK_SRC)/googlemock/src, \ INCLUDE_FILES := gtest-all.cc gmock-all.cc, \ - DISABLED_WARNINGS_gcc := undef unused-result format-nonliteral, \ + DISABLED_WARNINGS_gcc := undef unused-result format-nonliteral maybe-uninitialized, \ DISABLED_WARNINGS_clang := undef unused-result format-nonliteral, \ CFLAGS := $(JVM_CFLAGS) \ -I$(GTEST_FRAMEWORK_SRC)/googletest \ diff --git a/make/jdk/src/classes/build/tools/charsetmapping/SPI.java b/make/jdk/src/classes/build/tools/charsetmapping/SPI.java index 3ed09fd938bcf..15e0b9fcf68e4 100644 --- a/make/jdk/src/classes/build/tools/charsetmapping/SPI.java +++ b/make/jdk/src/classes/build/tools/charsetmapping/SPI.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -50,18 +50,19 @@ public static void genClass(String type, out.println(line); } else { charsets.values() - .stream() - .filter(cs -> cs.pkgName.equals("sun.nio.cs.ext") && - !cs.isInternal && - (cs.os == null || cs.os.equals(os))) - .forEach( cs -> { - out.printf(" charset(\"%s\", \"%s\",%n", cs.csName, cs.clzName); - out.printf(" new String[] {%n"); - for (String alias : cs.aliases) { - out.printf(" \"%s\",%n", alias); - } - out.printf(" });%n%n"); - }); + .stream() + .filter(cs -> cs.pkgName.equals("sun.nio.cs.ext") && + !cs.isInternal && + (cs.os == null || cs.os.equals(os))) + .forEach( cs -> { + out.printf(" charset(\"%s\", \"%s\",%n", cs.csName, cs.clzName); + out.printf(" new String[] {%n"); + for (String alias : cs.aliases) { + out.printf(" \"%s\",%n", + alias); + } + out.printf(" });%n%n"); + }); } } } else if (type.startsWith("stdcs")) { // StandardCharsets.java @@ -93,8 +94,15 @@ public static void genClass(String type, .filter(cs -> cs.pkgName.equals("sun.nio.cs")) .forEach( cs -> { if (cs.aliases == null || cs.aliases.length == 0) { - out.printf(" static String[] aliases_%s() { return null; }%n%n", - cs.clzName); + if (cs.csName.equals("GB18030")) { + out.printf(" static String[] aliases_GB18030() { return new String[] {%n"); + out.printf(" GB18030.IS_2000 ? \"gb18030-2000\" : \"gb18030-2022\"%n"); + out.printf(" };%n"); + out.printf(" }%n%n"); + } else { + out.printf(" static String[] aliases_%s() { return null; }%n%n", + cs.clzName); + } } else { boolean methodEnd = true; // non-final for SJIS and MS932 to support sun.nio.cs.map diff --git a/make/jdk/src/classes/build/tools/cldrconverter/Bundle.java b/make/jdk/src/classes/build/tools/cldrconverter/Bundle.java index 9f262fa010608..e7874e6e3b60a 100644 --- a/make/jdk/src/classes/build/tools/cldrconverter/Bundle.java +++ b/make/jdk/src/classes/build/tools/cldrconverter/Bundle.java @@ -197,9 +197,7 @@ Map getTargetMap() throws Exception { // parentsMap contains resources from id's parents. Map parentsMap = new HashMap<>(); for (int i = cldrBundles.length - 1; i > index; i--) { - if (!("no".equals(cldrBundles[i]) || cldrBundles[i].startsWith("no_"))) { - parentsMap.putAll(CLDRConverter.getCLDRBundle(cldrBundles[i])); - } + parentsMap.putAll(CLDRConverter.getCLDRBundle(cldrBundles[i])); } // Duplicate myMap as parentsMap for "root" so that the // fallback works. This is a hack, though. diff --git a/make/modules/java.base/lib/CoreLibraries.gmk b/make/modules/java.base/lib/CoreLibraries.gmk index e7188218df37d..bb09d8cf8a2ee 100644 --- a/make/modules/java.base/lib/CoreLibraries.gmk +++ b/make/modules/java.base/lib/CoreLibraries.gmk @@ -49,6 +49,7 @@ $(eval $(call SetupNativeCompilation, BUILD_LIBFDLIBM, \ CFLAGS_windows_debug := -DLOGGING, \ CFLAGS_aix := -qfloat=nomaf, \ DISABLED_WARNINGS_gcc := sign-compare misleading-indentation array-bounds, \ + DISABLED_WARNINGS_gcc_k_rem_pio2.c := maybe-uninitialized, \ DISABLED_WARNINGS_clang := sign-compare misleading-indentation, \ DISABLED_WARNINGS_microsoft := 4146 4244 4018, \ ARFLAGS := $(ARFLAGS), \ diff --git a/make/modules/java.desktop/lib/Awt2dLibraries.gmk b/make/modules/java.desktop/lib/Awt2dLibraries.gmk index 918a5884851e9..03d6c9c20e58a 100644 --- a/make/modules/java.desktop/lib/Awt2dLibraries.gmk +++ b/make/modules/java.desktop/lib/Awt2dLibraries.gmk @@ -265,7 +265,7 @@ endif ################################################################################ # The fast floor code loses precision. -LCMS_CFLAGS=-DCMS_DONT_USE_FAST_FLOOR +LCMS_CFLAGS=-DCMS_DONT_USE_FAST_FLOOR -DCMS_NO_HALF_SUPPORT ifeq ($(USE_EXTERNAL_LCMS), true) # If we're using an external library, we'll just need the wrapper part. @@ -446,7 +446,7 @@ else -DHB_NO_PRAGMA_GCC_DIAGNOSTIC endif ifeq ($(call isTargetOs, linux macosx), true) - HARFBUZZ_CFLAGS += -DHAVE_INTEL_ATOMIC_PRIMITIVES + HARFBUZZ_CFLAGS += -DHAVE_INTEL_ATOMIC_PRIMITIVES -DHB_NO_VISIBILITY endif # Early re-canonizing has to be disabled to workaround an internal XlC compiler error @@ -458,9 +458,13 @@ else # hb-ft.cc is not presently needed, and requires freetype 2.4.2 or later. LIBFONTMANAGER_EXCLUDE_FILES += libharfbuzz/hb-ft.cc - HARFBUZZ_DISABLED_WARNINGS_gcc := type-limits missing-field-initializers strict-aliasing + HARFBUZZ_DISABLED_WARNINGS_gcc := type-limits missing-field-initializers strict-aliasing \ + array-bounds + # noexcept-type required for GCC 7 builds. Not required for GCC 8+. + # expansion-to-defined required for GCC 9 builds. Not required for GCC 10+. HARFBUZZ_DISABLED_WARNINGS_CXX_gcc := reorder delete-non-virtual-dtor strict-overflow \ - maybe-uninitialized class-memaccess unused-result extra use-after-free + maybe-uninitialized class-memaccess unused-result extra use-after-free noexcept-type \ + expansion-to-defined dangling-reference HARFBUZZ_DISABLED_WARNINGS_clang := unused-value incompatible-pointer-types \ tautological-constant-out-of-range-compare int-to-pointer-cast \ undef missing-field-initializers range-loop-analysis \ diff --git a/make/modules/jdk.internal.vm.ci/Java.gmk b/make/modules/jdk.internal.vm.ci/Java.gmk index ad4991e541429..8720c97a36b92 100644 --- a/make/modules/jdk.internal.vm.ci/Java.gmk +++ b/make/modules/jdk.internal.vm.ci/Java.gmk @@ -26,13 +26,5 @@ # -parameters provides method's parameters information in class file, # JVMCI compilers make use of that information for various sanity checks. # Don't use Indy strings concatenation to have good JVMCI startup performance. -# The exports are needed since JVMCI is dynamically exported (see -# jdk.vm.ci.services.internal.ReflectionAccessJDK::openJVMCITo). JAVAC_FLAGS += -parameters -XDstringConcat=inline - -## WORKAROUND jdk.internal.vm.ci source structure issue -JVMCI_MODULESOURCEPATH := $(MODULESOURCEPATH) \ - $(subst /$(MODULE)/,/*/, $(filter-out %processor/src, \ - $(wildcard $(TOPDIR)/src/$(MODULE)/share/classes/*/src))) -MODULESOURCEPATH := $(call PathList, $(JVMCI_MODULESOURCEPATH)) diff --git a/make/test/JtregNativeHotspot.gmk b/make/test/JtregNativeHotspot.gmk index 58390110251d4..a55cce7ab08bc 100644 --- a/make/test/JtregNativeHotspot.gmk +++ b/make/test/JtregNativeHotspot.gmk @@ -871,7 +871,7 @@ BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exesigtest := -ljvm ifeq ($(call isTargetOs, windows), true) BUILD_HOTSPOT_JTREG_EXECUTABLES_CFLAGS_exeFPRegs := -MT - BUILD_HOTSPOT_JTREG_EXCLUDE += exesigtest.c libterminatedThread.c libTestJNI.c + BUILD_HOTSPOT_JTREG_EXCLUDE += exesigtest.c libterminatedThread.c libTestJNI.c libnativeStack.c BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libatExit := jvm.lib else BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libbootclssearch_agent += -lpthread @@ -1508,6 +1508,7 @@ else BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libgetphase002 += -lpthread BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libterminatedThread += -lpthread BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libatExit += -ljvm + BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libnativeStack += -lpthread endif # This evaluation is expensive and should only be done if this target was diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index 91ea50c002eff..bf03fbe0303a4 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -8646,6 +8646,7 @@ instruct membar_release() %{ instruct membar_storestore() %{ match(MemBarStoreStore); + match(StoreStoreFence); ins_cost(VOLATILE_REF_COST); format %{ "MEMBAR-store-store" %} @@ -11377,7 +11378,6 @@ instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{ // BEGIN This section of the file is automatically generated. Do not edit -------------- // This section is generated from aarch64_ad.m4 - // This pattern is automatically generated from aarch64_ad.m4 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct regL_not_reg(iRegLNoSp dst, @@ -13193,6 +13193,7 @@ instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk) // Rotations + // This pattern is automatically generated from aarch64_ad.m4 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr) @@ -13264,7 +13265,6 @@ instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, ins_pipe(ialu_reg_reg_extr); %} - // This pattern is automatically generated from aarch64_ad.m4 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift) @@ -13978,6 +13978,298 @@ instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, ins_pipe(ialu_reg_reg_shift); %} +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr) +%{ + effect(DEF dst, USE src1, USE src2, USE cr); + ins_cost(INSN_COST * 2); + format %{ "cselw $dst, $src1, $src2 lt\t" %} + + ins_encode %{ + __ cselw($dst$$Register, + $src1$$Register, + $src2$$Register, + Assembler::LT); + %} + ins_pipe(icond_reg_reg); +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr) +%{ + effect(DEF dst, USE src1, USE src2, USE cr); + ins_cost(INSN_COST * 2); + format %{ "cselw $dst, $src1, $src2 gt\t" %} + + ins_encode %{ + __ cselw($dst$$Register, + $src1$$Register, + $src2$$Register, + Assembler::GT); + %} + ins_pipe(icond_reg_reg); +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr) +%{ + effect(DEF dst, USE src1, USE cr); + ins_cost(INSN_COST * 2); + format %{ "cselw $dst, $src1, zr lt\t" %} + + ins_encode %{ + __ cselw($dst$$Register, + $src1$$Register, + zr, + Assembler::LT); + %} + ins_pipe(icond_reg); +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr) +%{ + effect(DEF dst, USE src1, USE cr); + ins_cost(INSN_COST * 2); + format %{ "cselw $dst, $src1, zr gt\t" %} + + ins_encode %{ + __ cselw($dst$$Register, + $src1$$Register, + zr, + Assembler::GT); + %} + ins_pipe(icond_reg); +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr) +%{ + effect(DEF dst, USE src1, USE cr); + ins_cost(INSN_COST * 2); + format %{ "csincw $dst, $src1, zr le\t" %} + + ins_encode %{ + __ csincw($dst$$Register, + $src1$$Register, + zr, + Assembler::LE); + %} + ins_pipe(icond_reg); +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr) +%{ + effect(DEF dst, USE src1, USE cr); + ins_cost(INSN_COST * 2); + format %{ "csincw $dst, $src1, zr gt\t" %} + + ins_encode %{ + __ csincw($dst$$Register, + $src1$$Register, + zr, + Assembler::GT); + %} + ins_pipe(icond_reg); +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr) +%{ + effect(DEF dst, USE src1, USE cr); + ins_cost(INSN_COST * 2); + format %{ "csinvw $dst, $src1, zr lt\t" %} + + ins_encode %{ + __ csinvw($dst$$Register, + $src1$$Register, + zr, + Assembler::LT); + %} + ins_pipe(icond_reg); +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr) +%{ + effect(DEF dst, USE src1, USE cr); + ins_cost(INSN_COST * 2); + format %{ "csinvw $dst, $src1, zr ge\t" %} + + ins_encode %{ + __ csinvw($dst$$Register, + $src1$$Register, + zr, + Assembler::GE); + %} + ins_pipe(icond_reg); +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm) +%{ + match(Set dst (MinI src imm)); + ins_cost(INSN_COST * 3); + expand %{ + rFlagsReg cr; + compI_reg_imm0(cr, src); + cmovI_reg_imm0_lt(dst, src, cr); + %} +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src) +%{ + match(Set dst (MinI imm src)); + ins_cost(INSN_COST * 3); + expand %{ + rFlagsReg cr; + compI_reg_imm0(cr, src); + cmovI_reg_imm0_lt(dst, src, cr); + %} +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm) +%{ + match(Set dst (MinI src imm)); + ins_cost(INSN_COST * 3); + expand %{ + rFlagsReg cr; + compI_reg_imm0(cr, src); + cmovI_reg_imm1_le(dst, src, cr); + %} +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src) +%{ + match(Set dst (MinI imm src)); + ins_cost(INSN_COST * 3); + expand %{ + rFlagsReg cr; + compI_reg_imm0(cr, src); + cmovI_reg_imm1_le(dst, src, cr); + %} +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm) +%{ + match(Set dst (MinI src imm)); + ins_cost(INSN_COST * 3); + expand %{ + rFlagsReg cr; + compI_reg_imm0(cr, src); + cmovI_reg_immM1_lt(dst, src, cr); + %} +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src) +%{ + match(Set dst (MinI imm src)); + ins_cost(INSN_COST * 3); + expand %{ + rFlagsReg cr; + compI_reg_imm0(cr, src); + cmovI_reg_immM1_lt(dst, src, cr); + %} +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm) +%{ + match(Set dst (MaxI src imm)); + ins_cost(INSN_COST * 3); + expand %{ + rFlagsReg cr; + compI_reg_imm0(cr, src); + cmovI_reg_imm0_gt(dst, src, cr); + %} +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src) +%{ + match(Set dst (MaxI imm src)); + ins_cost(INSN_COST * 3); + expand %{ + rFlagsReg cr; + compI_reg_imm0(cr, src); + cmovI_reg_imm0_gt(dst, src, cr); + %} +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm) +%{ + match(Set dst (MaxI src imm)); + ins_cost(INSN_COST * 3); + expand %{ + rFlagsReg cr; + compI_reg_imm0(cr, src); + cmovI_reg_imm1_gt(dst, src, cr); + %} +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src) +%{ + match(Set dst (MaxI imm src)); + ins_cost(INSN_COST * 3); + expand %{ + rFlagsReg cr; + compI_reg_imm0(cr, src); + cmovI_reg_imm1_gt(dst, src, cr); + %} +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm) +%{ + match(Set dst (MaxI src imm)); + ins_cost(INSN_COST * 3); + expand %{ + rFlagsReg cr; + compI_reg_imm0(cr, src); + cmovI_reg_immM1_ge(dst, src, cr); + %} +%} + +// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src) +%{ + match(Set dst (MaxI imm src)); + ins_cost(INSN_COST * 3); + expand %{ + rFlagsReg cr; + compI_reg_imm0(cr, src); + cmovI_reg_immM1_ge(dst, src, cr); + %} +%} + // END This section of the file is automatically generated. Do not edit -------------- @@ -15938,24 +16230,21 @@ instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg // ============================================================================ // Max and Min -instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr) -%{ - effect( DEF dst, USE src1, USE src2, USE cr ); +// Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter. - ins_cost(INSN_COST * 2); - format %{ "cselw $dst, $src1, $src2 lt\t" %} +instruct compI_reg_imm0(rFlagsReg cr, iRegI src) +%{ + effect(DEF cr, USE src); + ins_cost(INSN_COST); + format %{ "cmpw $src, 0" %} ins_encode %{ - __ cselw(as_Register($dst$$reg), - as_Register($src1$$reg), - as_Register($src2$$reg), - Assembler::LT); + __ cmpw($src$$Register, 0); %} - - ins_pipe(icond_reg_reg); + ins_pipe(icmp_reg_imm); %} -instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2) +instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{ match(Set dst (MinI src1 src2)); ins_cost(INSN_COST * 3); @@ -15965,31 +16254,13 @@ instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2) compI_reg_reg(cr, src1, src2); cmovI_reg_reg_lt(dst, src1, src2, cr); %} - %} -// FROM HERE -instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr) -%{ - effect( DEF dst, USE src1, USE src2, USE cr ); - - ins_cost(INSN_COST * 2); - format %{ "cselw $dst, $src1, $src2 gt\t" %} - - ins_encode %{ - __ cselw(as_Register($dst$$reg), - as_Register($src1$$reg), - as_Register($src2$$reg), - Assembler::GT); - %} - - ins_pipe(icond_reg_reg); -%} - -instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2) +instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{ match(Set dst (MaxI src1 src2)); ins_cost(INSN_COST * 3); + expand %{ rFlagsReg cr; compI_reg_reg(cr, src1, src2); @@ -15997,6 +16268,7 @@ instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2) %} %} + // ============================================================================ // Branch Instructions @@ -16769,14 +17041,17 @@ instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 %} instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2, - iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, - iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr) + iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, + iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, + vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr) %{ predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU); match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2))); effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, - TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr); - format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %} + TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, + TEMP vtmp0, TEMP vtmp1, KILL cr); + format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) " + "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %} ins_encode %{ __ string_indexof($str1$$Register, $str2$$Register, @@ -16790,14 +17065,17 @@ instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 %} instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2, - iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, - iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr) + iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, + iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, + vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr) %{ predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL); match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2))); effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, - TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr); - format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %} + TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, + TEMP vtmp0, TEMP vtmp1, KILL cr); + format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) " + "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %} ins_encode %{ __ string_indexof($str1$$Register, $str2$$Register, @@ -16811,14 +17089,17 @@ instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 %} instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2, - iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, - iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr) + iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3, + iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, + vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr) %{ predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL); match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2))); effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, - TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr); - format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %} + TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, + TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr); + format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) " + "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %} ins_encode %{ __ string_indexof($str1$$Register, $str2$$Register, @@ -16832,14 +17113,15 @@ instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 %} instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, - immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, - iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr) + immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, + iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr) %{ predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU); match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2))); effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr); - format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %} + format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) " + "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %} ins_encode %{ int icnt2 = (int)$int_cnt2$$constant; @@ -16853,14 +17135,15 @@ instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, %} instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, - immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, - iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr) + immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, + iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr) %{ predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL); match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2))); effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr); - format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %} + format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) " + "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %} ins_encode %{ int icnt2 = (int)$int_cnt2$$constant; @@ -16874,14 +17157,15 @@ instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, %} instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, - immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, - iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr) + immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, + iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr) %{ predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL); match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2))); effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr); - format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %} + format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) " + "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %} ins_encode %{ int icnt2 = (int)$int_cnt2$$constant; @@ -16966,13 +17250,17 @@ instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt, instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result, iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3, + vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3, + vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7, iRegP_R10 tmp, rFlagsReg cr) %{ predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL); match(Set result (AryEq ary1 ary2)); - effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); + effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, + TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, + TEMP vtmp6, TEMP vtmp7, KILL cr); - format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %} + format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %} ins_encode %{ address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register, $tmp1$$Register, $tmp2$$Register, $tmp3$$Register, @@ -16987,13 +17275,17 @@ instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result, instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result, iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3, + vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3, + vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7, iRegP_R10 tmp, rFlagsReg cr) %{ predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU); match(Set result (AryEq ary1 ary2)); - effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); + effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, + TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, + TEMP vtmp6, TEMP vtmp7, KILL cr); - format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %} + format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %} ins_encode %{ address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register, $tmp1$$Register, $tmp2$$Register, $tmp3$$Register, @@ -17023,35 +17315,39 @@ instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg c // fast char[] to byte[] compression instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len, - vRegD_V0 tmp1, vRegD_V1 tmp2, - vRegD_V2 tmp3, vRegD_V3 tmp4, + vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, + vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5, iRegI_R0 result, rFlagsReg cr) %{ match(Set result (StrCompressedCopy src (Binary dst len))); - effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr); + effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, + USE_KILL src, USE_KILL dst, USE len, KILL cr); - format %{ "String Compress $src,$dst -> $result // KILL R1, R2, R3, R4" %} + format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %} ins_encode %{ __ char_array_compress($src$$Register, $dst$$Register, $len$$Register, - $tmp1$$FloatRegister, $tmp2$$FloatRegister, - $tmp3$$FloatRegister, $tmp4$$FloatRegister, - $result$$Register); + $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister, + $vtmp2$$FloatRegister, $vtmp3$$FloatRegister, + $vtmp4$$FloatRegister, $vtmp5$$FloatRegister); %} - ins_pipe( pipe_slow ); + ins_pipe(pipe_slow); %} // fast byte[] to char[] inflation -instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, - vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr) +instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp, + vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3, + vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr) %{ match(Set dummy (StrInflatedCopy src (Binary dst len))); - effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr); + effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, + TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp, + USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr); - format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %} + format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %} ins_encode %{ address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register, - $tmp1$$FloatRegister, $tmp2$$FloatRegister, - $tmp3$$FloatRegister, $tmp4$$Register); + $vtmp0$$FloatRegister, $vtmp1$$FloatRegister, + $vtmp2$$FloatRegister, $tmp$$Register); if (tpc == NULL) { ciEnv::current()->record_failure("CodeCache is full"); return; @@ -17062,22 +17358,45 @@ instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len // encode char[] to byte[] in ISO_8859_1 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len, - vRegD_V0 Vtmp1, vRegD_V1 Vtmp2, - vRegD_V2 Vtmp3, vRegD_V3 Vtmp4, + vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, + vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5, iRegI_R0 result, rFlagsReg cr) %{ predicate(!((EncodeISOArrayNode*)n)->is_ascii()); match(Set result (EncodeISOArray src (Binary dst len))); - effect(USE_KILL src, USE_KILL dst, USE_KILL len, - KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr); + effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1, + KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr); - format %{ "Encode array $src,$dst,$len -> $result" %} + format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %} ins_encode %{ __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register, - $result$$Register, $Vtmp1$$FloatRegister, $Vtmp2$$FloatRegister, - $Vtmp3$$FloatRegister, $Vtmp4$$FloatRegister); + $result$$Register, false, + $vtmp0$$FloatRegister, $vtmp1$$FloatRegister, + $vtmp2$$FloatRegister, $vtmp3$$FloatRegister, + $vtmp4$$FloatRegister, $vtmp5$$FloatRegister); %} - ins_pipe( pipe_class_memory ); + ins_pipe(pipe_class_memory); +%} + +instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len, + vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, + vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5, + iRegI_R0 result, rFlagsReg cr) +%{ + predicate(((EncodeISOArrayNode*)n)->is_ascii()); + match(Set result (EncodeISOArray src (Binary dst len))); + effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1, + KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr); + + format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %} + ins_encode %{ + __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register, + $result$$Register, true, + $vtmp0$$FloatRegister, $vtmp1$$FloatRegister, + $vtmp2$$FloatRegister, $vtmp3$$FloatRegister, + $vtmp4$$FloatRegister, $vtmp5$$FloatRegister); + %} + ins_pipe(pipe_class_memory); %} // ============================================================================ diff --git a/src/hotspot/cpu/aarch64/aarch64_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_ad.m4 index ffb5c57195f3b..4b4d79a9b70e8 100644 --- a/src/hotspot/cpu/aarch64/aarch64_ad.m4 +++ b/src/hotspot/cpu/aarch64/aarch64_ad.m4 @@ -27,8 +27,10 @@ dnl 2. shift patterns dnl // BEGIN This section of the file is automatically generated. Do not edit -------------- // This section is generated from aarch64_ad.m4 -dnl -define(`ORL2I', `ifelse($1,I,orL2I)') + +define(`upcase', `translit(`$*', `a-z', `A-Z')')dnl +define(`downcase', `translit(`$*', `A-Z', `a-z')')dnl +define(`ORL2I', `ifelse($1,I,orL2I)')dnl dnl define(`BASE_SHIFT_INSN', `// This pattern is automatically generated from aarch64_ad.m4 @@ -189,7 +191,7 @@ ALL_SHIFT_KINDS_WITHOUT_ROR(Add, add) ALL_SHIFT_KINDS_WITHOUT_ROR(Sub, sub) dnl dnl EXTEND mode, rshift_op, src, lshift_count, rshift_count -define(`EXTEND', `($2$1 (LShift$1 $3 $4) $5)') dnl +define(`EXTEND', `($2$1 (LShift$1 $3 $4) $5)')dnl define(`BFM_INSN',`// This pattern is automatically generated from aarch64_ad.m4 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE @@ -212,7 +214,7 @@ instruct $4$1(iReg$1NoSp dst, iReg$1`'ORL2I($1) src, immI lshift_count, immI rsh ins_pipe(ialu_reg_shift); %} -') +')dnl BFM_INSN(L, 63, RShift, sbfm) BFM_INSN(I, 31, RShift, sbfmw) BFM_INSN(L, 63, URShift, ubfm) @@ -336,7 +338,7 @@ instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk) %} -// Rotations dnl +// Rotations define(`EXTRACT_INSN',` // This pattern is automatically generated from aarch64_ad.m4 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE @@ -539,4 +541,78 @@ dnl ADD_SUB_ZERO_EXTEND_SHIFT(I,255,Sub,subw,uxtb) ADD_SUB_ZERO_EXTEND_SHIFT(I,65535,Sub,subw,uxth) dnl +define(`CMOV_INSN', `// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct cmov$1_reg_reg_$3(iReg$1NoSp dst, iReg$1 src1, iReg$1 src2, rFlagsReg cr) +%{ + effect(DEF dst, USE src1, USE src2, USE cr); + ins_cost(INSN_COST * 2); + format %{ "$2 $dst, $src1, $src2 $3\t" %} + + ins_encode %{ + __ $2($dst$$Register, + $src1$$Register, + $src2$$Register, + Assembler::upcase($3)); + %} + ins_pipe(icond_reg_reg); +%} +')dnl +CMOV_INSN(I, cselw, lt) +CMOV_INSN(I, cselw, gt) +dnl +define(`CMOV_DRAW_INSN', `// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +instruct cmov$1_reg_imm$2_$4(iReg$1NoSp dst, iReg$1 src1, rFlagsReg cr) +%{ + effect(DEF dst, USE src1, USE cr); + ins_cost(INSN_COST * 2); + format %{ "$3 $dst, $src1, zr $4\t" %} + + ins_encode %{ + __ $3($dst$$Register, + $src1$$Register, + zr, + Assembler::upcase($4)); + %} + ins_pipe(icond_reg); +%} +')dnl +CMOV_DRAW_INSN(I, 0, cselw, lt) +CMOV_DRAW_INSN(I, 0, cselw, gt) +CMOV_DRAW_INSN(I, 1, csincw, le) +CMOV_DRAW_INSN(I, 1, csincw, gt) +CMOV_DRAW_INSN(I, M1, csinvw, lt) +CMOV_DRAW_INSN(I, M1, csinvw, ge) +dnl +define(`MINMAX_DRAW_INSN', `// This pattern is automatically generated from aarch64_ad.m4 +// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE +ifelse($6,, +instruct downcase($1)$2_reg_imm$4(iReg$2NoSp dst, iReg$2`'ORL2I($2) src, imm$2$3$4 imm), +instruct downcase($1)$2_imm$4_reg(iReg$2NoSp dst, imm$2$3$4 imm, iReg$2`'ORL2I($2) src)) +%{ + ifelse($6,, + match(Set dst ($1$2 src imm));, + match(Set dst ($1$2 imm src));) + ins_cost(INSN_COST * 3); + expand %{ + rFlagsReg cr; + comp$2_reg_imm0(cr, src); + cmov$2_reg_imm$4_$5(dst, src, cr); + %} +%} +')dnl +MINMAX_DRAW_INSN(Min, I, , 0, lt) +MINMAX_DRAW_INSN(Min, I, , 0, lt, rev) +MINMAX_DRAW_INSN(Min, I, _, 1, le) +MINMAX_DRAW_INSN(Min, I, _, 1, le, rev) +MINMAX_DRAW_INSN(Min, I, _, M1, lt) +MINMAX_DRAW_INSN(Min, I, _, M1, lt, rev) +dnl +MINMAX_DRAW_INSN(Max, I, , 0, gt) +MINMAX_DRAW_INSN(Max, I, , 0, gt, rev) +MINMAX_DRAW_INSN(Max, I, _, 1, gt) +MINMAX_DRAW_INSN(Max, I, _, 1, gt, rev) +MINMAX_DRAW_INSN(Max, I, _, M1, ge) +MINMAX_DRAW_INSN(Max, I, _, M1, ge, rev) diff --git a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp index 70f50dbe4ae94..1fea866cc1d29 100644 --- a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp @@ -2448,6 +2448,12 @@ void mvnw(Register Rd, Register Rm, INSN(cnt, 0, 0b100000010110, 0); // accepted arrangements: T8B, T16B INSN(uaddlp, 1, 0b100000001010, 2); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S INSN(uaddlv, 1, 0b110000001110, 1); // accepted arrangements: T8B, T16B, T4H, T8H, T4S + // Zero compare. + INSN(cmeq, 0, 0b100000100110, 3); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D + INSN(cmge, 1, 0b100000100010, 3); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D + INSN(cmgt, 0, 0b100000100010, 3); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D + INSN(cmle, 1, 0b100000100110, 3); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D + INSN(cmlt, 0, 0b100000101010, 3); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D #undef INSN diff --git a/src/hotspot/cpu/aarch64/atomic_aarch64.hpp b/src/hotspot/cpu/aarch64/atomic_aarch64.hpp index 6f9425e43ac14..f1e1f04c24422 100644 --- a/src/hotspot/cpu/aarch64/atomic_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/atomic_aarch64.hpp @@ -37,6 +37,8 @@ typedef uint64_t (*aarch64_atomic_stub_t)(volatile void *ptr, uint64_t arg1, uin // Pointers to stubs extern aarch64_atomic_stub_t aarch64_atomic_fetch_add_4_impl; extern aarch64_atomic_stub_t aarch64_atomic_fetch_add_8_impl; +extern aarch64_atomic_stub_t aarch64_atomic_fetch_add_4_relaxed_impl; +extern aarch64_atomic_stub_t aarch64_atomic_fetch_add_8_relaxed_impl; extern aarch64_atomic_stub_t aarch64_atomic_xchg_4_impl; extern aarch64_atomic_stub_t aarch64_atomic_xchg_8_impl; extern aarch64_atomic_stub_t aarch64_atomic_cmpxchg_1_impl; diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp index 5ce3ecf9e7ab6..d4de8aecc8bcf 100644 --- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp @@ -984,14 +984,7 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch __ ldr(dest->as_register(), as_Address(from_addr)); break; case T_ADDRESS: - // FIXME: OMG this is a horrible kludge. Any offset from an - // address that matches klass_offset_in_bytes() will be loaded - // as a word, not a long. - if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) { - __ ldrw(dest->as_register(), as_Address(from_addr)); - } else { - __ ldr(dest->as_register(), as_Address(from_addr)); - } + __ ldr(dest->as_register(), as_Address(from_addr)); break; case T_INT: __ ldrw(dest->as_register(), as_Address(from_addr)); @@ -1030,10 +1023,6 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch // Load barrier has not yet been applied, so ZGC can't verify the oop here __ verify_oop(dest->as_register()); } - } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) { - if (UseCompressedClassPointers) { - __ decode_klass_not_null(dest->as_register()); - } } } @@ -2595,6 +2584,22 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) { __ bind(*op->stub()->continuation()); } +void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { + Register obj = op->obj()->as_pointer_register(); + Register result = op->result_opr()->as_pointer_register(); + + CodeEmitInfo* info = op->info(); + if (info != NULL) { + add_debug_info_for_null_check_here(info); + } + + if (UseCompressedClassPointers) { + __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes())); + __ decode_klass_not_null(result); + } else { + __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes())); + } +} void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { ciMethod* method = op->profiled_method(); diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp index 9da93c99680a2..9a8d8c7625327 100644 --- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp @@ -43,6 +43,7 @@ typedef void (MacroAssembler::* chr_insn)(Register Rt, const Address &adr); // Search for str1 in str2 and return index or -1 +// Clobbers: rscratch1, rscratch2, rflags. May also clobber v0-v1, when icnt1==-1. void C2_MacroAssembler::string_indexof(Register str2, Register str1, Register cnt2, Register cnt1, Register tmp1, Register tmp2, diff --git a/src/hotspot/cpu/aarch64/jvmciCodeInstaller_aarch64.cpp b/src/hotspot/cpu/aarch64/jvmciCodeInstaller_aarch64.cpp index 17b9780129db0..c970070c45a6a 100644 --- a/src/hotspot/cpu/aarch64/jvmciCodeInstaller_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/jvmciCodeInstaller_aarch64.cpp @@ -138,21 +138,21 @@ void CodeInstaller::pd_relocate_JavaMethod(CodeBuffer &cbuf, JVMCIObject hotspot assert(method == NULL || !method->is_static(), "cannot call static method with invokeinterface"); NativeCall* call = nativeCall_at(_instructions->start() + pc_offset); _instructions->relocate(call->instruction_address(), virtual_call_Relocation::spec(_invoke_mark_pc)); - call->trampoline_jump(cbuf, SharedRuntime::get_resolve_virtual_call_stub()); + call->trampoline_jump(cbuf, SharedRuntime::get_resolve_virtual_call_stub(), JVMCI_CHECK); break; } case INVOKESTATIC: { assert(method == NULL || method->is_static(), "cannot call non-static method with invokestatic"); NativeCall* call = nativeCall_at(_instructions->start() + pc_offset); _instructions->relocate(call->instruction_address(), relocInfo::static_call_type); - call->trampoline_jump(cbuf, SharedRuntime::get_resolve_static_call_stub()); + call->trampoline_jump(cbuf, SharedRuntime::get_resolve_static_call_stub(), JVMCI_CHECK); break; } case INVOKESPECIAL: { assert(method == NULL || !method->is_static(), "cannot call static method with invokespecial"); NativeCall* call = nativeCall_at(_instructions->start() + pc_offset); _instructions->relocate(call->instruction_address(), relocInfo::opt_virtual_call_type); - call->trampoline_jump(cbuf, SharedRuntime::get_resolve_opt_virtual_call_stub()); + call->trampoline_jump(cbuf, SharedRuntime::get_resolve_opt_virtual_call_stub(), JVMCI_CHECK); break; } default: diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index 676a548d0399a..a35b8b844817b 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -4514,6 +4514,8 @@ address MacroAssembler::has_negatives(Register ary1, Register len, Register resu return pc(); } +// Clobbers: rscratch1, rscratch2, rflags +// May also clobber v0-v7 when (!UseSimpleArrayEquals && UseSIMDForArrayEquals) address MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3, Register tmp4, Register tmp5, Register result, Register cnt1, int elem_size) { @@ -5031,113 +5033,123 @@ void MacroAssembler::fill_words(Register base, Register cnt, Register value) bind(fini); } -// Intrinsic for sun/nio/cs/ISO_8859_1$Encoder.implEncodeISOArray and -// java/lang/StringUTF16.compress. +// Intrinsic for +// +// - sun/nio/cs/ISO_8859_1$Encoder.implEncodeISOArray +// return the number of characters copied. +// - java/lang/StringUTF16.compress +// return zero (0) if copy fails, otherwise 'len'. +// +// This version always returns the number of characters copied, and does not +// clobber the 'len' register. A successful copy will complete with the post- +// condition: 'res' == 'len', while an unsuccessful copy will exit with the +// post-condition: 0 <= 'res' < 'len'. +// +// NOTE: Attempts to use 'ld2' (and 'umaxv' in the ISO part) has proven to +// degrade performance (on Ampere Altra - Neoverse N1), to an extent +// beyond the acceptable, even though the footprint would be smaller. +// Using 'umaxv' in the ASCII-case comes with a small penalty but does +// avoid additional bloat. +// +// Clobbers: src, dst, res, rscratch1, rscratch2, rflags void MacroAssembler::encode_iso_array(Register src, Register dst, - Register len, Register result, - FloatRegister Vtmp1, FloatRegister Vtmp2, - FloatRegister Vtmp3, FloatRegister Vtmp4) + Register len, Register res, bool ascii, + FloatRegister vtmp0, FloatRegister vtmp1, + FloatRegister vtmp2, FloatRegister vtmp3, + FloatRegister vtmp4, FloatRegister vtmp5) { - Label DONE, SET_RESULT, NEXT_32, NEXT_32_PRFM, LOOP_8, NEXT_8, LOOP_1, NEXT_1, - NEXT_32_START, NEXT_32_PRFM_START; - Register tmp1 = rscratch1, tmp2 = rscratch2; - - mov(result, len); // Save initial len - - cmp(len, (u1)8); // handle shortest strings first - br(LT, LOOP_1); - cmp(len, (u1)32); - br(LT, NEXT_8); - // The following code uses the SIMD 'uzp1' and 'uzp2' instructions - // to convert chars to bytes - if (SoftwarePrefetchHintDistance >= 0) { - ld1(Vtmp1, Vtmp2, Vtmp3, Vtmp4, T8H, src); - subs(tmp2, len, SoftwarePrefetchHintDistance/2 + 16); - br(LE, NEXT_32_START); - b(NEXT_32_PRFM_START); - BIND(NEXT_32_PRFM); - ld1(Vtmp1, Vtmp2, Vtmp3, Vtmp4, T8H, src); - BIND(NEXT_32_PRFM_START); - prfm(Address(src, SoftwarePrefetchHintDistance)); - orr(v4, T16B, Vtmp1, Vtmp2); - orr(v5, T16B, Vtmp3, Vtmp4); - uzp1(Vtmp1, T16B, Vtmp1, Vtmp2); - uzp1(Vtmp3, T16B, Vtmp3, Vtmp4); - uzp2(v5, T16B, v4, v5); // high bytes - umov(tmp2, v5, D, 1); - fmovd(tmp1, v5); - orr(tmp1, tmp1, tmp2); - cbnz(tmp1, LOOP_8); - stpq(Vtmp1, Vtmp3, dst); - sub(len, len, 32); - add(dst, dst, 32); - add(src, src, 64); - subs(tmp2, len, SoftwarePrefetchHintDistance/2 + 16); - br(GE, NEXT_32_PRFM); - cmp(len, (u1)32); - br(LT, LOOP_8); - BIND(NEXT_32); - ld1(Vtmp1, Vtmp2, Vtmp3, Vtmp4, T8H, src); - BIND(NEXT_32_START); - } else { - BIND(NEXT_32); - ld1(Vtmp1, Vtmp2, Vtmp3, Vtmp4, T8H, src); - } - prfm(Address(src, SoftwarePrefetchHintDistance)); - uzp1(v4, T16B, Vtmp1, Vtmp2); - uzp1(v5, T16B, Vtmp3, Vtmp4); - orr(Vtmp1, T16B, Vtmp1, Vtmp2); - orr(Vtmp3, T16B, Vtmp3, Vtmp4); - uzp2(Vtmp1, T16B, Vtmp1, Vtmp3); // high bytes - umov(tmp2, Vtmp1, D, 1); - fmovd(tmp1, Vtmp1); - orr(tmp1, tmp1, tmp2); - cbnz(tmp1, LOOP_8); - stpq(v4, v5, dst); - sub(len, len, 32); - add(dst, dst, 32); - add(src, src, 64); - cmp(len, (u1)32); - br(GE, NEXT_32); - cbz(len, DONE); - - BIND(LOOP_8); - cmp(len, (u1)8); - br(LT, LOOP_1); - BIND(NEXT_8); - ld1(Vtmp1, T8H, src); - uzp1(Vtmp2, T16B, Vtmp1, Vtmp1); // low bytes - uzp2(Vtmp3, T16B, Vtmp1, Vtmp1); // high bytes - fmovd(tmp1, Vtmp3); - cbnz(tmp1, NEXT_1); - strd(Vtmp2, dst); - - sub(len, len, 8); - add(dst, dst, 8); - add(src, src, 16); - cmp(len, (u1)8); - br(GE, NEXT_8); - - BIND(LOOP_1); - - cbz(len, DONE); - BIND(NEXT_1); - ldrh(tmp1, Address(post(src, 2))); - tst(tmp1, 0xff00); - br(NE, SET_RESULT); - strb(tmp1, Address(post(dst, 1))); - subs(len, len, 1); - br(GT, NEXT_1); - - BIND(SET_RESULT); - sub(result, result, len); // Return index where we stopped - // Return len == 0 if we processed all - // characters - BIND(DONE); -} + Register cnt = res; + Register max = rscratch1; + Register chk = rscratch2; + + prfm(Address(src), PLDL1STRM); + movw(cnt, len); + +#define ASCII(insn) do { if (ascii) { insn; } } while (0) + Label LOOP_32, DONE_32, FAIL_32; + + BIND(LOOP_32); + { + cmpw(cnt, 32); + br(LT, DONE_32); + ld1(vtmp0, vtmp1, vtmp2, vtmp3, T8H, Address(post(src, 64))); + // Extract lower bytes. + FloatRegister vlo0 = vtmp4; + FloatRegister vlo1 = vtmp5; + uzp1(vlo0, T16B, vtmp0, vtmp1); + uzp1(vlo1, T16B, vtmp2, vtmp3); + // Merge bits... + orr(vtmp0, T16B, vtmp0, vtmp1); + orr(vtmp2, T16B, vtmp2, vtmp3); + // Extract merged upper bytes. + FloatRegister vhix = vtmp0; + uzp2(vhix, T16B, vtmp0, vtmp2); + // ISO-check on hi-parts (all zero). + // ASCII-check on lo-parts (no sign). + FloatRegister vlox = vtmp1; // Merge lower bytes. + ASCII(orr(vlox, T16B, vlo0, vlo1)); + umov(chk, vhix, D, 1); ASCII(cmlt(vlox, T16B, vlox)); + fmovd(max, vhix); ASCII(umaxv(vlox, T16B, vlox)); + orr(chk, chk, max); ASCII(umov(max, vlox, B, 0)); + ASCII(orr(chk, chk, max)); + cbnz(chk, FAIL_32); + subw(cnt, cnt, 32); + st1(vlo0, vlo1, T16B, Address(post(dst, 32))); + b(LOOP_32); + } + BIND(FAIL_32); + sub(src, src, 64); + BIND(DONE_32); + + Label LOOP_8, SKIP_8; + + BIND(LOOP_8); + { + cmpw(cnt, 8); + br(LT, SKIP_8); + FloatRegister vhi = vtmp0; + FloatRegister vlo = vtmp1; + ld1(vtmp3, T8H, src); + uzp1(vlo, T16B, vtmp3, vtmp3); + uzp2(vhi, T16B, vtmp3, vtmp3); + // ISO-check on hi-parts (all zero). + // ASCII-check on lo-parts (no sign). + ASCII(cmlt(vtmp2, T16B, vlo)); + fmovd(chk, vhi); ASCII(umaxv(vtmp2, T16B, vtmp2)); + ASCII(umov(max, vtmp2, B, 0)); + ASCII(orr(chk, chk, max)); + cbnz(chk, SKIP_8); + + strd(vlo, Address(post(dst, 8))); + subw(cnt, cnt, 8); + add(src, src, 16); + b(LOOP_8); + } + BIND(SKIP_8); + +#undef ASCII + + Label LOOP, DONE; + + cbz(cnt, DONE); + BIND(LOOP); + { + Register chr = rscratch1; + ldrh(chr, Address(post(src, 2))); + tst(chr, ascii ? 0xff80 : 0xff00); + br(NE, DONE); + strb(chr, Address(post(dst, 1))); + subs(cnt, cnt, 1); + br(GT, LOOP); + } + BIND(DONE); + // Return index where we stopped. + subw(res, len, cnt); +} // Inflate byte[] array to char[]. +// Clobbers: src, dst, len, rflags, rscratch1, v0-v6 address MacroAssembler::byte_array_inflate(Register src, Register dst, Register len, FloatRegister vtmp1, FloatRegister vtmp2, FloatRegister vtmp3, Register tmp4) { @@ -5244,13 +5256,14 @@ address MacroAssembler::byte_array_inflate(Register src, Register dst, Register // Compress char[] array to byte[]. void MacroAssembler::char_array_compress(Register src, Register dst, Register len, - FloatRegister tmp1Reg, FloatRegister tmp2Reg, - FloatRegister tmp3Reg, FloatRegister tmp4Reg, - Register result) { - encode_iso_array(src, dst, len, result, - tmp1Reg, tmp2Reg, tmp3Reg, tmp4Reg); - cmp(len, zr); - csel(result, result, zr, EQ); + Register res, + FloatRegister tmp0, FloatRegister tmp1, + FloatRegister tmp2, FloatRegister tmp3, + FloatRegister tmp4, FloatRegister tmp5) { + encode_iso_array(src, dst, len, res, false, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5); + // Adjust result: res == len ? len : 0 + cmp(len, res); + csel(res, res, zr, EQ); } // get_thread() can be called anywhere inside generated code so we diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp index e403289e22f4e..3f6ebb41f2ac2 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp @@ -1273,14 +1273,17 @@ class MacroAssembler: public Assembler { FloatRegister vtmp3, Register tmp4); void char_array_compress(Register src, Register dst, Register len, - FloatRegister tmp1Reg, FloatRegister tmp2Reg, - FloatRegister tmp3Reg, FloatRegister tmp4Reg, - Register result); + Register res, + FloatRegister vtmp0, FloatRegister vtmp1, + FloatRegister vtmp2, FloatRegister vtmp3, + FloatRegister vtmp4, FloatRegister vtmp5); void encode_iso_array(Register src, Register dst, - Register len, Register result, - FloatRegister Vtmp1, FloatRegister Vtmp2, - FloatRegister Vtmp3, FloatRegister Vtmp4); + Register len, Register res, bool ascii, + FloatRegister vtmp0, FloatRegister vtmp1, + FloatRegister vtmp2, FloatRegister vtmp3, + FloatRegister vtmp4, FloatRegister vtmp5); + void fast_log(FloatRegister vtmp0, FloatRegister vtmp1, FloatRegister vtmp2, FloatRegister vtmp3, FloatRegister vtmp4, FloatRegister vtmp5, FloatRegister tmpC1, FloatRegister tmpC2, FloatRegister tmpC3, diff --git a/src/hotspot/cpu/aarch64/matcher_aarch64.hpp b/src/hotspot/cpu/aarch64/matcher_aarch64.hpp index 5f93c841e4e86..cd4bb96e4ad9b 100644 --- a/src/hotspot/cpu/aarch64/matcher_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/matcher_aarch64.hpp @@ -156,6 +156,6 @@ } // Implements a variant of EncodeISOArrayNode that encode ASCII only - static const bool supports_encode_ascii_array = false; + static const bool supports_encode_ascii_array = true; #endif // CPU_AARCH64_MATCHER_AARCH64_HPP diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp index d808e4b5b53de..d5a5503213c65 100644 --- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp @@ -39,6 +39,9 @@ #ifdef COMPILER1 #include "c1/c1_Runtime1.hpp" #endif +#if INCLUDE_JVMCI +#include "jvmci/jvmciEnv.hpp" +#endif void NativeCall::verify() { assert(NativeCall::is_call_at((address)this), "unexpected code at call site"); @@ -524,23 +527,26 @@ void NativeCallTrampolineStub::set_destination(address new_destination) { OrderAccess::release(); } +#if INCLUDE_JVMCI // Generate a trampoline for a branch to dest. If there's no need for a // trampoline, simply patch the call directly to dest. -address NativeCall::trampoline_jump(CodeBuffer &cbuf, address dest) { +void NativeCall::trampoline_jump(CodeBuffer &cbuf, address dest, JVMCI_TRAPS) { MacroAssembler a(&cbuf); - address stub = NULL; - - if (a.far_branches() - && ! is_NativeCallTrampolineStub_at(instruction_address() + displacement())) { - stub = a.emit_trampoline_stub(instruction_address() - cbuf.insts()->start(), dest); - } - if (stub == NULL) { - // If we generated no stub, patch this call directly to dest. - // This will happen if we don't need far branches or if there - // already was a trampoline. + if (!a.far_branches()) { + // If not using far branches, patch this call directly to dest. set_destination(dest); + } else if (!is_NativeCallTrampolineStub_at(instruction_address() + displacement())) { + // If we want far branches and there isn't a trampoline stub, emit one. + address stub = a.emit_trampoline_stub(instruction_address() - cbuf.insts()->start(), dest); + if (stub == nullptr) { + JVMCI_ERROR("could not emit trampoline stub - code cache is full"); + } + // The relocation created while emitting the stub will ensure this + // call instruction is subsequently patched to call the stub. + } else { + // Not sure how this can be happen but be defensive + JVMCI_ERROR("single-use stub should not exist"); } - - return stub; } +#endif diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp index 75f2797c326fb..57366ad3c9c81 100644 --- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp @@ -29,6 +29,11 @@ #include "asm/assembler.hpp" #include "runtime/icache.hpp" #include "runtime/os.hpp" +#include "runtime/os.hpp" +#if INCLUDE_JVMCI +#include "jvmci/jvmciExceptions.hpp" +#endif + // We have interfaces for the following instructions: // - NativeInstruction @@ -251,7 +256,9 @@ class NativeCall: public NativeInstruction { void set_destination_mt_safe(address dest, bool assert_lock = true); address get_trampoline(); - address trampoline_jump(CodeBuffer &cbuf, address dest); +#if INCLUDE_JVMCI + void trampoline_jump(CodeBuffer &cbuf, address dest, JVMCI_TRAPS); +#endif }; inline NativeCall* nativeCall_at(address address) { diff --git a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp index da1584d796952..83e9be0b0317e 100644 --- a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp @@ -771,7 +771,6 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - __ flush(); return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); } @@ -2359,10 +2358,9 @@ void SharedRuntime::generate_deopt_blob() { Label retaddr; __ set_last_Java_frame(sp, noreg, retaddr, rscratch1); -#ifdef ASSERT0 +#ifdef ASSERT { Label L; - __ ldr(rscratch1, Address(rthread, - JavaThread::last_Java_fp_offset())); + __ ldr(rscratch1, Address(rthread, JavaThread::last_Java_fp_offset())); __ cbz(rscratch1, L); __ stop("SharedRuntime::generate_deopt_blob: last_Java_fp not cleared"); __ bind(L); diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 8c3e165ab8eff..c92f0cf05bd66 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -2964,6 +2964,22 @@ class StubGenerator: public StubCodeGenerator { return start; } + // Big-endian 128-bit + 64-bit -> 128-bit addition. + // Inputs: 128-bits. in is preserved. + // The least-significant 64-bit word is in the upper dword of the vector + // inc (the 64-bit increment) is preserved. Its lower dword must be zero + // Output: result + void be_add_128_64(FloatRegister result, FloatRegister in, + FloatRegister inc, FloatRegister tmp) { + assert_different_registers(result, tmp, inc); + + __ addv(result, __ T2D, in, inc); // Add inc to the least-significant dword of input + __ cmhi(tmp, __ T2D, inc, result); // Check for result overflowing + __ ins(tmp, __ D, tmp, 0, 1); // Move LSD of comparison result to MSD + __ ins(tmp, __ D, inc, 1, 0); // Move 0 to LSD of comparison result + __ subv(result, __ T2D, result, tmp); // Subtract -1 from MSD if there was an overflow + } + // CTR AES crypt. // Arguments: // @@ -3073,13 +3089,16 @@ class StubGenerator: public StubCodeGenerator { // Setup the counter __ movi(v4, __ T4S, 0); __ movi(v5, __ T4S, 1); - __ ins(v4, __ S, v5, 3, 3); // v4 contains { 0, 0, 0, 1 } + __ ins(v4, __ S, v5, 2, 2); // v4 contains { 0, 1 } - __ ld1(v0, __ T16B, counter); // Load the counter into v0 - __ rev32(v16, __ T16B, v0); - __ addv(v16, __ T4S, v16, v4); - __ rev32(v16, __ T16B, v16); - __ st1(v16, __ T16B, counter); // Save the incremented counter back + // 128-bit big-endian increment + __ ld1(v0, __ T16B, counter); + __ rev64(v16, __ T16B, v0); + be_add_128_64(v16, v16, v4, /*tmp*/v5); + __ rev64(v16, __ T16B, v16); + __ st1(v16, __ T16B, counter); + // Previous counter value is in v0 + // v4 contains { 0, 1 } { // We have fewer than bulk_width blocks of data left. Encrypt @@ -3111,9 +3130,9 @@ class StubGenerator: public StubCodeGenerator { // Increment the counter, store it back __ orr(v0, __ T16B, v16, v16); - __ rev32(v16, __ T16B, v16); - __ addv(v16, __ T4S, v16, v4); - __ rev32(v16, __ T16B, v16); + __ rev64(v16, __ T16B, v16); + be_add_128_64(v16, v16, v4, /*tmp*/v5); + __ rev64(v16, __ T16B, v16); __ st1(v16, __ T16B, counter); // Save the incremented counter back __ b(inner_loop); @@ -3161,7 +3180,7 @@ class StubGenerator: public StubCodeGenerator { // Keys should already be loaded into the correct registers __ ld1(v0, __ T16B, counter); // v0 contains the first counter - __ rev32(v16, __ T16B, v0); // v16 contains byte-reversed counter + __ rev64(v16, __ T16B, v0); // v16 contains byte-reversed counter // AES/CTR loop { @@ -3171,11 +3190,11 @@ class StubGenerator: public StubCodeGenerator { // Setup the counters __ movi(v8, __ T4S, 0); __ movi(v9, __ T4S, 1); - __ ins(v8, __ S, v9, 3, 3); // v8 contains { 0, 0, 0, 1 } + __ ins(v8, __ S, v9, 2, 2); // v8 contains { 0, 1 } for (FloatRegister f = v0; f < v0 + bulk_width; f++) { - __ rev32(f, __ T16B, v16); - __ addv(v16, __ T4S, v16, v8); + __ rev64(f, __ T16B, v16); + be_add_128_64(v16, v16, v8, /*tmp*/v9); } __ ld1(v8, v9, v10, v11, __ T16B, __ post(in, 4 * 16)); @@ -3203,7 +3222,7 @@ class StubGenerator: public StubCodeGenerator { } // Save the counter back where it goes - __ rev32(v16, __ T16B, v16); + __ rev64(v16, __ T16B, v16); __ st1(v16, __ T16B, counter); __ pop(saved_regs, sp); @@ -3265,19 +3284,19 @@ class StubGenerator: public StubCodeGenerator { __ addw(rscratch4, r1, rscratch2); \ __ ldrw(rscratch1, Address(buf, k*4)); \ __ eorw(rscratch3, rscratch3, r4); \ - __ addw(rscratch3, rscratch3, rscratch1); \ + __ addw(rscratch4, rscratch4, rscratch1); \ __ addw(rscratch3, rscratch3, rscratch4); \ __ rorw(rscratch2, rscratch3, 32 - s); \ __ addw(r1, rscratch2, r2); #define GG(r1, r2, r3, r4, k, s, t) \ - __ eorw(rscratch2, r2, r3); \ + __ andw(rscratch3, r2, r4); \ + __ bicw(rscratch4, r3, r4); \ __ ldrw(rscratch1, Address(buf, k*4)); \ - __ andw(rscratch3, rscratch2, r4); \ __ movw(rscratch2, t); \ - __ eorw(rscratch3, rscratch3, r3); \ + __ orrw(rscratch3, rscratch3, rscratch4); \ __ addw(rscratch4, r1, rscratch2); \ - __ addw(rscratch3, rscratch3, rscratch1); \ + __ addw(rscratch4, rscratch4, rscratch1); \ __ addw(rscratch3, rscratch3, rscratch4); \ __ rorw(rscratch2, rscratch3, 32 - s); \ __ addw(r1, rscratch2, r2); @@ -3288,7 +3307,7 @@ class StubGenerator: public StubCodeGenerator { __ addw(rscratch4, r1, rscratch2); \ __ ldrw(rscratch1, Address(buf, k*4)); \ __ eorw(rscratch3, rscratch3, r2); \ - __ addw(rscratch3, rscratch3, rscratch1); \ + __ addw(rscratch4, rscratch4, rscratch1); \ __ addw(rscratch3, rscratch3, rscratch4); \ __ rorw(rscratch2, rscratch3, 32 - s); \ __ addw(r1, rscratch2, r2); @@ -3299,7 +3318,7 @@ class StubGenerator: public StubCodeGenerator { __ addw(rscratch4, r1, rscratch3); \ __ ldrw(rscratch1, Address(buf, k*4)); \ __ eorw(rscratch3, rscratch2, r3); \ - __ addw(rscratch3, rscratch3, rscratch1); \ + __ addw(rscratch4, rscratch4, rscratch1); \ __ addw(rscratch3, rscratch3, rscratch4); \ __ rorw(rscratch2, rscratch3, 32 - s); \ __ addw(r1, rscratch2, r2); @@ -4002,46 +4021,6 @@ class StubGenerator: public StubCodeGenerator { return start; } - // Safefetch stubs. - void generate_safefetch(const char* name, int size, address* entry, - address* fault_pc, address* continuation_pc) { - // safefetch signatures: - // int SafeFetch32(int* adr, int errValue); - // intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue); - // - // arguments: - // c_rarg0 = adr - // c_rarg1 = errValue - // - // result: - // PPC_RET = *adr or errValue - - StubCodeMark mark(this, "StubRoutines", name); - - // Entry point, pc or function descriptor. - *entry = __ pc(); - - // Load *adr into c_rarg1, may fault. - *fault_pc = __ pc(); - switch (size) { - case 4: - // int32_t - __ ldrw(c_rarg1, Address(c_rarg0, 0)); - break; - case 8: - // int64_t - __ ldr(c_rarg1, Address(c_rarg0, 0)); - break; - default: - ShouldNotReachHere(); - } - - // return errValue or *adr - *continuation_pc = __ pc(); - __ mov(r0, c_rarg1); - __ ret(lr); - } - /** * Arguments: * @@ -4982,6 +4961,7 @@ class StubGenerator: public StubCodeGenerator { // result = r0 - return value. Already contains "false" // cnt1 = r10 - amount of elements left to check, reduced by wordSize // r3-r5 are reserved temporary registers + // Clobbers: v0-v7 when UseSIMDForArrayEquals, rscratch1, rscratch2 address generate_large_array_equals() { Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1, tmp2 = rscratch2, tmp3 = r3, tmp4 = r4, tmp5 = r5, tmp6 = r11, @@ -5428,6 +5408,8 @@ class StubGenerator: public StubCodeGenerator { // R2 = cnt1 // R3 = str1 // R4 = cnt2 + // Clobbers: rscratch1, rscratch2, v0, v1, rflags + // // This generic linear code use few additional ideas, which makes it faster: // 1) we can safely keep at least 1st register of pattern(since length >= 8) // in order to skip initial loading(help in systems with 1 ld pipeline) @@ -5742,6 +5724,7 @@ class StubGenerator: public StubCodeGenerator { // R3 = len >> 3 // V0 = 0 // v1 = loaded 8 bytes + // Clobbers: r0, r1, r3, rscratch1, rflags, v0-v6 address generate_large_byte_array_inflate() { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", "large_byte_array_inflate"); @@ -6481,10 +6464,16 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); } - void gen_ldaddal_entry(Assembler::operand_size size) { + void gen_ldadd_entry(Assembler::operand_size size, atomic_memory_order order) { Register prev = r2, addr = c_rarg0, incr = c_rarg1; - __ ldaddal(size, incr, prev, addr); - __ membar(Assembler::StoreStore|Assembler::StoreLoad); + // If not relaxed, then default to conservative. Relaxed is the only + // case we use enough to be worth specializing. + if (order == memory_order_relaxed) { + __ ldadd(size, incr, prev, addr); + } else { + __ ldaddal(size, incr, prev, addr); + __ membar(Assembler::StoreStore|Assembler::StoreLoad); + } if (size == Assembler::xword) { __ mov(r0, prev); } else { @@ -6514,12 +6503,21 @@ class StubGenerator: public StubCodeGenerator { StubCodeMark mark(this, "StubRoutines", "atomic entry points"); address first_entry = __ pc(); - // All memory_order_conservative + // ADD, memory_order_conservative AtomicStubMark mark_fetch_add_4(_masm, &aarch64_atomic_fetch_add_4_impl); - gen_ldaddal_entry(Assembler::word); + gen_ldadd_entry(Assembler::word, memory_order_conservative); AtomicStubMark mark_fetch_add_8(_masm, &aarch64_atomic_fetch_add_8_impl); - gen_ldaddal_entry(Assembler::xword); + gen_ldadd_entry(Assembler::xword, memory_order_conservative); + + // ADD, memory_order_relaxed + AtomicStubMark mark_fetch_add_4_relaxed + (_masm, &aarch64_atomic_fetch_add_4_relaxed_impl); + gen_ldadd_entry(MacroAssembler::word, memory_order_relaxed); + AtomicStubMark mark_fetch_add_8_relaxed + (_masm, &aarch64_atomic_fetch_add_8_relaxed_impl); + gen_ldadd_entry(MacroAssembler::xword, memory_order_relaxed); + // XCHG, memory_order_conservative AtomicStubMark mark_xchg_4(_masm, &aarch64_atomic_xchg_4_impl); gen_swpal_entry(Assembler::word); AtomicStubMark mark_xchg_8_impl(_masm, &aarch64_atomic_xchg_8_impl); @@ -7544,14 +7542,6 @@ class StubGenerator: public StubCodeGenerator { if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) { StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true); } - - // Safefetch stubs. - generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry, - &StubRoutines::_safefetch32_fault_pc, - &StubRoutines::_safefetch32_continuation_pc); - generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry, - &StubRoutines::_safefetchN_fault_pc, - &StubRoutines::_safefetchN_continuation_pc); } void generate_all() { @@ -7730,6 +7720,8 @@ void StubGenerator_generate(CodeBuffer* code, bool all) { DEFAULT_ATOMIC_OP(fetch_add, 4, ) DEFAULT_ATOMIC_OP(fetch_add, 8, ) +DEFAULT_ATOMIC_OP(fetch_add, 4, _relaxed) +DEFAULT_ATOMIC_OP(fetch_add, 8, _relaxed) DEFAULT_ATOMIC_OP(xchg, 4, ) DEFAULT_ATOMIC_OP(xchg, 8, ) DEFAULT_ATOMIC_OP(cmpxchg, 1, ) diff --git a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp index 7cdaa89f2a3a0..4666b42b933b9 100644 --- a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp @@ -1748,12 +1748,6 @@ void TemplateTable::float_cmp(bool is_float, int unordered_result) void TemplateTable::branch(bool is_jsr, bool is_wide) { - // We might be moving to a safepoint. The thread which calls - // Interpreter::notice_safepoints() will effectively flush its cache - // when it makes a system call, but we need to do something to - // ensure that we see the changed dispatch table. - __ membar(MacroAssembler::LoadLoad); - __ profile_taken_branch(r0, r1); const ByteSize be_offset = MethodCounters::backedge_counter_offset() + InvocationCounter::counter_offset(); @@ -1969,12 +1963,6 @@ void TemplateTable::if_acmp(Condition cc) void TemplateTable::ret() { transition(vtos, vtos); - // We might be moving to a safepoint. The thread which calls - // Interpreter::notice_safepoints() will effectively flush its cache - // when it makes a system call, but we need to do something to - // ensure that we see the changed dispatch table. - __ membar(MacroAssembler::LoadLoad); - locals_index(r1); __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp __ profile_ret(r1, r2); diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp index 31dfb77275974..9f33e7526fee7 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp @@ -493,36 +493,52 @@ void VM_Version::initialize() { UNSUPPORTED_OPTION(CriticalJNINatives); } -void VM_Version::check_virtualizations() { #if defined(LINUX) - const char* info_file = "/sys/devices/virtual/dmi/id/product_name"; - // check for various strings in the dmi data indicating virtualizations +static bool check_info_file(const char* fpath, + const char* virt1, VirtualizationType vt1, + const char* virt2, VirtualizationType vt2) { char line[500]; - FILE* fp = os::fopen(info_file, "r"); + FILE* fp = os::fopen(fpath, "r"); if (fp == nullptr) { - return; + return false; } while (fgets(line, sizeof(line), fp) != nullptr) { - if (strcasestr(line, "KVM") != 0) { - Abstract_VM_Version::_detected_virtualization = KVM; - break; + if (strcasestr(line, virt1) != 0) { + Abstract_VM_Version::_detected_virtualization = vt1; + fclose(fp); + return true; } - if (strcasestr(line, "VMware") != 0) { - Abstract_VM_Version::_detected_virtualization = VMWare; - break; + if (virt2 != NULL && strcasestr(line, virt2) != 0) { + Abstract_VM_Version::_detected_virtualization = vt2; + fclose(fp); + return true; } } fclose(fp); + return false; +} +#endif + +void VM_Version::check_virtualizations() { +#if defined(LINUX) + const char* pname_file = "/sys/devices/virtual/dmi/id/product_name"; + const char* tname_file = "/sys/hypervisor/type"; + if (check_info_file(pname_file, "KVM", KVM, "VMWare", VMWare)) { + return; + } + check_info_file(tname_file, "Xen", XenPVHVM, NULL, NoDetectedVirtualization); #endif } void VM_Version::print_platform_virtualization_info(outputStream* st) { #if defined(LINUX) - VirtualizationType vrt = VM_Version::get_detected_virtualization(); - if (vrt == KVM) { - st->print_cr("KVM virtualization detected"); - } else if (vrt == VMWare) { - st->print_cr("VMWare virtualization detected"); - } + VirtualizationType vrt = VM_Version::get_detected_virtualization(); + if (vrt == KVM) { + st->print_cr("KVM virtualization detected"); + } else if (vrt == VMWare) { + st->print_cr("VMWare virtualization detected"); + } else if (vrt == XenPVHVM) { + st->print_cr("Xen virtualization detected"); + } #endif } diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad index 0626704602825..d9ade024b911b 100644 --- a/src/hotspot/cpu/arm/arm.ad +++ b/src/hotspot/cpu/arm/arm.ad @@ -4511,6 +4511,7 @@ instruct storeF( memoryF mem, regF src) %{ // pattern-match out unnecessary membars instruct membar_storestore() %{ match(MemBarStoreStore); + match(StoreStoreFence); ins_cost(4*MEMORY_REF_COST); size(4); diff --git a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp index 1c21f835f7568..38ac26be464c4 100644 --- a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp +++ b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp @@ -721,11 +721,7 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, break; case T_ADDRESS: - if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) { - __ ldr_u32(dest->as_pointer_register(), as_Address(addr)); - } else { - __ ldr(dest->as_pointer_register(), as_Address(addr)); - } + __ ldr(dest->as_pointer_register(), as_Address(addr)); break; case T_INT: @@ -2454,6 +2450,21 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) { __ bind(*op->stub()->continuation()); } +void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { + Register obj = op->obj()->as_pointer_register(); + Register result = op->result_opr()->as_pointer_register(); + + CodeEmitInfo* info = op->info(); + if (info != NULL) { + add_debug_info_for_null_check_here(info); + } + + if (UseCompressedClassPointers) { // On 32 bit arm?? + __ ldr_u32(result, Address(obj, oopDesc::klass_offset_in_bytes())); + } else { + __ ldr(result, Address(obj, oopDesc::klass_offset_in_bytes())); + } +} void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { ciMethod* method = op->profiled_method(); diff --git a/src/hotspot/cpu/arm/c2_MacroAssembler_arm.cpp b/src/hotspot/cpu/arm/c2_MacroAssembler_arm.cpp index 2211d5c5fa338..3d205fe7062f2 100644 --- a/src/hotspot/cpu/arm/c2_MacroAssembler_arm.cpp +++ b/src/hotspot/cpu/arm/c2_MacroAssembler_arm.cpp @@ -160,7 +160,7 @@ void C2_MacroAssembler::fast_unlock(Register Roop, Register Rbox, Register Rscra // Restore the object header bool allow_fallthrough_on_failure = true; bool one_shot = true; - cas_for_lock_release(Rmark, Rbox, Roop, Rscratch, done, allow_fallthrough_on_failure, one_shot); + cas_for_lock_release(Rbox, Rmark, Roop, Rscratch, done, allow_fallthrough_on_failure, one_shot); bind(done); } diff --git a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp index 30b5ab93ce46a..2a37b934df8eb 100644 --- a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp +++ b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp @@ -644,7 +644,6 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm address c2i_entry = __ pc(); gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - __ flush(); return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry); } diff --git a/src/hotspot/cpu/arm/stubGenerator_arm.cpp b/src/hotspot/cpu/arm/stubGenerator_arm.cpp index a202a7f09490b..60cc825cb255c 100644 --- a/src/hotspot/cpu/arm/stubGenerator_arm.cpp +++ b/src/hotspot/cpu/arm/stubGenerator_arm.cpp @@ -2837,46 +2837,6 @@ class StubGenerator: public StubCodeGenerator { return start; } - // Safefetch stubs. - void generate_safefetch(const char* name, int size, address* entry, address* fault_pc, address* continuation_pc) { - // safefetch signatures: - // int SafeFetch32(int* adr, int errValue); - // intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue); - // - // arguments: - // R0 = adr - // R1 = errValue - // - // result: - // R0 = *adr or errValue - - StubCodeMark mark(this, "StubRoutines", name); - - // Entry point, pc or function descriptor. - *entry = __ pc(); - - // Load *adr into c_rarg2, may fault. - *fault_pc = __ pc(); - - switch (size) { - case 4: // int32_t - __ ldr_s32(R1, Address(R0)); - break; - - case 8: // int64_t - Unimplemented(); - break; - - default: - ShouldNotReachHere(); - } - - // return errValue or *adr - *continuation_pc = __ pc(); - __ mov(R0, R1); - __ ret(); - } - void generate_arraycopy_stubs() { // Note: the disjoint stubs must be generated first, some of @@ -3029,16 +2989,9 @@ class StubGenerator: public StubCodeGenerator { StubRoutines::_atomic_load_long_entry = generate_atomic_load_long(); StubRoutines::_atomic_store_long_entry = generate_atomic_store_long(); - // Safefetch stubs. - generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry, - &StubRoutines::_safefetch32_fault_pc, - &StubRoutines::_safefetch32_continuation_pc); - assert (sizeof(int) == wordSize, "32-bit architecture"); - StubRoutines::_safefetchN_entry = StubRoutines::_safefetch32_entry; - StubRoutines::_safefetchN_fault_pc = StubRoutines::_safefetch32_fault_pc; - StubRoutines::_safefetchN_continuation_pc = StubRoutines::_safefetch32_continuation_pc; } + void generate_all() { // Generates all stubs and initializes the entry points diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp index af9a856ae087d..10ecd0eee3a3e 100644 --- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp @@ -812,12 +812,7 @@ int LIR_Assembler::load(Register base, int offset, LIR_Opr to_reg, BasicType typ case T_LONG : __ ld(to_reg->as_register_lo(), offset, base); break; case T_METADATA: __ ld(to_reg->as_register(), offset, base); break; case T_ADDRESS: - if (offset == oopDesc::klass_offset_in_bytes() && UseCompressedClassPointers) { - __ lwz(to_reg->as_register(), offset, base); - __ decode_klass_not_null(to_reg->as_register()); - } else { - __ ld(to_reg->as_register(), offset, base); - } + __ ld(to_reg->as_register(), offset, base); break; case T_ARRAY : // fall through case T_OBJECT: @@ -828,7 +823,6 @@ int LIR_Assembler::load(Register base, int offset, LIR_Opr to_reg, BasicType typ } else { __ ld(to_reg->as_register(), offset, base); } - __ verify_oop(to_reg->as_register(), FILE_AND_LINE); break; } case T_FLOAT: __ lfs(to_reg->as_float_reg(), offset, base); break; @@ -859,7 +853,6 @@ int LIR_Assembler::load(Register base, Register disp, LIR_Opr to_reg, BasicType } else { __ ldx(to_reg->as_register(), base, disp); } - __ verify_oop(to_reg->as_register(), FILE_AND_LINE); break; } case T_FLOAT: __ lfsx(to_reg->as_float_reg() , base, disp); break; @@ -2735,6 +2728,26 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) { __ bind(*op->stub()->continuation()); } +void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { + Register obj = op->obj()->as_pointer_register(); + Register result = op->result_opr()->as_pointer_register(); + + CodeEmitInfo* info = op->info(); + if (info != NULL) { + if (!os::zero_page_read_protected() || !ImplicitNullChecks) { + explicit_null_check(obj, info); + } else { + add_debug_info_for_null_check_here(info); + } + } + + if (UseCompressedClassPointers) { + __ lwz(result, oopDesc::klass_offset_in_bytes(), obj); + __ decode_klass_not_null(result); + } else { + __ ld(result, oopDesc::klass_offset_in_bytes(), obj); + } +} void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { ciMethod* method = op->profiled_method(); diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad index 1b6c9c6ba9631..067114616669c 100644 --- a/src/hotspot/cpu/ppc/ppc.ad +++ b/src/hotspot/cpu/ppc/ppc.ad @@ -7152,6 +7152,7 @@ instruct membar_release() %{ instruct membar_storestore() %{ match(MemBarStoreStore); + match(StoreStoreFence); ins_cost(4*MEMORY_REF_COST); format %{ "MEMBAR-store-store" %} diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp index 94869ae7ca2dd..b53a4ba3ab3c6 100644 --- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp +++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2021 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -3390,8 +3390,9 @@ void SharedRuntime::montgomery_multiply(jint *a_ints, jint *b_ints, jint *n_ints // Make very sure we don't use so much space that the stack might // overflow. 512 jints corresponds to an 16384-bit integer and // will use here a total of 8k bytes of stack space. + int divisor = sizeof(unsigned long) * 4; + guarantee(longwords <= 8192 / divisor, "must be"); int total_allocation = longwords * sizeof (unsigned long) * 4; - guarantee(total_allocation <= 8192, "must be"); unsigned long *scratch = (unsigned long *)alloca(total_allocation); // Local scratch arrays @@ -3420,8 +3421,9 @@ void SharedRuntime::montgomery_square(jint *a_ints, jint *n_ints, // Make very sure we don't use so much space that the stack might // overflow. 512 jints corresponds to an 16384-bit integer and // will use here a total of 6k bytes of stack space. + int divisor = sizeof(unsigned long) * 3; + guarantee(longwords <= (8192 / divisor), "must be"); int total_allocation = longwords * sizeof (unsigned long) * 3; - guarantee(total_allocation <= 8192, "must be"); unsigned long *scratch = (unsigned long *)alloca(total_allocation); // Local scratch arrays diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp index 8daa0070dbb54..912be97c96b43 100644 --- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp +++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp @@ -3159,45 +3159,6 @@ class StubGenerator: public StubCodeGenerator { #endif } - // Safefetch stubs. - void generate_safefetch(const char* name, int size, address* entry, address* fault_pc, address* continuation_pc) { - // safefetch signatures: - // int SafeFetch32(int* adr, int errValue); - // intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue); - // - // arguments: - // R3_ARG1 = adr - // R4_ARG2 = errValue - // - // result: - // R3_RET = *adr or errValue - - StubCodeMark mark(this, "StubRoutines", name); - - // Entry point, pc or function descriptor. - *entry = __ function_entry(); - - // Load *adr into R4_ARG2, may fault. - *fault_pc = __ pc(); - switch (size) { - case 4: - // int32_t, signed extended - __ lwa(R4_ARG2, 0, R3_ARG1); - break; - case 8: - // int64_t - __ ld(R4_ARG2, 0, R3_ARG1); - break; - default: - ShouldNotReachHere(); - } - - // return errValue or *adr - *continuation_pc = __ pc(); - __ mr(R3_RET, R4_ARG2); - __ blr(); - } - // Stub for BigInteger::multiplyToLen() // // Arguments: @@ -4534,14 +4495,6 @@ class StubGenerator: public StubCodeGenerator { StubRoutines::_crc32c_table_addr = StubRoutines::ppc::generate_crc_constants(REVERSE_CRC32C_POLY); StubRoutines::_updateBytesCRC32C = generate_CRC32_updateBytes(true); } - - // Safefetch stubs. - generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry, - &StubRoutines::_safefetch32_fault_pc, - &StubRoutines::_safefetch32_continuation_pc); - generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry, - &StubRoutines::_safefetchN_fault_pc, - &StubRoutines::_safefetchN_continuation_pc); } void generate_all() { diff --git a/src/hotspot/cpu/riscv/abstractInterpreter_riscv.cpp b/src/hotspot/cpu/riscv/abstractInterpreter_riscv.cpp new file mode 100644 index 0000000000000..31c63abe71d1a --- /dev/null +++ b/src/hotspot/cpu/riscv/abstractInterpreter_riscv.cpp @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2003, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "interpreter/interpreter.hpp" +#include "oops/constMethod.hpp" +#include "oops/klass.inline.hpp" +#include "oops/method.hpp" +#include "runtime/frame.inline.hpp" +#include "utilities/align.hpp" +#include "utilities/debug.hpp" +#include "utilities/macros.hpp" + +int AbstractInterpreter::BasicType_as_index(BasicType type) { + int i = 0; + switch (type) { + case T_BOOLEAN: i = 0; break; + case T_CHAR : i = 1; break; + case T_BYTE : i = 2; break; + case T_SHORT : i = 3; break; + case T_INT : i = 4; break; + case T_LONG : i = 5; break; + case T_VOID : i = 6; break; + case T_FLOAT : i = 7; break; + case T_DOUBLE : i = 8; break; + case T_OBJECT : i = 9; break; + case T_ARRAY : i = 9; break; + default : ShouldNotReachHere(); + } + assert(0 <= i && i < AbstractInterpreter::number_of_result_handlers, + "index out of bounds"); + return i; +} + +// How much stack a method activation needs in words. +int AbstractInterpreter::size_top_interpreter_activation(Method* method) { + const int entry_size = frame::interpreter_frame_monitor_size(); + + // total overhead size: entry_size + (saved fp thru expr stack + // bottom). be sure to change this if you add/subtract anything + // to/from the overhead area + const int overhead_size = + -(frame::interpreter_frame_initial_sp_offset) + entry_size; + + const int stub_code = frame::entry_frame_after_call_words; + assert_cond(method != NULL); + const int method_stack = (method->max_locals() + method->max_stack()) * + Interpreter::stackElementWords; + return (overhead_size + method_stack + stub_code); +} + +// asm based interpreter deoptimization helpers +int AbstractInterpreter::size_activation(int max_stack, + int temps, + int extra_args, + int monitors, + int callee_params, + int callee_locals, + bool is_top_frame) { + // Note: This calculation must exactly parallel the frame setup + // in TemplateInterpreterGenerator::generate_method_entry. + + // fixed size of an interpreter frame: + int overhead = frame::sender_sp_offset - + frame::interpreter_frame_initial_sp_offset; + // Our locals were accounted for by the caller (or last_frame_adjust + // on the transistion) Since the callee parameters already account + // for the callee's params we only need to account for the extra + // locals. + int size = overhead + + (callee_locals - callee_params) + + monitors * frame::interpreter_frame_monitor_size() + + // On the top frame, at all times SP <= ESP, and SP is + // 16-aligned. We ensure this by adjusting SP on method + // entry and re-entry to allow room for the maximum size of + // the expression stack. When we call another method we bump + // SP so that no stack space is wasted. So, only on the top + // frame do we need to allow max_stack words. + (is_top_frame ? max_stack : temps + extra_args); + + // On riscv we always keep the stack pointer 16-aligned, so we + // must round up here. + size = align_up(size, 2); + + return size; +} + +void AbstractInterpreter::layout_activation(Method* method, + int tempcount, + int popframe_extra_args, + int moncount, + int caller_actual_parameters, + int callee_param_count, + int callee_locals, + frame* caller, + frame* interpreter_frame, + bool is_top_frame, + bool is_bottom_frame) { + // The frame interpreter_frame is guaranteed to be the right size, + // as determined by a previous call to the size_activation() method. + // It is also guaranteed to be walkable even though it is in a + // skeletal state + assert_cond(method != NULL && caller != NULL && interpreter_frame != NULL); + int max_locals = method->max_locals() * Interpreter::stackElementWords; + int extra_locals = (method->max_locals() - method->size_of_parameters()) * + Interpreter::stackElementWords; + +#ifdef ASSERT + assert(caller->sp() == interpreter_frame->sender_sp(), "Frame not properly walkable"); +#endif + + interpreter_frame->interpreter_frame_set_method(method); + // NOTE the difference in using sender_sp and interpreter_frame_sender_sp + // interpreter_frame_sender_sp is the original sp of the caller (the unextended_sp) + // and sender_sp is fp + intptr_t* locals = NULL; + if (caller->is_interpreted_frame()) { + locals = caller->interpreter_frame_last_sp() + caller_actual_parameters - 1; + } else { + locals = interpreter_frame->sender_sp() + max_locals - 1; + } + +#ifdef ASSERT + if (caller->is_interpreted_frame()) { + assert(locals < caller->fp() + frame::interpreter_frame_initial_sp_offset, "bad placement"); + } +#endif + + interpreter_frame->interpreter_frame_set_locals(locals); + BasicObjectLock* montop = interpreter_frame->interpreter_frame_monitor_begin(); + BasicObjectLock* monbot = montop - moncount; + interpreter_frame->interpreter_frame_set_monitor_end(monbot); + + // Set last_sp + intptr_t* last_sp = (intptr_t*) monbot - + tempcount*Interpreter::stackElementWords - + popframe_extra_args; + interpreter_frame->interpreter_frame_set_last_sp(last_sp); + + // All frames but the initial (oldest) interpreter frame we fill in have + // a value for sender_sp that allows walking the stack but isn't + // truly correct. Correct the value here. + if (extra_locals != 0 && + interpreter_frame->sender_sp() == + interpreter_frame->interpreter_frame_sender_sp()) { + interpreter_frame->set_interpreter_frame_sender_sp(caller->sp() + + extra_locals); + } + + *interpreter_frame->interpreter_frame_cache_addr() = + method->constants()->cache(); + *interpreter_frame->interpreter_frame_mirror_addr() = + method->method_holder()->java_mirror(); +} diff --git a/src/hotspot/cpu/riscv/assembler_riscv.cpp b/src/hotspot/cpu/riscv/assembler_riscv.cpp new file mode 100644 index 0000000000000..2fb75c802cf0d --- /dev/null +++ b/src/hotspot/cpu/riscv/assembler_riscv.cpp @@ -0,0 +1,78 @@ +/* + * Copyright (c) 1997, 2012, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include +#include + +#include "precompiled.hpp" +#include "asm/assembler.hpp" +#include "asm/assembler.inline.hpp" +#include "compiler/disassembler.hpp" +#include "interpreter/interpreter.hpp" +#include "memory/resourceArea.hpp" +#include "runtime/interfaceSupport.inline.hpp" +#include "runtime/sharedRuntime.hpp" + +int AbstractAssembler::code_fill_byte() { + return 0; +} + +Address::Address(address target, relocInfo::relocType rtype) : _base(noreg), _offset(0), _mode(literal) { + _target = target; + switch (rtype) { + case relocInfo::oop_type: + case relocInfo::metadata_type: + // Oops are a special case. Normally they would be their own section + // but in cases like icBuffer they are literals in the code stream that + // we don't have a section for. We use none so that we get a literal address + // which is always patchable. + break; + case relocInfo::external_word_type: + _rspec = external_word_Relocation::spec(target); + break; + case relocInfo::internal_word_type: + _rspec = internal_word_Relocation::spec(target); + break; + case relocInfo::opt_virtual_call_type: + _rspec = opt_virtual_call_Relocation::spec(); + break; + case relocInfo::static_call_type: + _rspec = static_call_Relocation::spec(); + break; + case relocInfo::runtime_call_type: + _rspec = runtime_call_Relocation::spec(); + break; + case relocInfo::poll_type: + case relocInfo::poll_return_type: + _rspec = Relocation::spec_simple(rtype); + break; + case relocInfo::none: + _rspec = RelocationHolder::none; + break; + default: + ShouldNotReachHere(); + } +} diff --git a/src/hotspot/cpu/riscv/assembler_riscv.hpp b/src/hotspot/cpu/riscv/assembler_riscv.hpp new file mode 100644 index 0000000000000..ca7a0a8b1c2f8 --- /dev/null +++ b/src/hotspot/cpu/riscv/assembler_riscv.hpp @@ -0,0 +1,2776 @@ +/* + * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_ASSEMBLER_RISCV_HPP +#define CPU_RISCV_ASSEMBLER_RISCV_HPP + +#include "asm/register.hpp" +#include "code/codeCache.hpp" +#include "metaprogramming/enableIf.hpp" + +#define XLEN 64 + +// definitions of various symbolic names for machine registers + +// First intercalls between C and Java which use 8 general registers +// and 8 floating registers + +class Argument { + public: + enum { + n_int_register_parameters_c = 8, // x10, x11, ... x17 (c_rarg0, c_rarg1, ...) + n_float_register_parameters_c = 8, // f10, f11, ... f17 (c_farg0, c_farg1, ... ) + + n_int_register_parameters_j = 8, // x11, ... x17, x10 (j_rarg0, j_rarg1, ...) + n_float_register_parameters_j = 8 // f10, f11, ... f17 (j_farg0, j_farg1, ...) + }; +}; + +// function argument(caller-save registers) +REGISTER_DECLARATION(Register, c_rarg0, x10); +REGISTER_DECLARATION(Register, c_rarg1, x11); +REGISTER_DECLARATION(Register, c_rarg2, x12); +REGISTER_DECLARATION(Register, c_rarg3, x13); +REGISTER_DECLARATION(Register, c_rarg4, x14); +REGISTER_DECLARATION(Register, c_rarg5, x15); +REGISTER_DECLARATION(Register, c_rarg6, x16); +REGISTER_DECLARATION(Register, c_rarg7, x17); + +REGISTER_DECLARATION(FloatRegister, c_farg0, f10); +REGISTER_DECLARATION(FloatRegister, c_farg1, f11); +REGISTER_DECLARATION(FloatRegister, c_farg2, f12); +REGISTER_DECLARATION(FloatRegister, c_farg3, f13); +REGISTER_DECLARATION(FloatRegister, c_farg4, f14); +REGISTER_DECLARATION(FloatRegister, c_farg5, f15); +REGISTER_DECLARATION(FloatRegister, c_farg6, f16); +REGISTER_DECLARATION(FloatRegister, c_farg7, f17); + +// Symbolically name the register arguments used by the Java calling convention. +// We have control over the convention for java so we can do what we please. +// What pleases us is to offset the java calling convention so that when +// we call a suitable jni method the arguments are lined up and we don't +// have to do much shuffling. A suitable jni method is non-static and a +// small number of arguments. +// +// |------------------------------------------------------------------------| +// | c_rarg0 c_rarg1 c_rarg2 c_rarg3 c_rarg4 c_rarg5 c_rarg6 c_rarg7 | +// |------------------------------------------------------------------------| +// | x10 x11 x12 x13 x14 x15 x16 x17 | +// |------------------------------------------------------------------------| +// | j_rarg7 j_rarg0 j_rarg1 j_rarg2 j_rarg3 j_rarg4 j_rarg5 j_rarg6 | +// |------------------------------------------------------------------------| + +REGISTER_DECLARATION(Register, j_rarg0, c_rarg1); +REGISTER_DECLARATION(Register, j_rarg1, c_rarg2); +REGISTER_DECLARATION(Register, j_rarg2, c_rarg3); +REGISTER_DECLARATION(Register, j_rarg3, c_rarg4); +REGISTER_DECLARATION(Register, j_rarg4, c_rarg5); +REGISTER_DECLARATION(Register, j_rarg5, c_rarg6); +REGISTER_DECLARATION(Register, j_rarg6, c_rarg7); +REGISTER_DECLARATION(Register, j_rarg7, c_rarg0); + +// Java floating args are passed as per C + +REGISTER_DECLARATION(FloatRegister, j_farg0, f10); +REGISTER_DECLARATION(FloatRegister, j_farg1, f11); +REGISTER_DECLARATION(FloatRegister, j_farg2, f12); +REGISTER_DECLARATION(FloatRegister, j_farg3, f13); +REGISTER_DECLARATION(FloatRegister, j_farg4, f14); +REGISTER_DECLARATION(FloatRegister, j_farg5, f15); +REGISTER_DECLARATION(FloatRegister, j_farg6, f16); +REGISTER_DECLARATION(FloatRegister, j_farg7, f17); + +// zero rigster +REGISTER_DECLARATION(Register, zr, x0); +// global pointer +REGISTER_DECLARATION(Register, gp, x3); +// thread pointer +REGISTER_DECLARATION(Register, tp, x4); + +// registers used to hold VM data either temporarily within a method +// or across method calls + +// volatile (caller-save) registers + +// current method -- must be in a call-clobbered register +REGISTER_DECLARATION(Register, xmethod, x31); +// return address +REGISTER_DECLARATION(Register, ra, x1); + +// non-volatile (callee-save) registers + +// stack pointer +REGISTER_DECLARATION(Register, sp, x2); +// frame pointer +REGISTER_DECLARATION(Register, fp, x8); +// base of heap +REGISTER_DECLARATION(Register, xheapbase, x27); +// constant pool cache +REGISTER_DECLARATION(Register, xcpool, x26); +// monitors allocated on stack +REGISTER_DECLARATION(Register, xmonitors, x25); +// locals on stack +REGISTER_DECLARATION(Register, xlocals, x24); + +// java thread pointer +REGISTER_DECLARATION(Register, xthread, x23); +// bytecode pointer +REGISTER_DECLARATION(Register, xbcp, x22); +// Dispatch table base +REGISTER_DECLARATION(Register, xdispatch, x21); +// Java stack pointer +REGISTER_DECLARATION(Register, esp, x20); + +// temporary register(caller-save registers) +REGISTER_DECLARATION(Register, t0, x5); +REGISTER_DECLARATION(Register, t1, x6); +REGISTER_DECLARATION(Register, t2, x7); + +const Register g_INTArgReg[Argument::n_int_register_parameters_c] = { + c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5, c_rarg6, c_rarg7 +}; + +const FloatRegister g_FPArgReg[Argument::n_float_register_parameters_c] = { + c_farg0, c_farg1, c_farg2, c_farg3, c_farg4, c_farg5, c_farg6, c_farg7 +}; + +#define assert_cond(ARG1) assert(ARG1, #ARG1) + +// Addressing modes +class Address { + public: + + enum mode { no_mode, base_plus_offset, pcrel, literal }; + + private: + Register _base; + Register _index; + int64_t _offset; + enum mode _mode; + + RelocationHolder _rspec; + + // If the target is far we'll need to load the ea of this to a + // register to reach it. Otherwise if near we can do PC-relative + // addressing. + address _target; + + public: + Address() + : _base(noreg), _index(noreg), _offset(0), _mode(no_mode), _target(NULL) { } + + Address(Register r) + : _base(r), _index(noreg), _offset(0), _mode(base_plus_offset), _target(NULL) { } + + template::value)> + Address(Register r, T o) + : _base(r), _index(noreg), _offset(o), _mode(base_plus_offset), _target(NULL) {} + + Address(Register r, ByteSize disp) + : Address(r, in_bytes(disp)) {} + + Address(address target, RelocationHolder const& rspec) + : _base(noreg), + _index(noreg), + _offset(0), + _mode(literal), + _rspec(rspec), + _target(target) { } + + Address(address target, relocInfo::relocType rtype = relocInfo::external_word_type); + + const Register base() const { + guarantee((_mode == base_plus_offset | _mode == pcrel | _mode == literal), "wrong mode"); + return _base; + } + long offset() const { + return _offset; + } + Register index() const { + return _index; + } + mode getMode() const { + return _mode; + } + + bool uses(Register reg) const { return _base == reg; } + const address target() const { return _target; } + const RelocationHolder& rspec() const { return _rspec; } + ~Address() { + _target = NULL; + _base = NULL; + } +}; + +// Convience classes +class RuntimeAddress: public Address { + + public: + + RuntimeAddress(address target) : Address(target, relocInfo::runtime_call_type) {} + ~RuntimeAddress() {} +}; + +class OopAddress: public Address { + + public: + + OopAddress(address target) : Address(target, relocInfo::oop_type) {} + ~OopAddress() {} +}; + +class ExternalAddress: public Address { + private: + static relocInfo::relocType reloc_for_target(address target) { + // Sometimes ExternalAddress is used for values which aren't + // exactly addresses, like the card table base. + // external_word_type can't be used for values in the first page + // so just skip the reloc in that case. + return external_word_Relocation::can_be_relocated(target) ? relocInfo::external_word_type : relocInfo::none; + } + + public: + + ExternalAddress(address target) : Address(target, reloc_for_target(target)) {} + ~ExternalAddress() {} +}; + +class InternalAddress: public Address { + + public: + + InternalAddress(address target) : Address(target, relocInfo::internal_word_type) {} + ~InternalAddress() {} +}; + +class Assembler : public AbstractAssembler { +public: + + enum { + instruction_size = 4, + compressed_instruction_size = 2, + }; + + // instruction must start at passed address + static bool is_compressed_instr(address instr) { + // The RISC-V ISA Manual, Section 'Base Instruction-Length Encoding': + // Instructions are stored in memory as a sequence of 16-bit little-endian parcels, regardless of + // memory system endianness. Parcels forming one instruction are stored at increasing halfword + // addresses, with the lowest-addressed parcel holding the lowest-numbered bits in the instruction + // specification. + if (UseRVC && (((uint16_t *)instr)[0] & 0b11) != 0b11) { + // 16-bit instructions have their lowest two bits equal to 0b00, 0b01, or 0b10 + return true; + } + // 32-bit instructions have their lowest two bits set to 0b11 + return false; + } + + //---< calculate length of instruction >--- + // We just use the values set above. + // instruction must start at passed address + static unsigned int instr_len(address instr) { + return is_compressed_instr(instr) ? compressed_instruction_size : instruction_size; + } + + //---< longest instructions >--- + static unsigned int instr_maxlen() { return instruction_size; } + + enum RoundingMode { + rne = 0b000, // round to Nearest, ties to Even + rtz = 0b001, // round towards Zero + rdn = 0b010, // round Down (towards eegative infinity) + rup = 0b011, // round Up (towards infinity) + rmm = 0b100, // round to Nearest, ties to Max Magnitude + rdy = 0b111, // in instruction's rm field, selects dynamic rounding mode.In Rounding Mode register, Invalid. + }; + + static inline uint32_t extract(uint32_t val, unsigned msb, unsigned lsb) { + assert_cond(msb >= lsb && msb <= 31); + unsigned nbits = msb - lsb + 1; + uint32_t mask = (1U << nbits) - 1; + uint32_t result = val >> lsb; + result &= mask; + return result; + } + + static inline int32_t sextract(uint32_t val, unsigned msb, unsigned lsb) { + assert_cond(msb >= lsb && msb <= 31); + int32_t result = val << (31 - msb); + result >>= (31 - msb + lsb); + return result; + } + + static void patch(address a, unsigned msb, unsigned lsb, unsigned val) { + assert_cond(a != NULL); + assert_cond(msb >= lsb && msb <= 31); + unsigned nbits = msb - lsb + 1; + guarantee(val < (1U << nbits), "Field too big for insn"); + unsigned mask = (1U << nbits) - 1; + val <<= lsb; + mask <<= lsb; + unsigned target = *(unsigned *)a; + target &= ~mask; + target |= val; + *(unsigned *)a = target; + } + + static void patch(address a, unsigned bit, unsigned val) { + patch(a, bit, bit, val); + } + + static void patch_reg(address a, unsigned lsb, Register reg) { + patch(a, lsb + 4, lsb, reg->encoding_nocheck()); + } + + static void patch_reg(address a, unsigned lsb, FloatRegister reg) { + patch(a, lsb + 4, lsb, reg->encoding_nocheck()); + } + + static void patch_reg(address a, unsigned lsb, VectorRegister reg) { + patch(a, lsb + 4, lsb, reg->encoding_nocheck()); + } + + void emit(unsigned insn) { + emit_int32((jint)insn); + } + + enum csr { + cycle = 0xc00, + time, + instret, + hpmcounter3, + hpmcounter4, + hpmcounter5, + hpmcounter6, + hpmcounter7, + hpmcounter8, + hpmcounter9, + hpmcounter10, + hpmcounter11, + hpmcounter12, + hpmcounter13, + hpmcounter14, + hpmcounter15, + hpmcounter16, + hpmcounter17, + hpmcounter18, + hpmcounter19, + hpmcounter20, + hpmcounter21, + hpmcounter22, + hpmcounter23, + hpmcounter24, + hpmcounter25, + hpmcounter26, + hpmcounter27, + hpmcounter28, + hpmcounter29, + hpmcounter30, + hpmcounter31 = 0xc1f + }; + + // Emit an illegal instruction that's known to trap, with 32 read-only CSR + // to choose as the input operand. + // According to the RISC-V Assembly Programmer's Manual, a de facto implementation + // of this instruction is the UNIMP pseduo-instruction, 'CSRRW x0, cycle, x0', + // attempting to write zero to a read-only CSR 'cycle' (0xC00). + // RISC-V ISAs provide a set of up to 32 read-only CSR registers 0xC00-0xC1F, + // and an attempt to write into any read-only CSR (whether it exists or not) + // will generate an illegal instruction exception. + void illegal_instruction(csr csr_reg) { + csrrw(x0, (unsigned)csr_reg, x0); + } + +// Register Instruction +#define INSN(NAME, op, funct3, funct7) \ + void NAME(Register Rd, Register Rs1, Register Rs2) { \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch((address)&insn, 31, 25, funct7); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + patch_reg((address)&insn, 20, Rs2); \ + emit(insn); \ + } + + INSN(_add, 0b0110011, 0b000, 0b0000000); + INSN(_sub, 0b0110011, 0b000, 0b0100000); + INSN(_andr, 0b0110011, 0b111, 0b0000000); + INSN(_orr, 0b0110011, 0b110, 0b0000000); + INSN(_xorr, 0b0110011, 0b100, 0b0000000); + INSN(sll, 0b0110011, 0b001, 0b0000000); + INSN(sra, 0b0110011, 0b101, 0b0100000); + INSN(srl, 0b0110011, 0b101, 0b0000000); + INSN(slt, 0b0110011, 0b010, 0b0000000); + INSN(sltu, 0b0110011, 0b011, 0b0000000); + INSN(_addw, 0b0111011, 0b000, 0b0000000); + INSN(_subw, 0b0111011, 0b000, 0b0100000); + INSN(sllw, 0b0111011, 0b001, 0b0000000); + INSN(sraw, 0b0111011, 0b101, 0b0100000); + INSN(srlw, 0b0111011, 0b101, 0b0000000); + INSN(mul, 0b0110011, 0b000, 0b0000001); + INSN(mulh, 0b0110011, 0b001, 0b0000001); + INSN(mulhsu,0b0110011, 0b010, 0b0000001); + INSN(mulhu, 0b0110011, 0b011, 0b0000001); + INSN(mulw, 0b0111011, 0b000, 0b0000001); + INSN(div, 0b0110011, 0b100, 0b0000001); + INSN(divu, 0b0110011, 0b101, 0b0000001); + INSN(divw, 0b0111011, 0b100, 0b0000001); + INSN(divuw, 0b0111011, 0b101, 0b0000001); + INSN(rem, 0b0110011, 0b110, 0b0000001); + INSN(remu, 0b0110011, 0b111, 0b0000001); + INSN(remw, 0b0111011, 0b110, 0b0000001); + INSN(remuw, 0b0111011, 0b111, 0b0000001); + +#undef INSN + +// Load/store register (all modes) +#define INSN(NAME, op, funct3) \ + void NAME(Register Rd, Register Rs, const int32_t offset) { \ + guarantee(is_simm12(offset), "offset is invalid."); \ + unsigned insn = 0; \ + int32_t val = offset & 0xfff; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch_reg((address)&insn, 15, Rs); \ + patch_reg((address)&insn, 7, Rd); \ + patch((address)&insn, 31, 20, val); \ + emit(insn); \ + } + + INSN(lb, 0b0000011, 0b000); + INSN(lbu, 0b0000011, 0b100); + INSN(lh, 0b0000011, 0b001); + INSN(lhu, 0b0000011, 0b101); + INSN(_lw, 0b0000011, 0b010); + INSN(lwu, 0b0000011, 0b110); + INSN(_ld, 0b0000011, 0b011); + +#undef INSN + +#define INSN(NAME, op, funct3) \ + void NAME(FloatRegister Rd, Register Rs, const int32_t offset) { \ + guarantee(is_simm12(offset), "offset is invalid."); \ + unsigned insn = 0; \ + uint32_t val = offset & 0xfff; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch_reg((address)&insn, 15, Rs); \ + patch_reg((address)&insn, 7, Rd); \ + patch((address)&insn, 31, 20, val); \ + emit(insn); \ + } + + INSN(flw, 0b0000111, 0b010); + INSN(_fld, 0b0000111, 0b011); + +#undef INSN + +#define INSN(NAME, op, funct3) \ + void NAME(Register Rs1, Register Rs2, const int64_t offset) { \ + guarantee(is_simm13(offset) && ((offset % 2) == 0), "offset is invalid."); \ + unsigned insn = 0; \ + uint32_t val = offset & 0x1fff; \ + uint32_t val11 = (val >> 11) & 0x1; \ + uint32_t val12 = (val >> 12) & 0x1; \ + uint32_t low = (val >> 1) & 0xf; \ + uint32_t high = (val >> 5) & 0x3f; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch_reg((address)&insn, 15, Rs1); \ + patch_reg((address)&insn, 20, Rs2); \ + patch((address)&insn, 7, val11); \ + patch((address)&insn, 11, 8, low); \ + patch((address)&insn, 30, 25, high); \ + patch((address)&insn, 31, val12); \ + emit(insn); \ + } + + INSN(beq, 0b1100011, 0b000); + INSN(bne, 0b1100011, 0b001); + INSN(bge, 0b1100011, 0b101); + INSN(bgeu, 0b1100011, 0b111); + INSN(blt, 0b1100011, 0b100); + INSN(bltu, 0b1100011, 0b110); + +#undef INSN + +#define INSN(NAME, REGISTER, op, funct3) \ + void NAME(REGISTER Rs1, Register Rs2, const int32_t offset) { \ + guarantee(is_simm12(offset), "offset is invalid."); \ + unsigned insn = 0; \ + uint32_t val = offset & 0xfff; \ + uint32_t low = val & 0x1f; \ + uint32_t high = (val >> 5) & 0x7f; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch_reg((address)&insn, 15, Rs2); \ + patch_reg((address)&insn, 20, Rs1); \ + patch((address)&insn, 11, 7, low); \ + patch((address)&insn, 31, 25, high); \ + emit(insn); \ + } \ + + INSN(sb, Register, 0b0100011, 0b000); + INSN(sh, Register, 0b0100011, 0b001); + INSN(_sw, Register, 0b0100011, 0b010); + INSN(_sd, Register, 0b0100011, 0b011); + INSN(fsw, FloatRegister, 0b0100111, 0b010); + INSN(_fsd, FloatRegister, 0b0100111, 0b011); + +#undef INSN + +#define INSN(NAME, op, funct3) \ + void NAME(Register Rd, const uint32_t csr, Register Rs1) { \ + guarantee(is_uimm12(csr), "csr is invalid"); \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + patch((address)&insn, 31, 20, csr); \ + emit(insn); \ + } + + INSN(csrrw, 0b1110011, 0b001); + INSN(csrrs, 0b1110011, 0b010); + INSN(csrrc, 0b1110011, 0b011); + +#undef INSN + +#define INSN(NAME, op, funct3) \ + void NAME(Register Rd, const uint32_t csr, const uint32_t uimm) { \ + guarantee(is_uimm12(csr), "csr is invalid"); \ + guarantee(is_uimm5(uimm), "uimm is invalid"); \ + unsigned insn = 0; \ + uint32_t val = uimm & 0x1f; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch_reg((address)&insn, 7, Rd); \ + patch((address)&insn, 19, 15, val); \ + patch((address)&insn, 31, 20, csr); \ + emit(insn); \ + } + + INSN(csrrwi, 0b1110011, 0b101); + INSN(csrrsi, 0b1110011, 0b110); + INSN(csrrci, 0b1110011, 0b111); + +#undef INSN + +#define INSN(NAME, op) \ + void NAME(Register Rd, const int32_t offset) { \ + guarantee(is_simm21(offset) && ((offset % 2) == 0), "offset is invalid."); \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch_reg((address)&insn, 7, Rd); \ + patch((address)&insn, 19, 12, (uint32_t)((offset >> 12) & 0xff)); \ + patch((address)&insn, 20, (uint32_t)((offset >> 11) & 0x1)); \ + patch((address)&insn, 30, 21, (uint32_t)((offset >> 1) & 0x3ff)); \ + patch((address)&insn, 31, (uint32_t)((offset >> 20) & 0x1)); \ + emit(insn); \ + } + + INSN(jal, 0b1101111); + +#undef INSN + +#define INSN(NAME, op, funct) \ + void NAME(Register Rd, Register Rs, const int32_t offset) { \ + guarantee(is_simm12(offset), "offset is invalid."); \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch_reg((address)&insn, 7, Rd); \ + patch((address)&insn, 14, 12, funct); \ + patch_reg((address)&insn, 15, Rs); \ + int32_t val = offset & 0xfff; \ + patch((address)&insn, 31, 20, val); \ + emit(insn); \ + } + + INSN(_jalr, 0b1100111, 0b000); + +#undef INSN + + enum barrier { + i = 0b1000, o = 0b0100, r = 0b0010, w = 0b0001, + ir = i | r, ow = o | w, iorw = i | o | r | w + }; + + void fence(const uint32_t predecessor, const uint32_t successor) { + unsigned insn = 0; + guarantee(predecessor < 16, "predecessor is invalid"); + guarantee(successor < 16, "successor is invalid"); + patch((address)&insn, 6, 0, 0b001111); + patch((address)&insn, 11, 7, 0b00000); + patch((address)&insn, 14, 12, 0b000); + patch((address)&insn, 19, 15, 0b00000); + patch((address)&insn, 23, 20, successor); + patch((address)&insn, 27, 24, predecessor); + patch((address)&insn, 31, 28, 0b0000); + emit(insn); + } + +#define INSN(NAME, op, funct3, funct7) \ + void NAME() { \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 11, 7, 0b00000); \ + patch((address)&insn, 14, 12, funct3); \ + patch((address)&insn, 19, 15, 0b00000); \ + patch((address)&insn, 31, 20, funct7); \ + emit(insn); \ + } + + INSN(ecall, 0b1110011, 0b000, 0b000000000000); + INSN(_ebreak, 0b1110011, 0b000, 0b000000000001); + +#undef INSN + +enum Aqrl {relaxed = 0b00, rl = 0b01, aq = 0b10, aqrl = 0b11}; + +#define INSN(NAME, op, funct3, funct7) \ + void NAME(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + patch_reg((address)&insn, 20, Rs2); \ + patch((address)&insn, 31, 27, funct7); \ + patch((address)&insn, 26, 25, memory_order); \ + emit(insn); \ + } + + INSN(amoswap_w, 0b0101111, 0b010, 0b00001); + INSN(amoadd_w, 0b0101111, 0b010, 0b00000); + INSN(amoxor_w, 0b0101111, 0b010, 0b00100); + INSN(amoand_w, 0b0101111, 0b010, 0b01100); + INSN(amoor_w, 0b0101111, 0b010, 0b01000); + INSN(amomin_w, 0b0101111, 0b010, 0b10000); + INSN(amomax_w, 0b0101111, 0b010, 0b10100); + INSN(amominu_w, 0b0101111, 0b010, 0b11000); + INSN(amomaxu_w, 0b0101111, 0b010, 0b11100); + INSN(amoswap_d, 0b0101111, 0b011, 0b00001); + INSN(amoadd_d, 0b0101111, 0b011, 0b00000); + INSN(amoxor_d, 0b0101111, 0b011, 0b00100); + INSN(amoand_d, 0b0101111, 0b011, 0b01100); + INSN(amoor_d, 0b0101111, 0b011, 0b01000); + INSN(amomin_d, 0b0101111, 0b011, 0b10000); + INSN(amomax_d , 0b0101111, 0b011, 0b10100); + INSN(amominu_d, 0b0101111, 0b011, 0b11000); + INSN(amomaxu_d, 0b0101111, 0b011, 0b11100); +#undef INSN + +enum operand_size { int8, int16, int32, uint32, int64 }; + +#define INSN(NAME, op, funct3, funct7) \ + void NAME(Register Rd, Register Rs1, Aqrl memory_order = relaxed) { \ + unsigned insn = 0; \ + uint32_t val = memory_order & 0x3; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + patch((address)&insn, 25, 20, 0b00000); \ + patch((address)&insn, 31, 27, funct7); \ + patch((address)&insn, 26, 25, val); \ + emit(insn); \ + } + + INSN(lr_w, 0b0101111, 0b010, 0b00010); + INSN(lr_d, 0b0101111, 0b011, 0b00010); + +#undef INSN + +#define INSN(NAME, op, funct3, funct7) \ + void NAME(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = relaxed) { \ + unsigned insn = 0; \ + uint32_t val = memory_order & 0x3; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs2); \ + patch_reg((address)&insn, 20, Rs1); \ + patch((address)&insn, 31, 27, funct7); \ + patch((address)&insn, 26, 25, val); \ + emit(insn); \ + } + + INSN(sc_w, 0b0101111, 0b010, 0b00011); + INSN(sc_d, 0b0101111, 0b011, 0b00011); +#undef INSN + +#define INSN(NAME, op, funct5, funct7) \ + void NAME(FloatRegister Rd, FloatRegister Rs1, RoundingMode rm = rne) { \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, rm); \ + patch((address)&insn, 24, 20, funct5); \ + patch((address)&insn, 31, 25, funct7); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + emit(insn); \ + } + + INSN(fsqrt_s, 0b1010011, 0b00000, 0b0101100); + INSN(fsqrt_d, 0b1010011, 0b00000, 0b0101101); + INSN(fcvt_s_d, 0b1010011, 0b00001, 0b0100000); + INSN(fcvt_d_s, 0b1010011, 0b00000, 0b0100001); +#undef INSN + +// Immediate Instruction +#define INSN(NAME, op, funct3) \ + void NAME(Register Rd, Register Rs1, int32_t imm) { \ + guarantee(is_simm12(imm), "Immediate is out of validity"); \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch((address)&insn, 31, 20, imm & 0x00000fff); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + emit(insn); \ + } + + INSN(_addi, 0b0010011, 0b000); + INSN(slti, 0b0010011, 0b010); + INSN(_addiw, 0b0011011, 0b000); + INSN(_and_imm12, 0b0010011, 0b111); + INSN(ori, 0b0010011, 0b110); + INSN(xori, 0b0010011, 0b100); + +#undef INSN + +#define INSN(NAME, op, funct3) \ + void NAME(Register Rd, Register Rs1, uint32_t imm) { \ + guarantee(is_uimm12(imm), "Immediate is out of validity"); \ + unsigned insn = 0; \ + patch((address)&insn,6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch((address)&insn, 31, 20, imm & 0x00000fff); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + emit(insn); \ + } + + INSN(sltiu, 0b0010011, 0b011); + +#undef INSN + +// Shift Immediate Instruction +#define INSN(NAME, op, funct3, funct6) \ + void NAME(Register Rd, Register Rs1, unsigned shamt) { \ + guarantee(shamt <= 0x3f, "Shamt is invalid"); \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch((address)&insn, 25, 20, shamt); \ + patch((address)&insn, 31, 26, funct6); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + emit(insn); \ + } + + INSN(_slli, 0b0010011, 0b001, 0b000000); + INSN(_srai, 0b0010011, 0b101, 0b010000); + INSN(_srli, 0b0010011, 0b101, 0b000000); + +#undef INSN + +// Shift Word Immediate Instruction +#define INSN(NAME, op, funct3, funct7) \ + void NAME(Register Rd, Register Rs1, unsigned shamt) { \ + guarantee(shamt <= 0x1f, "Shamt is invalid"); \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch((address)&insn, 24, 20, shamt); \ + patch((address)&insn, 31, 25, funct7); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + emit(insn); \ + } + + INSN(slliw, 0b0011011, 0b001, 0b0000000); + INSN(sraiw, 0b0011011, 0b101, 0b0100000); + INSN(srliw, 0b0011011, 0b101, 0b0000000); + +#undef INSN + +// Upper Immediate Instruction +#define INSN(NAME, op) \ + void NAME(Register Rd, int32_t imm) { \ + int32_t upperImm = imm >> 12; \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch_reg((address)&insn, 7, Rd); \ + upperImm &= 0x000fffff; \ + patch((address)&insn, 31, 12, upperImm); \ + emit(insn); \ + } + + INSN(_lui, 0b0110111); + INSN(auipc, 0b0010111); + +#undef INSN + +// Float and Double Rigster Instruction +#define INSN(NAME, op, funct2) \ + void NAME(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, FloatRegister Rs3, RoundingMode rm = rne) { \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, rm); \ + patch((address)&insn, 26, 25, funct2); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + patch_reg((address)&insn, 20, Rs2); \ + patch_reg((address)&insn, 27, Rs3); \ + emit(insn); \ + } + + INSN(fmadd_s, 0b1000011, 0b00); + INSN(fmsub_s, 0b1000111, 0b00); + INSN(fnmsub_s, 0b1001011, 0b00); + INSN(fnmadd_s, 0b1001111, 0b00); + INSN(fmadd_d, 0b1000011, 0b01); + INSN(fmsub_d, 0b1000111, 0b01); + INSN(fnmsub_d, 0b1001011, 0b01); + INSN(fnmadd_d, 0b1001111, 0b01); + +#undef INSN + +// Float and Double Rigster Instruction +#define INSN(NAME, op, funct3, funct7) \ + void NAME(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) { \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch((address)&insn, 31, 25, funct7); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + patch_reg((address)&insn, 20, Rs2); \ + emit(insn); \ + } + + INSN(fsgnj_s, 0b1010011, 0b000, 0b0010000); + INSN(fsgnjn_s, 0b1010011, 0b001, 0b0010000); + INSN(fsgnjx_s, 0b1010011, 0b010, 0b0010000); + INSN(fmin_s, 0b1010011, 0b000, 0b0010100); + INSN(fmax_s, 0b1010011, 0b001, 0b0010100); + INSN(fsgnj_d, 0b1010011, 0b000, 0b0010001); + INSN(fsgnjn_d, 0b1010011, 0b001, 0b0010001); + INSN(fsgnjx_d, 0b1010011, 0b010, 0b0010001); + INSN(fmin_d, 0b1010011, 0b000, 0b0010101); + INSN(fmax_d, 0b1010011, 0b001, 0b0010101); + +#undef INSN + +// Float and Double Rigster Arith Instruction +#define INSN(NAME, op, funct3, funct7) \ + void NAME(Register Rd, FloatRegister Rs1, FloatRegister Rs2) { \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch((address)&insn, 31, 25, funct7); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + patch_reg((address)&insn, 20, Rs2); \ + emit(insn); \ + } + + INSN(feq_s, 0b1010011, 0b010, 0b1010000); + INSN(flt_s, 0b1010011, 0b001, 0b1010000); + INSN(fle_s, 0b1010011, 0b000, 0b1010000); + INSN(feq_d, 0b1010011, 0b010, 0b1010001); + INSN(fle_d, 0b1010011, 0b000, 0b1010001); + INSN(flt_d, 0b1010011, 0b001, 0b1010001); +#undef INSN + +// Float and Double Arith Instruction +#define INSN(NAME, op, funct7) \ + void NAME(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, RoundingMode rm = rne) { \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, rm); \ + patch((address)&insn, 31, 25, funct7); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + patch_reg((address)&insn, 20, Rs2); \ + emit(insn); \ + } + + INSN(fadd_s, 0b1010011, 0b0000000); + INSN(fsub_s, 0b1010011, 0b0000100); + INSN(fmul_s, 0b1010011, 0b0001000); + INSN(fdiv_s, 0b1010011, 0b0001100); + INSN(fadd_d, 0b1010011, 0b0000001); + INSN(fsub_d, 0b1010011, 0b0000101); + INSN(fmul_d, 0b1010011, 0b0001001); + INSN(fdiv_d, 0b1010011, 0b0001101); + +#undef INSN + +// Whole Float and Double Conversion Instruction +#define INSN(NAME, op, funct5, funct7) \ + void NAME(FloatRegister Rd, Register Rs1, RoundingMode rm = rne) { \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, rm); \ + patch((address)&insn, 24, 20, funct5); \ + patch((address)&insn, 31, 25, funct7); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + emit(insn); \ + } + + INSN(fcvt_s_w, 0b1010011, 0b00000, 0b1101000); + INSN(fcvt_s_wu, 0b1010011, 0b00001, 0b1101000); + INSN(fcvt_s_l, 0b1010011, 0b00010, 0b1101000); + INSN(fcvt_s_lu, 0b1010011, 0b00011, 0b1101000); + INSN(fcvt_d_w, 0b1010011, 0b00000, 0b1101001); + INSN(fcvt_d_wu, 0b1010011, 0b00001, 0b1101001); + INSN(fcvt_d_l, 0b1010011, 0b00010, 0b1101001); + INSN(fcvt_d_lu, 0b1010011, 0b00011, 0b1101001); + +#undef INSN + +// Float and Double Conversion Instruction +#define INSN(NAME, op, funct5, funct7) \ + void NAME(Register Rd, FloatRegister Rs1, RoundingMode rm = rtz) { \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, rm); \ + patch((address)&insn, 24, 20, funct5); \ + patch((address)&insn, 31, 25, funct7); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + emit(insn); \ + } + + INSN(fcvt_w_s, 0b1010011, 0b00000, 0b1100000); + INSN(fcvt_l_s, 0b1010011, 0b00010, 0b1100000); + INSN(fcvt_wu_s, 0b1010011, 0b00001, 0b1100000); + INSN(fcvt_lu_s, 0b1010011, 0b00011, 0b1100000); + INSN(fcvt_w_d, 0b1010011, 0b00000, 0b1100001); + INSN(fcvt_wu_d, 0b1010011, 0b00001, 0b1100001); + INSN(fcvt_l_d, 0b1010011, 0b00010, 0b1100001); + INSN(fcvt_lu_d, 0b1010011, 0b00011, 0b1100001); + +#undef INSN + +// Float and Double Move Instruction +#define INSN(NAME, op, funct3, funct5, funct7) \ + void NAME(FloatRegister Rd, Register Rs1) { \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch((address)&insn, 20, funct5); \ + patch((address)&insn, 31, 25, funct7); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + emit(insn); \ + } + + INSN(fmv_w_x, 0b1010011, 0b000, 0b00000, 0b1111000); + INSN(fmv_d_x, 0b1010011, 0b000, 0b00000, 0b1111001); + +#undef INSN + +// Float and Double Conversion Instruction +#define INSN(NAME, op, funct3, funct5, funct7) \ + void NAME(Register Rd, FloatRegister Rs1) { \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch((address)&insn, 20, funct5); \ + patch((address)&insn, 31, 25, funct7); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + emit(insn); \ + } + + INSN(fclass_s, 0b1010011, 0b001, 0b00000, 0b1110000); + INSN(fclass_d, 0b1010011, 0b001, 0b00000, 0b1110001); + INSN(fmv_x_w, 0b1010011, 0b000, 0b00000, 0b1110000); + INSN(fmv_x_d, 0b1010011, 0b000, 0b00000, 0b1110001); + +#undef INSN + +// ========================== +// RISC-V Vector Extension +// ========================== +enum SEW { + e8, + e16, + e32, + e64, + RESERVED, +}; + +enum LMUL { + mf8 = 0b101, + mf4 = 0b110, + mf2 = 0b111, + m1 = 0b000, + m2 = 0b001, + m4 = 0b010, + m8 = 0b011, +}; + +enum VMA { + mu, // undisturbed + ma, // agnostic +}; + +enum VTA { + tu, // undisturbed + ta, // agnostic +}; + +static Assembler::SEW elembytes_to_sew(int ebytes) { + assert(ebytes > 0 && ebytes <= 8, "unsupported element size"); + return (Assembler::SEW) exact_log2(ebytes); +} + +static Assembler::SEW elemtype_to_sew(BasicType etype) { + return Assembler::elembytes_to_sew(type2aelembytes(etype)); +} + +#define patch_vtype(hsb, lsb, vlmul, vsew, vta, vma, vill) \ + if (vill == 1) { \ + guarantee((vlmul | vsew | vta | vma == 0), \ + "the other bits in vtype shall be zero"); \ + } \ + patch((address)&insn, lsb + 2, lsb, vlmul); \ + patch((address)&insn, lsb + 5, lsb + 3, vsew); \ + patch((address)&insn, lsb + 6, vta); \ + patch((address)&insn, lsb + 7, vma); \ + patch((address)&insn, hsb - 1, lsb + 8, 0); \ + patch((address)&insn, hsb, vill) + +#define INSN(NAME, op, funct3) \ + void NAME(Register Rd, Register Rs1, SEW sew, LMUL lmul = m1, \ + VMA vma = mu, VTA vta = tu, bool vill = false) { \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch_vtype(30, 20, lmul, sew, vta, vma, vill); \ + patch((address)&insn, 31, 0); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + emit(insn); \ + } + + INSN(vsetvli, 0b1010111, 0b111); + +#undef INSN + +#define INSN(NAME, op, funct3) \ + void NAME(Register Rd, uint32_t imm, SEW sew, LMUL lmul = m1, \ + VMA vma = mu, VTA vta = tu, bool vill = false) { \ + unsigned insn = 0; \ + guarantee(is_uimm5(imm), "imm is invalid"); \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch((address)&insn, 19, 15, imm); \ + patch_vtype(29, 20, lmul, sew, vta, vma, vill); \ + patch((address)&insn, 31, 30, 0b11); \ + patch_reg((address)&insn, 7, Rd); \ + emit(insn); \ + } + + INSN(vsetivli, 0b1010111, 0b111); + +#undef INSN + +#undef patch_vtype + +#define INSN(NAME, op, funct3, funct7) \ + void NAME(Register Rd, Register Rs1, Register Rs2) { \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch((address)&insn, 31, 25, funct7); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + patch_reg((address)&insn, 20, Rs2); \ + emit(insn); \ + } + + // Vector Configuration Instruction + INSN(vsetvl, 0b1010111, 0b111, 0b1000000); + +#undef INSN + +enum VectorMask { + v0_t = 0b0, + unmasked = 0b1 +}; + +#define patch_VArith(op, Reg, funct3, Reg_or_Imm5, Vs2, vm, funct6) \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch((address)&insn, 19, 15, Reg_or_Imm5); \ + patch((address)&insn, 25, vm); \ + patch((address)&insn, 31, 26, funct6); \ + patch_reg((address)&insn, 7, Reg); \ + patch_reg((address)&insn, 20, Vs2); \ + emit(insn) + +// r2_vm +#define INSN(NAME, op, funct3, Vs1, funct6) \ + void NAME(Register Rd, VectorRegister Vs2, VectorMask vm = unmasked) { \ + patch_VArith(op, Rd, funct3, Vs1, Vs2, vm, funct6); \ + } + + // Vector Mask + INSN(vcpop_m, 0b1010111, 0b010, 0b10000, 0b010000); + INSN(vfirst_m, 0b1010111, 0b010, 0b10001, 0b010000); +#undef INSN + +#define INSN(NAME, op, funct3, Vs1, funct6) \ + void NAME(VectorRegister Vd, VectorRegister Vs2, VectorMask vm = unmasked) { \ + patch_VArith(op, Vd, funct3, Vs1, Vs2, vm, funct6); \ + } + + // Vector Integer Extension + INSN(vzext_vf2, 0b1010111, 0b010, 0b00110, 0b010010); + INSN(vzext_vf4, 0b1010111, 0b010, 0b00100, 0b010010); + INSN(vzext_vf8, 0b1010111, 0b010, 0b00010, 0b010010); + INSN(vsext_vf2, 0b1010111, 0b010, 0b00111, 0b010010); + INSN(vsext_vf4, 0b1010111, 0b010, 0b00101, 0b010010); + INSN(vsext_vf8, 0b1010111, 0b010, 0b00011, 0b010010); + + // Vector Mask + INSN(vmsbf_m, 0b1010111, 0b010, 0b00001, 0b010100); + INSN(vmsif_m, 0b1010111, 0b010, 0b00011, 0b010100); + INSN(vmsof_m, 0b1010111, 0b010, 0b00010, 0b010100); + INSN(viota_m, 0b1010111, 0b010, 0b10000, 0b010100); + + // Vector Single-Width Floating-Point/Integer Type-Convert Instructions + INSN(vfcvt_xu_f_v, 0b1010111, 0b001, 0b00000, 0b010010); + INSN(vfcvt_x_f_v, 0b1010111, 0b001, 0b00001, 0b010010); + INSN(vfcvt_f_xu_v, 0b1010111, 0b001, 0b00010, 0b010010); + INSN(vfcvt_f_x_v, 0b1010111, 0b001, 0b00011, 0b010010); + INSN(vfcvt_rtz_xu_f_v, 0b1010111, 0b001, 0b00110, 0b010010); + INSN(vfcvt_rtz_x_f_v, 0b1010111, 0b001, 0b00111, 0b010010); + + // Vector Floating-Point Instruction + INSN(vfsqrt_v, 0b1010111, 0b001, 0b00000, 0b010011); + INSN(vfclass_v, 0b1010111, 0b001, 0b10000, 0b010011); + +#undef INSN + +// r2rd +#define INSN(NAME, op, funct3, simm5, vm, funct6) \ + void NAME(VectorRegister Vd, VectorRegister Vs2) { \ + patch_VArith(op, Vd, funct3, simm5, Vs2, vm, funct6); \ + } + + // Vector Whole Vector Register Move + INSN(vmv1r_v, 0b1010111, 0b011, 0b00000, 0b1, 0b100111); + INSN(vmv2r_v, 0b1010111, 0b011, 0b00001, 0b1, 0b100111); + INSN(vmv4r_v, 0b1010111, 0b011, 0b00011, 0b1, 0b100111); + INSN(vmv8r_v, 0b1010111, 0b011, 0b00111, 0b1, 0b100111); + +#undef INSN + +#define INSN(NAME, op, funct3, Vs1, vm, funct6) \ + void NAME(FloatRegister Rd, VectorRegister Vs2) { \ + patch_VArith(op, Rd, funct3, Vs1, Vs2, vm, funct6); \ + } + + // Vector Floating-Point Move Instruction + INSN(vfmv_f_s, 0b1010111, 0b001, 0b00000, 0b1, 0b010000); + +#undef INSN + +#define INSN(NAME, op, funct3, Vs1, vm, funct6) \ + void NAME(Register Rd, VectorRegister Vs2) { \ + patch_VArith(op, Rd, funct3, Vs1, Vs2, vm, funct6); \ + } + + // Vector Integer Scalar Move Instructions + INSN(vmv_x_s, 0b1010111, 0b010, 0b00000, 0b1, 0b010000); + +#undef INSN + +// r_vm +#define INSN(NAME, op, funct3, funct6) \ + void NAME(VectorRegister Vd, VectorRegister Vs2, uint32_t imm, VectorMask vm = unmasked) { \ + guarantee(is_uimm5(imm), "imm is invalid"); \ + patch_VArith(op, Vd, funct3, (uint32_t)(imm & 0x1f), Vs2, vm, funct6); \ + } + + // Vector Single-Width Bit Shift Instructions + INSN(vsra_vi, 0b1010111, 0b011, 0b101001); + INSN(vsrl_vi, 0b1010111, 0b011, 0b101000); + INSN(vsll_vi, 0b1010111, 0b011, 0b100101); + +#undef INSN + +#define INSN(NAME, op, funct3, funct6) \ + void NAME(VectorRegister Vd, VectorRegister Vs1, VectorRegister Vs2, VectorMask vm = unmasked) { \ + patch_VArith(op, Vd, funct3, Vs1->encoding_nocheck(), Vs2, vm, funct6); \ + } + + // Vector Single-Width Floating-Point Fused Multiply-Add Instructions + INSN(vfnmsub_vv, 0b1010111, 0b001, 0b101011); + INSN(vfmsub_vv, 0b1010111, 0b001, 0b101010); + INSN(vfnmadd_vv, 0b1010111, 0b001, 0b101001); + INSN(vfmadd_vv, 0b1010111, 0b001, 0b101000); + INSN(vfnmsac_vv, 0b1010111, 0b001, 0b101111); + INSN(vfmsac_vv, 0b1010111, 0b001, 0b101110); + INSN(vfmacc_vv, 0b1010111, 0b001, 0b101100); + INSN(vfnmacc_vv, 0b1010111, 0b001, 0b101101); + + // Vector Single-Width Integer Multiply-Add Instructions + INSN(vnmsub_vv, 0b1010111, 0b010, 0b101011); + INSN(vmadd_vv, 0b1010111, 0b010, 0b101001); + INSN(vnmsac_vv, 0b1010111, 0b010, 0b101111); + INSN(vmacc_vv, 0b1010111, 0b010, 0b101101); + +#undef INSN + +#define INSN(NAME, op, funct3, funct6) \ + void NAME(VectorRegister Vd, Register Rs1, VectorRegister Vs2, VectorMask vm = unmasked) { \ + patch_VArith(op, Vd, funct3, Rs1->encoding_nocheck(), Vs2, vm, funct6); \ + } + + // Vector Single-Width Integer Multiply-Add Instructions + INSN(vnmsub_vx, 0b1010111, 0b110, 0b101011); + INSN(vmadd_vx, 0b1010111, 0b110, 0b101001); + INSN(vnmsac_vx, 0b1010111, 0b110, 0b101111); + INSN(vmacc_vx, 0b1010111, 0b110, 0b101101); + +#undef INSN + +#define INSN(NAME, op, funct3, funct6) \ + void NAME(VectorRegister Vd, FloatRegister Rs1, VectorRegister Vs2, VectorMask vm = unmasked) { \ + patch_VArith(op, Vd, funct3, Rs1->encoding_nocheck(), Vs2, vm, funct6); \ + } + + // Vector Single-Width Floating-Point Fused Multiply-Add Instructions + INSN(vfnmsub_vf, 0b1010111, 0b101, 0b101011); + INSN(vfmsub_vf, 0b1010111, 0b101, 0b101010); + INSN(vfnmadd_vf, 0b1010111, 0b101, 0b101001); + INSN(vfmadd_vf, 0b1010111, 0b101, 0b101000); + INSN(vfnmsac_vf, 0b1010111, 0b101, 0b101111); + INSN(vfmsac_vf, 0b1010111, 0b101, 0b101110); + INSN(vfmacc_vf, 0b1010111, 0b101, 0b101100); + INSN(vfnmacc_vf, 0b1010111, 0b101, 0b101101); + +#undef INSN + +#define INSN(NAME, op, funct3, funct6) \ + void NAME(VectorRegister Vd, VectorRegister Vs2, VectorRegister Vs1, VectorMask vm = unmasked) { \ + patch_VArith(op, Vd, funct3, Vs1->encoding_nocheck(), Vs2, vm, funct6); \ + } + + // Vector Single-Width Floating-Point Reduction Instructions + INSN(vfredusum_vs, 0b1010111, 0b001, 0b000001); + INSN(vfredosum_vs, 0b1010111, 0b001, 0b000011); + INSN(vfredmin_vs, 0b1010111, 0b001, 0b000101); + INSN(vfredmax_vs, 0b1010111, 0b001, 0b000111); + + // Vector Single-Width Integer Reduction Instructions + INSN(vredsum_vs, 0b1010111, 0b010, 0b000000); + INSN(vredand_vs, 0b1010111, 0b010, 0b000001); + INSN(vredor_vs, 0b1010111, 0b010, 0b000010); + INSN(vredxor_vs, 0b1010111, 0b010, 0b000011); + INSN(vredminu_vs, 0b1010111, 0b010, 0b000100); + INSN(vredmin_vs, 0b1010111, 0b010, 0b000101); + INSN(vredmaxu_vs, 0b1010111, 0b010, 0b000110); + INSN(vredmax_vs, 0b1010111, 0b010, 0b000111); + + // Vector Floating-Point Compare Instructions + INSN(vmfle_vv, 0b1010111, 0b001, 0b011001); + INSN(vmflt_vv, 0b1010111, 0b001, 0b011011); + INSN(vmfne_vv, 0b1010111, 0b001, 0b011100); + INSN(vmfeq_vv, 0b1010111, 0b001, 0b011000); + + // Vector Floating-Point Sign-Injection Instructions + INSN(vfsgnjx_vv, 0b1010111, 0b001, 0b001010); + INSN(vfsgnjn_vv, 0b1010111, 0b001, 0b001001); + INSN(vfsgnj_vv, 0b1010111, 0b001, 0b001000); + + // Vector Floating-Point MIN/MAX Instructions + INSN(vfmax_vv, 0b1010111, 0b001, 0b000110); + INSN(vfmin_vv, 0b1010111, 0b001, 0b000100); + + // Vector Single-Width Floating-Point Multiply/Divide Instructions + INSN(vfdiv_vv, 0b1010111, 0b001, 0b100000); + INSN(vfmul_vv, 0b1010111, 0b001, 0b100100); + + // Vector Single-Width Floating-Point Add/Subtract Instructions + INSN(vfsub_vv, 0b1010111, 0b001, 0b000010); + INSN(vfadd_vv, 0b1010111, 0b001, 0b000000); + + // Vector Single-Width Fractional Multiply with Rounding and Saturation + INSN(vsmul_vv, 0b1010111, 0b000, 0b100111); + + // Vector Integer Divide Instructions + INSN(vrem_vv, 0b1010111, 0b010, 0b100011); + INSN(vremu_vv, 0b1010111, 0b010, 0b100010); + INSN(vdiv_vv, 0b1010111, 0b010, 0b100001); + INSN(vdivu_vv, 0b1010111, 0b010, 0b100000); + + // Vector Single-Width Integer Multiply Instructions + INSN(vmulhsu_vv, 0b1010111, 0b010, 0b100110); + INSN(vmulhu_vv, 0b1010111, 0b010, 0b100100); + INSN(vmulh_vv, 0b1010111, 0b010, 0b100111); + INSN(vmul_vv, 0b1010111, 0b010, 0b100101); + + // Vector Integer Min/Max Instructions + INSN(vmax_vv, 0b1010111, 0b000, 0b000111); + INSN(vmaxu_vv, 0b1010111, 0b000, 0b000110); + INSN(vmin_vv, 0b1010111, 0b000, 0b000101); + INSN(vminu_vv, 0b1010111, 0b000, 0b000100); + + // Vector Integer Comparison Instructions + INSN(vmsle_vv, 0b1010111, 0b000, 0b011101); + INSN(vmsleu_vv, 0b1010111, 0b000, 0b011100); + INSN(vmslt_vv, 0b1010111, 0b000, 0b011011); + INSN(vmsltu_vv, 0b1010111, 0b000, 0b011010); + INSN(vmsne_vv, 0b1010111, 0b000, 0b011001); + INSN(vmseq_vv, 0b1010111, 0b000, 0b011000); + + // Vector Single-Width Bit Shift Instructions + INSN(vsra_vv, 0b1010111, 0b000, 0b101001); + INSN(vsrl_vv, 0b1010111, 0b000, 0b101000); + INSN(vsll_vv, 0b1010111, 0b000, 0b100101); + + // Vector Bitwise Logical Instructions + INSN(vxor_vv, 0b1010111, 0b000, 0b001011); + INSN(vor_vv, 0b1010111, 0b000, 0b001010); + INSN(vand_vv, 0b1010111, 0b000, 0b001001); + + // Vector Single-Width Integer Add and Subtract + INSN(vsub_vv, 0b1010111, 0b000, 0b000010); + INSN(vadd_vv, 0b1010111, 0b000, 0b000000); + +#undef INSN + + +#define INSN(NAME, op, funct3, funct6) \ + void NAME(VectorRegister Vd, VectorRegister Vs2, Register Rs1, VectorMask vm = unmasked) { \ + patch_VArith(op, Vd, funct3, Rs1->encoding_nocheck(), Vs2, vm, funct6); \ + } + + // Vector Integer Divide Instructions + INSN(vrem_vx, 0b1010111, 0b110, 0b100011); + INSN(vremu_vx, 0b1010111, 0b110, 0b100010); + INSN(vdiv_vx, 0b1010111, 0b110, 0b100001); + INSN(vdivu_vx, 0b1010111, 0b110, 0b100000); + + // Vector Single-Width Integer Multiply Instructions + INSN(vmulhsu_vx, 0b1010111, 0b110, 0b100110); + INSN(vmulhu_vx, 0b1010111, 0b110, 0b100100); + INSN(vmulh_vx, 0b1010111, 0b110, 0b100111); + INSN(vmul_vx, 0b1010111, 0b110, 0b100101); + + // Vector Integer Min/Max Instructions + INSN(vmax_vx, 0b1010111, 0b100, 0b000111); + INSN(vmaxu_vx, 0b1010111, 0b100, 0b000110); + INSN(vmin_vx, 0b1010111, 0b100, 0b000101); + INSN(vminu_vx, 0b1010111, 0b100, 0b000100); + + // Vector Integer Comparison Instructions + INSN(vmsgt_vx, 0b1010111, 0b100, 0b011111); + INSN(vmsgtu_vx, 0b1010111, 0b100, 0b011110); + INSN(vmsle_vx, 0b1010111, 0b100, 0b011101); + INSN(vmsleu_vx, 0b1010111, 0b100, 0b011100); + INSN(vmslt_vx, 0b1010111, 0b100, 0b011011); + INSN(vmsltu_vx, 0b1010111, 0b100, 0b011010); + INSN(vmsne_vx, 0b1010111, 0b100, 0b011001); + INSN(vmseq_vx, 0b1010111, 0b100, 0b011000); + + // Vector Narrowing Integer Right Shift Instructions + INSN(vnsra_wx, 0b1010111, 0b100, 0b101101); + INSN(vnsrl_wx, 0b1010111, 0b100, 0b101100); + + // Vector Single-Width Bit Shift Instructions + INSN(vsra_vx, 0b1010111, 0b100, 0b101001); + INSN(vsrl_vx, 0b1010111, 0b100, 0b101000); + INSN(vsll_vx, 0b1010111, 0b100, 0b100101); + + // Vector Bitwise Logical Instructions + INSN(vxor_vx, 0b1010111, 0b100, 0b001011); + INSN(vor_vx, 0b1010111, 0b100, 0b001010); + INSN(vand_vx, 0b1010111, 0b100, 0b001001); + + // Vector Single-Width Integer Add and Subtract + INSN(vsub_vx, 0b1010111, 0b100, 0b000010); + INSN(vadd_vx, 0b1010111, 0b100, 0b000000); + INSN(vrsub_vx, 0b1010111, 0b100, 0b000011); + +#undef INSN + +#define INSN(NAME, op, funct3, funct6) \ + void NAME(VectorRegister Vd, VectorRegister Vs2, FloatRegister Rs1, VectorMask vm = unmasked) { \ + patch_VArith(op, Vd, funct3, Rs1->encoding_nocheck(), Vs2, vm, funct6); \ + } + + // Vector Floating-Point Compare Instructions + INSN(vmfge_vf, 0b1010111, 0b101, 0b011111); + INSN(vmfgt_vf, 0b1010111, 0b101, 0b011101); + INSN(vmfle_vf, 0b1010111, 0b101, 0b011001); + INSN(vmflt_vf, 0b1010111, 0b101, 0b011011); + INSN(vmfne_vf, 0b1010111, 0b101, 0b011100); + INSN(vmfeq_vf, 0b1010111, 0b101, 0b011000); + + // Vector Floating-Point Sign-Injection Instructions + INSN(vfsgnjx_vf, 0b1010111, 0b101, 0b001010); + INSN(vfsgnjn_vf, 0b1010111, 0b101, 0b001001); + INSN(vfsgnj_vf, 0b1010111, 0b101, 0b001000); + + // Vector Floating-Point MIN/MAX Instructions + INSN(vfmax_vf, 0b1010111, 0b101, 0b000110); + INSN(vfmin_vf, 0b1010111, 0b101, 0b000100); + + // Vector Single-Width Floating-Point Multiply/Divide Instructions + INSN(vfdiv_vf, 0b1010111, 0b101, 0b100000); + INSN(vfmul_vf, 0b1010111, 0b101, 0b100100); + INSN(vfrdiv_vf, 0b1010111, 0b101, 0b100001); + + // Vector Single-Width Floating-Point Add/Subtract Instructions + INSN(vfsub_vf, 0b1010111, 0b101, 0b000010); + INSN(vfadd_vf, 0b1010111, 0b101, 0b000000); + INSN(vfrsub_vf, 0b1010111, 0b101, 0b100111); + +#undef INSN + +#define INSN(NAME, op, funct3, funct6) \ + void NAME(VectorRegister Vd, VectorRegister Vs2, int32_t imm, VectorMask vm = unmasked) { \ + guarantee(is_simm5(imm), "imm is invalid"); \ + patch_VArith(op, Vd, funct3, (uint32_t)(imm & 0x1f), Vs2, vm, funct6); \ + } + + INSN(vmsgt_vi, 0b1010111, 0b011, 0b011111); + INSN(vmsgtu_vi, 0b1010111, 0b011, 0b011110); + INSN(vmsle_vi, 0b1010111, 0b011, 0b011101); + INSN(vmsleu_vi, 0b1010111, 0b011, 0b011100); + INSN(vmsne_vi, 0b1010111, 0b011, 0b011001); + INSN(vmseq_vi, 0b1010111, 0b011, 0b011000); + INSN(vxor_vi, 0b1010111, 0b011, 0b001011); + INSN(vor_vi, 0b1010111, 0b011, 0b001010); + INSN(vand_vi, 0b1010111, 0b011, 0b001001); + INSN(vadd_vi, 0b1010111, 0b011, 0b000000); + INSN(vrsub_vi, 0b1010111, 0b011, 0b000011); + +#undef INSN + +#define INSN(NAME, op, funct3, vm, funct6) \ + void NAME(VectorRegister Vd, VectorRegister Vs2, VectorRegister Vs1) { \ + patch_VArith(op, Vd, funct3, Vs1->encoding_nocheck(), Vs2, vm, funct6); \ + } + + // Vector Compress Instruction + INSN(vcompress_vm, 0b1010111, 0b010, 0b1, 0b010111); + + // Vector Mask-Register Logical Instructions + INSN(vmxnor_mm, 0b1010111, 0b010, 0b1, 0b011111); + INSN(vmorn_mm, 0b1010111, 0b010, 0b1, 0b011100); + INSN(vmnor_mm, 0b1010111, 0b010, 0b1, 0b011110); + INSN(vmor_mm, 0b1010111, 0b010, 0b1, 0b011010); + INSN(vmxor_mm, 0b1010111, 0b010, 0b1, 0b011011); + INSN(vmandn_mm, 0b1010111, 0b010, 0b1, 0b011000); + INSN(vmnand_mm, 0b1010111, 0b010, 0b1, 0b011101); + INSN(vmand_mm, 0b1010111, 0b010, 0b1, 0b011001); + +#undef INSN + +#define INSN(NAME, op, funct3, Vs2, vm, funct6) \ + void NAME(VectorRegister Vd, int32_t imm) { \ + guarantee(is_simm5(imm), "imm is invalid"); \ + patch_VArith(op, Vd, funct3, (uint32_t)(imm & 0x1f), Vs2, vm, funct6); \ + } + + // Vector Integer Move Instructions + INSN(vmv_v_i, 0b1010111, 0b011, v0, 0b1, 0b010111); + +#undef INSN + +#define INSN(NAME, op, funct3, Vs2, vm, funct6) \ + void NAME(VectorRegister Vd, FloatRegister Rs1) { \ + patch_VArith(op, Vd, funct3, Rs1->encoding_nocheck(), Vs2, vm, funct6); \ + } + + // Floating-Point Scalar Move Instructions + INSN(vfmv_s_f, 0b1010111, 0b101, v0, 0b1, 0b010000); + // Vector Floating-Point Move Instruction + INSN(vfmv_v_f, 0b1010111, 0b101, v0, 0b1, 0b010111); + +#undef INSN + +#define INSN(NAME, op, funct3, Vs2, vm, funct6) \ + void NAME(VectorRegister Vd, VectorRegister Vs1) { \ + patch_VArith(op, Vd, funct3, Vs1->encoding_nocheck(), Vs2, vm, funct6); \ + } + + // Vector Integer Move Instructions + INSN(vmv_v_v, 0b1010111, 0b000, v0, 0b1, 0b010111); + +#undef INSN + +#define INSN(NAME, op, funct3, Vs2, vm, funct6) \ + void NAME(VectorRegister Vd, Register Rs1) { \ + patch_VArith(op, Vd, funct3, Rs1->encoding_nocheck(), Vs2, vm, funct6); \ + } + + // Integer Scalar Move Instructions + INSN(vmv_s_x, 0b1010111, 0b110, v0, 0b1, 0b010000); + + // Vector Integer Move Instructions + INSN(vmv_v_x, 0b1010111, 0b100, v0, 0b1, 0b010111); + +#undef INSN +#undef patch_VArith + +#define INSN(NAME, op, funct13, funct6) \ + void NAME(VectorRegister Vd, VectorMask vm = unmasked) { \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 24, 12, funct13); \ + patch((address)&insn, 25, vm); \ + patch((address)&insn, 31, 26, funct6); \ + patch_reg((address)&insn, 7, Vd); \ + emit(insn); \ + } + + // Vector Element Index Instruction + INSN(vid_v, 0b1010111, 0b0000010001010, 0b010100); + +#undef INSN + +enum Nf { + g1 = 0b000, + g2 = 0b001, + g3 = 0b010, + g4 = 0b011, + g5 = 0b100, + g6 = 0b101, + g7 = 0b110, + g8 = 0b111 +}; + +#define patch_VLdSt(op, VReg, width, Rs1, Reg_or_umop, vm, mop, mew, nf) \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, width); \ + patch((address)&insn, 24, 20, Reg_or_umop); \ + patch((address)&insn, 25, vm); \ + patch((address)&insn, 27, 26, mop); \ + patch((address)&insn, 28, mew); \ + patch((address)&insn, 31, 29, nf); \ + patch_reg((address)&insn, 7, VReg); \ + patch_reg((address)&insn, 15, Rs1); \ + emit(insn) + +#define INSN(NAME, op, lumop, vm, mop, nf) \ + void NAME(VectorRegister Vd, Register Rs1, uint32_t width = 0, bool mew = false) { \ + guarantee(is_uimm3(width), "width is invalid"); \ + patch_VLdSt(op, Vd, width, Rs1, lumop, vm, mop, mew, nf); \ + } + + // Vector Load/Store Instructions + INSN(vl1re8_v, 0b0000111, 0b01000, 0b1, 0b00, g1); + +#undef INSN + +#define INSN(NAME, op, width, sumop, vm, mop, mew, nf) \ + void NAME(VectorRegister Vs3, Register Rs1) { \ + patch_VLdSt(op, Vs3, width, Rs1, sumop, vm, mop, mew, nf); \ + } + + // Vector Load/Store Instructions + INSN(vs1r_v, 0b0100111, 0b000, 0b01000, 0b1, 0b00, 0b0, g1); + +#undef INSN + +// r2_nfvm +#define INSN(NAME, op, width, umop, mop, mew) \ + void NAME(VectorRegister Vd_or_Vs3, Register Rs1, Nf nf = g1) { \ + patch_VLdSt(op, Vd_or_Vs3, width, Rs1, umop, 1, mop, mew, nf); \ + } + + // Vector Unit-Stride Instructions + INSN(vlm_v, 0b0000111, 0b000, 0b01011, 0b00, 0b0); + INSN(vsm_v, 0b0100111, 0b000, 0b01011, 0b00, 0b0); + +#undef INSN + +#define INSN(NAME, op, width, umop, mop, mew) \ + void NAME(VectorRegister Vd_or_Vs3, Register Rs1, VectorMask vm = unmasked, Nf nf = g1) { \ + patch_VLdSt(op, Vd_or_Vs3, width, Rs1, umop, vm, mop, mew, nf); \ + } + + // Vector Unit-Stride Instructions + INSN(vle8_v, 0b0000111, 0b000, 0b00000, 0b00, 0b0); + INSN(vle16_v, 0b0000111, 0b101, 0b00000, 0b00, 0b0); + INSN(vle32_v, 0b0000111, 0b110, 0b00000, 0b00, 0b0); + INSN(vle64_v, 0b0000111, 0b111, 0b00000, 0b00, 0b0); + + // Vector unit-stride fault-only-first Instructions + INSN(vle8ff_v, 0b0000111, 0b000, 0b10000, 0b00, 0b0); + INSN(vle16ff_v, 0b0000111, 0b101, 0b10000, 0b00, 0b0); + INSN(vle32ff_v, 0b0000111, 0b110, 0b10000, 0b00, 0b0); + INSN(vle64ff_v, 0b0000111, 0b111, 0b10000, 0b00, 0b0); + + INSN(vse8_v, 0b0100111, 0b000, 0b00000, 0b00, 0b0); + INSN(vse16_v, 0b0100111, 0b101, 0b00000, 0b00, 0b0); + INSN(vse32_v, 0b0100111, 0b110, 0b00000, 0b00, 0b0); + INSN(vse64_v, 0b0100111, 0b111, 0b00000, 0b00, 0b0); + +#undef INSN + +#define INSN(NAME, op, width, mop, mew) \ + void NAME(VectorRegister Vd, Register Rs1, VectorRegister Vs2, VectorMask vm = unmasked, Nf nf = g1) { \ + patch_VLdSt(op, Vd, width, Rs1, Vs2->encoding_nocheck(), vm, mop, mew, nf); \ + } + + // Vector unordered indexed load instructions + INSN(vluxei8_v, 0b0000111, 0b000, 0b01, 0b0); + INSN(vluxei16_v, 0b0000111, 0b101, 0b01, 0b0); + INSN(vluxei32_v, 0b0000111, 0b110, 0b01, 0b0); + INSN(vluxei64_v, 0b0000111, 0b111, 0b01, 0b0); + + // Vector ordered indexed load instructions + INSN(vloxei8_v, 0b0000111, 0b000, 0b11, 0b0); + INSN(vloxei16_v, 0b0000111, 0b101, 0b11, 0b0); + INSN(vloxei32_v, 0b0000111, 0b110, 0b11, 0b0); + INSN(vloxei64_v, 0b0000111, 0b111, 0b11, 0b0); +#undef INSN + +#define INSN(NAME, op, width, mop, mew) \ + void NAME(VectorRegister Vd, Register Rs1, Register Rs2, VectorMask vm = unmasked, Nf nf = g1) { \ + patch_VLdSt(op, Vd, width, Rs1, Rs2->encoding_nocheck(), vm, mop, mew, nf); \ + } + + // Vector Strided Instructions + INSN(vlse8_v, 0b0000111, 0b000, 0b10, 0b0); + INSN(vlse16_v, 0b0000111, 0b101, 0b10, 0b0); + INSN(vlse32_v, 0b0000111, 0b110, 0b10, 0b0); + INSN(vlse64_v, 0b0000111, 0b111, 0b10, 0b0); + +#undef INSN +#undef patch_VLdSt + +// ==================================== +// RISC-V Bit-Manipulation Extension +// Currently only support Zba, Zbb and Zbs bitmanip extensions. +// ==================================== +#define INSN(NAME, op, funct3, funct7) \ + void NAME(Register Rd, Register Rs1, Register Rs2) { \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch((address)&insn, 31, 25, funct7); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + patch_reg((address)&insn, 20, Rs2); \ + emit(insn); \ + } + + INSN(add_uw, 0b0111011, 0b000, 0b0000100); + INSN(rol, 0b0110011, 0b001, 0b0110000); + INSN(rolw, 0b0111011, 0b001, 0b0110000); + INSN(ror, 0b0110011, 0b101, 0b0110000); + INSN(rorw, 0b0111011, 0b101, 0b0110000); + INSN(sh1add, 0b0110011, 0b010, 0b0010000); + INSN(sh2add, 0b0110011, 0b100, 0b0010000); + INSN(sh3add, 0b0110011, 0b110, 0b0010000); + INSN(sh1add_uw, 0b0111011, 0b010, 0b0010000); + INSN(sh2add_uw, 0b0111011, 0b100, 0b0010000); + INSN(sh3add_uw, 0b0111011, 0b110, 0b0010000); + INSN(andn, 0b0110011, 0b111, 0b0100000); + INSN(orn, 0b0110011, 0b110, 0b0100000); + INSN(xnor, 0b0110011, 0b100, 0b0100000); + INSN(max, 0b0110011, 0b110, 0b0000101); + INSN(maxu, 0b0110011, 0b111, 0b0000101); + INSN(min, 0b0110011, 0b100, 0b0000101); + INSN(minu, 0b0110011, 0b101, 0b0000101); + +#undef INSN + +#define INSN(NAME, op, funct3, funct12) \ + void NAME(Register Rd, Register Rs1) { \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch((address)&insn, 31, 20, funct12); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + emit(insn); \ + } + + INSN(rev8, 0b0010011, 0b101, 0b011010111000); + INSN(sext_b, 0b0010011, 0b001, 0b011000000100); + INSN(sext_h, 0b0010011, 0b001, 0b011000000101); + INSN(zext_h, 0b0111011, 0b100, 0b000010000000); + INSN(clz, 0b0010011, 0b001, 0b011000000000); + INSN(clzw, 0b0011011, 0b001, 0b011000000000); + INSN(ctz, 0b0010011, 0b001, 0b011000000001); + INSN(ctzw, 0b0011011, 0b001, 0b011000000001); + INSN(cpop, 0b0010011, 0b001, 0b011000000010); + INSN(cpopw, 0b0011011, 0b001, 0b011000000010); + INSN(orc_b, 0b0010011, 0b101, 0b001010000111); + +#undef INSN + +#define INSN(NAME, op, funct3, funct6) \ + void NAME(Register Rd, Register Rs1, unsigned shamt) {\ + guarantee(shamt <= 0x3f, "Shamt is invalid"); \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch((address)&insn, 25, 20, shamt); \ + patch((address)&insn, 31, 26, funct6); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + emit(insn); \ + } + + INSN(rori, 0b0010011, 0b101, 0b011000); + INSN(slli_uw, 0b0011011, 0b001, 0b000010); + INSN(bexti, 0b0010011, 0b101, 0b010010); + +#undef INSN + +#define INSN(NAME, op, funct3, funct7) \ + void NAME(Register Rd, Register Rs1, unsigned shamt) {\ + guarantee(shamt <= 0x1f, "Shamt is invalid"); \ + unsigned insn = 0; \ + patch((address)&insn, 6, 0, op); \ + patch((address)&insn, 14, 12, funct3); \ + patch((address)&insn, 24, 20, shamt); \ + patch((address)&insn, 31, 25, funct7); \ + patch_reg((address)&insn, 7, Rd); \ + patch_reg((address)&insn, 15, Rs1); \ + emit(insn); \ + } + + INSN(roriw, 0b0011011, 0b101, 0b0110000); + +#undef INSN + +// ======================================== +// RISC-V Compressed Instructions Extension +// ======================================== +// Note: +// 1. Assembler functions encoding 16-bit compressed instructions always begin with a 'c_' +// prefix, such as 'c_add'. Correspondingly, assembler functions encoding normal 32-bit +// instructions with begin with a '_' prefix, such as "_add". Most of time users have no +// need to explicitly emit these compressed instructions. Instead, they still use unified +// wrappers such as 'add' which do the compressing work through 'c_add' depending on the +// the operands of the instruction and availability of the RVC hardware extension. +// +// 2. 'CompressibleRegion' and 'IncompressibleRegion' are introduced to mark assembler scopes +// within which instructions are qualified or unqualified to be compressed into their 16-bit +// versions. An example: +// +// CompressibleRegion cr(_masm); +// __ add(...); // this instruction will be compressed into 'c.add' when possible +// { +// IncompressibleRegion ir(_masm); +// __ add(...); // this instruction will not be compressed +// { +// CompressibleRegion cr(_masm); +// __ add(...); // this instruction will be compressed into 'c.add' when possible +// } +// } +// +// 3. When printing JIT assembly code, using -XX:PrintAssemblyOptions=no-aliases could help +// distinguish compressed 16-bit instructions from normal 32-bit ones. + +private: + bool _in_compressible_region; +public: + bool in_compressible_region() const { return _in_compressible_region; } + void set_in_compressible_region(bool b) { _in_compressible_region = b; } +public: + + // an abstract compressible region + class AbstractCompressibleRegion : public StackObj { + protected: + Assembler *_masm; + bool _saved_in_compressible_region; + protected: + AbstractCompressibleRegion(Assembler *_masm) + : _masm(_masm) + , _saved_in_compressible_region(_masm->in_compressible_region()) {} + }; + // a compressible region + class CompressibleRegion : public AbstractCompressibleRegion { + public: + CompressibleRegion(Assembler *_masm) : AbstractCompressibleRegion(_masm) { + _masm->set_in_compressible_region(true); + } + ~CompressibleRegion() { + _masm->set_in_compressible_region(_saved_in_compressible_region); + } + }; + // an incompressible region + class IncompressibleRegion : public AbstractCompressibleRegion { + public: + IncompressibleRegion(Assembler *_masm) : AbstractCompressibleRegion(_masm) { + _masm->set_in_compressible_region(false); + } + ~IncompressibleRegion() { + _masm->set_in_compressible_region(_saved_in_compressible_region); + } + }; + +public: + // Emit a relocation. + void relocate(RelocationHolder const& rspec, int format = 0) { + AbstractAssembler::relocate(rspec, format); + } + void relocate(relocInfo::relocType rtype, int format = 0) { + AbstractAssembler::relocate(rtype, format); + } + template + void relocate(RelocationHolder const& rspec, Callback emit_insts, int format = 0) { + AbstractAssembler::relocate(rspec, format); + IncompressibleRegion ir(this); // relocations + emit_insts(); + } + template + void relocate(relocInfo::relocType rtype, Callback emit_insts, int format = 0) { + AbstractAssembler::relocate(rtype, format); + IncompressibleRegion ir(this); // relocations + emit_insts(); + } + + // patch a 16-bit instruction. + static void c_patch(address a, unsigned msb, unsigned lsb, uint16_t val) { + assert_cond(a != NULL); + assert_cond(msb >= lsb && msb <= 15); + unsigned nbits = msb - lsb + 1; + guarantee(val < (1U << nbits), "Field too big for insn"); + uint16_t mask = (1U << nbits) - 1; + val <<= lsb; + mask <<= lsb; + uint16_t target = *(uint16_t *)a; + target &= ~mask; + target |= val; + *(uint16_t *)a = target; + } + + static void c_patch(address a, unsigned bit, uint16_t val) { + c_patch(a, bit, bit, val); + } + + // patch a 16-bit instruction with a general purpose register ranging [0, 31] (5 bits) + static void c_patch_reg(address a, unsigned lsb, Register reg) { + c_patch(a, lsb + 4, lsb, reg->encoding_nocheck()); + } + + // patch a 16-bit instruction with a general purpose register ranging [8, 15] (3 bits) + static void c_patch_compressed_reg(address a, unsigned lsb, Register reg) { + c_patch(a, lsb + 2, lsb, reg->compressed_encoding_nocheck()); + } + + // patch a 16-bit instruction with a float register ranging [0, 31] (5 bits) + static void c_patch_reg(address a, unsigned lsb, FloatRegister reg) { + c_patch(a, lsb + 4, lsb, reg->encoding_nocheck()); + } + + // patch a 16-bit instruction with a float register ranging [8, 15] (3 bits) + static void c_patch_compressed_reg(address a, unsigned lsb, FloatRegister reg) { + c_patch(a, lsb + 2, lsb, reg->compressed_encoding_nocheck()); + } + +// -------------- RVC Instruction Definitions -------------- + + void c_nop() { + c_addi(x0, 0); + } + +#define INSN(NAME, funct3, op) \ + void NAME(Register Rd_Rs1, int32_t imm) { \ + assert_cond(is_simm6(imm)); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch((address)&insn, 6, 2, (imm & right_n_bits(5))); \ + c_patch_reg((address)&insn, 7, Rd_Rs1); \ + c_patch((address)&insn, 12, 12, (imm & nth_bit(5)) >> 5); \ + c_patch((address)&insn, 15, 13, funct3); \ + emit_int16(insn); \ + } + + INSN(c_addi, 0b000, 0b01); + INSN(c_addiw, 0b001, 0b01); + +#undef INSN + +#define INSN(NAME, funct3, op) \ + void NAME(int32_t imm) { \ + assert_cond(is_simm10(imm)); \ + assert_cond((imm & 0b1111) == 0); \ + assert_cond(imm != 0); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch((address)&insn, 2, 2, (imm & nth_bit(5)) >> 5); \ + c_patch((address)&insn, 4, 3, (imm & right_n_bits(9)) >> 7); \ + c_patch((address)&insn, 5, 5, (imm & nth_bit(6)) >> 6); \ + c_patch((address)&insn, 6, 6, (imm & nth_bit(4)) >> 4); \ + c_patch_reg((address)&insn, 7, sp); \ + c_patch((address)&insn, 12, 12, (imm & nth_bit(9)) >> 9); \ + c_patch((address)&insn, 15, 13, funct3); \ + emit_int16(insn); \ + } + + INSN(c_addi16sp, 0b011, 0b01); + +#undef INSN + +#define INSN(NAME, funct3, op) \ + void NAME(Register Rd, uint32_t uimm) { \ + assert_cond(is_uimm10(uimm)); \ + assert_cond((uimm & 0b11) == 0); \ + assert_cond(uimm != 0); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch_compressed_reg((address)&insn, 2, Rd); \ + c_patch((address)&insn, 5, 5, (uimm & nth_bit(3)) >> 3); \ + c_patch((address)&insn, 6, 6, (uimm & nth_bit(2)) >> 2); \ + c_patch((address)&insn, 10, 7, (uimm & right_n_bits(10)) >> 6); \ + c_patch((address)&insn, 12, 11, (uimm & right_n_bits(6)) >> 4); \ + c_patch((address)&insn, 15, 13, funct3); \ + emit_int16(insn); \ + } + + INSN(c_addi4spn, 0b000, 0b00); + +#undef INSN + +#define INSN(NAME, funct3, op) \ + void NAME(Register Rd_Rs1, uint32_t shamt) { \ + assert_cond(is_uimm6(shamt)); \ + assert_cond(shamt != 0); \ + assert_cond(Rd_Rs1 != x0); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch((address)&insn, 6, 2, (shamt & right_n_bits(5))); \ + c_patch_reg((address)&insn, 7, Rd_Rs1); \ + c_patch((address)&insn, 12, 12, (shamt & nth_bit(5)) >> 5); \ + c_patch((address)&insn, 15, 13, funct3); \ + emit_int16(insn); \ + } + + INSN(c_slli, 0b000, 0b10); + +#undef INSN + +#define INSN(NAME, funct3, funct2, op) \ + void NAME(Register Rd_Rs1, uint32_t shamt) { \ + assert_cond(is_uimm6(shamt)); \ + assert_cond(shamt != 0); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch((address)&insn, 6, 2, (shamt & right_n_bits(5))); \ + c_patch_compressed_reg((address)&insn, 7, Rd_Rs1); \ + c_patch((address)&insn, 11, 10, funct2); \ + c_patch((address)&insn, 12, 12, (shamt & nth_bit(5)) >> 5); \ + c_patch((address)&insn, 15, 13, funct3); \ + emit_int16(insn); \ + } + + INSN(c_srli, 0b100, 0b00, 0b01); + INSN(c_srai, 0b100, 0b01, 0b01); + +#undef INSN + +#define INSN(NAME, funct3, funct2, op) \ + void NAME(Register Rd_Rs1, int32_t imm) { \ + assert_cond(is_simm6(imm)); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch((address)&insn, 6, 2, (imm & right_n_bits(5))); \ + c_patch_compressed_reg((address)&insn, 7, Rd_Rs1); \ + c_patch((address)&insn, 11, 10, funct2); \ + c_patch((address)&insn, 12, 12, (imm & nth_bit(5)) >> 5); \ + c_patch((address)&insn, 15, 13, funct3); \ + emit_int16(insn); \ + } + + INSN(c_andi, 0b100, 0b10, 0b01); + +#undef INSN + +#define INSN(NAME, funct6, funct2, op) \ + void NAME(Register Rd_Rs1, Register Rs2) { \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch_compressed_reg((address)&insn, 2, Rs2); \ + c_patch((address)&insn, 6, 5, funct2); \ + c_patch_compressed_reg((address)&insn, 7, Rd_Rs1); \ + c_patch((address)&insn, 15, 10, funct6); \ + emit_int16(insn); \ + } + + INSN(c_sub, 0b100011, 0b00, 0b01); + INSN(c_xor, 0b100011, 0b01, 0b01); + INSN(c_or, 0b100011, 0b10, 0b01); + INSN(c_and, 0b100011, 0b11, 0b01); + INSN(c_subw, 0b100111, 0b00, 0b01); + INSN(c_addw, 0b100111, 0b01, 0b01); + +#undef INSN + +#define INSN(NAME, funct4, op) \ + void NAME(Register Rd_Rs1, Register Rs2) { \ + assert_cond(Rd_Rs1 != x0); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch_reg((address)&insn, 2, Rs2); \ + c_patch_reg((address)&insn, 7, Rd_Rs1); \ + c_patch((address)&insn, 15, 12, funct4); \ + emit_int16(insn); \ + } + + INSN(c_mv, 0b1000, 0b10); + INSN(c_add, 0b1001, 0b10); + +#undef INSN + +#define INSN(NAME, funct4, op) \ + void NAME(Register Rs1) { \ + assert_cond(Rs1 != x0); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch_reg((address)&insn, 2, x0); \ + c_patch_reg((address)&insn, 7, Rs1); \ + c_patch((address)&insn, 15, 12, funct4); \ + emit_int16(insn); \ + } + + INSN(c_jr, 0b1000, 0b10); + INSN(c_jalr, 0b1001, 0b10); + +#undef INSN + + typedef void (Assembler::* j_c_insn)(address dest); + typedef void (Assembler::* compare_and_branch_c_insn)(Register Rs1, address dest); + + void wrap_label(Label &L, j_c_insn insn) { + if (L.is_bound()) { + (this->*insn)(target(L)); + } else { + L.add_patch_at(code(), locator()); + (this->*insn)(pc()); + } + } + + void wrap_label(Label &L, Register r, compare_and_branch_c_insn insn) { + if (L.is_bound()) { + (this->*insn)(r, target(L)); + } else { + L.add_patch_at(code(), locator()); + (this->*insn)(r, pc()); + } + } + +#define INSN(NAME, funct3, op) \ + void NAME(int32_t offset) { \ + assert(is_simm12(offset) && ((offset % 2) == 0), "invalid encoding"); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch((address)&insn, 2, 2, (offset & nth_bit(5)) >> 5); \ + c_patch((address)&insn, 5, 3, (offset & right_n_bits(4)) >> 1); \ + c_patch((address)&insn, 6, 6, (offset & nth_bit(7)) >> 7); \ + c_patch((address)&insn, 7, 7, (offset & nth_bit(6)) >> 6); \ + c_patch((address)&insn, 8, 8, (offset & nth_bit(10)) >> 10); \ + c_patch((address)&insn, 10, 9, (offset & right_n_bits(10)) >> 8); \ + c_patch((address)&insn, 11, 11, (offset & nth_bit(4)) >> 4); \ + c_patch((address)&insn, 12, 12, (offset & nth_bit(11)) >> 11); \ + c_patch((address)&insn, 15, 13, funct3); \ + emit_int16(insn); \ + } \ + void NAME(address dest) { \ + assert_cond(dest != NULL); \ + int64_t distance = dest - pc(); \ + assert(is_simm12(distance) && ((distance % 2) == 0), "invalid encoding"); \ + c_j(distance); \ + } \ + void NAME(Label &L) { \ + wrap_label(L, &Assembler::NAME); \ + } + + INSN(c_j, 0b101, 0b01); + +#undef INSN + +#define INSN(NAME, funct3, op) \ + void NAME(Register Rs1, int32_t imm) { \ + assert(is_simm9(imm) && ((imm % 2) == 0), "invalid encoding"); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch((address)&insn, 2, 2, (imm & nth_bit(5)) >> 5); \ + c_patch((address)&insn, 4, 3, (imm & right_n_bits(3)) >> 1); \ + c_patch((address)&insn, 6, 5, (imm & right_n_bits(8)) >> 6); \ + c_patch_compressed_reg((address)&insn, 7, Rs1); \ + c_patch((address)&insn, 11, 10, (imm & right_n_bits(5)) >> 3); \ + c_patch((address)&insn, 12, 12, (imm & nth_bit(8)) >> 8); \ + c_patch((address)&insn, 15, 13, funct3); \ + emit_int16(insn); \ + } \ + void NAME(Register Rs1, address dest) { \ + assert_cond(dest != NULL); \ + int64_t distance = dest - pc(); \ + assert(is_simm9(distance) && ((distance % 2) == 0), "invalid encoding"); \ + NAME(Rs1, distance); \ + } \ + void NAME(Register Rs1, Label &L) { \ + wrap_label(L, Rs1, &Assembler::NAME); \ + } + + INSN(c_beqz, 0b110, 0b01); + INSN(c_bnez, 0b111, 0b01); + +#undef INSN + +#define INSN(NAME, funct3, op) \ + void NAME(Register Rd, int32_t imm) { \ + assert_cond(is_simm18(imm)); \ + assert_cond((imm & 0xfff) == 0); \ + assert_cond(imm != 0); \ + assert_cond(Rd != x0 && Rd != x2); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch((address)&insn, 6, 2, (imm & right_n_bits(17)) >> 12); \ + c_patch_reg((address)&insn, 7, Rd); \ + c_patch((address)&insn, 12, 12, (imm & nth_bit(17)) >> 17); \ + c_patch((address)&insn, 15, 13, funct3); \ + emit_int16(insn); \ + } + + INSN(c_lui, 0b011, 0b01); + +#undef INSN + +#define INSN(NAME, funct3, op) \ + void NAME(Register Rd, int32_t imm) { \ + assert_cond(is_simm6(imm)); \ + assert_cond(Rd != x0); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch((address)&insn, 6, 2, (imm & right_n_bits(5))); \ + c_patch_reg((address)&insn, 7, Rd); \ + c_patch((address)&insn, 12, 12, (imm & right_n_bits(6)) >> 5); \ + c_patch((address)&insn, 15, 13, funct3); \ + emit_int16(insn); \ + } + + INSN(c_li, 0b010, 0b01); + +#undef INSN + +#define INSN(NAME, funct3, op) \ + void NAME(Register Rd, uint32_t uimm) { \ + assert_cond(is_uimm9(uimm)); \ + assert_cond((uimm & 0b111) == 0); \ + assert_cond(Rd != x0); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch((address)&insn, 4, 2, (uimm & right_n_bits(9)) >> 6); \ + c_patch((address)&insn, 6, 5, (uimm & right_n_bits(5)) >> 3); \ + c_patch_reg((address)&insn, 7, Rd); \ + c_patch((address)&insn, 12, 12, (uimm & nth_bit(5)) >> 5); \ + c_patch((address)&insn, 15, 13, funct3); \ + emit_int16(insn); \ + } + + INSN(c_ldsp, 0b011, 0b10); + +#undef INSN + +#define INSN(NAME, funct3, op) \ + void NAME(FloatRegister Rd, uint32_t uimm) { \ + assert_cond(is_uimm9(uimm)); \ + assert_cond((uimm & 0b111) == 0); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch((address)&insn, 4, 2, (uimm & right_n_bits(9)) >> 6); \ + c_patch((address)&insn, 6, 5, (uimm & right_n_bits(5)) >> 3); \ + c_patch_reg((address)&insn, 7, Rd); \ + c_patch((address)&insn, 12, 12, (uimm & nth_bit(5)) >> 5); \ + c_patch((address)&insn, 15, 13, funct3); \ + emit_int16(insn); \ + } + + INSN(c_fldsp, 0b001, 0b10); + +#undef INSN + +#define INSN(NAME, funct3, op, REGISTER_TYPE) \ + void NAME(REGISTER_TYPE Rd_Rs2, Register Rs1, uint32_t uimm) { \ + assert_cond(is_uimm8(uimm)); \ + assert_cond((uimm & 0b111) == 0); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch_compressed_reg((address)&insn, 2, Rd_Rs2); \ + c_patch((address)&insn, 6, 5, (uimm & right_n_bits(8)) >> 6); \ + c_patch_compressed_reg((address)&insn, 7, Rs1); \ + c_patch((address)&insn, 12, 10, (uimm & right_n_bits(6)) >> 3); \ + c_patch((address)&insn, 15, 13, funct3); \ + emit_int16(insn); \ + } + + INSN(c_ld, 0b011, 0b00, Register); + INSN(c_sd, 0b111, 0b00, Register); + INSN(c_fld, 0b001, 0b00, FloatRegister); + INSN(c_fsd, 0b101, 0b00, FloatRegister); + +#undef INSN + +#define INSN(NAME, funct3, op, REGISTER_TYPE) \ + void NAME(REGISTER_TYPE Rs2, uint32_t uimm) { \ + assert_cond(is_uimm9(uimm)); \ + assert_cond((uimm & 0b111) == 0); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch_reg((address)&insn, 2, Rs2); \ + c_patch((address)&insn, 9, 7, (uimm & right_n_bits(9)) >> 6); \ + c_patch((address)&insn, 12, 10, (uimm & right_n_bits(6)) >> 3); \ + c_patch((address)&insn, 15, 13, funct3); \ + emit_int16(insn); \ + } + + INSN(c_sdsp, 0b111, 0b10, Register); + INSN(c_fsdsp, 0b101, 0b10, FloatRegister); + +#undef INSN + +#define INSN(NAME, funct3, op) \ + void NAME(Register Rs2, uint32_t uimm) { \ + assert_cond(is_uimm8(uimm)); \ + assert_cond((uimm & 0b11) == 0); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch_reg((address)&insn, 2, Rs2); \ + c_patch((address)&insn, 8, 7, (uimm & right_n_bits(8)) >> 6); \ + c_patch((address)&insn, 12, 9, (uimm & right_n_bits(6)) >> 2); \ + c_patch((address)&insn, 15, 13, funct3); \ + emit_int16(insn); \ + } + + INSN(c_swsp, 0b110, 0b10); + +#undef INSN + +#define INSN(NAME, funct3, op) \ + void NAME(Register Rd, uint32_t uimm) { \ + assert_cond(is_uimm8(uimm)); \ + assert_cond((uimm & 0b11) == 0); \ + assert_cond(Rd != x0); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch((address)&insn, 3, 2, (uimm & right_n_bits(8)) >> 6); \ + c_patch((address)&insn, 6, 4, (uimm & right_n_bits(5)) >> 2); \ + c_patch_reg((address)&insn, 7, Rd); \ + c_patch((address)&insn, 12, 12, (uimm & nth_bit(5)) >> 5); \ + c_patch((address)&insn, 15, 13, funct3); \ + emit_int16(insn); \ + } + + INSN(c_lwsp, 0b010, 0b10); + +#undef INSN + +#define INSN(NAME, funct3, op) \ + void NAME(Register Rd_Rs2, Register Rs1, uint32_t uimm) { \ + assert_cond(is_uimm7(uimm)); \ + assert_cond((uimm & 0b11) == 0); \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch_compressed_reg((address)&insn, 2, Rd_Rs2); \ + c_patch((address)&insn, 5, 5, (uimm & nth_bit(6)) >> 6); \ + c_patch((address)&insn, 6, 6, (uimm & nth_bit(2)) >> 2); \ + c_patch_compressed_reg((address)&insn, 7, Rs1); \ + c_patch((address)&insn, 12, 10, (uimm & right_n_bits(6)) >> 3); \ + c_patch((address)&insn, 15, 13, funct3); \ + emit_int16(insn); \ + } + + INSN(c_lw, 0b010, 0b00); + INSN(c_sw, 0b110, 0b00); + +#undef INSN + +#define INSN(NAME, funct3, op) \ + void NAME() { \ + uint16_t insn = 0; \ + c_patch((address)&insn, 1, 0, op); \ + c_patch((address)&insn, 11, 2, 0x0); \ + c_patch((address)&insn, 12, 12, 0b1); \ + c_patch((address)&insn, 15, 13, funct3); \ + emit_int16(insn); \ + } + + INSN(c_ebreak, 0b100, 0b10); + +#undef INSN + +// -------------- RVC Transformation Functions -------------- + +// -------------------------- +// Register instructions +// -------------------------- +#define INSN(NAME) \ + void NAME(Register Rd, Register Rs1, Register Rs2) { \ + /* add -> c.add */ \ + if (do_compress()) { \ + Register src = noreg; \ + if (Rs1 != x0 && Rs2 != x0 && ((src = Rs1, Rs2 == Rd) || (src = Rs2, Rs1 == Rd))) { \ + c_add(Rd, src); \ + return; \ + } \ + } \ + _add(Rd, Rs1, Rs2); \ + } + + INSN(add); + +#undef INSN + +// -------------------------- +#define INSN(NAME, C_NAME, NORMAL_NAME) \ + void NAME(Register Rd, Register Rs1, Register Rs2) { \ + /* sub/subw -> c.sub/c.subw */ \ + if (do_compress() && \ + (Rd == Rs1 && Rd->is_compressed_valid() && Rs2->is_compressed_valid())) { \ + C_NAME(Rd, Rs2); \ + return; \ + } \ + NORMAL_NAME(Rd, Rs1, Rs2); \ + } + + INSN(sub, c_sub, _sub); + INSN(subw, c_subw, _subw); + +#undef INSN + +// -------------------------- +#define INSN(NAME, C_NAME, NORMAL_NAME) \ + void NAME(Register Rd, Register Rs1, Register Rs2) { \ + /* and/or/xor/addw -> c.and/c.or/c.xor/c.addw */ \ + if (do_compress()) { \ + Register src = noreg; \ + if (Rs1->is_compressed_valid() && Rs2->is_compressed_valid() && \ + ((src = Rs1, Rs2 == Rd) || (src = Rs2, Rs1 == Rd))) { \ + C_NAME(Rd, src); \ + return; \ + } \ + } \ + NORMAL_NAME(Rd, Rs1, Rs2); \ + } + + INSN(andr, c_and, _andr); + INSN(orr, c_or, _orr); + INSN(xorr, c_xor, _xorr); + INSN(addw, c_addw, _addw); + +#undef INSN + +private: +// some helper functions +#define FUNC(NAME, funct3, bits) \ + bool NAME(Register rs1, Register rd_rs2, int32_t imm12, bool ld) { \ + return rs1 == sp && \ + is_uimm(imm12, bits) && \ + (intx(imm12) & funct3) == 0x0 && \ + (!ld || rd_rs2 != x0); \ + } \ + + FUNC(is_c_ldsdsp, 0b111, 9); + FUNC(is_c_lwswsp, 0b011, 8); + +#undef FUNC + +#define FUNC(NAME, funct3, bits) \ + bool NAME(Register rs1, int32_t imm12) { \ + return rs1 == sp && \ + is_uimm(imm12, bits) && \ + (intx(imm12) & funct3) == 0x0; \ + } \ + + FUNC(is_c_fldsdsp, 0b111, 9); + +#undef FUNC + +#define FUNC(NAME, REG_TYPE, funct3, bits) \ + bool NAME(Register rs1, REG_TYPE rd_rs2, int32_t imm12) { \ + return rs1->is_compressed_valid() && \ + rd_rs2->is_compressed_valid() && \ + is_uimm(imm12, bits) && \ + (intx(imm12) & funct3) == 0x0; \ + } \ + + FUNC(is_c_ldsd, Register, 0b111, 8); + FUNC(is_c_lwsw, Register, 0b011, 7); + FUNC(is_c_fldsd, FloatRegister, 0b111, 8); + +#undef FUNC + +public: + bool do_compress() const { + return UseRVC && in_compressible_region(); + } + +// -------------------------- +// Load/store register +// -------------------------- +#define INSN(NAME) \ + void NAME(Register Rd, Register Rs, const int32_t offset) { \ + /* lw -> c.lwsp/c.lw */ \ + if (do_compress()) { \ + if (is_c_lwswsp(Rs, Rd, offset, true)) { \ + c_lwsp(Rd, offset); \ + return; \ + } else if (is_c_lwsw(Rs, Rd, offset)) { \ + c_lw(Rd, Rs, offset); \ + return; \ + } \ + } \ + _lw(Rd, Rs, offset); \ + } + + INSN(lw); + +#undef INSN + +// -------------------------- +#define INSN(NAME) \ + void NAME(Register Rd, Register Rs, const int32_t offset) { \ + /* ld -> c.ldsp/c.ld */ \ + if (do_compress()) { \ + if (is_c_ldsdsp(Rs, Rd, offset, true)) { \ + c_ldsp(Rd, offset); \ + return; \ + } else if (is_c_ldsd(Rs, Rd, offset)) { \ + c_ld(Rd, Rs, offset); \ + return; \ + } \ + } \ + _ld(Rd, Rs, offset); \ + } + + INSN(ld); + +#undef INSN + +// -------------------------- +#define INSN(NAME) \ + void NAME(FloatRegister Rd, Register Rs, const int32_t offset) { \ + /* fld -> c.fldsp/c.fld */ \ + if (do_compress()) { \ + if (is_c_fldsdsp(Rs, offset)) { \ + c_fldsp(Rd, offset); \ + return; \ + } else if (is_c_fldsd(Rs, Rd, offset)) { \ + c_fld(Rd, Rs, offset); \ + return; \ + } \ + } \ + _fld(Rd, Rs, offset); \ + } + + INSN(fld); + +#undef INSN + +// -------------------------- +#define INSN(NAME) \ + void NAME(Register Rd, Register Rs, const int32_t offset) { \ + /* sd -> c.sdsp/c.sd */ \ + if (do_compress()) { \ + if (is_c_ldsdsp(Rs, Rd, offset, false)) { \ + c_sdsp(Rd, offset); \ + return; \ + } else if (is_c_ldsd(Rs, Rd, offset)) { \ + c_sd(Rd, Rs, offset); \ + return; \ + } \ + } \ + _sd(Rd, Rs, offset); \ + } + + INSN(sd); + +#undef INSN + +// -------------------------- +#define INSN(NAME) \ + void NAME(Register Rd, Register Rs, const int32_t offset) { \ + /* sw -> c.swsp/c.sw */ \ + if (do_compress()) { \ + if (is_c_lwswsp(Rs, Rd, offset, false)) { \ + c_swsp(Rd, offset); \ + return; \ + } else if (is_c_lwsw(Rs, Rd, offset)) { \ + c_sw(Rd, Rs, offset); \ + return; \ + } \ + } \ + _sw(Rd, Rs, offset); \ + } + + INSN(sw); + +#undef INSN + +// -------------------------- +#define INSN(NAME) \ + void NAME(FloatRegister Rd, Register Rs, const int32_t offset) { \ + /* fsd -> c.fsdsp/c.fsd */ \ + if (do_compress()) { \ + if (is_c_fldsdsp(Rs, offset)) { \ + c_fsdsp(Rd, offset); \ + return; \ + } else if (is_c_fldsd(Rs, Rd, offset)) { \ + c_fsd(Rd, Rs, offset); \ + return; \ + } \ + } \ + _fsd(Rd, Rs, offset); \ + } + + INSN(fsd); + +#undef INSN + +// -------------------------- +// Unconditional branch instructions +// -------------------------- +#define INSN(NAME) \ + void NAME(Register Rd, Register Rs, const int32_t offset) { \ + /* jalr -> c.jr/c.jalr */ \ + if (do_compress() && (offset == 0 && Rs != x0)) { \ + if (Rd == x1) { \ + c_jalr(Rs); \ + return; \ + } else if (Rd == x0) { \ + c_jr(Rs); \ + return; \ + } \ + } \ + _jalr(Rd, Rs, offset); \ + } + + INSN(jalr); + +#undef INSN + +// -------------------------- +// Miscellaneous Instructions +// -------------------------- +#define INSN(NAME) \ + void NAME() { \ + /* ebreak -> c.ebreak */ \ + if (do_compress()) { \ + c_ebreak(); \ + return; \ + } \ + _ebreak(); \ + } + + INSN(ebreak); + +#undef INSN + +// -------------------------- +// Immediate Instructions +// -------------------------- +#define INSN(NAME) \ + void NAME(Register Rd, Register Rs1, int32_t imm) { \ + /* addi -> c.addi/c.nop/c.mv/c.addi16sp/c.addi4spn */ \ + if (do_compress()) { \ + if (Rd == Rs1 && is_simm6(imm)) { \ + c_addi(Rd, imm); \ + return; \ + } else if (imm == 0 && Rd != x0 && Rs1 != x0) { \ + c_mv(Rd, Rs1); \ + return; \ + } else if (Rs1 == sp && imm != 0) { \ + if (Rd == Rs1 && (imm & 0b1111) == 0x0 && is_simm10(imm)) { \ + c_addi16sp(imm); \ + return; \ + } else if (Rd->is_compressed_valid() && (imm & 0b11) == 0x0 && is_uimm10(imm)) { \ + c_addi4spn(Rd, imm); \ + return; \ + } \ + } \ + } \ + _addi(Rd, Rs1, imm); \ + } + + INSN(addi); + +#undef INSN + +// -------------------------- +#define INSN(NAME) \ + void NAME(Register Rd, Register Rs1, int32_t imm) { \ + /* addiw -> c.addiw */ \ + if (do_compress() && (Rd == Rs1 && Rd != x0 && is_simm6(imm))) { \ + c_addiw(Rd, imm); \ + return; \ + } \ + _addiw(Rd, Rs1, imm); \ + } + + INSN(addiw); + +#undef INSN + +// -------------------------- +#define INSN(NAME) \ + void NAME(Register Rd, Register Rs1, int32_t imm) { \ + /* and_imm12 -> c.andi */ \ + if (do_compress() && \ + (Rd == Rs1 && Rd->is_compressed_valid() && is_simm6(imm))) { \ + c_andi(Rd, imm); \ + return; \ + } \ + _and_imm12(Rd, Rs1, imm); \ + } + + INSN(and_imm12); + +#undef INSN + +// -------------------------- +// Shift Immediate Instructions +// -------------------------- +#define INSN(NAME) \ + void NAME(Register Rd, Register Rs1, unsigned shamt) { \ + /* slli -> c.slli */ \ + if (do_compress() && (Rd == Rs1 && Rd != x0 && shamt != 0)) { \ + c_slli(Rd, shamt); \ + return; \ + } \ + if (shamt != 0) { \ + _slli(Rd, Rs1, shamt); \ + } else { \ + if (Rd != Rs1) { \ + addi(Rd, Rs1, 0); \ + } \ + } \ + } + + INSN(slli); + +#undef INSN + +// -------------------------- +#define INSN(NAME, C_NAME, NORMAL_NAME) \ + void NAME(Register Rd, Register Rs1, unsigned shamt) { \ + /* srai/srli -> c.srai/c.srli */ \ + if (do_compress() && (Rd == Rs1 && Rd->is_compressed_valid() && shamt != 0)) { \ + C_NAME(Rd, shamt); \ + return; \ + } \ + if (shamt != 0) { \ + NORMAL_NAME(Rd, Rs1, shamt); \ + } else { \ + if (Rd != Rs1) { \ + addi(Rd, Rs1, 0); \ + } \ + } \ + } + + INSN(srai, c_srai, _srai); + INSN(srli, c_srli, _srli); + +#undef INSN + +// -------------------------- +// Upper Immediate Instruction +// -------------------------- +#define INSN(NAME) \ + void NAME(Register Rd, int32_t imm) { \ + /* lui -> c.lui */ \ + if (do_compress() && (Rd != x0 && Rd != x2 && imm != 0 && is_simm18(imm))) { \ + c_lui(Rd, imm); \ + return; \ + } \ + _lui(Rd, imm); \ + } + + INSN(lui); + +#undef INSN + +// --------------------------------------------------------------------------------------- + +#define INSN(NAME, REGISTER) \ + void NAME(Register Rs) { \ + jalr(REGISTER, Rs, 0); \ + } + + INSN(jr, x0); + INSN(jalr, x1); + +#undef INSN + + // Stack overflow checking + virtual void bang_stack_with_offset(int offset) { Unimplemented(); } + + static bool is_simm5(int64_t x); + static bool is_simm6(int64_t x); + static bool is_simm12(int64_t x); + static bool is_simm13(int64_t x); + static bool is_simm18(int64_t x); + static bool is_simm21(int64_t x); + + static bool is_uimm3(uint64_t x); + static bool is_uimm5(uint64_t x); + static bool is_uimm6(uint64_t x); + static bool is_uimm7(uint64_t x); + static bool is_uimm8(uint64_t x); + static bool is_uimm9(uint64_t x); + static bool is_uimm10(uint64_t x); + + // The maximum range of a branch is fixed for the RISCV architecture. + static const unsigned long branch_range = 1 * M; + + static bool reachable_from_branch_at(address branch, address target) { + return uabs(target - branch) < branch_range; + } + + Assembler(CodeBuffer* code) : AbstractAssembler(code), _in_compressible_region(false) {} + + virtual ~Assembler() {} +}; + +class BiasedLockingCounters; + +#endif // CPU_RISCV_ASSEMBLER_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/assembler_riscv.inline.hpp b/src/hotspot/cpu/riscv/assembler_riscv.inline.hpp new file mode 100644 index 0000000000000..c51650881fc73 --- /dev/null +++ b/src/hotspot/cpu/riscv/assembler_riscv.inline.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_ASSEMBLER_RISCV_INLINE_HPP +#define CPU_RISCV_ASSEMBLER_RISCV_INLINE_HPP + +#include "asm/assembler.inline.hpp" +#include "asm/codeBuffer.hpp" +#include "code/codeCache.hpp" + +inline bool Assembler::is_simm5(int64_t x) { return is_simm(x, 5); } +inline bool Assembler::is_simm6(int64_t x) { return is_simm(x, 6); } +inline bool Assembler::is_simm12(int64_t x) { return is_simm(x, 12); } +inline bool Assembler::is_simm13(int64_t x) { return is_simm(x, 13); } +inline bool Assembler::is_simm18(int64_t x) { return is_simm(x, 18); } +inline bool Assembler::is_simm21(int64_t x) { return is_simm(x, 21); } + +inline bool Assembler::is_uimm3(uint64_t x) { return is_uimm(x, 3); } +inline bool Assembler::is_uimm5(uint64_t x) { return is_uimm(x, 5); } +inline bool Assembler::is_uimm6(uint64_t x) { return is_uimm(x, 6); } +inline bool Assembler::is_uimm7(uint64_t x) { return is_uimm(x, 7); } +inline bool Assembler::is_uimm8(uint64_t x) { return is_uimm(x, 8); } +inline bool Assembler::is_uimm9(uint64_t x) { return is_uimm(x, 9); } +inline bool Assembler::is_uimm10(uint64_t x) { return is_uimm(x, 10); } + +#endif // CPU_RISCV_ASSEMBLER_RISCV_INLINE_HPP diff --git a/src/hotspot/cpu/riscv/bytes_riscv.hpp b/src/hotspot/cpu/riscv/bytes_riscv.hpp new file mode 100644 index 0000000000000..34143c265bfb0 --- /dev/null +++ b/src/hotspot/cpu/riscv/bytes_riscv.hpp @@ -0,0 +1,169 @@ +/* + * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2016 SAP SE. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_BYTES_RISCV_HPP +#define CPU_RISCV_BYTES_RISCV_HPP + +#include "memory/allStatic.hpp" + +class Bytes: AllStatic { + public: + // Efficient reading and writing of unaligned unsigned data in platform-specific byte ordering + // RISCV needs to check for alignment. + + // Forward declarations of the compiler-dependent implementation + static inline u2 swap_u2(u2 x); + static inline u4 swap_u4(u4 x); + static inline u8 swap_u8(u8 x); + + static inline u2 get_native_u2(address p) { + if ((intptr_t(p) & 1) == 0) { + return *(u2*)p; + } else { + return ((u2)(p[1]) << 8) | + ((u2)(p[0])); + } + } + + static inline u4 get_native_u4(address p) { + switch (intptr_t(p) & 3) { + case 0: + return *(u4*)p; + + case 2: + return ((u4)(((u2*)p)[1]) << 16) | + ((u4)(((u2*)p)[0])); + + default: + return ((u4)(p[3]) << 24) | + ((u4)(p[2]) << 16) | + ((u4)(p[1]) << 8) | + ((u4)(p[0])); + } + } + + static inline u8 get_native_u8(address p) { + switch (intptr_t(p) & 7) { + case 0: + return *(u8*)p; + + case 4: + return ((u8)(((u4*)p)[1]) << 32) | + ((u8)(((u4*)p)[0])); + + case 2: + case 6: + return ((u8)(((u2*)p)[3]) << 48) | + ((u8)(((u2*)p)[2]) << 32) | + ((u8)(((u2*)p)[1]) << 16) | + ((u8)(((u2*)p)[0])); + + default: + return ((u8)(p[7]) << 56) | + ((u8)(p[6]) << 48) | + ((u8)(p[5]) << 40) | + ((u8)(p[4]) << 32) | + ((u8)(p[3]) << 24) | + ((u8)(p[2]) << 16) | + ((u8)(p[1]) << 8) | + ((u8)(p[0])); + } + } + + static inline void put_native_u2(address p, u2 x) { + if ((intptr_t(p) & 1) == 0) { + *(u2*)p = x; + } else { + p[1] = x >> 8; + p[0] = x; + } + } + + static inline void put_native_u4(address p, u4 x) { + switch (intptr_t(p) & 3) { + case 0: + *(u4*)p = x; + break; + + case 2: + ((u2*)p)[1] = x >> 16; + ((u2*)p)[0] = x; + break; + + default: + ((u1*)p)[3] = x >> 24; + ((u1*)p)[2] = x >> 16; + ((u1*)p)[1] = x >> 8; + ((u1*)p)[0] = x; + break; + } + } + + static inline void put_native_u8(address p, u8 x) { + switch (intptr_t(p) & 7) { + case 0: + *(u8*)p = x; + break; + + case 4: + ((u4*)p)[1] = x >> 32; + ((u4*)p)[0] = x; + break; + + case 2: + case 6: + ((u2*)p)[3] = x >> 48; + ((u2*)p)[2] = x >> 32; + ((u2*)p)[1] = x >> 16; + ((u2*)p)[0] = x; + break; + + default: + ((u1*)p)[7] = x >> 56; + ((u1*)p)[6] = x >> 48; + ((u1*)p)[5] = x >> 40; + ((u1*)p)[4] = x >> 32; + ((u1*)p)[3] = x >> 24; + ((u1*)p)[2] = x >> 16; + ((u1*)p)[1] = x >> 8; + ((u1*)p)[0] = x; + break; + } + } + + // Efficient reading and writing of unaligned unsigned data in Java byte ordering (i.e. big-endian ordering) + static inline u2 get_Java_u2(address p) { return swap_u2(get_native_u2(p)); } + static inline u4 get_Java_u4(address p) { return swap_u4(get_native_u4(p)); } + static inline u8 get_Java_u8(address p) { return swap_u8(get_native_u8(p)); } + + static inline void put_Java_u2(address p, u2 x) { put_native_u2(p, swap_u2(x)); } + static inline void put_Java_u4(address p, u4 x) { put_native_u4(p, swap_u4(x)); } + static inline void put_Java_u8(address p, u8 x) { put_native_u8(p, swap_u8(x)); } +}; + +#include OS_CPU_HEADER(bytes) + +#endif // CPU_RISCV_BYTES_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp b/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp new file mode 100644 index 0000000000000..788e605c43f95 --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp @@ -0,0 +1,357 @@ +/* + * Copyright (c) 1999, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "c1/c1_CodeStubs.hpp" +#include "c1/c1_FrameMap.hpp" +#include "c1/c1_LIRAssembler.hpp" +#include "c1/c1_MacroAssembler.hpp" +#include "c1/c1_Runtime1.hpp" +#include "classfile/javaClasses.hpp" +#include "nativeInst_riscv.hpp" +#include "runtime/sharedRuntime.hpp" +#include "vmreg_riscv.inline.hpp" + + +#define __ ce->masm()-> + +void C1SafepointPollStub::emit_code(LIR_Assembler* ce) { + __ bind(_entry); + InternalAddress safepoint_pc(__ pc() - __ offset() + safepoint_offset()); + __ relocate(safepoint_pc.rspec(), [&] { + __ la(t0, safepoint_pc.target()); + }); + __ sd(t0, Address(xthread, JavaThread::saved_exception_pc_offset())); + + assert(SharedRuntime::polling_page_return_handler_blob() != NULL, + "polling page return stub not created yet"); + address stub = SharedRuntime::polling_page_return_handler_blob()->entry_point(); + + __ far_jump(RuntimeAddress(stub)); +} + +void CounterOverflowStub::emit_code(LIR_Assembler* ce) { + __ bind(_entry); + Metadata *m = _method->as_constant_ptr()->as_metadata(); + __ mov_metadata(t0, m); + ce->store_parameter(t0, 1); + ce->store_parameter(_bci, 0); + __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::counter_overflow_id))); + ce->add_call_info_here(_info); + ce->verify_oop_map(_info); + __ j(_continuation); +} + +RangeCheckStub::RangeCheckStub(CodeEmitInfo* info, LIR_Opr index, LIR_Opr array) + : _index(index), _array(array), _throw_index_out_of_bounds_exception(false) { + assert(info != NULL, "must have info"); + _info = new CodeEmitInfo(info); +} + +RangeCheckStub::RangeCheckStub(CodeEmitInfo* info, LIR_Opr index) + : _index(index), _array(NULL), _throw_index_out_of_bounds_exception(true) { + assert(info != NULL, "must have info"); + _info = new CodeEmitInfo(info); +} + +void RangeCheckStub::emit_code(LIR_Assembler* ce) { + __ bind(_entry); + if (_info->deoptimize_on_exception()) { + address a = Runtime1::entry_for(Runtime1::predicate_failed_trap_id); + __ far_call(RuntimeAddress(a)); + ce->add_call_info_here(_info); + ce->verify_oop_map(_info); + debug_only(__ should_not_reach_here()); + return; + } + + if (_index->is_cpu_register()) { + __ mv(t0, _index->as_register()); + } else { + __ mv(t0, _index->as_jint()); + } + Runtime1::StubID stub_id; + if (_throw_index_out_of_bounds_exception) { + stub_id = Runtime1::throw_index_exception_id; + } else { + assert(_array != NULL, "sanity"); + __ mv(t1, _array->as_pointer_register()); + stub_id = Runtime1::throw_range_check_failed_id; + } + RuntimeAddress target(Runtime1::entry_for(stub_id)); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(ra, target, offset); + __ jalr(ra, ra, offset); + }); + ce->add_call_info_here(_info); + ce->verify_oop_map(_info); + debug_only(__ should_not_reach_here()); +} + +PredicateFailedStub::PredicateFailedStub(CodeEmitInfo* info) { + _info = new CodeEmitInfo(info); +} + +void PredicateFailedStub::emit_code(LIR_Assembler* ce) { + __ bind(_entry); + address a = Runtime1::entry_for(Runtime1::predicate_failed_trap_id); + __ far_call(RuntimeAddress(a)); + ce->add_call_info_here(_info); + ce->verify_oop_map(_info); + debug_only(__ should_not_reach_here()); +} + +void DivByZeroStub::emit_code(LIR_Assembler* ce) { + if (_offset != -1) { + ce->compilation()->implicit_exception_table()->append(_offset, __ offset()); + } + __ bind(_entry); + __ far_call(Address(Runtime1::entry_for(Runtime1::throw_div0_exception_id), relocInfo::runtime_call_type)); + ce->add_call_info_here(_info); + ce->verify_oop_map(_info); +#ifdef ASSERT + __ should_not_reach_here(); +#endif +} + +// Implementation of NewInstanceStub +NewInstanceStub::NewInstanceStub(LIR_Opr klass_reg, LIR_Opr result, ciInstanceKlass* klass, CodeEmitInfo* info, Runtime1::StubID stub_id) { + _result = result; + _klass = klass; + _klass_reg = klass_reg; + _info = new CodeEmitInfo(info); + assert(stub_id == Runtime1::new_instance_id || + stub_id == Runtime1::fast_new_instance_id || + stub_id == Runtime1::fast_new_instance_init_check_id, + "need new_instance id"); + _stub_id = stub_id; +} + +void NewInstanceStub::emit_code(LIR_Assembler* ce) { + assert(__ rsp_offset() == 0, "frame size should be fixed"); + __ bind(_entry); + __ mv(x13, _klass_reg->as_register()); + __ far_call(RuntimeAddress(Runtime1::entry_for(_stub_id))); + ce->add_call_info_here(_info); + ce->verify_oop_map(_info); + assert(_result->as_register() == x10, "result must in x10"); + __ j(_continuation); +} + +// Implementation of NewTypeArrayStub +NewTypeArrayStub::NewTypeArrayStub(LIR_Opr klass_reg, LIR_Opr length, LIR_Opr result, CodeEmitInfo* info) { + _klass_reg = klass_reg; + _length = length; + _result = result; + _info = new CodeEmitInfo(info); +} + +void NewTypeArrayStub::emit_code(LIR_Assembler* ce) { + assert(__ rsp_offset() == 0, "frame size should be fixed"); + __ bind(_entry); + assert(_length->as_register() == x9, "length must in x9"); + assert(_klass_reg->as_register() == x13, "klass_reg must in x13"); + __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::new_type_array_id))); + ce->add_call_info_here(_info); + ce->verify_oop_map(_info); + assert(_result->as_register() == x10, "result must in x10"); + __ j(_continuation); +} + +// Implementation of NewObjectArrayStub +NewObjectArrayStub::NewObjectArrayStub(LIR_Opr klass_reg, LIR_Opr length, LIR_Opr result, CodeEmitInfo* info) { + _klass_reg = klass_reg; + _result = result; + _length = length; + _info = new CodeEmitInfo(info); +} + +void NewObjectArrayStub::emit_code(LIR_Assembler* ce) { + assert(__ rsp_offset() == 0, "frame size should be fixed"); + __ bind(_entry); + assert(_length->as_register() == x9, "length must in x9"); + assert(_klass_reg->as_register() == x13, "klass_reg must in x13"); + __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::new_object_array_id))); + ce->add_call_info_here(_info); + ce->verify_oop_map(_info); + assert(_result->as_register() == x10, "result must in x10"); + __ j(_continuation); +} + +// Implementation of MonitorAccessStubs +MonitorEnterStub::MonitorEnterStub(LIR_Opr obj_reg, LIR_Opr lock_reg, CodeEmitInfo* info) +: MonitorAccessStub(obj_reg, lock_reg) { + _info = new CodeEmitInfo(info); +} + +void MonitorEnterStub::emit_code(LIR_Assembler* ce) { + assert(__ rsp_offset() == 0, "frame size should be fixed"); + __ bind(_entry); + ce->store_parameter(_obj_reg->as_register(), 1); + ce->store_parameter(_lock_reg->as_register(), 0); + Runtime1::StubID enter_id; + if (ce->compilation()->has_fpu_code()) { + enter_id = Runtime1::monitorenter_id; + } else { + enter_id = Runtime1::monitorenter_nofpu_id; + } + __ far_call(RuntimeAddress(Runtime1::entry_for(enter_id))); + ce->add_call_info_here(_info); + ce->verify_oop_map(_info); + __ j(_continuation); +} + +void MonitorExitStub::emit_code(LIR_Assembler* ce) { + __ bind(_entry); + if (_compute_lock) { + // lock_reg was destroyed by fast unlocking attempt => recompute it + ce->monitor_address(_monitor_ix, _lock_reg); + } + ce->store_parameter(_lock_reg->as_register(), 0); + // note: non-blocking leaf routine => no call info needed + Runtime1::StubID exit_id; + if (ce->compilation()->has_fpu_code()) { + exit_id = Runtime1::monitorexit_id; + } else { + exit_id = Runtime1::monitorexit_nofpu_id; + } + __ la(ra, _continuation); + __ far_jump(RuntimeAddress(Runtime1::entry_for(exit_id))); +} + +// Implementation of patching: +// - Copy the code at given offset to an inlined buffer (first the bytes, then the number of bytes) +// - Replace original code with a call to the stub +// At Runtime: +// - call to stub, jump to runtime +// - in runtime: preserve all registers (rspecially objects, i.e., source and destination object) +// - in runtime: after initializing class, restore original code, reexecute instruction + +int PatchingStub::_patch_info_offset = -NativeGeneralJump::instruction_size; + +void PatchingStub::align_patch_site(MacroAssembler* masm) {} + +void PatchingStub::emit_code(LIR_Assembler* ce) { + assert(false, "RISCV should not use C1 runtime patching"); +} + +void DeoptimizeStub::emit_code(LIR_Assembler* ce) { + __ bind(_entry); + ce->store_parameter(_trap_request, 0); + __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::deoptimize_id))); + ce->add_call_info_here(_info); + DEBUG_ONLY(__ should_not_reach_here()); +} + +void ImplicitNullCheckStub::emit_code(LIR_Assembler* ce) { + address a = NULL; + if (_info->deoptimize_on_exception()) { + // Deoptimize, do not throw the exception, because it is probably wrong to do it here. + a = Runtime1::entry_for(Runtime1::predicate_failed_trap_id); + } else { + a = Runtime1::entry_for(Runtime1::throw_null_pointer_exception_id); + } + + ce->compilation()->implicit_exception_table()->append(_offset, __ offset()); + __ bind(_entry); + __ far_call(RuntimeAddress(a)); + ce->add_call_info_here(_info); + ce->verify_oop_map(_info); + debug_only(__ should_not_reach_here()); +} + +void SimpleExceptionStub::emit_code(LIR_Assembler* ce) { + assert(__ rsp_offset() == 0, "frame size should be fixed"); + + __ bind(_entry); + // pass the object in a tmp register because all other registers + // must be preserved + if (_obj->is_cpu_register()) { + __ mv(t0, _obj->as_register()); + } + __ far_call(RuntimeAddress(Runtime1::entry_for(_stub)), NULL, t1); + ce->add_call_info_here(_info); + debug_only(__ should_not_reach_here()); +} + +void ArrayCopyStub::emit_code(LIR_Assembler* ce) { + // ---------------slow case: call to native----------------- + __ bind(_entry); + // Figure out where the args should go + // This should really convert the IntrinsicID to the Method* and signature + // but I don't know how to do that. + const int args_num = 5; + VMRegPair args[args_num]; + BasicType signature[args_num] = { T_OBJECT, T_INT, T_OBJECT, T_INT, T_INT }; + SharedRuntime::java_calling_convention(signature, args, args_num); + + // push parameters + Register r[args_num]; + r[0] = src()->as_register(); + r[1] = src_pos()->as_register(); + r[2] = dst()->as_register(); + r[3] = dst_pos()->as_register(); + r[4] = length()->as_register(); + + // next registers will get stored on the stack + for (int j = 0; j < args_num; j++) { + VMReg r_1 = args[j].first(); + if (r_1->is_stack()) { + int st_off = r_1->reg2stack() * wordSize; + __ sd(r[j], Address(sp, st_off)); + } else { + assert(r[j] == args[j].first()->as_Register(), "Wrong register for arg"); + } + } + + ce->align_call(lir_static_call); + + ce->emit_static_call_stub(); + if (ce->compilation()->bailed_out()) { + return; // CodeCache is full + } + Address resolve(SharedRuntime::get_resolve_static_call_stub(), + relocInfo::static_call_type); + address call = __ trampoline_call(resolve); + if (call == NULL) { + ce->bailout("trampoline stub overflow"); + return; + } + ce->add_call_info_here(info()); + +#ifndef PRODUCT + if (PrintC1Statistics) { + __ la(t1, ExternalAddress((address)&Runtime1::_arraycopy_slowcase_cnt)); + __ incrementw(Address(t1)); + } +#endif + + __ j(_continuation); +} + +#undef __ diff --git a/src/hotspot/cpu/riscv/c1_Defs_riscv.hpp b/src/hotspot/cpu/riscv/c1_Defs_riscv.hpp new file mode 100644 index 0000000000000..4417ad6309124 --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_Defs_riscv.hpp @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_C1_DEFS_RISCV_HPP +#define CPU_RISCV_C1_DEFS_RISCV_HPP + +// native word offsets from memory address (little endian) +enum { + pd_lo_word_offset_in_bytes = 0, + pd_hi_word_offset_in_bytes = BytesPerWord +}; + +// explicit rounding operations are required to implement the strictFP mode +enum { + pd_strict_fp_requires_explicit_rounding = false +}; + +// registers +enum { + pd_nof_cpu_regs_frame_map = RegisterImpl::number_of_registers, // number of registers used during code emission + pd_nof_fpu_regs_frame_map = FloatRegisterImpl::number_of_registers, // number of float registers used during code emission + + // caller saved + pd_nof_caller_save_cpu_regs_frame_map = 13, // number of registers killed by calls + pd_nof_caller_save_fpu_regs_frame_map = 32, // number of float registers killed by calls + + pd_first_callee_saved_reg = pd_nof_caller_save_cpu_regs_frame_map, + pd_last_callee_saved_reg = 21, + + pd_last_allocatable_cpu_reg = pd_nof_caller_save_cpu_regs_frame_map - 1, + + pd_nof_cpu_regs_reg_alloc + = pd_nof_caller_save_cpu_regs_frame_map, // number of registers that are visible to register allocator + pd_nof_fpu_regs_reg_alloc = 32, // number of float registers that are visible to register allocator + + pd_nof_cpu_regs_linearscan = 32, // number of registers visible to linear scan + pd_nof_fpu_regs_linearscan = pd_nof_fpu_regs_frame_map, // number of float registers visible to linear scan + pd_nof_xmm_regs_linearscan = 0, // don't have vector registers + + pd_first_cpu_reg = 0, + pd_last_cpu_reg = pd_nof_cpu_regs_reg_alloc - 1, + pd_first_byte_reg = 0, + pd_last_byte_reg = pd_nof_cpu_regs_reg_alloc - 1, + + pd_first_fpu_reg = pd_nof_cpu_regs_frame_map, + pd_last_fpu_reg = pd_first_fpu_reg + 31, + + pd_first_callee_saved_fpu_reg_1 = 8 + pd_first_fpu_reg, + pd_last_callee_saved_fpu_reg_1 = 9 + pd_first_fpu_reg, + pd_first_callee_saved_fpu_reg_2 = 18 + pd_first_fpu_reg, + pd_last_callee_saved_fpu_reg_2 = 27 + pd_first_fpu_reg +}; + + +// Encoding of float value in debug info. This is true on x86 where +// floats are extended to doubles when stored in the stack, false for +// RISCV where floats and doubles are stored in their native form. +enum { + pd_float_saved_as_double = false +}; + +#endif // CPU_RISCV_C1_DEFS_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/c1_FpuStackSim_riscv.cpp b/src/hotspot/cpu/riscv/c1_FpuStackSim_riscv.cpp new file mode 100644 index 0000000000000..e3a2606c5323b --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_FpuStackSim_riscv.cpp @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +//-------------------------------------------------------- +// FpuStackSim +//-------------------------------------------------------- + +// No FPU stack on RISCV diff --git a/src/hotspot/cpu/riscv/c1_FpuStackSim_riscv.hpp b/src/hotspot/cpu/riscv/c1_FpuStackSim_riscv.hpp new file mode 100644 index 0000000000000..7bc3d3115018e --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_FpuStackSim_riscv.hpp @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2005, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_C1_FPUSTACKSIM_RISCV_HPP +#define CPU_RISCV_C1_FPUSTACKSIM_RISCV_HPP + +// No FPU stack on RISCV +class FpuStackSim; + +#endif // CPU_RISCV_C1_FPUSTACKSIM_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/c1_FrameMap_riscv.cpp b/src/hotspot/cpu/riscv/c1_FrameMap_riscv.cpp new file mode 100644 index 0000000000000..1f8b2b5510001 --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_FrameMap_riscv.cpp @@ -0,0 +1,388 @@ +/* + * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "c1/c1_FrameMap.hpp" +#include "c1/c1_LIR.hpp" +#include "runtime/sharedRuntime.hpp" +#include "vmreg_riscv.inline.hpp" + +LIR_Opr FrameMap::map_to_opr(BasicType type, VMRegPair* reg, bool) { + LIR_Opr opr = LIR_OprFact::illegalOpr; + VMReg r_1 = reg->first(); + VMReg r_2 = reg->second(); + if (r_1->is_stack()) { + // Convert stack slot to an SP offset + // The calling convention does not count the SharedRuntime::out_preserve_stack_slots() value + // so we must add it in here. + int st_off = (r_1->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size; + opr = LIR_OprFact::address(new LIR_Address(sp_opr, st_off, type)); + } else if (r_1->is_Register()) { + Register reg1 = r_1->as_Register(); + if (r_2->is_Register() && (type == T_LONG || type == T_DOUBLE)) { + Register reg2 = r_2->as_Register(); + assert(reg2 == reg1, "must be same register"); + opr = as_long_opr(reg1); + } else if (is_reference_type(type)) { + opr = as_oop_opr(reg1); + } else if (type == T_METADATA) { + opr = as_metadata_opr(reg1); + } else if (type == T_ADDRESS) { + opr = as_address_opr(reg1); + } else { + opr = as_opr(reg1); + } + } else if (r_1->is_FloatRegister()) { + assert(type == T_DOUBLE || type == T_FLOAT, "wrong type"); + int num = r_1->as_FloatRegister()->encoding(); + if (type == T_FLOAT) { + opr = LIR_OprFact::single_fpu(num); + } else { + opr = LIR_OprFact::double_fpu(num); + } + } else { + ShouldNotReachHere(); + } + return opr; +} + +LIR_Opr FrameMap::zr_opr; +LIR_Opr FrameMap::r1_opr; +LIR_Opr FrameMap::r2_opr; +LIR_Opr FrameMap::r3_opr; +LIR_Opr FrameMap::r4_opr; +LIR_Opr FrameMap::r5_opr; +LIR_Opr FrameMap::r6_opr; +LIR_Opr FrameMap::r7_opr; +LIR_Opr FrameMap::r8_opr; +LIR_Opr FrameMap::r9_opr; +LIR_Opr FrameMap::r10_opr; +LIR_Opr FrameMap::r11_opr; +LIR_Opr FrameMap::r12_opr; +LIR_Opr FrameMap::r13_opr; +LIR_Opr FrameMap::r14_opr; +LIR_Opr FrameMap::r15_opr; +LIR_Opr FrameMap::r16_opr; +LIR_Opr FrameMap::r17_opr; +LIR_Opr FrameMap::r18_opr; +LIR_Opr FrameMap::r19_opr; +LIR_Opr FrameMap::r20_opr; +LIR_Opr FrameMap::r21_opr; +LIR_Opr FrameMap::r22_opr; +LIR_Opr FrameMap::r23_opr; +LIR_Opr FrameMap::r24_opr; +LIR_Opr FrameMap::r25_opr; +LIR_Opr FrameMap::r26_opr; +LIR_Opr FrameMap::r27_opr; +LIR_Opr FrameMap::r28_opr; +LIR_Opr FrameMap::r29_opr; +LIR_Opr FrameMap::r30_opr; +LIR_Opr FrameMap::r31_opr; + +LIR_Opr FrameMap::fp_opr; +LIR_Opr FrameMap::sp_opr; + +LIR_Opr FrameMap::receiver_opr; + +LIR_Opr FrameMap::zr_oop_opr; +LIR_Opr FrameMap::r1_oop_opr; +LIR_Opr FrameMap::r2_oop_opr; +LIR_Opr FrameMap::r3_oop_opr; +LIR_Opr FrameMap::r4_oop_opr; +LIR_Opr FrameMap::r5_oop_opr; +LIR_Opr FrameMap::r6_oop_opr; +LIR_Opr FrameMap::r7_oop_opr; +LIR_Opr FrameMap::r8_oop_opr; +LIR_Opr FrameMap::r9_oop_opr; +LIR_Opr FrameMap::r10_oop_opr; +LIR_Opr FrameMap::r11_oop_opr; +LIR_Opr FrameMap::r12_oop_opr; +LIR_Opr FrameMap::r13_oop_opr; +LIR_Opr FrameMap::r14_oop_opr; +LIR_Opr FrameMap::r15_oop_opr; +LIR_Opr FrameMap::r16_oop_opr; +LIR_Opr FrameMap::r17_oop_opr; +LIR_Opr FrameMap::r18_oop_opr; +LIR_Opr FrameMap::r19_oop_opr; +LIR_Opr FrameMap::r20_oop_opr; +LIR_Opr FrameMap::r21_oop_opr; +LIR_Opr FrameMap::r22_oop_opr; +LIR_Opr FrameMap::r23_oop_opr; +LIR_Opr FrameMap::r24_oop_opr; +LIR_Opr FrameMap::r25_oop_opr; +LIR_Opr FrameMap::r26_oop_opr; +LIR_Opr FrameMap::r27_oop_opr; +LIR_Opr FrameMap::r28_oop_opr; +LIR_Opr FrameMap::r29_oop_opr; +LIR_Opr FrameMap::r30_oop_opr; +LIR_Opr FrameMap::r31_oop_opr; + +LIR_Opr FrameMap::t0_opr; +LIR_Opr FrameMap::t1_opr; +LIR_Opr FrameMap::t0_long_opr; +LIR_Opr FrameMap::t1_long_opr; + +LIR_Opr FrameMap::r10_metadata_opr; +LIR_Opr FrameMap::r11_metadata_opr; +LIR_Opr FrameMap::r12_metadata_opr; +LIR_Opr FrameMap::r13_metadata_opr; +LIR_Opr FrameMap::r14_metadata_opr; +LIR_Opr FrameMap::r15_metadata_opr; + +LIR_Opr FrameMap::long10_opr; +LIR_Opr FrameMap::long11_opr; +LIR_Opr FrameMap::fpu10_float_opr; +LIR_Opr FrameMap::fpu10_double_opr; + +LIR_Opr FrameMap::_caller_save_cpu_regs[] = { 0, }; +LIR_Opr FrameMap::_caller_save_fpu_regs[] = { 0, }; + +//-------------------------------------------------------- +// FrameMap +//-------------------------------------------------------- +// |---f31--| +// |---..---| +// |---f28--| +// |---f27--|<---pd_last_callee_saved_fpu_reg_2 +// |---..---| +// |---f18--|<---pd_first_callee_saved_fpu_reg_2 +// |---f17--| +// |---..---| +// |---f10--| +// |---f9---|<---pd_last_callee_saved_fpu_reg_1 +// |---f8---|<---pd_first_callee_saved_fpu_reg_1 +// |---f7---| +// |---..---| +// |---f0---| +// |---x27--| +// |---x23--| +// |---x8---| +// |---x4---| +// |---x3---| +// |---x2---| +// |---x1---| +// |---x0---| +// |---x26--|<---pd_last_callee_saved_reg +// |---..---| +// |---x18--| +// |---x9---|<---pd_first_callee_saved_reg +// |---x31--| +// |---..---| +// |---x28--| +// |---x17--| +// |---..---| +// |---x10--| +// |---x7---| + +void FrameMap::initialize() { + assert(!_init_done, "once"); + + int i = 0; + + // caller save register + map_register(i, x7); r7_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x10); r10_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x11); r11_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x12); r12_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x13); r13_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x14); r14_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x15); r15_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x16); r16_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x17); r17_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x28); r28_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x29); r29_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x30); r30_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x31); r31_opr = LIR_OprFact::single_cpu(i); i++; + + // callee save register + map_register(i, x9); r9_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x18); r18_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x19); r19_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x20); r20_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x21); r21_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x22); r22_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x24); r24_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x25); r25_opr = LIR_OprFact::single_cpu(i); i++; + map_register(i, x26); r26_opr = LIR_OprFact::single_cpu(i); i++; + + // special register + map_register(i, x0); zr_opr = LIR_OprFact::single_cpu(i); i++; // zr + map_register(i, x1); r1_opr = LIR_OprFact::single_cpu(i); i++; // ra + map_register(i, x2); r2_opr = LIR_OprFact::single_cpu(i); i++; // sp + map_register(i, x3); r3_opr = LIR_OprFact::single_cpu(i); i++; // gp + map_register(i, x4); r4_opr = LIR_OprFact::single_cpu(i); i++; // thread + map_register(i, x8); r8_opr = LIR_OprFact::single_cpu(i); i++; // fp + map_register(i, x23); r23_opr = LIR_OprFact::single_cpu(i); i++; // java thread + map_register(i, x27); r27_opr = LIR_OprFact::single_cpu(i); i++; // heapbase + + // tmp register + map_register(i, x5); r5_opr = LIR_OprFact::single_cpu(i); i++; // t0 + map_register(i, x6); r6_opr = LIR_OprFact::single_cpu(i); i++; // t1 + + t0_opr = r5_opr; + t1_opr = r6_opr; + t0_long_opr = LIR_OprFact::double_cpu(r5_opr->cpu_regnr(), r5_opr->cpu_regnr()); + t1_long_opr = LIR_OprFact::double_cpu(r6_opr->cpu_regnr(), r6_opr->cpu_regnr()); + + long10_opr = LIR_OprFact::double_cpu(r10_opr->cpu_regnr(), r10_opr->cpu_regnr()); + long11_opr = LIR_OprFact::double_cpu(r11_opr->cpu_regnr(), r11_opr->cpu_regnr()); + + fpu10_float_opr = LIR_OprFact::single_fpu(10); + fpu10_double_opr = LIR_OprFact::double_fpu(10); + + i = 0; + _caller_save_cpu_regs[i++] = r7_opr; + _caller_save_cpu_regs[i++] = r10_opr; + _caller_save_cpu_regs[i++] = r11_opr; + _caller_save_cpu_regs[i++] = r12_opr; + _caller_save_cpu_regs[i++] = r13_opr; + _caller_save_cpu_regs[i++] = r14_opr; + _caller_save_cpu_regs[i++] = r15_opr; + _caller_save_cpu_regs[i++] = r16_opr; + _caller_save_cpu_regs[i++] = r17_opr; + _caller_save_cpu_regs[i++] = r28_opr; + _caller_save_cpu_regs[i++] = r29_opr; + _caller_save_cpu_regs[i++] = r30_opr; + _caller_save_cpu_regs[i++] = r31_opr; + + _init_done = true; + + zr_oop_opr = as_oop_opr(x0); + r1_oop_opr = as_oop_opr(x1); + r2_oop_opr = as_oop_opr(x2); + r3_oop_opr = as_oop_opr(x3); + r4_oop_opr = as_oop_opr(x4); + r5_oop_opr = as_oop_opr(x5); + r6_oop_opr = as_oop_opr(x6); + r7_oop_opr = as_oop_opr(x7); + r8_oop_opr = as_oop_opr(x8); + r9_oop_opr = as_oop_opr(x9); + r10_oop_opr = as_oop_opr(x10); + r11_oop_opr = as_oop_opr(x11); + r12_oop_opr = as_oop_opr(x12); + r13_oop_opr = as_oop_opr(x13); + r14_oop_opr = as_oop_opr(x14); + r15_oop_opr = as_oop_opr(x15); + r16_oop_opr = as_oop_opr(x16); + r17_oop_opr = as_oop_opr(x17); + r18_oop_opr = as_oop_opr(x18); + r19_oop_opr = as_oop_opr(x19); + r20_oop_opr = as_oop_opr(x20); + r21_oop_opr = as_oop_opr(x21); + r22_oop_opr = as_oop_opr(x22); + r23_oop_opr = as_oop_opr(x23); + r24_oop_opr = as_oop_opr(x24); + r25_oop_opr = as_oop_opr(x25); + r26_oop_opr = as_oop_opr(x26); + r27_oop_opr = as_oop_opr(x27); + r28_oop_opr = as_oop_opr(x28); + r29_oop_opr = as_oop_opr(x29); + r30_oop_opr = as_oop_opr(x30); + r31_oop_opr = as_oop_opr(x31); + + r10_metadata_opr = as_metadata_opr(x10); + r11_metadata_opr = as_metadata_opr(x11); + r12_metadata_opr = as_metadata_opr(x12); + r13_metadata_opr = as_metadata_opr(x13); + r14_metadata_opr = as_metadata_opr(x14); + r15_metadata_opr = as_metadata_opr(x15); + + sp_opr = as_pointer_opr(sp); + fp_opr = as_pointer_opr(fp); + + VMRegPair regs; + BasicType sig_bt = T_OBJECT; + SharedRuntime::java_calling_convention(&sig_bt, ®s, 1); + receiver_opr = as_oop_opr(regs.first()->as_Register()); + + for (i = 0; i < nof_caller_save_fpu_regs; i++) { + _caller_save_fpu_regs[i] = LIR_OprFact::single_fpu(i); + } +} + + +Address FrameMap::make_new_address(ByteSize sp_offset) const { + return Address(sp, in_bytes(sp_offset)); +} + + +// ----------------mapping----------------------- +// all mapping is based on fp addressing, except for simple leaf methods where we access +// the locals sp based (and no frame is built) + + +// Frame for simple leaf methods (quick entries) +// +// +----------+ +// | ret addr | <- TOS +// +----------+ +// | args | +// | ...... | + +// Frame for standard methods +// +// | .........| <- TOS +// | locals | +// +----------+ +// | old fp, | +// +----------+ +// | ret addr | +// +----------+ +// | args | <- FP +// | .........| + + +// For OopMaps, map a local variable or spill index to an VMRegImpl name. +// This is the offset from sp() in the frame of the slot for the index, +// skewed by VMRegImpl::stack0 to indicate a stack location (vs.a register.) +// +// framesize + +// stack0 stack0 0 <- VMReg +// | | | +// ...........|..............|.............| +// 0 1 2 3 x x 4 5 6 ... | <- local indices +// ^ ^ sp() ( x x indicate link +// | | and return addr) +// arguments non-argument locals + + +VMReg FrameMap::fpu_regname (int n) { + // Return the OptoReg name for the fpu stack slot "n" + // A spilled fpu stack slot comprises to two single-word OptoReg's. + return as_FloatRegister(n)->as_VMReg(); +} + +LIR_Opr FrameMap::stack_pointer() { + return FrameMap::sp_opr; +} + +// JSR 292 +LIR_Opr FrameMap::method_handle_invoke_SP_save_opr() { + return LIR_OprFact::illegalOpr; // Not needed on riscv +} + +bool FrameMap::validate_frame() { + return true; +} diff --git a/src/hotspot/cpu/riscv/c1_FrameMap_riscv.hpp b/src/hotspot/cpu/riscv/c1_FrameMap_riscv.hpp new file mode 100644 index 0000000000000..01281f5c9e161 --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_FrameMap_riscv.hpp @@ -0,0 +1,148 @@ +/* + * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_C1_FRAMEMAP_RISCV_HPP +#define CPU_RISCV_C1_FRAMEMAP_RISCV_HPP + +// On RISCV the frame looks as follows: +// +// +-----------------------------+---------+----------------------------------------+----------------+----------- +// | size_arguments-nof_reg_args | 2 words | size_locals-size_arguments+numreg_args | _size_monitors | spilling . +// +-----------------------------+---------+----------------------------------------+----------------+----------- + + public: + static const int pd_c_runtime_reserved_arg_size; + + enum { + first_available_sp_in_frame = 0, + frame_pad_in_bytes = 16, + nof_reg_args = 8 + }; + + public: + static LIR_Opr receiver_opr; + + static LIR_Opr zr_opr; + static LIR_Opr r1_opr; + static LIR_Opr r2_opr; + static LIR_Opr r3_opr; + static LIR_Opr r4_opr; + static LIR_Opr r5_opr; + static LIR_Opr r6_opr; + static LIR_Opr r7_opr; + static LIR_Opr r8_opr; + static LIR_Opr r9_opr; + static LIR_Opr r10_opr; + static LIR_Opr r11_opr; + static LIR_Opr r12_opr; + static LIR_Opr r13_opr; + static LIR_Opr r14_opr; + static LIR_Opr r15_opr; + static LIR_Opr r16_opr; + static LIR_Opr r17_opr; + static LIR_Opr r18_opr; + static LIR_Opr r19_opr; + static LIR_Opr r20_opr; + static LIR_Opr r21_opr; + static LIR_Opr r22_opr; + static LIR_Opr r23_opr; + static LIR_Opr r24_opr; + static LIR_Opr r25_opr; + static LIR_Opr r26_opr; + static LIR_Opr r27_opr; + static LIR_Opr r28_opr; + static LIR_Opr r29_opr; + static LIR_Opr r30_opr; + static LIR_Opr r31_opr; + static LIR_Opr fp_opr; + static LIR_Opr sp_opr; + + static LIR_Opr zr_oop_opr; + static LIR_Opr r1_oop_opr; + static LIR_Opr r2_oop_opr; + static LIR_Opr r3_oop_opr; + static LIR_Opr r4_oop_opr; + static LIR_Opr r5_oop_opr; + static LIR_Opr r6_oop_opr; + static LIR_Opr r7_oop_opr; + static LIR_Opr r8_oop_opr; + static LIR_Opr r9_oop_opr; + static LIR_Opr r10_oop_opr; + static LIR_Opr r11_oop_opr; + static LIR_Opr r12_oop_opr; + static LIR_Opr r13_oop_opr; + static LIR_Opr r14_oop_opr; + static LIR_Opr r15_oop_opr; + static LIR_Opr r16_oop_opr; + static LIR_Opr r17_oop_opr; + static LIR_Opr r18_oop_opr; + static LIR_Opr r19_oop_opr; + static LIR_Opr r20_oop_opr; + static LIR_Opr r21_oop_opr; + static LIR_Opr r22_oop_opr; + static LIR_Opr r23_oop_opr; + static LIR_Opr r24_oop_opr; + static LIR_Opr r25_oop_opr; + static LIR_Opr r26_oop_opr; + static LIR_Opr r27_oop_opr; + static LIR_Opr r28_oop_opr; + static LIR_Opr r29_oop_opr; + static LIR_Opr r30_oop_opr; + static LIR_Opr r31_oop_opr; + + static LIR_Opr t0_opr; + static LIR_Opr t1_opr; + static LIR_Opr t0_long_opr; + static LIR_Opr t1_long_opr; + + static LIR_Opr r10_metadata_opr; + static LIR_Opr r11_metadata_opr; + static LIR_Opr r12_metadata_opr; + static LIR_Opr r13_metadata_opr; + static LIR_Opr r14_metadata_opr; + static LIR_Opr r15_metadata_opr; + + static LIR_Opr long10_opr; + static LIR_Opr long11_opr; + static LIR_Opr fpu10_float_opr; + static LIR_Opr fpu10_double_opr; + + static LIR_Opr as_long_opr(Register r) { + return LIR_OprFact::double_cpu(cpu_reg2rnr(r), cpu_reg2rnr(r)); + } + static LIR_Opr as_pointer_opr(Register r) { + return LIR_OprFact::double_cpu(cpu_reg2rnr(r), cpu_reg2rnr(r)); + } + + // VMReg name for spilled physical FPU stack slot n + static VMReg fpu_regname(int n); + + static bool is_caller_save_register(LIR_Opr opr) { return true; } + static bool is_caller_save_register(Register r) { return true; } + + static int nof_caller_save_cpu_regs() { return pd_nof_caller_save_cpu_regs_frame_map; } + static int last_cpu_reg() { return pd_last_cpu_reg; } + +#endif // CPU_RISCV_C1_FRAMEMAP_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_arith_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_arith_riscv.cpp new file mode 100644 index 0000000000000..c2ce48814084d --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_arith_riscv.cpp @@ -0,0 +1,281 @@ +/* + * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/assembler.hpp" +#include "c1/c1_LIRAssembler.hpp" +#include "c1/c1_MacroAssembler.hpp" + +#ifndef PRODUCT +#define COMMENT(x) do { __ block_comment(x); } while (0) +#else +#define COMMENT(x) +#endif + +#define __ _masm-> + +void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr illegal, + LIR_Opr result, CodeEmitInfo* info) { + // opcode check + assert((code == lir_idiv) || (code == lir_irem), "opcode must be idiv or irem"); + bool is_irem = (code == lir_irem); + // opreand check + assert(left->is_single_cpu(), "left must be a register"); + assert(right->is_single_cpu() || right->is_constant(), "right must be a register or constant"); + assert(result->is_single_cpu(), "result must be a register"); + Register lreg = left->as_register(); + Register dreg = result->as_register(); + + // power-of-2 constant check and codegen + if (right->is_constant()) { + int c = right->as_constant_ptr()->as_jint(); + assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant"); + if (is_irem) { + if (c == 1) { + // move 0 to dreg if divisor is 1 + __ mv(dreg, zr); + } else { + unsigned int shift = exact_log2(c); + __ sraiw(t0, lreg, 0x1f); + __ srliw(t0, t0, BitsPerInt - shift); + __ addw(t1, lreg, t0); + if (Assembler::is_simm12(c - 1)) { + __ andi(t1, t1, c - 1); + } else { + __ zero_extend(t1, t1, shift); + } + __ subw(dreg, t1, t0); + } + } else { + if (c == 1) { + // move lreg to dreg if divisor is 1 + __ mv(dreg, lreg); + } else { + unsigned int shift = exact_log2(c); + __ sraiw(t0, lreg, 0x1f); + if (Assembler::is_simm12(c - 1)) { + __ andi(t0, t0, c - 1); + } else { + __ zero_extend(t0, t0, shift); + } + __ addw(dreg, t0, lreg); + __ sraiw(dreg, dreg, shift); + } + } + } else { + Register rreg = right->as_register(); + __ corrected_idivl(dreg, lreg, rreg, is_irem); + } +} + +void LIR_Assembler::arith_op_single_cpu_right_constant(LIR_Code code, LIR_Opr left, LIR_Opr right, + Register lreg, Register dreg) { + // cpu register - constant + jlong c; + + switch (right->type()) { + case T_LONG: + c = right->as_constant_ptr()->as_jlong(); break; + case T_INT: // fall through + case T_ADDRESS: + c = right->as_constant_ptr()->as_jint(); break; + default: + ShouldNotReachHere(); + c = 0; // unreachable + } + + assert(code == lir_add || code == lir_sub, "mismatched arithmetic op"); + if (c == 0 && dreg == lreg) { + COMMENT("effective nop elided"); + return; + } + switch (left->type()) { + case T_INT: + switch (code) { + case lir_add: __ addw(dreg, lreg, c); break; + case lir_sub: __ subw(dreg, lreg, c); break; + default: ShouldNotReachHere(); + } + break; + case T_OBJECT: // fall through + case T_ADDRESS: + switch (code) { + case lir_add: __ add(dreg, lreg, c); break; + case lir_sub: __ sub(dreg, lreg, c); break; + default: ShouldNotReachHere(); + } + break; + default: + ShouldNotReachHere(); + } +} + +void LIR_Assembler::arith_op_single_cpu(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest) { + Register lreg = left->as_register(); + Register dreg = as_reg(dest); + + if (right->is_single_cpu()) { + // cpu register - cpu register + assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT, "should be"); + Register rreg = right->as_register(); + switch (code) { + case lir_add: __ addw(dest->as_register(), lreg, rreg); break; + case lir_sub: __ subw(dest->as_register(), lreg, rreg); break; + case lir_mul: __ mulw(dest->as_register(), lreg, rreg); break; + default: ShouldNotReachHere(); + } + } else if (right->is_double_cpu()) { + Register rreg = right->as_register_lo(); + // sigle_cpu + double_cpu; can happen with obj_long + assert(code == lir_add || code == lir_sub, "mismatched arithmetic op"); + switch (code) { + case lir_add: __ add(dreg, lreg, rreg); break; + case lir_sub: __ sub(dreg, lreg, rreg); break; + default: ShouldNotReachHere(); + } + } else if (right->is_constant()) { + arith_op_single_cpu_right_constant(code, left, right, lreg, dreg); + } else { + ShouldNotReachHere(); + } +} + +void LIR_Assembler::arith_op_double_cpu(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest) { + Register lreg_lo = left->as_register_lo(); + + if (right->is_double_cpu()) { + // cpu register - cpu register + Register rreg_lo = right->as_register_lo(); + switch (code) { + case lir_add: __ add(dest->as_register_lo(), lreg_lo, rreg_lo); break; + case lir_sub: __ sub(dest->as_register_lo(), lreg_lo, rreg_lo); break; + case lir_mul: __ mul(dest->as_register_lo(), lreg_lo, rreg_lo); break; + case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false); break; + case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true); break; + default: + ShouldNotReachHere(); + } + } else if (right->is_constant()) { + jlong c = right->as_constant_ptr()->as_jlong(); + Register dreg = as_reg(dest); + switch (code) { + case lir_add: // fall through + case lir_sub: + if (c == 0 && dreg == lreg_lo) { + COMMENT("effective nop elided"); + return; + } + code == lir_add ? __ add(dreg, lreg_lo, c) : __ sub(dreg, lreg_lo, c); + break; + case lir_div: + assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant"); + if (c == 1) { + // move lreg_lo to dreg if divisor is 1 + __ mv(dreg, lreg_lo); + } else { + unsigned int shift = exact_log2_long(c); + // use t0 as intermediate result register + __ srai(t0, lreg_lo, 0x3f); + if (Assembler::is_simm12(c - 1)) { + __ andi(t0, t0, c - 1); + } else { + __ zero_extend(t0, t0, shift); + } + __ add(dreg, t0, lreg_lo); + __ srai(dreg, dreg, shift); + } + break; + case lir_rem: + assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant"); + if (c == 1) { + // move 0 to dreg if divisor is 1 + __ mv(dreg, zr); + } else { + unsigned int shift = exact_log2_long(c); + __ srai(t0, lreg_lo, 0x3f); + __ srli(t0, t0, BitsPerLong - shift); + __ add(t1, lreg_lo, t0); + if (Assembler::is_simm12(c - 1)) { + __ andi(t1, t1, c - 1); + } else { + __ zero_extend(t1, t1, shift); + } + __ sub(dreg, t1, t0); + } + break; + default: + ShouldNotReachHere(); + } + } else { + ShouldNotReachHere(); + } +} + +void LIR_Assembler::arith_op_single_fpu(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest) { + assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register"); + switch (code) { + case lir_add: __ fadd_s(dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; + case lir_sub: __ fsub_s(dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; + case lir_mul: __ fmul_s(dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; + case lir_div: __ fdiv_s(dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; + default: + ShouldNotReachHere(); + } +} + +void LIR_Assembler::arith_op_double_fpu(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest) { + if (right->is_double_fpu()) { + // fpu register - fpu register + switch (code) { + case lir_add: __ fadd_d(dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; + case lir_sub: __ fsub_d(dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; + case lir_mul: __ fmul_d(dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; + case lir_div: __ fdiv_d(dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; + default: + ShouldNotReachHere(); + } + } else { + ShouldNotReachHere(); + } +} + +void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, + CodeEmitInfo* info, bool pop_fpu_stack) { + assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method"); + + if (left->is_single_cpu()) { + arith_op_single_cpu(code, left, right, dest); + } else if (left->is_double_cpu()) { + arith_op_double_cpu(code, left, right, dest); + } else if (left->is_single_fpu()) { + arith_op_single_fpu(code, left, right, dest); + } else if (left->is_double_fpu()) { + arith_op_double_fpu(code, left, right, dest); + } else { + ShouldNotReachHere(); + } +} + +#undef __ diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_arith_riscv.hpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_arith_riscv.hpp new file mode 100644 index 0000000000000..ab0a9963fc113 --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_arith_riscv.hpp @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_C1_LIRASSEMBLER_ARITH_RISCV_HPP +#define CPU_RISCV_C1_LIRASSEMBLER_ARITH_RISCV_HPP + + // arith_op sub functions + void arith_op_single_cpu(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest); + void arith_op_double_cpu(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest); + void arith_op_single_fpu(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest); + void arith_op_double_fpu(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest); + void arith_op_single_cpu_right_constant(LIR_Code code, LIR_Opr left, LIR_Opr right, Register lreg, Register dreg); + void arithmetic_idiv(LIR_Op3* op, bool is_irem); + +#endif // CPU_RISCV_C1_LIRASSEMBLER_ARITH_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp new file mode 100644 index 0000000000000..81ee91e44811b --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp @@ -0,0 +1,387 @@ +/* + * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/assembler.hpp" +#include "c1/c1_LIRAssembler.hpp" +#include "c1/c1_MacroAssembler.hpp" +#include "ci/ciArrayKlass.hpp" +#include "oops/objArrayKlass.hpp" +#include "runtime/stubRoutines.hpp" + +#define __ _masm-> + + +void LIR_Assembler::generic_arraycopy(Register src, Register src_pos, Register length, + Register dst, Register dst_pos, CodeStub *stub) { + assert(src == x11 && src_pos == x12, "mismatch in calling convention"); + // Save the arguments in case the generic arraycopy fails and we + // have to fall back to the JNI stub + arraycopy_store_args(src, src_pos, length, dst, dst_pos); + + address copyfunc_addr = StubRoutines::generic_arraycopy(); + assert(copyfunc_addr != NULL, "generic arraycopy stub required"); + + // The arguments are in java calling convention so we shift them + // to C convention + assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4); + __ mv(c_rarg0, j_rarg0); + assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4); + __ mv(c_rarg1, j_rarg1); + assert_different_registers(c_rarg2, j_rarg3, j_rarg4); + __ mv(c_rarg2, j_rarg2); + assert_different_registers(c_rarg3, j_rarg4); + __ mv(c_rarg3, j_rarg3); + __ mv(c_rarg4, j_rarg4); +#ifndef PRODUCT + if (PrintC1Statistics) { + __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt)); + } +#endif + __ far_call(RuntimeAddress(copyfunc_addr)); + __ beqz(x10, *stub->continuation()); + // Reload values from the stack so they are where the stub + // expects them. + arraycopy_load_args(src, src_pos, length, dst, dst_pos); + + // x10 is -1^K where K == partial copied count + __ xori(t0, x10, -1); + // adjust length down and src/end pos up by partial copied count + __ subw(length, length, t0); + __ addw(src_pos, src_pos, t0); + __ addw(dst_pos, dst_pos, t0); + __ j(*stub->entry()); + + __ bind(*stub->continuation()); +} + +void LIR_Assembler::arraycopy_simple_check(Register src, Register src_pos, Register length, + Register dst, Register dst_pos, Register tmp, + CodeStub *stub, int flags) { + // test for NULL + if (flags & LIR_OpArrayCopy::src_null_check) { + __ beqz(src, *stub->entry(), /* is_far */ true); + } + if (flags & LIR_OpArrayCopy::dst_null_check) { + __ beqz(dst, *stub->entry(), /* is_far */ true); + } + + // If the compiler was not able to prove that exact type of the source or the destination + // of the arraycopy is an array type, check at runtime if the source or the destination is + // an instance type. + if (flags & LIR_OpArrayCopy::type_check) { + assert(Klass::_lh_neutral_value == 0, "or replace bgez instructions"); + if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) { + __ load_klass(tmp, dst); + __ lw(t0, Address(tmp, in_bytes(Klass::layout_helper_offset()))); + __ bgez(t0, *stub->entry(), /* is_far */ true); + } + + if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) { + __ load_klass(tmp, src); + __ lw(t0, Address(tmp, in_bytes(Klass::layout_helper_offset()))); + __ bgez(t0, *stub->entry(), /* is_far */ true); + } + } + + // check if negative + if (flags & LIR_OpArrayCopy::src_pos_positive_check) { + __ bltz(src_pos, *stub->entry(), /* is_far */ true); + } + if (flags & LIR_OpArrayCopy::dst_pos_positive_check) { + __ bltz(dst_pos, *stub->entry(), /* is_far */ true); + } + if (flags & LIR_OpArrayCopy::length_positive_check) { + __ bltz(length, *stub->entry(), /* is_far */ true); + } + + if (flags & LIR_OpArrayCopy::src_range_check) { + __ addw(tmp, src_pos, length); + __ lwu(t0, Address(src, arrayOopDesc::length_offset_in_bytes())); + __ bgtu(tmp, t0, *stub->entry(), /* is_far */ true); + } + if (flags & LIR_OpArrayCopy::dst_range_check) { + __ addw(tmp, dst_pos, length); + __ lwu(t0, Address(dst, arrayOopDesc::length_offset_in_bytes())); + __ bgtu(tmp, t0, *stub->entry(), /* is_far */ true); + } +} + +void LIR_Assembler::arraycopy_checkcast(Register src, Register src_pos, Register length, + Register dst, Register dst_pos, Register tmp, + CodeStub *stub, BasicType basic_type, + address copyfunc_addr, int flags) { + // src is not a sub class of dst so we have to do a + // per-element check. + int mask = LIR_OpArrayCopy::src_objarray | LIR_OpArrayCopy::dst_objarray; + if ((flags & mask) != mask) { + // Check that at least both of them object arrays. + assert(flags & mask, "one of the two should be known to be an object array"); + + if (!(flags & LIR_OpArrayCopy::src_objarray)) { + __ load_klass(tmp, src); + } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) { + __ load_klass(tmp, dst); + } + int lh_offset = in_bytes(Klass::layout_helper_offset()); + Address klass_lh_addr(tmp, lh_offset); + jint objArray_lh = Klass::array_layout_helper(T_OBJECT); + __ lw(t0, klass_lh_addr); + __ mv(t1, objArray_lh); + __ bne(t0, t1, *stub->entry(), /* is_far */ true); + } + + // Spill because stubs can use any register they like and it's + // easier to restore just those that we care about. + arraycopy_store_args(src, src_pos, length, dst, dst_pos); + arraycopy_checkcast_prepare_params(src, src_pos, length, dst, dst_pos, basic_type); + __ far_call(RuntimeAddress(copyfunc_addr)); + +#ifndef PRODUCT + if (PrintC1Statistics) { + Label failed; + __ bnez(x10, failed); + __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt)); + __ bind(failed); + } +#endif + + __ beqz(x10, *stub->continuation()); + +#ifndef PRODUCT + if (PrintC1Statistics) { + __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt)); + } +#endif + assert_different_registers(dst, dst_pos, length, src_pos, src, x10, t0); + + // Restore previously spilled arguments + arraycopy_load_args(src, src_pos, length, dst, dst_pos); + + // return value is -1^K where K is partial copied count + __ xori(t0, x10, -1); + // adjust length down and src/end pos up by partial copied count + __ subw(length, length, t0); + __ addw(src_pos, src_pos, t0); + __ addw(dst_pos, dst_pos, t0); +} + +void LIR_Assembler::arraycopy_type_check(Register src, Register src_pos, Register length, + Register dst, Register dst_pos, Register tmp, + CodeStub *stub, BasicType basic_type, int flags) { + // We don't know the array types are compatible + if (basic_type != T_OBJECT) { + // Simple test for basic type arrays + if (UseCompressedClassPointers) { + __ lwu(tmp, Address(src, oopDesc::klass_offset_in_bytes())); + __ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes())); + } else { + __ ld(tmp, Address(src, oopDesc::klass_offset_in_bytes())); + __ ld(t0, Address(dst, oopDesc::klass_offset_in_bytes())); + } + __ bne(tmp, t0, *stub->entry(), /* is_far */ true); + } else { + // For object arrays, if src is a sub class of dst then we can + // safely do the copy. + Label cont, slow; + +#define PUSH(r1, r2) \ + __ addi(sp, sp, -2 * wordSize); \ + __ sd(r1, Address(sp, 1 * wordSize)); \ + __ sd(r2, Address(sp, 0)); + +#define POP(r1, r2) \ + __ ld(r1, Address(sp, 1 * wordSize)); \ + __ ld(r2, Address(sp, 0)); \ + __ addi(sp, sp, 2 * wordSize); + + PUSH(src, dst); + __ load_klass(src, src); + __ load_klass(dst, dst); + __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL); + + PUSH(src, dst); + __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); + POP(src, dst); + __ bnez(dst, cont); + + __ bind(slow); + POP(src, dst); + + address copyfunc_addr = StubRoutines::checkcast_arraycopy(); + if (copyfunc_addr != NULL) { // use stub if available + arraycopy_checkcast(src, src_pos, length, dst, dst_pos, tmp, stub, basic_type, copyfunc_addr, flags); + } + + __ j(*stub->entry()); + __ bind(cont); + POP(src, dst); + } +} + +void LIR_Assembler::arraycopy_assert(Register src, Register dst, Register tmp, ciArrayKlass *default_type, int flags) { + assert(default_type != NULL, "NULL default_type!"); + BasicType basic_type = default_type->element_type()->basic_type(); + + if (basic_type == T_ARRAY) { basic_type = T_OBJECT; } + if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) { + // Sanity check the known type with the incoming class. For the + // primitive case the types must match exactly with src.klass and + // dst.klass each exactly matching the default type. For the + // object array case, if no type check is needed then either the + // dst type is exactly the expected type and the src type is a + // subtype which we can't check or src is the same array as dst + // but not necessarily exactly of type default_type. + Label known_ok, halt; + __ mov_metadata(tmp, default_type->constant_encoding()); + if (UseCompressedClassPointers) { + __ encode_klass_not_null(tmp); + } + + if (basic_type != T_OBJECT) { + if (UseCompressedClassPointers) { + __ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes())); + } else { + __ ld(t0, Address(dst, oopDesc::klass_offset_in_bytes())); + } + __ bne(tmp, t0, halt); + if (UseCompressedClassPointers) { + __ lwu(t0, Address(src, oopDesc::klass_offset_in_bytes())); + } else { + __ ld(t0, Address(src, oopDesc::klass_offset_in_bytes())); + } + __ beq(tmp, t0, known_ok); + } else { + if (UseCompressedClassPointers) { + __ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes())); + } else { + __ ld(t0, Address(dst, oopDesc::klass_offset_in_bytes())); + } + __ beq(tmp, t0, known_ok); + __ beq(src, dst, known_ok); + } + __ bind(halt); + __ stop("incorrect type information in arraycopy"); + __ bind(known_ok); + } +} + +void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { + ciArrayKlass *default_type = op->expected_type(); + Register src = op->src()->as_register(); + Register dst = op->dst()->as_register(); + Register src_pos = op->src_pos()->as_register(); + Register dst_pos = op->dst_pos()->as_register(); + Register length = op->length()->as_register(); + Register tmp = op->tmp()->as_register(); + + CodeStub* stub = op->stub(); + int flags = op->flags(); + BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL; + if (is_reference_type(basic_type)) { basic_type = T_OBJECT; } + + // if we don't know anything, just go through the generic arraycopy + if (default_type == NULL) { + generic_arraycopy(src, src_pos, length, dst, dst_pos, stub); + return; + } + + assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), + "must be true at this point"); + + arraycopy_simple_check(src, src_pos, length, dst, dst_pos, tmp, stub, flags); + + if (flags & LIR_OpArrayCopy::type_check) { + arraycopy_type_check(src, src_pos, length, dst, dst_pos, tmp, stub, basic_type, flags); + } + +#ifdef ASSERT + arraycopy_assert(src, dst, tmp, default_type, flags); +#endif + +#ifndef PRODUCT + if (PrintC1Statistics) { + __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type))); + } +#endif + arraycopy_prepare_params(src, src_pos, length, dst, dst_pos, basic_type); + + bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0; + bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0; + const char *name = NULL; + address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false); + + CodeBlob *cb = CodeCache::find_blob(entry); + if (cb != NULL) { + __ far_call(RuntimeAddress(entry)); + } else { + const int args_num = 3; + __ call_VM_leaf(entry, args_num); + } + + __ bind(*stub->continuation()); +} + + +void LIR_Assembler::arraycopy_prepare_params(Register src, Register src_pos, Register length, + Register dst, Register dst_pos, BasicType basic_type) { + int scale = array_element_size(basic_type); + __ shadd(c_rarg0, src_pos, src, t0, scale); + __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type)); + assert_different_registers(c_rarg0, dst, dst_pos, length); + __ shadd(c_rarg1, dst_pos, dst, t0, scale); + __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type)); + assert_different_registers(c_rarg1, dst, length); + __ mv(c_rarg2, length); + assert_different_registers(c_rarg2, dst); +} + +void LIR_Assembler::arraycopy_checkcast_prepare_params(Register src, Register src_pos, Register length, + Register dst, Register dst_pos, BasicType basic_type) { + arraycopy_prepare_params(src, src_pos, length, dst, dst_pos, basic_type); + __ load_klass(c_rarg4, dst); + __ ld(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset())); + __ lwu(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset())); +} + +void LIR_Assembler::arraycopy_store_args(Register src, Register src_pos, Register length, + Register dst, Register dst_pos) { + __ sd(dst_pos, Address(sp, 0)); // 0: dst_pos sp offset + __ sd(dst, Address(sp, 1 * BytesPerWord)); // 1: dst sp offset + __ sd(length, Address(sp, 2 * BytesPerWord)); // 2: length sp offset + __ sd(src_pos, Address(sp, 3 * BytesPerWord)); // 3: src_pos sp offset + __ sd(src, Address(sp, 4 * BytesPerWord)); // 4: src sp offset +} + +void LIR_Assembler::arraycopy_load_args(Register src, Register src_pos, Register length, + Register dst, Register dst_pos) { + __ ld(dst_pos, Address(sp, 0)); // 0: dst_pos sp offset + __ ld(dst, Address(sp, 1 * BytesPerWord)); // 1: dst sp offset + __ ld(length, Address(sp, 2 * BytesPerWord)); // 2: length sp offset + __ ld(src_pos, Address(sp, 3 * BytesPerWord)); // 3: src_pos sp offset + __ ld(src, Address(sp, 4 * BytesPerWord)); // 4: src sp offset +} + +#undef __ diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.hpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.hpp new file mode 100644 index 0000000000000..06a0f248ca690 --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.hpp @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_C1_LIRASSEMBLER_ARRAYCOPY_RISCV_HPP +#define CPU_RISCV_C1_LIRASSEMBLER_ARRAYCOPY_RISCV_HPP + + // arraycopy sub functions + void generic_arraycopy(Register src, Register src_pos, Register length, + Register dst, Register dst_pos, CodeStub *stub); + void arraycopy_simple_check(Register src, Register src_pos, Register length, + Register dst, Register dst_pos, Register tmp, + CodeStub *stub, int flags); + void arraycopy_checkcast(Register src, Register src_pos, Register length, + Register dst, Register dst_pos, Register tmp, + CodeStub *stub, BasicType basic_type, + address copyfunc_addr, int flags); + void arraycopy_type_check(Register src, Register src_pos, Register length, + Register dst, Register dst_pos, Register tmp, + CodeStub *stub, BasicType basic_type, int flags); + void arraycopy_assert(Register src, Register dst, Register tmp, ciArrayKlass *default_type, int flags); + void arraycopy_prepare_params(Register src, Register src_pos, Register length, + Register dst, Register dst_pos, BasicType basic_type); + void arraycopy_checkcast_prepare_params(Register src, Register src_pos, Register length, + Register dst, Register dst_pos, BasicType basic_type); + void arraycopy_store_args(Register src, Register src_pos, Register length, + Register dst, Register dst_pos); + void arraycopy_load_args(Register src, Register src_pos, Register length, + Register dst, Register dst_pos); + +#endif // CPU_RISCV_C1_LIRASSEMBLER_ARRAYCOPY_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp new file mode 100644 index 0000000000000..9c9b6acb1c591 --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp @@ -0,0 +1,2263 @@ +/* + * Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/assembler.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "c1/c1_CodeStubs.hpp" +#include "c1/c1_Compilation.hpp" +#include "c1/c1_LIRAssembler.hpp" +#include "c1/c1_MacroAssembler.hpp" +#include "c1/c1_Runtime1.hpp" +#include "c1/c1_ValueStack.hpp" +#include "ci/ciArrayKlass.hpp" +#include "ci/ciInstance.hpp" +#include "code/compiledIC.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "nativeInst_riscv.hpp" +#include "oops/objArrayKlass.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/sharedRuntime.hpp" +#include "utilities/powerOfTwo.hpp" +#include "vmreg_riscv.inline.hpp" + +#ifndef PRODUCT +#define COMMENT(x) do { __ block_comment(x); } while (0) +#else +#define COMMENT(x) +#endif + +NEEDS_CLEANUP // remove this definitions ? +const Register IC_Klass = t1; // where the IC klass is cached +const Register SYNC_header = x10; // synchronization header +const Register SHIFT_count = x10; // where count for shift operations must be + +#define __ _masm-> + +static void select_different_registers(Register preserve, + Register extra, + Register &tmp1, + Register &tmp2) { + if (tmp1 == preserve) { + assert_different_registers(tmp1, tmp2, extra); + tmp1 = extra; + } else if (tmp2 == preserve) { + assert_different_registers(tmp1, tmp2, extra); + tmp2 = extra; + } + assert_different_registers(preserve, tmp1, tmp2); +} + +static void select_different_registers(Register preserve, + Register extra, + Register &tmp1, + Register &tmp2, + Register &tmp3) { + if (tmp1 == preserve) { + assert_different_registers(tmp1, tmp2, tmp3, extra); + tmp1 = extra; + } else if (tmp2 == preserve) { + assert_different_registers(tmp1, tmp2, tmp3, extra); + tmp2 = extra; + } else if (tmp3 == preserve) { + assert_different_registers(tmp1, tmp2, tmp3, extra); + tmp3 = extra; + } + assert_different_registers(preserve, tmp1, tmp2, tmp3); +} + +bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; } + +void LIR_Assembler::clinit_barrier(ciMethod* method) { + assert(VM_Version::supports_fast_class_init_checks(), "sanity"); + assert(!method->holder()->is_not_initialized(), "initialization should have been started"); + + Label L_skip_barrier; + + __ mov_metadata(t1, method->holder()->constant_encoding()); + __ clinit_barrier(t1, t0, &L_skip_barrier /* L_fast_path */); + __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); + __ bind(L_skip_barrier); +} + +LIR_Opr LIR_Assembler::receiverOpr() { + return FrameMap::receiver_opr; +} + +LIR_Opr LIR_Assembler::osrBufferPointer() { + return FrameMap::as_pointer_opr(receiverOpr()->as_register()); +} + +void LIR_Assembler::breakpoint() { Unimplemented(); } + +void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); } + +void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); } + +static jlong as_long(LIR_Opr data) { + jlong result; + switch (data->type()) { + case T_INT: + result = (data->as_jint()); + break; + case T_LONG: + result = (data->as_jlong()); + break; + default: + ShouldNotReachHere(); + result = 0; // unreachable + } + return result; +} + +Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) { + if (addr->base()->is_illegal()) { + assert(addr->index()->is_illegal(), "must be illegal too"); + __ movptr(tmp, addr->disp()); + return Address(tmp, 0); + } + + Register base = addr->base()->as_pointer_register(); + LIR_Opr index_opr = addr->index(); + + if (index_opr->is_illegal()) { + return Address(base, addr->disp()); + } + + int scale = addr->scale(); + if (index_opr->is_cpu_register()) { + Register index; + if (index_opr->is_single_cpu()) { + index = index_opr->as_register(); + } else { + index = index_opr->as_register_lo(); + } + if (scale != 0) { + __ shadd(tmp, index, base, tmp, scale); + } else { + __ add(tmp, base, index); + } + return Address(tmp, addr->disp()); + } else if (index_opr->is_constant()) { + intptr_t addr_offset = (((intptr_t)index_opr->as_constant_ptr()->as_jint()) << scale) + addr->disp(); + return Address(base, addr_offset); + } + + Unimplemented(); + return Address(); +} + +Address LIR_Assembler::as_Address_hi(LIR_Address* addr) { + ShouldNotReachHere(); + return Address(); +} + +Address LIR_Assembler::as_Address(LIR_Address* addr) { + return as_Address(addr, t0); +} + +Address LIR_Assembler::as_Address_lo(LIR_Address* addr) { + return as_Address(addr); +} + +// Ensure a valid Address (base + offset) to a stack-slot. If stack access is +// not encodable as a base + (immediate) offset, generate an explicit address +// calculation to hold the address in t0. +Address LIR_Assembler::stack_slot_address(int index, uint size, int adjust) { + precond(size == 4 || size == 8); + Address addr = frame_map()->address_for_slot(index, adjust); + precond(addr.getMode() == Address::base_plus_offset); + precond(addr.base() == sp); + precond(addr.offset() > 0); + uint mask = size - 1; + assert((addr.offset() & mask) == 0, "scaled offsets only"); + + return addr; +} + +void LIR_Assembler::osr_entry() { + offsets()->set_value(CodeOffsets::OSR_Entry, code_offset()); + BlockBegin* osr_entry = compilation()->hir()->osr_entry(); + guarantee(osr_entry != NULL, "NULL osr_entry!"); + ValueStack* entry_state = osr_entry->state(); + int number_of_locks = entry_state->locks_size(); + + // we jump here if osr happens with the interpreter + // state set up to continue at the beginning of the + // loop that triggered osr - in particular, we have + // the following registers setup: + // + // x12: osr buffer + // + + //build frame + ciMethod* m = compilation()->method(); + __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes()); + + // OSR buffer is + // + // locals[nlocals-1..0] + // monitors[0..number_of_locks] + // + // locals is a direct copy of the interpreter frame so in the osr buffer + // so first slot in the local array is the last local from the interpreter + // and last slot is local[0] (receiver) from the interpreter + // + // Similarly with locks. The first lock slot in the osr buffer is the nth lock + // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock + // in the interpreter frame (the method lock if a sync method) + + // Initialize monitors in the compiled activation. + // x12: pointer to osr buffer + // All other registers are dead at this point and the locals will be + // copied into place by code emitted in the IR. + + Register OSR_buf = osrBufferPointer()->as_pointer_register(); + { + assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below"); + int monitor_offset = BytesPerWord * method()->max_locals() + + (2 * BytesPerWord) * (number_of_locks - 1); + // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in + // the OSR buffer using 2 word entries: first the lock and then + // the oop. + for (int i = 0; i < number_of_locks; i++) { + int slot_offset = monitor_offset - ((i * 2) * BytesPerWord); +#ifdef ASSERT + // verify the interpreter's monitor has a non-null object + { + Label L; + __ ld(t0, Address(OSR_buf, slot_offset + 1 * BytesPerWord)); + __ bnez(t0, L); + __ stop("locked object is NULL"); + __ bind(L); + } +#endif // ASSERT + __ ld(x9, Address(OSR_buf, slot_offset + 0)); + __ sd(x9, frame_map()->address_for_monitor_lock(i)); + __ ld(x9, Address(OSR_buf, slot_offset + 1 * BytesPerWord)); + __ sd(x9, frame_map()->address_for_monitor_object(i)); + } + } +} + +// inline cache check; done before the frame is built. +int LIR_Assembler::check_icache() { + Register receiver = FrameMap::receiver_opr->as_register(); + Register ic_klass = IC_Klass; + int start_offset = __ offset(); + Label dont; + __ inline_cache_check(receiver, ic_klass, dont); + + // if icache check fails, then jump to runtime routine + // Note: RECEIVER must still contain the receiver! + __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub())); + + // We align the verified entry point unless the method body + // (including its inline cache check) will fit in a single 64-byte + // icache line. + if (!method()->is_accessor() || __ offset() - start_offset > 4 * 4) { + // force alignment after the cache check. + __ align(CodeEntryAlignment); + } + + __ bind(dont); + return start_offset; +} + +void LIR_Assembler::jobject2reg(jobject o, Register reg) { + if (o == NULL) { + __ mv(reg, zr); + } else { + __ movoop(reg, o, /* immediate */ true); + } +} + +void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) { + deoptimize_trap(info); +} + +// This specifies the rsp decrement needed to build the frame +int LIR_Assembler::initial_frame_size_in_bytes() const { + // if rounding, must let FrameMap know! + + return in_bytes(frame_map()->framesize_in_bytes()); +} + +int LIR_Assembler::emit_exception_handler() { + // if the last instruction is a call (typically to do a throw which + // is coming at the end after block reordering) the return address + // must still point into the code area in order to avoid assertion + // failures when searching for the corresponding bci ==> add a nop + // (was bug 5/14/1999 -gri) + __ nop(); + + // generate code for exception handler + address handler_base = __ start_a_stub(exception_handler_size()); + if (handler_base == NULL) { + // not enough space left for the handler + bailout("exception handler overflow"); + return -1; + } + + int offset = code_offset(); + + // the exception oop and pc are in x10, and x13 + // no other registers need to be preserved, so invalidate them + __ invalidate_registers(false, true, true, false, true, true); + + // check that there is really an exception + __ verify_not_null_oop(x10); + + // search an exception handler (x10: exception oop, x13: throwing pc) + __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); + __ should_not_reach_here(); + guarantee(code_offset() - offset <= exception_handler_size(), "overflow"); + __ end_a_stub(); + + return offset; +} + +// Emit the code to remove the frame from the stack in the exception +// unwind path. +int LIR_Assembler::emit_unwind_handler() { +#ifndef PRODUCT + if (CommentedAssembly) { + _masm->block_comment("Unwind handler"); + } +#endif // PRODUCT + + int offset = code_offset(); + + // Fetch the exception from TLS and clear out exception related thread state + __ ld(x10, Address(xthread, JavaThread::exception_oop_offset())); + __ sd(zr, Address(xthread, JavaThread::exception_oop_offset())); + __ sd(zr, Address(xthread, JavaThread::exception_pc_offset())); + + __ bind(_unwind_handler_entry); + __ verify_not_null_oop(x10); + if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { + __ mv(x9, x10); // Perserve the exception + } + + // Preform needed unlocking + MonitorExitStub* stub = NULL; + if (method()->is_synchronized()) { + monitor_address(0, FrameMap::r10_opr); + stub = new MonitorExitStub(FrameMap::r10_opr, true, 0); + __ unlock_object(x15, x14, x10, *stub->entry()); + __ bind(*stub->continuation()); + } + + if (compilation()->env()->dtrace_method_probes()) { + __ mv(c_rarg0, xthread); + __ mov_metadata(c_rarg1, method()->constant_encoding()); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1); + } + + if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { + __ mv(x10, x9); // Restore the exception + } + + // remove the activation and dispatch to the unwind handler + __ block_comment("remove_frame and dispatch to the unwind handler"); + __ remove_frame(initial_frame_size_in_bytes()); + __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id))); + + // Emit the slow path assembly + if (stub != NULL) { + stub->emit_code(this); + } + + return offset; +} + +int LIR_Assembler::emit_deopt_handler() { + // if the last instruciton is a call (typically to do a throw which + // is coming at the end after block reordering) the return address + // must still point into the code area in order to avoid assertion + // failures when searching for the corresponding bck => add a nop + // (was bug 5/14/1999 - gri) + __ nop(); + + // generate code for exception handler + address handler_base = __ start_a_stub(deopt_handler_size()); + if (handler_base == NULL) { + // not enough space left for the handler + bailout("deopt handler overflow"); + return -1; + } + + int offset = code_offset(); + + __ auipc(ra, 0); + __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); + guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); + __ end_a_stub(); + + return offset; +} + +void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) { + assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == x10, "word returns are in x10"); + + // Pop the stack before the safepoint code + __ remove_frame(initial_frame_size_in_bytes()); + + if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) { + __ reserved_stack_check(); + } + + code_stub->set_safepoint_offset(__ offset()); + __ relocate(relocInfo::poll_return_type); + __ safepoint_poll(*code_stub->entry(), true /* at_return */, false /* acquire */, true /* in_nmethod */); + __ ret(); +} + +int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) { + guarantee(info != NULL, "Shouldn't be NULL"); + __ get_polling_page(t0, relocInfo::poll_type); + add_debug_info_for_branch(info); // This isn't just debug info: + // it's the oop map + __ read_polling_page(t0, 0, relocInfo::poll_type); + return __ offset(); +} + +void LIR_Assembler::move_regs(Register from_reg, Register to_reg) { + __ mv(to_reg, from_reg); +} + +void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); } + +void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { + assert(src->is_constant(), "should not call otherwise"); + assert(dest->is_register(), "should not call otherwise"); + LIR_Const* c = src->as_constant_ptr(); + address const_addr = NULL; + + switch (c->type()) { + case T_INT: + assert(patch_code == lir_patch_none, "no patching handled here"); + __ mv(dest->as_register(), c->as_jint()); + break; + + case T_ADDRESS: + assert(patch_code == lir_patch_none, "no patching handled here"); + __ mv(dest->as_register(), c->as_jint()); + break; + + case T_LONG: + assert(patch_code == lir_patch_none, "no patching handled here"); + __ mv(dest->as_register_lo(), (intptr_t)c->as_jlong()); + break; + + case T_OBJECT: + case T_ARRAY: + if (patch_code == lir_patch_none) { + jobject2reg(c->as_jobject(), dest->as_register()); + } else { + jobject2reg_with_patching(dest->as_register(), info); + } + break; + + case T_METADATA: + if (patch_code != lir_patch_none) { + klass2reg_with_patching(dest->as_register(), info); + } else { + __ mov_metadata(dest->as_register(), c->as_metadata()); + } + break; + + case T_FLOAT: + const_addr = float_constant(c->as_jfloat()); + assert(const_addr != NULL, "must create float constant in the constant table"); + __ flw(dest->as_float_reg(), InternalAddress(const_addr)); + break; + + case T_DOUBLE: + const_addr = double_constant(c->as_jdouble()); + assert(const_addr != NULL, "must create double constant in the constant table"); + __ fld(dest->as_double_reg(), InternalAddress(const_addr)); + break; + + default: + ShouldNotReachHere(); + } +} + +void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) { + assert(src->is_constant(), "should not call otherwise"); + assert(dest->is_stack(), "should not call otherwise"); + LIR_Const* c = src->as_constant_ptr(); + switch (c->type()) { + case T_OBJECT: + if (c->as_jobject() == NULL) { + __ sd(zr, frame_map()->address_for_slot(dest->single_stack_ix())); + } else { + const2reg(src, FrameMap::t1_opr, lir_patch_none, NULL); + reg2stack(FrameMap::t1_opr, dest, c->type(), false); + } + break; + case T_ADDRESS: // fall through + const2reg(src, FrameMap::t1_opr, lir_patch_none, NULL); + reg2stack(FrameMap::t1_opr, dest, c->type(), false); + case T_INT: // fall through + case T_FLOAT: + if (c->as_jint_bits() == 0) { + __ sw(zr, frame_map()->address_for_slot(dest->single_stack_ix())); + } else { + __ mv(t1, c->as_jint_bits()); + __ sw(t1, frame_map()->address_for_slot(dest->single_stack_ix())); + } + break; + case T_LONG: // fall through + case T_DOUBLE: + if (c->as_jlong_bits() == 0) { + __ sd(zr, frame_map()->address_for_slot(dest->double_stack_ix(), + lo_word_offset_in_bytes)); + } else { + __ mv(t1, (intptr_t)c->as_jlong_bits()); + __ sd(t1, frame_map()->address_for_slot(dest->double_stack_ix(), + lo_word_offset_in_bytes)); + } + break; + default: + ShouldNotReachHere(); + } +} + +void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) { + assert(src->is_constant(), "should not call otherwise"); + assert(dest->is_address(), "should not call otherwise"); + LIR_Const* c = src->as_constant_ptr(); + LIR_Address* to_addr = dest->as_address_ptr(); + void (MacroAssembler::* insn)(Register Rt, const Address &adr, Register temp); + switch (type) { + case T_ADDRESS: + assert(c->as_jint() == 0, "should be"); + insn = &MacroAssembler::sd; break; + case T_LONG: + assert(c->as_jlong() == 0, "should be"); + insn = &MacroAssembler::sd; break; + case T_DOUBLE: + assert(c->as_jdouble() == 0.0, "should be"); + insn = &MacroAssembler::sd; break; + case T_INT: + assert(c->as_jint() == 0, "should be"); + insn = &MacroAssembler::sw; break; + case T_FLOAT: + assert(c->as_jfloat() == 0.0f, "should be"); + insn = &MacroAssembler::sw; break; + case T_OBJECT: // fall through + case T_ARRAY: + assert(c->as_jobject() == 0, "should be"); + if (UseCompressedOops && !wide) { + insn = &MacroAssembler::sw; + } else { + insn = &MacroAssembler::sd; + } + break; + case T_CHAR: // fall through + case T_SHORT: + assert(c->as_jint() == 0, "should be"); + insn = &MacroAssembler::sh; + break; + case T_BOOLEAN: // fall through + case T_BYTE: + assert(c->as_jint() == 0, "should be"); + insn = &MacroAssembler::sb; break; + default: + ShouldNotReachHere(); + insn = &MacroAssembler::sd; // unreachable + } + if (info != NULL) { + add_debug_info_for_null_check_here(info); + } + (_masm->*insn)(zr, as_Address(to_addr), t0); +} + +void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) { + assert(src->is_register(), "should not call otherwise"); + assert(dest->is_register(), "should not call otherwise"); + + // move between cpu-registers + if (dest->is_single_cpu()) { + if (src->type() == T_LONG) { + // Can do LONG -> OBJECT + move_regs(src->as_register_lo(), dest->as_register()); + return; + } + assert(src->is_single_cpu(), "must match"); + if (src->type() == T_OBJECT) { + __ verify_oop(src->as_register()); + } + move_regs(src->as_register(), dest->as_register()); + } else if (dest->is_double_cpu()) { + if (is_reference_type(src->type())) { + __ verify_oop(src->as_register()); + move_regs(src->as_register(), dest->as_register_lo()); + return; + } + assert(src->is_double_cpu(), "must match"); + Register f_lo = src->as_register_lo(); + Register f_hi = src->as_register_hi(); + Register t_lo = dest->as_register_lo(); + Register t_hi = dest->as_register_hi(); + assert(f_hi == f_lo, "must be same"); + assert(t_hi == t_lo, "must be same"); + move_regs(f_lo, t_lo); + } else if (dest->is_single_fpu()) { + assert(src->is_single_fpu(), "expect single fpu"); + __ fmv_s(dest->as_float_reg(), src->as_float_reg()); + } else if (dest->is_double_fpu()) { + assert(src->is_double_fpu(), "expect double fpu"); + __ fmv_d(dest->as_double_reg(), src->as_double_reg()); + } else { + ShouldNotReachHere(); + } +} + +void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) { + precond(src->is_register() && dest->is_stack()); + + uint const c_sz32 = sizeof(uint32_t); + uint const c_sz64 = sizeof(uint64_t); + + assert(src->is_register(), "should not call otherwise"); + assert(dest->is_stack(), "should not call otherwise"); + if (src->is_single_cpu()) { + int index = dest->single_stack_ix(); + if (is_reference_type(type)) { + __ sd(src->as_register(), stack_slot_address(index, c_sz64)); + __ verify_oop(src->as_register()); + } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) { + __ sd(src->as_register(), stack_slot_address(index, c_sz64)); + } else { + __ sw(src->as_register(), stack_slot_address(index, c_sz32)); + } + } else if (src->is_double_cpu()) { + int index = dest->double_stack_ix(); + Address dest_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes); + __ sd(src->as_register_lo(), dest_addr_LO); + } else if (src->is_single_fpu()) { + int index = dest->single_stack_ix(); + __ fsw(src->as_float_reg(), stack_slot_address(index, c_sz32)); + } else if (src->is_double_fpu()) { + int index = dest->double_stack_ix(); + __ fsd(src->as_double_reg(), stack_slot_address(index, c_sz64)); + } else { + ShouldNotReachHere(); + } +} + +void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) { + LIR_Address* to_addr = dest->as_address_ptr(); + // t0 was used as tmp reg in as_Address, so we use t1 as compressed_src + Register compressed_src = t1; + + if (patch_code != lir_patch_none) { + deoptimize_trap(info); + return; + } + + if (is_reference_type(type)) { + __ verify_oop(src->as_register()); + + if (UseCompressedOops && !wide) { + __ encode_heap_oop(compressed_src, src->as_register()); + } else { + compressed_src = src->as_register(); + } + } + + int null_check_here = code_offset(); + + switch (type) { + case T_FLOAT: + __ fsw(src->as_float_reg(), as_Address(to_addr)); + break; + + case T_DOUBLE: + __ fsd(src->as_double_reg(), as_Address(to_addr)); + break; + + case T_ARRAY: // fall through + case T_OBJECT: + if (UseCompressedOops && !wide) { + __ sw(compressed_src, as_Address(to_addr)); + } else { + __ sd(compressed_src, as_Address(to_addr)); + } + break; + case T_METADATA: + // We get here to store a method pointer to the stack to pass to + // a dtrace runtime call. This can't work on 64 bit with + // compressed klass ptrs: T_METADATA can be compressed klass + // ptr or a 64 bit method pointer. + ShouldNotReachHere(); + __ sd(src->as_register(), as_Address(to_addr)); + break; + case T_ADDRESS: + __ sd(src->as_register(), as_Address(to_addr)); + break; + case T_INT: + __ sw(src->as_register(), as_Address(to_addr)); + break; + case T_LONG: + __ sd(src->as_register_lo(), as_Address(to_addr)); + break; + case T_BYTE: // fall through + case T_BOOLEAN: + __ sb(src->as_register(), as_Address(to_addr)); + break; + case T_CHAR: // fall through + case T_SHORT: + __ sh(src->as_register(), as_Address(to_addr)); + break; + default: + ShouldNotReachHere(); + } + + if (info != NULL) { + add_debug_info_for_null_check(null_check_here, info); + } +} + +void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) { + precond(src->is_stack() && dest->is_register()); + + uint const c_sz32 = sizeof(uint32_t); + uint const c_sz64 = sizeof(uint64_t); + + if (dest->is_single_cpu()) { + int index = src->single_stack_ix(); + if (type == T_INT) { + __ lw(dest->as_register(), stack_slot_address(index, c_sz32)); + } else if (is_reference_type(type)) { + __ ld(dest->as_register(), stack_slot_address(index, c_sz64)); + __ verify_oop(dest->as_register()); + } else if (type == T_METADATA || type == T_ADDRESS) { + __ ld(dest->as_register(), stack_slot_address(index, c_sz64)); + } else { + __ lwu(dest->as_register(), stack_slot_address(index, c_sz32)); + } + } else if (dest->is_double_cpu()) { + int index = src->double_stack_ix(); + Address src_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes); + __ ld(dest->as_register_lo(), src_addr_LO); + } else if (dest->is_single_fpu()) { + int index = src->single_stack_ix(); + __ flw(dest->as_float_reg(), stack_slot_address(index, c_sz32)); + } else if (dest->is_double_fpu()) { + int index = src->double_stack_ix(); + __ fld(dest->as_double_reg(), stack_slot_address(index, c_sz64)); + } else { + ShouldNotReachHere(); + } +} + +void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) { + deoptimize_trap(info); +} + +void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) { + LIR_Opr temp; + if (type == T_LONG || type == T_DOUBLE) { + temp = FrameMap::t1_long_opr; + } else { + temp = FrameMap::t1_opr; + } + + stack2reg(src, temp, src->type()); + reg2stack(temp, dest, dest->type(), false); +} + +void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) { + assert(src->is_address(), "should not call otherwise"); + assert(dest->is_register(), "should not call otherwise"); + + LIR_Address* addr = src->as_address_ptr(); + LIR_Address* from_addr = src->as_address_ptr(); + + if (addr->base()->type() == T_OBJECT) { + __ verify_oop(addr->base()->as_pointer_register()); + } + + if (patch_code != lir_patch_none) { + deoptimize_trap(info); + return; + } + + if (info != NULL) { + add_debug_info_for_null_check_here(info); + } + + int null_check_here = code_offset(); + switch (type) { + case T_FLOAT: + __ flw(dest->as_float_reg(), as_Address(from_addr)); + break; + case T_DOUBLE: + __ fld(dest->as_double_reg(), as_Address(from_addr)); + break; + case T_ARRAY: // fall through + case T_OBJECT: + if (UseCompressedOops && !wide) { + __ lwu(dest->as_register(), as_Address(from_addr)); + } else { + __ ld(dest->as_register(), as_Address(from_addr)); + } + break; + case T_METADATA: + // We get here to store a method pointer to the stack to pass to + // a dtrace runtime call. This can't work on 64 bit with + // compressed klass ptrs: T_METADATA can be a compressed klass + // ptr or a 64 bit method pointer. + ShouldNotReachHere(); + __ ld(dest->as_register(), as_Address(from_addr)); + break; + case T_ADDRESS: + __ ld(dest->as_register(), as_Address(from_addr)); + break; + case T_INT: + __ lw(dest->as_register(), as_Address(from_addr)); + break; + case T_LONG: + __ ld(dest->as_register_lo(), as_Address_lo(from_addr)); + break; + case T_BYTE: + __ lb(dest->as_register(), as_Address(from_addr)); + break; + case T_BOOLEAN: + __ lbu(dest->as_register(), as_Address(from_addr)); + break; + case T_CHAR: + __ lhu(dest->as_register(), as_Address(from_addr)); + break; + case T_SHORT: + __ lh(dest->as_register(), as_Address(from_addr)); + break; + default: + ShouldNotReachHere(); + } + + if (is_reference_type(type)) { + if (UseCompressedOops && !wide) { + __ decode_heap_oop(dest->as_register()); + } + + if (!UseZGC) { + // Load barrier has not yet been applied, so ZGC can't verify the oop here + __ verify_oop(dest->as_register()); + } + } +} + +void LIR_Assembler::emit_op3(LIR_Op3* op) { + switch (op->code()) { + case lir_idiv: // fall through + case lir_irem: + arithmetic_idiv(op->code(), + op->in_opr1(), + op->in_opr2(), + op->in_opr3(), + op->result_opr(), + op->info()); + break; + case lir_fmad: + __ fmadd_d(op->result_opr()->as_double_reg(), + op->in_opr1()->as_double_reg(), + op->in_opr2()->as_double_reg(), + op->in_opr3()->as_double_reg()); + break; + case lir_fmaf: + __ fmadd_s(op->result_opr()->as_float_reg(), + op->in_opr1()->as_float_reg(), + op->in_opr2()->as_float_reg(), + op->in_opr3()->as_float_reg()); + break; + default: + ShouldNotReachHere(); + } +} + +void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type, + LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) { + Label label; + + emit_branch(condition, cmp_opr1, cmp_opr2, label, /* is_far */ false, + /* is_unordered */ (condition == lir_cond_greaterEqual || condition == lir_cond_greater) ? false : true); + + Label done; + move_op(opr2, result, type, lir_patch_none, NULL, + false, // pop_fpu_stack + false, // wide + false); // unaligned + __ j(done); + __ bind(label); + move_op(opr1, result, type, lir_patch_none, NULL, + false, // pop_fpu_stack + false, // wide + false); // unaligned + __ bind(done); +} + +void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) { + LIR_Condition condition = op->cond(); + if (condition == lir_cond_always) { + if (op->info() != NULL) { + add_debug_info_for_branch(op->info()); + } + } else { + assert(op->in_opr1() != LIR_OprFact::illegalOpr && op->in_opr2() != LIR_OprFact::illegalOpr, "conditional branches must have legal operands"); + } + bool is_unordered = (op->ublock() == op->block()); + emit_branch(condition, op->in_opr1(), op->in_opr2(), *op->label(), /* is_far */ true, is_unordered); +} + +void LIR_Assembler::emit_branch(LIR_Condition cmp_flag, LIR_Opr cmp1, LIR_Opr cmp2, Label& label, + bool is_far, bool is_unordered) { + + if (cmp_flag == lir_cond_always) { + __ j(label); + return; + } + + if (cmp1->is_cpu_register()) { + Register reg1 = as_reg(cmp1); + if (cmp2->is_cpu_register()) { + Register reg2 = as_reg(cmp2); + __ c1_cmp_branch(cmp_flag, reg1, reg2, label, cmp1->type(), is_far); + } else if (cmp2->is_constant()) { + const2reg_helper(cmp2); + __ c1_cmp_branch(cmp_flag, reg1, t0, label, cmp2->type(), is_far); + } else { + ShouldNotReachHere(); + } + } else if (cmp1->is_single_fpu()) { + assert(cmp2->is_single_fpu(), "expect single float register"); + __ c1_float_cmp_branch(cmp_flag, cmp1->as_float_reg(), cmp2->as_float_reg(), label, is_far, is_unordered); + } else if (cmp1->is_double_fpu()) { + assert(cmp2->is_double_fpu(), "expect double float register"); + __ c1_float_cmp_branch(cmp_flag | C1_MacroAssembler::c1_double_branch_mask, + cmp1->as_double_reg(), cmp2->as_double_reg(), label, is_far, is_unordered); + } else { + ShouldNotReachHere(); + } +} + +void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) { + LIR_Opr src = op->in_opr(); + LIR_Opr dest = op->result_opr(); + + switch (op->bytecode()) { + case Bytecodes::_i2f: + __ fcvt_s_w(dest->as_float_reg(), src->as_register()); break; + case Bytecodes::_i2d: + __ fcvt_d_w(dest->as_double_reg(), src->as_register()); break; + case Bytecodes::_l2d: + __ fcvt_d_l(dest->as_double_reg(), src->as_register_lo()); break; + case Bytecodes::_l2f: + __ fcvt_s_l(dest->as_float_reg(), src->as_register_lo()); break; + case Bytecodes::_f2d: + __ fcvt_d_s(dest->as_double_reg(), src->as_float_reg()); break; + case Bytecodes::_d2f: + __ fcvt_s_d(dest->as_float_reg(), src->as_double_reg()); break; + case Bytecodes::_i2c: + __ zero_extend(dest->as_register(), src->as_register(), 16); break; + case Bytecodes::_i2l: + __ sign_extend(dest->as_register_lo(), src->as_register(), 32); break; + case Bytecodes::_i2s: + __ sign_extend(dest->as_register(), src->as_register(), 16); break; + case Bytecodes::_i2b: + __ sign_extend(dest->as_register(), src->as_register(), 8); break; + case Bytecodes::_l2i: + __ sign_extend(dest->as_register(), src->as_register_lo(), 32); break; + case Bytecodes::_d2l: + __ fcvt_l_d_safe(dest->as_register_lo(), src->as_double_reg()); break; + case Bytecodes::_f2i: + __ fcvt_w_s_safe(dest->as_register(), src->as_float_reg()); break; + case Bytecodes::_f2l: + __ fcvt_l_s_safe(dest->as_register_lo(), src->as_float_reg()); break; + case Bytecodes::_d2i: + __ fcvt_w_d_safe(dest->as_register(), src->as_double_reg()); break; + default: + ShouldNotReachHere(); + } +} + +void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) { + if (op->init_check()) { + __ lbu(t0, Address(op->klass()->as_register(), + InstanceKlass::init_state_offset())); + __ mv(t1, (u1)InstanceKlass::fully_initialized); + add_debug_info_for_null_check_here(op->stub()->info()); + __ bne(t0, t1, *op->stub()->entry(), /* is_far */ true); + } + + __ allocate_object(op->obj()->as_register(), + op->tmp1()->as_register(), + op->tmp2()->as_register(), + op->header_size(), + op->object_size(), + op->klass()->as_register(), + *op->stub()->entry()); + + __ bind(*op->stub()->continuation()); +} + +void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) { + Register len = op->len()->as_register(); + + if (UseSlowPath || + (!UseFastNewObjectArray && is_reference_type(op->type())) || + (!UseFastNewTypeArray && !is_reference_type(op->type()))) { + __ j(*op->stub()->entry()); + } else { + Register tmp1 = op->tmp1()->as_register(); + Register tmp2 = op->tmp2()->as_register(); + Register tmp3 = op->tmp3()->as_register(); + if (len == tmp1) { + tmp1 = tmp3; + } else if (len == tmp2) { + tmp2 = tmp3; + } else if (len == tmp3) { + // everything is ok + } else { + __ mv(tmp3, len); + } + __ allocate_array(op->obj()->as_register(), + len, + tmp1, + tmp2, + arrayOopDesc::header_size(op->type()), + array_element_size(op->type()), + op->klass()->as_register(), + *op->stub()->entry()); + } + __ bind(*op->stub()->continuation()); +} + +void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md, ciProfileData *data, + Register recv, Label* update_done) { + for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { + Label next_test; + // See if the receiver is receiver[n]. + __ ld(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)))); + __ bne(recv, t1, next_test); + Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))); + __ increment(data_addr, DataLayout::counter_increment); + __ j(*update_done); + __ bind(next_test); + } + + // Didn't find receiver; find next empty slot and fill it in + for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { + Label next_test; + Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))); + __ ld(t1, recv_addr); + __ bnez(t1, next_test); + __ sd(recv, recv_addr); + __ mv(t1, DataLayout::counter_increment); + __ sd(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)))); + __ j(*update_done); + __ bind(next_test); + } +} + +void LIR_Assembler::data_check(LIR_OpTypeCheck *op, ciMethodData **md, ciProfileData **data) { + ciMethod* method = op->profiled_method(); + assert(method != NULL, "Should have method"); + int bci = op->profiled_bci(); + *md = method->method_data_or_null(); + guarantee(*md != NULL, "Sanity"); + *data = ((*md)->bci_to_data(bci)); + assert(*data != NULL, "need data for type check"); + assert((*data)->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); +} + +void LIR_Assembler::typecheck_helper_slowcheck(ciKlass *k, Register obj, Register Rtmp1, + Register k_RInfo, Register klass_RInfo, + Label *failure_target, Label *success_target) { + // get object class + // not a safepoint as obj null check happens earlier + __ load_klass(klass_RInfo, obj); + if (k->is_loaded()) { + // See if we get an immediate positive hit + __ ld(t0, Address(klass_RInfo, int64_t(k->super_check_offset()))); + if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) { + __ bne(k_RInfo, t0, *failure_target, /* is_far */ true); + // successful cast, fall through to profile or jump + } else { + // See if we get an immediate positive hit + __ beq(k_RInfo, t0, *success_target); + // check for self + __ beq(klass_RInfo, k_RInfo, *success_target); + + __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo + __ sd(k_RInfo, Address(sp, 0)); // sub klass + __ sd(klass_RInfo, Address(sp, wordSize)); // super klass + __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); + // load result to k_RInfo + __ ld(k_RInfo, Address(sp, 0)); + __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo + // result is a boolean + __ beqz(k_RInfo, *failure_target, /* is_far */ true); + // successful cast, fall through to profile or jump + } + } else { + // perform the fast part of the checking logic + __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL); + // call out-of-line instance of __ check_klass_subtytpe_slow_path(...) + __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo + __ sd(klass_RInfo, Address(sp, wordSize)); // sub klass + __ sd(k_RInfo, Address(sp, 0)); // super klass + __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); + // load result to k_RInfo + __ ld(k_RInfo, Address(sp, 0)); + __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo + // result is a boolean + __ beqz(k_RInfo, *failure_target, /* is_far */ true); + // successful cast, fall thriugh to profile or jump + } +} + +void LIR_Assembler::profile_object(ciMethodData* md, ciProfileData* data, Register obj, + Register klass_RInfo, Label* obj_is_null) { + Label not_null; + __ bnez(obj, not_null); + // Object is null, update MDO and exit + Register mdo = klass_RInfo; + __ mov_metadata(mdo, md->constant_encoding()); + Address data_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset())); + __ lbu(t0, data_addr); + __ ori(t0, t0, BitData::null_seen_byte_constant()); + __ sb(t0, data_addr); + __ j(*obj_is_null); + __ bind(not_null); +} + +void LIR_Assembler::typecheck_loaded(LIR_OpTypeCheck *op, ciKlass* k, Register k_RInfo) { + if (!k->is_loaded()) { + klass2reg_with_patching(k_RInfo, op->info_for_patch()); + } else { + __ mov_metadata(k_RInfo, k->constant_encoding()); + } +} + +void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) { + Register obj = op->object()->as_register(); + Register k_RInfo = op->tmp1()->as_register(); + Register klass_RInfo = op->tmp2()->as_register(); + Register dst = op->result_opr()->as_register(); + ciKlass* k = op->klass(); + Register Rtmp1 = noreg; + + // check if it needs to be profiled + ciMethodData* md = NULL; + ciProfileData* data = NULL; + + const bool should_profile = op->should_profile(); + if (should_profile) { + data_check(op, &md, &data); + } + Label profile_cast_success, profile_cast_failure; + Label *success_target = should_profile ? &profile_cast_success : success; + Label *failure_target = should_profile ? &profile_cast_failure : failure; + + if (obj == k_RInfo) { + k_RInfo = dst; + } else if (obj == klass_RInfo) { + klass_RInfo = dst; + } + if (k->is_loaded() && !UseCompressedClassPointers) { + select_different_registers(obj, dst, k_RInfo, klass_RInfo); + } else { + Rtmp1 = op->tmp3()->as_register(); + select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1); + } + + assert_different_registers(obj, k_RInfo, klass_RInfo); + + if (should_profile) { + profile_object(md, data, obj, klass_RInfo, obj_is_null); + } else { + __ beqz(obj, *obj_is_null); + } + + typecheck_loaded(op, k, k_RInfo); + __ verify_oop(obj); + + if (op->fast_check()) { + // get object class + // not a safepoint as obj null check happens earlier + __ load_klass(t0, obj, t1); + __ bne(t0, k_RInfo, *failure_target, /* is_far */ true); + // successful cast, fall through to profile or jump + } else { + typecheck_helper_slowcheck(k, obj, Rtmp1, k_RInfo, klass_RInfo, failure_target, success_target); + } + if (should_profile) { + type_profile(obj, md, klass_RInfo, k_RInfo, data, success, failure, profile_cast_success, profile_cast_failure); + } + __ j(*success); +} + +void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) { + const bool should_profile = op->should_profile(); + + LIR_Code code = op->code(); + if (code == lir_store_check) { + typecheck_lir_store(op, should_profile); + } else if (code == lir_checkcast) { + Register obj = op->object()->as_register(); + Register dst = op->result_opr()->as_register(); + Label success; + emit_typecheck_helper(op, &success, op->stub()->entry(), &success); + __ bind(success); + if (dst != obj) { + __ mv(dst, obj); + } + } else if (code == lir_instanceof) { + Register obj = op->object()->as_register(); + Register dst = op->result_opr()->as_register(); + Label success, failure, done; + emit_typecheck_helper(op, &success, &failure, &failure); + __ bind(failure); + __ mv(dst, zr); + __ j(done); + __ bind(success); + __ mv(dst, 1); + __ bind(done); + } else { + ShouldNotReachHere(); + } +} + +void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) { + assert(VM_Version::supports_cx8(), "wrong machine"); + Register addr; + if (op->addr()->is_register()) { + addr = as_reg(op->addr()); + } else { + assert(op->addr()->is_address(), "what else?"); + LIR_Address* addr_ptr = op->addr()->as_address_ptr(); + assert(addr_ptr->disp() == 0, "need 0 disp"); + assert(addr_ptr->index() == LIR_OprDesc::illegalOpr(), "need 0 index"); + addr = as_reg(addr_ptr->base()); + } + Register newval = as_reg(op->new_value()); + Register cmpval = as_reg(op->cmp_value()); + + if (op->code() == lir_cas_obj) { + if (UseCompressedOops) { + Register tmp1 = op->tmp1()->as_register(); + assert(op->tmp1()->is_valid(), "must be"); + __ encode_heap_oop(tmp1, cmpval); + cmpval = tmp1; + __ encode_heap_oop(t1, newval); + newval = t1; + caswu(addr, newval, cmpval); + } else { + casl(addr, newval, cmpval); + } + } else if (op->code() == lir_cas_int) { + casw(addr, newval, cmpval); + } else { + casl(addr, newval, cmpval); + } +} + +void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) { + switch (code) { + case lir_abs: __ fabs_d(dest->as_double_reg(), value->as_double_reg()); break; + case lir_sqrt: __ fsqrt_d(dest->as_double_reg(), value->as_double_reg()); break; + default: ShouldNotReachHere(); + } +} + +void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) { + assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register"); + Register Rleft = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); + if (dst->is_single_cpu()) { + Register Rdst = dst->as_register(); + if (right->is_constant()) { + int right_const = right->as_jint(); + if (Assembler::is_simm12(right_const)) { + logic_op_imm(Rdst, Rleft, right_const, code); + __ sign_extend(Rdst, Rdst, 32); + } else { + __ mv(t0, right_const); + logic_op_reg32(Rdst, Rleft, t0, code); + } + } else { + Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo(); + logic_op_reg32(Rdst, Rleft, Rright, code); + } + } else { + Register Rdst = dst->as_register_lo(); + if (right->is_constant()) { + long right_const = right->as_jlong(); + if (Assembler::is_simm12(right_const)) { + logic_op_imm(Rdst, Rleft, right_const, code); + } else { + __ mv(t0, right_const); + logic_op_reg(Rdst, Rleft, t0, code); + } + } else { + Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo(); + logic_op_reg(Rdst, Rleft, Rright, code); + } + } +} + +void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr src, LIR_Opr result, LIR_Op2* op) { + ShouldNotCallThis(); +} + +void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) { + if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) { + bool is_unordered_less = (code == lir_ucmp_fd2i); + if (left->is_single_fpu()) { + __ float_cmp(true, is_unordered_less ? -1 : 1, + left->as_float_reg(), right->as_float_reg(), dst->as_register()); + } else if (left->is_double_fpu()) { + __ float_cmp(false, is_unordered_less ? -1 : 1, + left->as_double_reg(), right->as_double_reg(), dst->as_register()); + } else { + ShouldNotReachHere(); + } + } else if (code == lir_cmp_l2i) { + __ cmp_l2i(dst->as_register(), left->as_register_lo(), right->as_register_lo()); + } else { + ShouldNotReachHere(); + } +} + +void LIR_Assembler::align_call(LIR_Code code) { + // With RVC a call instruction may get 2-byte aligned. + // The address of the call instruction needs to be 4-byte aligned to + // ensure that it does not span a cache line so that it can be patched. + __ align(NativeInstruction::instruction_size); +} + +void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { + address call = __ trampoline_call(Address(op->addr(), rtype)); + if (call == NULL) { + bailout("trampoline stub overflow"); + return; + } + add_call_info(code_offset(), op->info()); +} + +void LIR_Assembler::ic_call(LIR_OpJavaCall* op) { + address call = __ ic_call(op->addr()); + if (call == NULL) { + bailout("trampoline stub overflow"); + return; + } + add_call_info(code_offset(), op->info()); +} + +void LIR_Assembler::emit_static_call_stub() { + address call_pc = __ pc(); + MacroAssembler::assert_alignment(call_pc); + address stub = __ start_a_stub(call_stub_size()); + if (stub == NULL) { + bailout("static call stub overflow"); + return; + } + + int start = __ offset(); + + __ relocate(static_stub_Relocation::spec(call_pc)); + __ emit_static_call_stub(); + + assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size() + <= call_stub_size(), "stub too big"); + __ end_a_stub(); +} + +void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) { + assert(exceptionOop->as_register() == x10, "must match"); + assert(exceptionPC->as_register() == x13, "must match"); + + // exception object is not added to oop map by LinearScan + // (LinearScan assumes that no oops are in fixed registers) + info->add_register_oop(exceptionOop); + Runtime1::StubID unwind_id; + + // get current pc information + // pc is only needed if the method has an exception handler, the unwind code does not need it. + if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) { + // As no instructions have been generated yet for this LIR node it's + // possible that an oop map already exists for the current offset. + // In that case insert an dummy NOP here to ensure all oop map PCs + // are unique. See JDK-8237483. + __ nop(); + } + int pc_for_athrow_offset = __ offset(); + InternalAddress pc_for_athrow(__ pc()); + __ relocate(pc_for_athrow.rspec(), [&] { + int32_t offset; + __ la_patchable(exceptionPC->as_register(), pc_for_athrow, offset); + __ addi(exceptionPC->as_register(), exceptionPC->as_register(), offset); + }); + add_call_info(pc_for_athrow_offset, info); // for exception handler + + __ verify_not_null_oop(x10); + // search an exception handler (x10: exception oop, x13: throwing pc) + if (compilation()->has_fpu_code()) { + unwind_id = Runtime1::handle_exception_id; + } else { + unwind_id = Runtime1::handle_exception_nofpu_id; + } + __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id))); + __ nop(); +} + +void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) { + assert(exceptionOop->as_register() == x10, "must match"); + __ j(_unwind_handler_entry); +} + +void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) { + Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); + Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo(); + Register count_reg = count->as_register(); + if (dest->is_single_cpu()) { + assert (dest->type() == T_INT, "unexpected result type"); + assert (left->type() == T_INT, "unexpected left type"); + __ andi(t0, count_reg, 31); // should not shift more than 31 bits + switch (code) { + case lir_shl: __ sllw(dest_reg, left_reg, t0); break; + case lir_shr: __ sraw(dest_reg, left_reg, t0); break; + case lir_ushr: __ srlw(dest_reg, left_reg, t0); break; + default: ShouldNotReachHere(); + } + } else if (dest->is_double_cpu()) { + __ andi(t0, count_reg, 63); // should not shift more than 63 bits + switch (code) { + case lir_shl: __ sll(dest_reg, left_reg, t0); break; + case lir_shr: __ sra(dest_reg, left_reg, t0); break; + case lir_ushr: __ srl(dest_reg, left_reg, t0); break; + default: ShouldNotReachHere(); + } + } else { + ShouldNotReachHere(); + } +} + +void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) { + Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); + Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo(); + if (dest->is_single_cpu()) { + assert (dest->type() == T_INT, "unexpected result type"); + assert (left->type() == T_INT, "unexpected left type"); + count &= 0x1f; + if (count != 0) { + switch (code) { + case lir_shl: __ slliw(dest_reg, left_reg, count); break; + case lir_shr: __ sraiw(dest_reg, left_reg, count); break; + case lir_ushr: __ srliw(dest_reg, left_reg, count); break; + default: ShouldNotReachHere(); + } + } else { + move_regs(left_reg, dest_reg); + } + } else if (dest->is_double_cpu()) { + count &= 0x3f; + if (count != 0) { + switch (code) { + case lir_shl: __ slli(dest_reg, left_reg, count); break; + case lir_shr: __ srai(dest_reg, left_reg, count); break; + case lir_ushr: __ srli(dest_reg, left_reg, count); break; + default: ShouldNotReachHere(); + } + } else { + move_regs(left->as_register_lo(), dest->as_register_lo()); + } + } else { + ShouldNotReachHere(); + } +} + +void LIR_Assembler::emit_lock(LIR_OpLock* op) { + Register obj = op->obj_opr()->as_register(); // may not be an oop + Register hdr = op->hdr_opr()->as_register(); + Register lock = op->lock_opr()->as_register(); + if (!UseFastLocking) { + __ j(*op->stub()->entry()); + } else if (op->code() == lir_lock) { + Register tmp = noreg; + if (UseBiasedLocking) { + tmp = op->scratch_opr()->as_register(); + } + assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); + // add debug info for NullPointerException only if one is possible + int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry()); + if (op->info() != NULL) { + add_debug_info_for_null_check(null_check_offset, op->info()); + } + } else if (op->code() == lir_unlock) { + assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); + __ unlock_object(hdr, obj, lock, *op->stub()->entry()); + } else { + Unimplemented(); + } + __ bind(*op->stub()->continuation()); +} + +void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { + Register obj = op->obj()->as_pointer_register(); + Register result = op->result_opr()->as_pointer_register(); + + CodeEmitInfo* info = op->info(); + if (info != NULL) { + add_debug_info_for_null_check_here(info); + } + + if (UseCompressedClassPointers) { + __ lwu(result, Address(obj, oopDesc::klass_offset_in_bytes())); + __ decode_klass_not_null(result); + } else { + __ ld(result, Address(obj, oopDesc::klass_offset_in_bytes())); + } +} + +void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { + ciMethod* method = op->profiled_method(); + int bci = op->profiled_bci(); + + // Update counter for all call types + ciMethodData* md = method->method_data_or_null(); + guarantee(md != NULL, "Sanity"); + ciProfileData* data = md->bci_to_data(bci); + assert(data != NULL && data->is_CounterData(), "need CounterData for calls"); + assert(op->mdo()->is_single_cpu(), "mdo must be allocated"); + Register mdo = op->mdo()->as_register(); + __ mov_metadata(mdo, md->constant_encoding()); + Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); + // Perform additional virtual call profiling for invokevirtual and + // invokeinterface bytecodes + if (op->should_profile_receiver_type()) { + assert(op->recv()->is_single_cpu(), "recv must be allocated"); + Register recv = op->recv()->as_register(); + assert_different_registers(mdo, recv); + assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls"); + ciKlass* known_klass = op->known_holder(); + if (C1OptimizeVirtualCallProfiling && known_klass != NULL) { + // We know the type that will be seen at this call site; we can + // statically update the MethodData* rather than needing to do + // dynamic tests on the receiver type + // NOTE: we should probably put a lock around this search to + // avoid collisions by concurrent compilations + ciVirtualCallData* vc_data = (ciVirtualCallData*) data; + uint i; + for (i = 0; i < VirtualCallData::row_limit(); i++) { + ciKlass* receiver = vc_data->receiver(i); + if (known_klass->equals(receiver)) { + Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); + __ increment(data_addr, DataLayout::counter_increment); + return; + } + } + + // Receiver type not found in profile data; select an empty slot + // Note that this is less efficient than it should be because it + // always does a write to the receiver part of the + // VirtualCallData rather than just the first time + for (i = 0; i < VirtualCallData::row_limit(); i++) { + ciKlass* receiver = vc_data->receiver(i); + if (receiver == NULL) { + Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i))); + __ mov_metadata(t1, known_klass->constant_encoding()); + __ sd(t1, recv_addr); + Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); + __ increment(data_addr, DataLayout::counter_increment); + return; + } + } + } else { + __ load_klass(recv, recv); + Label update_done; + type_profile_helper(mdo, md, data, recv, &update_done); + // Receiver did not match any saved receiver and there is no empty row for it. + // Increment total counter to indicate polymorphic case. + __ increment(counter_addr, DataLayout::counter_increment); + + __ bind(update_done); + } + } else { + // Static call + __ increment(counter_addr, DataLayout::counter_increment); + } +} + +void LIR_Assembler::emit_delay(LIR_OpDelay*) { Unimplemented(); } + +void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) { + __ la(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no)); +} + +void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { Unimplemented(); } + +void LIR_Assembler::check_conflict(ciKlass* exact_klass, intptr_t current_klass, + Register tmp, Label &next, Label &none, + Address mdo_addr) { + if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) { + if (exact_klass != NULL) { + __ mov_metadata(tmp, exact_klass->constant_encoding()); + } else { + __ load_klass(tmp, tmp); + } + + __ ld(t1, mdo_addr); + __ xorr(tmp, tmp, t1); + __ andi(t0, tmp, TypeEntries::type_klass_mask); + // klass seen before, nothing to do. The unknown bit may have been + // set already but no need to check. + __ beqz(t0, next); + + // already unknown. Nothing to do anymore. + __ test_bit(t0, tmp, exact_log2(TypeEntries::type_unknown)); + __ bnez(t0, next); + + if (TypeEntries::is_type_none(current_klass)) { + __ beqz(t1, none); + __ mv(t0, (u1)TypeEntries::null_seen); + __ beq(t0, t1, none); + // There is a chance that the checks above (re-reading profiling + // data from memory) fail if another thread has just set the + // profiling to this obj's klass + __ membar(MacroAssembler::LoadLoad); + __ ld(t1, mdo_addr); + __ xorr(tmp, tmp, t1); + __ andi(t0, tmp, TypeEntries::type_klass_mask); + __ beqz(t0, next); + } + } else { + assert(ciTypeEntries::valid_ciklass(current_klass) != NULL && + ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only"); + + __ ld(tmp, mdo_addr); + // already unknown. Nothing to do anymore. + __ test_bit(t0, tmp, exact_log2(TypeEntries::type_unknown)); + __ bnez(t0, next); + } + + // different than before. Cannot keep accurate profile. + __ ld(t1, mdo_addr); + __ ori(t1, t1, TypeEntries::type_unknown); + __ sd(t1, mdo_addr); + + if (TypeEntries::is_type_none(current_klass)) { + __ j(next); + + __ bind(none); + // first time here. Set profile type. + __ sd(tmp, mdo_addr); + } +} + +void LIR_Assembler::check_no_conflict(ciKlass* exact_klass, intptr_t current_klass, Register tmp, + Address mdo_addr, Label &next) { + // There's a single possible klass at this profile point + assert(exact_klass != NULL, "should be"); + if (TypeEntries::is_type_none(current_klass)) { + __ mov_metadata(tmp, exact_klass->constant_encoding()); + __ ld(t1, mdo_addr); + __ xorr(tmp, tmp, t1); + __ andi(t0, tmp, TypeEntries::type_klass_mask); + __ beqz(t0, next); +#ifdef ASSERT + { + Label ok; + __ ld(t0, mdo_addr); + __ beqz(t0, ok); + __ mv(t1, (u1)TypeEntries::null_seen); + __ beq(t0, t1, ok); + // may have been set by another thread + __ membar(MacroAssembler::LoadLoad); + __ mov_metadata(t0, exact_klass->constant_encoding()); + __ ld(t1, mdo_addr); + __ xorr(t1, t0, t1); + __ andi(t1, t1, TypeEntries::type_mask); + __ beqz(t1, ok); + + __ stop("unexpected profiling mismatch"); + __ bind(ok); + } +#endif + // first time here. Set profile type. + __ sd(tmp, mdo_addr); + } else { + assert(ciTypeEntries::valid_ciklass(current_klass) != NULL && + ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent"); + + __ ld(tmp, mdo_addr); + // already unknown. Nothing to do anymore. + __ test_bit(t0, tmp, exact_log2(TypeEntries::type_unknown)); + __ bnez(t0, next); + + __ ori(tmp, tmp, TypeEntries::type_unknown); + __ sd(tmp, mdo_addr); + } +} + +void LIR_Assembler::check_null(Register tmp, Label &update, intptr_t current_klass, + Address mdo_addr, bool do_update, Label &next) { + __ bnez(tmp, update); + if (!TypeEntries::was_null_seen(current_klass)) { + __ ld(t1, mdo_addr); + __ ori(t1, t1, TypeEntries::null_seen); + __ sd(t1, mdo_addr); + } + if (do_update) { + __ j(next); + } +} + +void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) { + COMMENT("emit_profile_type {"); + Register obj = op->obj()->as_register(); + Register tmp = op->tmp()->as_pointer_register(); + Address mdo_addr = as_Address(op->mdp()->as_address_ptr()); + ciKlass* exact_klass = op->exact_klass(); + intptr_t current_klass = op->current_klass(); + bool not_null = op->not_null(); + bool no_conflict = op->no_conflict(); + + Label update, next, none; + + bool do_null = !not_null; + bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass; + bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set; + + assert(do_null || do_update, "why are we here?"); + assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?"); + assert_different_registers(tmp, t0, t1, mdo_addr.base()); + + __ verify_oop(obj); + + if (tmp != obj) { + __ mv(tmp, obj); + } + if (do_null) { + check_null(tmp, update, current_klass, mdo_addr, do_update, next); +#ifdef ASSERT + } else { + __ bnez(tmp, update); + __ stop("unexpected null obj"); +#endif + } + + __ bind(update); + + if (do_update) { +#ifdef ASSERT + if (exact_klass != NULL) { + check_exact_klass(tmp, exact_klass); + } +#endif + if (!no_conflict) { + check_conflict(exact_klass, current_klass, tmp, next, none, mdo_addr); + } else { + check_no_conflict(exact_klass, current_klass, tmp, mdo_addr, next); + } + + __ bind(next); + } + COMMENT("} emit_profile_type"); +} + +void LIR_Assembler::align_backward_branch_target() { } + +void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) { + // tmp must be unused + assert(tmp->is_illegal(), "wasting a register if tmp is allocated"); + + if (left->is_single_cpu()) { + assert(dest->is_single_cpu(), "expect single result reg"); + __ negw(dest->as_register(), left->as_register()); + } else if (left->is_double_cpu()) { + assert(dest->is_double_cpu(), "expect double result reg"); + __ neg(dest->as_register_lo(), left->as_register_lo()); + } else if (left->is_single_fpu()) { + assert(dest->is_single_fpu(), "expect single float result reg"); + __ fneg_s(dest->as_float_reg(), left->as_float_reg()); + } else { + assert(left->is_double_fpu(), "expect double float operand reg"); + assert(dest->is_double_fpu(), "expect double float result reg"); + __ fneg_d(dest->as_double_reg(), left->as_double_reg()); + } +} + + +void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { + if (patch_code != lir_patch_none) { + deoptimize_trap(info); + return; + } + + LIR_Address* adr = addr->as_address_ptr(); + Register dst = dest->as_register_lo(); + + assert_different_registers(dst, t0); + if (adr->base()->is_valid() && dst == adr->base()->as_pointer_register() && (!adr->index()->is_cpu_register())) { + int scale = adr->scale(); + intptr_t offset = adr->disp(); + LIR_Opr index_op = adr->index(); + if (index_op->is_constant()) { + offset += ((intptr_t)index_op->as_constant_ptr()->as_jint()) << scale; + } + + if (!Assembler::is_simm12(offset)) { + __ la(t0, as_Address(adr)); + __ mv(dst, t0); + return; + } + } + + __ la(dst, as_Address(adr)); +} + + +void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) { + assert(!tmp->is_valid(), "don't need temporary"); + + CodeBlob *cb = CodeCache::find_blob(dest); + if (cb != NULL) { + __ far_call(RuntimeAddress(dest)); + } else { + RuntimeAddress target(dest); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ jalr(x1, t0, offset); + }); + } + + if (info != NULL) { + add_call_info_here(info); + } +} + +void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) { + if (dest->is_address() || src->is_address()) { + move_op(src, dest, type, lir_patch_none, info, /* pop_fpu_stack */ false, /* wide */ false, /* unaligned */ false); + } else { + ShouldNotReachHere(); + } +} + +#ifdef ASSERT +// emit run-time assertion +void LIR_Assembler::emit_assert(LIR_OpAssert* op) { + assert(op->code() == lir_assert, "must be"); + + Label ok; + if (op->in_opr1()->is_valid()) { + assert(op->in_opr2()->is_valid(), "both operands must be valid"); + bool is_unordered = false; + LIR_Condition cond = op->condition(); + emit_branch(cond, op->in_opr1(), op->in_opr2(), ok, /* is_far */ false, + /* is_unordered */(cond == lir_cond_greaterEqual || cond == lir_cond_greater) ? false : true); + } else { + assert(op->in_opr2()->is_illegal(), "both operands must be illegal"); + assert(op->condition() == lir_cond_always, "no other conditions allowed"); + } + + if (op->halt()) { + const char* str = __ code_string(op->msg()); + __ stop(str); + } else { + breakpoint(); + } + __ bind(ok); +} +#endif + +#ifndef PRODUCT +#define COMMENT(x) do { __ block_comment(x); } while (0) +#else +#define COMMENT(x) +#endif + +void LIR_Assembler::membar() { + COMMENT("membar"); + __ membar(MacroAssembler::AnyAny); +} + +void LIR_Assembler::membar_acquire() { + __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); +} + +void LIR_Assembler::membar_release() { + __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore); +} + +void LIR_Assembler::membar_loadload() { + __ membar(MacroAssembler::LoadLoad); +} + +void LIR_Assembler::membar_storestore() { + __ membar(MacroAssembler::StoreStore); +} + +void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); } + +void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); } + +void LIR_Assembler::on_spin_wait() { + Unimplemented(); +} + +void LIR_Assembler::get_thread(LIR_Opr result_reg) { + __ mv(result_reg->as_register(), xthread); +} + +void LIR_Assembler::peephole(LIR_List *lir) {} + +void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) { + Address addr = as_Address(src->as_address_ptr()); + BasicType type = src->type(); + bool is_oop = is_reference_type(type); + + get_op(type); + + switch (code) { + case lir_xadd: + { + RegisterOrConstant inc; + Register tmp = as_reg(tmp_op); + Register dst = as_reg(dest); + if (data->is_constant()) { + inc = RegisterOrConstant(as_long(data)); + assert_different_registers(dst, addr.base(), tmp); + assert_different_registers(tmp, t0); + } else { + inc = RegisterOrConstant(as_reg(data)); + assert_different_registers(inc.as_register(), dst, addr.base(), tmp); + } + __ la(tmp, addr); + (_masm->*add)(dst, inc, tmp); + break; + } + case lir_xchg: + { + Register tmp = tmp_op->as_register(); + Register obj = as_reg(data); + Register dst = as_reg(dest); + if (is_oop && UseCompressedOops) { + __ encode_heap_oop(t0, obj); + obj = t0; + } + assert_different_registers(obj, addr.base(), tmp, dst); + __ la(tmp, addr); + (_masm->*xchg)(dst, obj, tmp); + if (is_oop && UseCompressedOops) { + __ decode_heap_oop(dst); + } + } + break; + default: + ShouldNotReachHere(); + } + __ membar(MacroAssembler::AnyAny); +} + +int LIR_Assembler::array_element_size(BasicType type) const { + int elem_size = type2aelembytes(type); + return exact_log2(elem_size); +} + +// helper functions which checks for overflow and sets bailout if it +// occurs. Always returns a valid embeddable pointer but in the +// bailout case the pointer won't be to unique storage. +address LIR_Assembler::float_constant(float f) { + address const_addr = __ float_constant(f); + if (const_addr == NULL) { + bailout("const section overflow"); + return __ code()->consts()->start(); + } else { + return const_addr; + } +} + +address LIR_Assembler::double_constant(double d) { + address const_addr = __ double_constant(d); + if (const_addr == NULL) { + bailout("const section overflow"); + return __ code()->consts()->start(); + } else { + return const_addr; + } +} + +address LIR_Assembler::int_constant(jlong n) { + address const_addr = __ long_constant(n); + if (const_addr == NULL) { + bailout("const section overflow"); + return __ code()->consts()->start(); + } else { + return const_addr; + } +} + +void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) { + __ cmpxchg(addr, cmpval, newval, Assembler::int32, Assembler::aq /* acquire */, + Assembler::rl /* release */, t0, true /* result as bool */); + __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1 + __ membar(MacroAssembler::AnyAny); +} + +void LIR_Assembler::caswu(Register addr, Register newval, Register cmpval) { + __ cmpxchg(addr, cmpval, newval, Assembler::uint32, Assembler::aq /* acquire */, + Assembler::rl /* release */, t0, true /* result as bool */); + __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1 + __ membar(MacroAssembler::AnyAny); +} + +void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) { + __ cmpxchg(addr, cmpval, newval, Assembler::int64, Assembler::aq /* acquire */, + Assembler::rl /* release */, t0, true /* result as bool */); + __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1 + __ membar(MacroAssembler::AnyAny); +} + +void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) { + address target = NULL; + + switch (patching_id(info)) { + case PatchingStub::access_field_id: + target = Runtime1::entry_for(Runtime1::access_field_patching_id); + break; + case PatchingStub::load_klass_id: + target = Runtime1::entry_for(Runtime1::load_klass_patching_id); + break; + case PatchingStub::load_mirror_id: + target = Runtime1::entry_for(Runtime1::load_mirror_patching_id); + break; + case PatchingStub::load_appendix_id: + target = Runtime1::entry_for(Runtime1::load_appendix_patching_id); + break; + default: ShouldNotReachHere(); + } + + __ far_call(RuntimeAddress(target)); + add_call_info_here(info); +} + +void LIR_Assembler::check_exact_klass(Register tmp, ciKlass* exact_klass) { + Label ok; + __ load_klass(tmp, tmp); + __ mov_metadata(t0, exact_klass->constant_encoding()); + __ beq(tmp, t0, ok); + __ stop("exact klass and actual klass differ"); + __ bind(ok); +} + +void LIR_Assembler::get_op(BasicType type) { + switch (type) { + case T_INT: + xchg = &MacroAssembler::atomic_xchgalw; + add = &MacroAssembler::atomic_addalw; + break; + case T_LONG: + xchg = &MacroAssembler::atomic_xchgal; + add = &MacroAssembler::atomic_addal; + break; + case T_OBJECT: + case T_ARRAY: + if (UseCompressedOops) { + xchg = &MacroAssembler::atomic_xchgalwu; + add = &MacroAssembler::atomic_addalw; + } else { + xchg = &MacroAssembler::atomic_xchgal; + add = &MacroAssembler::atomic_addal; + } + break; + default: + ShouldNotReachHere(); + } +} + +// emit_opTypeCheck sub functions +void LIR_Assembler::typecheck_lir_store(LIR_OpTypeCheck* op, bool should_profile) { + Register value = op->object()->as_register(); + Register array = op->array()->as_register(); + Register k_RInfo = op->tmp1()->as_register(); + Register klass_RInfo = op->tmp2()->as_register(); + Register Rtmp1 = op->tmp3()->as_register(); + + CodeStub* stub = op->stub(); + + // check if it needs to be profiled + ciMethodData* md = NULL; + ciProfileData* data = NULL; + + if (should_profile) { + data_check(op, &md, &data); + } + Label profile_cast_success, profile_cast_failure, done; + Label *success_target = should_profile ? &profile_cast_success : &done; + Label *failure_target = should_profile ? &profile_cast_failure : stub->entry(); + + if (should_profile) { + profile_object(md, data, value, klass_RInfo, &done); + } else { + __ beqz(value, done); + } + + add_debug_info_for_null_check_here(op->info_for_exception()); + __ load_klass(k_RInfo, array); + __ load_klass(klass_RInfo, value); + + lir_store_slowcheck(k_RInfo, klass_RInfo, Rtmp1, success_target, failure_target); + + // fall through to the success case + if (should_profile) { + Register mdo = klass_RInfo; + Register recv = k_RInfo; + __ bind(profile_cast_success); + __ mov_metadata(mdo, md->constant_encoding()); + __ load_klass(recv, value); + type_profile_helper(mdo, md, data, recv, &done); + __ j(done); + + __ bind(profile_cast_failure); + __ mov_metadata(mdo, md->constant_encoding()); + Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); + __ ld(t1, counter_addr); + __ addi(t1, t1, -DataLayout::counter_increment); + __ sd(t1, counter_addr); + __ j(*stub->entry()); + } + + __ bind(done); +} + +void LIR_Assembler::type_profile(Register obj, ciMethodData* md, Register klass_RInfo, Register k_RInfo, + ciProfileData* data, Label* success, Label* failure, + Label& profile_cast_success, Label& profile_cast_failure) { + Register mdo = klass_RInfo; + Register recv = k_RInfo; + __ bind(profile_cast_success); + __ mov_metadata(mdo, md->constant_encoding()); + __ load_klass(recv, obj); + Label update_done; + type_profile_helper(mdo, md, data, recv, success); + __ j(*success); + + __ bind(profile_cast_failure); + __ mov_metadata(mdo, md->constant_encoding()); + Address counter_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); + __ ld(t0, counter_addr); + __ addi(t0, t0, -DataLayout::counter_increment); + __ sd(t0, counter_addr); + __ j(*failure); +} + +void LIR_Assembler::lir_store_slowcheck(Register k_RInfo, Register klass_RInfo, Register Rtmp1, + Label* success_target, Label* failure_target) { + // get instance klass (it's already uncompressed) + __ ld(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset())); + // perform the fast part of the checking logic + __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL); + // call out-of-line instance of __ check_klass_subtype_slow_path(...) + __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo + __ sd(klass_RInfo, Address(sp, wordSize)); // sub klass + __ sd(k_RInfo, Address(sp, 0)); // super klass + __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); + // load result to k_RInfo + __ ld(k_RInfo, Address(sp, 0)); + __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo + // result is a boolean + __ beqz(k_RInfo, *failure_target, /* is_far */ true); +} + +void LIR_Assembler::const2reg_helper(LIR_Opr src) { + switch (src->as_constant_ptr()->type()) { + case T_INT: + case T_ADDRESS: + case T_OBJECT: + case T_ARRAY: + case T_METADATA: + const2reg(src, FrameMap::t0_opr, lir_patch_none, NULL); + break; + case T_LONG: + const2reg(src, FrameMap::t0_long_opr, lir_patch_none, NULL); + break; + case T_FLOAT: + case T_DOUBLE: + default: + ShouldNotReachHere(); + } +} + +void LIR_Assembler::logic_op_reg32(Register dst, Register left, Register right, LIR_Code code) { + switch (code) { + case lir_logic_and: __ andrw(dst, left, right); break; + case lir_logic_or: __ orrw (dst, left, right); break; + case lir_logic_xor: __ xorrw(dst, left, right); break; + default: ShouldNotReachHere(); + } +} + +void LIR_Assembler::logic_op_reg(Register dst, Register left, Register right, LIR_Code code) { + switch (code) { + case lir_logic_and: __ andr(dst, left, right); break; + case lir_logic_or: __ orr (dst, left, right); break; + case lir_logic_xor: __ xorr(dst, left, right); break; + default: ShouldNotReachHere(); + } +} + +void LIR_Assembler::logic_op_imm(Register dst, Register left, int right, LIR_Code code) { + switch (code) { + case lir_logic_and: __ andi(dst, left, right); break; + case lir_logic_or: __ ori (dst, left, right); break; + case lir_logic_xor: __ xori(dst, left, right); break; + default: ShouldNotReachHere(); + } +} + +void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) { + assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); + int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; + assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); + __ sd(r, Address(sp, offset_from_rsp_in_bytes)); +} + +void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) { + assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); + int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; + assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); + __ mv(t0, c); + __ sd(t0, Address(sp, offset_from_rsp_in_bytes)); +} + +#undef __ diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp new file mode 100644 index 0000000000000..8e85a5f372930 --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_C1_LIRASSEMBLER_RISCV_HPP +#define CPU_RISCV_C1_LIRASSEMBLER_RISCV_HPP + +// ArrayCopyStub needs access to bailout +friend class ArrayCopyStub; + +private: + +#include "c1_LIRAssembler_arith_riscv.hpp" +#include "c1_LIRAssembler_arraycopy_riscv.hpp" + + int array_element_size(BasicType type) const; + + static Register as_reg(LIR_Opr op) { + return op->is_double_cpu() ? op->as_register_lo() : op->as_register(); + } + + Address as_Address(LIR_Address* addr, Register tmp); + + // helper functions which checks for overflow and sets bailout if it + // occurs. Always returns a valid embeddable pointer but in the + // bailout case the pointer won't be to unique storage. + address float_constant(float f); + address double_constant(double d); + address int_constant(jlong n); + + // Ensure we have a valid Address (base + offset) to a stack-slot. + Address stack_slot_address(int index, uint shift, int adjust = 0); + + // Record the type of the receiver in ReceiverTypeData + void type_profile_helper(Register mdo, + ciMethodData *md, ciProfileData *data, + Register recv, Label* update_done); + + void casw(Register addr, Register newval, Register cmpval); + void caswu(Register addr, Register newval, Register cmpval); + void casl(Register addr, Register newval, Register cmpval); + + void poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info = NULL); + + void deoptimize_trap(CodeEmitInfo *info); + + enum { + // See emit_static_call_stub for detail + // CompiledStaticCall::to_interp_stub_size() (14) + CompiledStaticCall::to_trampoline_stub_size() (1 + 3 + address) + _call_stub_size = 14 * NativeInstruction::instruction_size + + (NativeInstruction::instruction_size + NativeCallTrampolineStub::instruction_size), + // See emit_exception_handler for detail + // verify_not_null_oop + far_call + should_not_reach_here + invalidate_registers(DEBUG_ONLY) + _exception_handler_size = DEBUG_ONLY(584) NOT_DEBUG(548), // or smaller + // See emit_deopt_handler for detail + // auipc (1) + far_jump (6 or 2) + _deopt_handler_size = 1 * NativeInstruction::instruction_size + + 6 * NativeInstruction::instruction_size // or smaller + }; + + void check_conflict(ciKlass* exact_klass, intptr_t current_klass, Register tmp, + Label &next, Label &none, Address mdo_addr); + void check_no_conflict(ciKlass* exact_klass, intptr_t current_klass, Register tmp, Address mdo_addr, Label &next); + + void check_exact_klass(Register tmp, ciKlass* exact_klass); + + void check_null(Register tmp, Label &update, intptr_t current_klass, Address mdo_addr, bool do_update, Label &next); + + void (MacroAssembler::*add)(Register prev, RegisterOrConstant incr, Register addr); + void (MacroAssembler::*xchg)(Register prev, Register newv, Register addr); + + void get_op(BasicType type); + + // emit_typecheck_helper sub functions + void data_check(LIR_OpTypeCheck *op, ciMethodData **md, ciProfileData **data); + void typecheck_helper_slowcheck(ciKlass* k, Register obj, Register Rtmp1, + Register k_RInfo, Register klass_RInfo, + Label* failure_target, Label* success_target); + void profile_object(ciMethodData* md, ciProfileData* data, Register obj, + Register klass_RInfo, Label* obj_is_null); + void typecheck_loaded(LIR_OpTypeCheck* op, ciKlass* k, Register k_RInfo); + + // emit_opTypeCheck sub functions + void typecheck_lir_store(LIR_OpTypeCheck* op, bool should_profile); + + void type_profile(Register obj, ciMethodData* md, Register klass_RInfo, Register k_RInfo, + ciProfileData* data, Label* success, Label* failure, + Label& profile_cast_success, Label& profile_cast_failure); + + void lir_store_slowcheck(Register k_RInfo, Register klass_RInfo, Register Rtmp1, + Label* success_target, Label* failure_target); + + void const2reg_helper(LIR_Opr src); + + void emit_branch(LIR_Condition cmp_flag, LIR_Opr cmp1, LIR_Opr cmp2, Label& label, bool is_far, bool is_unordered); + + void logic_op_reg32(Register dst, Register left, Register right, LIR_Code code); + void logic_op_reg(Register dst, Register left, Register right, LIR_Code code); + void logic_op_imm(Register dst, Register left, int right, LIR_Code code); + +public: + + void emit_cmove(LIR_Op4* op); + + void store_parameter(Register r, int offset_from_rsp_in_words); + void store_parameter(jint c, int offset_from_rsp_in_words); + +#endif // CPU_RISCV_C1_LIRASSEMBLER_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp new file mode 100644 index 0000000000000..55a309f24ba07 --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp @@ -0,0 +1,1086 @@ +/* + * Copyright (c) 2005, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "c1/c1_Compilation.hpp" +#include "c1/c1_FrameMap.hpp" +#include "c1/c1_Instruction.hpp" +#include "c1/c1_LIRAssembler.hpp" +#include "c1/c1_LIRGenerator.hpp" +#include "c1/c1_Runtime1.hpp" +#include "c1/c1_ValueStack.hpp" +#include "ci/ciArray.hpp" +#include "ci/ciObjArrayKlass.hpp" +#include "ci/ciTypeArrayKlass.hpp" +#include "runtime/sharedRuntime.hpp" +#include "runtime/stubRoutines.hpp" +#include "utilities/powerOfTwo.hpp" +#include "vmreg_riscv.inline.hpp" + +#ifdef ASSERT +#define __ gen()->lir(__FILE__, __LINE__)-> +#else +#define __ gen()->lir()-> +#endif + +// Item will be loaded into a byte register; Intel only +void LIRItem::load_byte_item() { + load_item(); +} + + +void LIRItem::load_nonconstant() { + LIR_Opr r = value()->operand(); + if (r->is_constant()) { + _result = r; + } else { + load_item(); + } +} + +//-------------------------------------------------------------- +// LIRGenerator +//-------------------------------------------------------------- + + +LIR_Opr LIRGenerator::exceptionOopOpr() { return FrameMap::r10_oop_opr; } +LIR_Opr LIRGenerator::exceptionPcOpr() { return FrameMap::r13_opr; } +LIR_Opr LIRGenerator::divInOpr() { Unimplemented(); return LIR_OprFact::illegalOpr; } +LIR_Opr LIRGenerator::divOutOpr() { Unimplemented(); return LIR_OprFact::illegalOpr; } +LIR_Opr LIRGenerator::remOutOpr() { Unimplemented(); return LIR_OprFact::illegalOpr; } +LIR_Opr LIRGenerator::shiftCountOpr() { Unimplemented(); return LIR_OprFact::illegalOpr; } +LIR_Opr LIRGenerator::syncLockOpr() { return new_register(T_INT); } +LIR_Opr LIRGenerator::syncTempOpr() { return FrameMap::r10_opr; } +LIR_Opr LIRGenerator::getThreadTemp() { return LIR_OprFact::illegalOpr; } + + +LIR_Opr LIRGenerator::result_register_for(ValueType* type, bool callee) { + LIR_Opr opr; + switch (type->tag()) { + case intTag: opr = FrameMap::r10_opr; break; + case objectTag: opr = FrameMap::r10_oop_opr; break; + case longTag: opr = FrameMap::long10_opr; break; + case floatTag: opr = FrameMap::fpu10_float_opr; break; + case doubleTag: opr = FrameMap::fpu10_double_opr; break; + + case addressTag: // fall through + default: + ShouldNotReachHere(); + return LIR_OprFact::illegalOpr; + } + + assert(opr->type_field() == as_OprType(as_BasicType(type)), "type mismatch"); + return opr; +} + + +LIR_Opr LIRGenerator::rlock_byte(BasicType type) { + LIR_Opr reg = new_register(T_INT); + set_vreg_flag(reg, LIRGenerator::byte_reg); + return reg; +} + +//--------- loading items into registers -------------------------------- + + +bool LIRGenerator::can_store_as_constant(Value v, BasicType type) const { + if (v->type()->as_IntConstant() != NULL) { + return v->type()->as_IntConstant()->value() == 0; + } else if (v->type()->as_LongConstant() != NULL) { + return v->type()->as_LongConstant()->value() == 0; + } else if (v->type()->as_ObjectConstant() != NULL) { + return v->type()->as_ObjectConstant()->value()->is_null_object(); + } else if (v->type()->as_FloatConstant() != NULL) { + return jint_cast(v->type()->as_FloatConstant()->value()) == 0.0f; + } else if (v->type()->as_DoubleConstant() != NULL) { + return jlong_cast(v->type()->as_DoubleConstant()->value()) == 0.0; + } + return false; +} + +bool LIRGenerator::can_inline_as_constant(Value v) const { + if (v->type()->as_IntConstant() != NULL) { + int value = v->type()->as_IntConstant()->value(); + // "-value" must be defined for value may be used for sub + return Assembler::is_simm12(value) && Assembler::is_simm12(- value); + } else if (v->type()->as_ObjectConstant() != NULL) { + return v->type()->as_ObjectConstant()->value()->is_null_object(); + } else if (v->type()->as_LongConstant() != NULL) { + long value = v->type()->as_LongConstant()->value(); + // "-value" must be defined for value may be used for sub + return Assembler::is_simm12(value) && Assembler::is_simm12(- value); + } else if (v->type()->as_FloatConstant() != NULL) { + return v->type()->as_FloatConstant()->value() == 0.0f; + } else if (v->type()->as_DoubleConstant() != NULL) { + return v->type()->as_DoubleConstant()->value() == 0.0; + } + return false; +} + +bool LIRGenerator::can_inline_as_constant(LIR_Const* c) const { + if (c->as_constant() != NULL) { + long constant = 0; + switch (c->type()) { + case T_INT: constant = c->as_jint(); break; + case T_LONG: constant = c->as_jlong(); break; + default: return false; + } + // "-constant" must be defined for c may be used for sub + return Assembler::is_simm12(constant) && Assembler::is_simm12(- constant); + } + return false; +} + +LIR_Opr LIRGenerator::safepoint_poll_register() { + return LIR_OprFact::illegalOpr; +} + +LIR_Address* LIRGenerator::generate_address(LIR_Opr base, LIR_Opr index, + int shift, int disp, BasicType type) { + assert(base->is_register(), "must be"); + + if (index->is_constant()) { + LIR_Const *constant = index->as_constant_ptr(); + jlong c; + if (constant->type() == T_INT) { + c = (jlong(index->as_jint()) << shift) + disp; + } else { + assert(constant->type() == T_LONG, "should be"); + c = (index->as_jlong() << shift) + disp; + } + if ((jlong)((jint)c) == c) { + return new LIR_Address(base, (jint)c, type); + } else { + LIR_Opr tmp = new_register(T_LONG); + __ move(index, tmp); + return new LIR_Address(base, tmp, type); + } + } + + return new LIR_Address(base, index, (LIR_Address::Scale)shift, disp, type); +} + +LIR_Address* LIRGenerator::emit_array_address(LIR_Opr array_opr, LIR_Opr index_opr, + BasicType type) { + int offset_in_bytes = arrayOopDesc::base_offset_in_bytes(type); + int elem_size = type2aelembytes(type); + int shift = exact_log2(elem_size); + return generate_address(array_opr, index_opr, shift, offset_in_bytes, type); +} + +LIR_Opr LIRGenerator::load_immediate(int x, BasicType type) { + LIR_Opr r; + switch (type) { + case T_LONG: + r = LIR_OprFact::longConst(x); + break; + case T_INT: + r = LIR_OprFact::intConst(x); + break; + default: + ShouldNotReachHere(); + } + return r; +} + +void LIRGenerator::increment_counter(address counter, BasicType type, int step) { + LIR_Opr pointer = new_pointer_register(); + __ move(LIR_OprFact::intptrConst(counter), pointer); + LIR_Address* addr = new LIR_Address(pointer, type); + increment_counter(addr, step); +} + +void LIRGenerator::increment_counter(LIR_Address* addr, int step) { + LIR_Opr reg = new_register(addr->type()); + __ load(addr, reg); + __ add(reg, load_immediate(step, addr->type()), reg); + __ store(reg, addr); +} + +void LIRGenerator::cmp_mem_int(LIR_Condition condition, LIR_Opr base, int disp, int c, CodeEmitInfo* info) { + LIR_Opr reg = new_register(T_INT); + __ load(generate_address(base, disp, T_INT), reg, info); + __ cmp(condition, reg, LIR_OprFact::intConst(c)); +} + +void LIRGenerator::cmp_reg_mem(LIR_Condition condition, LIR_Opr reg, LIR_Opr base, int disp, BasicType type, CodeEmitInfo* info) { + LIR_Opr reg1 = new_register(T_INT); + __ load(generate_address(base, disp, type), reg1, info); + __ cmp(condition, reg, reg1); +} + +bool LIRGenerator::strength_reduce_multiply(LIR_Opr left, jint c, LIR_Opr result, LIR_Opr tmp) { + if (tmp->is_valid() && c > 0 && c < max_jint) { + if (is_power_of_2(c - 1)) { + __ shift_left(left, exact_log2(c - 1), tmp); + __ add(tmp, left, result); + return true; + } else if (is_power_of_2(c + 1)) { + __ shift_left(left, exact_log2(c + 1), tmp); + __ sub(tmp, left, result); + return true; + } + } + return false; +} + +void LIRGenerator::store_stack_parameter (LIR_Opr item, ByteSize offset_from_sp) { + BasicType type = item->type(); + __ store(item, new LIR_Address(FrameMap::sp_opr, in_bytes(offset_from_sp), type)); +} + +void LIRGenerator::array_store_check(LIR_Opr value, LIR_Opr array, CodeEmitInfo* store_check_info, + ciMethod* profiled_method, int profiled_bci) { + LIR_Opr tmp1 = new_register(objectType); + LIR_Opr tmp2 = new_register(objectType); + LIR_Opr tmp3 = new_register(objectType); + __ store_check(value, array, tmp1, tmp2, tmp3, store_check_info, profiled_method, profiled_bci); +} + +//---------------------------------------------------------------------- +// visitor functions +//---------------------------------------------------------------------- + +void LIRGenerator::do_MonitorEnter(MonitorEnter* x) { + assert(x->is_pinned(), ""); + LIRItem obj(x->obj(), this); + obj.load_item(); + + set_no_result(x); + + // "lock" stores the address of the monitor stack slot, so this is not an oop + LIR_Opr lock = new_register(T_INT); + // Need a tmp register for biased locking + LIR_Opr tmp = LIR_OprFact::illegalOpr; + if (UseBiasedLocking) { + tmp = new_register(T_INT); + } + + CodeEmitInfo* info_for_exception = NULL; + if (x->needs_null_check()) { + info_for_exception = state_for(x); + } + // this CodeEmitInfo must not have the xhandlers because here the + // object is already locked (xhandlers expect object to be unlocked) + CodeEmitInfo* info = state_for(x, x->state(), true); + monitor_enter(obj.result(), lock, syncTempOpr(), tmp, + x->monitor_no(), info_for_exception, info); +} + +void LIRGenerator::do_MonitorExit(MonitorExit* x) { + assert(x->is_pinned(), ""); + + LIRItem obj(x->obj(), this); + obj.dont_load_item(); + + LIR_Opr lock = new_register(T_INT); + LIR_Opr obj_temp = new_register(T_INT); + set_no_result(x); + monitor_exit(obj_temp, lock, syncTempOpr(), LIR_OprFact::illegalOpr, x->monitor_no()); +} + +// neg +void LIRGenerator::do_NegateOp(NegateOp* x) { + LIRItem from(x->x(), this); + from.load_item(); + LIR_Opr result = rlock_result(x); + __ negate(from.result(), result); +} + +// for _fadd, _fmul, _fsub, _fdiv, _frem +// _dadd, _dmul, _dsub, _ddiv, _drem +void LIRGenerator::do_ArithmeticOp_FPU(ArithmeticOp* x) { + LIRItem left(x->x(), this); + LIRItem right(x->y(), this); + + if (x->op() == Bytecodes::_frem || x->op() == Bytecodes::_drem) { + + // float remainder is implemented as a direct call into the runtime + BasicTypeList signature(2); + if (x->op() == Bytecodes::_frem) { + signature.append(T_FLOAT); + signature.append(T_FLOAT); + } else { + signature.append(T_DOUBLE); + signature.append(T_DOUBLE); + } + CallingConvention* cc = frame_map()->c_calling_convention(&signature); + + const LIR_Opr result_reg = result_register_for(x->type()); + + left.load_item(); + __ move(left.result(), cc->at(0)); + right.load_item_force(cc->at(1)); + + address entry; + if (x->op() == Bytecodes::_frem) { + entry = CAST_FROM_FN_PTR(address, SharedRuntime::frem); + } else { + entry = CAST_FROM_FN_PTR(address, SharedRuntime::drem); + } + + LIR_Opr result = rlock_result(x); + __ call_runtime_leaf(entry, getThreadTemp(), result_reg, cc->args()); + __ move(result_reg, result); + + return; + } + + if (!left.is_register()) { + left.load_item(); + } + // Always load right hand side. + right.load_item(); + + LIR_Opr reg = rlock(x); + arithmetic_op_fpu(x->op(), reg, left.result(), right.result()); + + set_result(x, round_item(reg)); +} + +// for _ladd, _lmul, _lsub, _ldiv, _lrem +void LIRGenerator::do_ArithmeticOp_Long(ArithmeticOp* x) { + + // missing test if instr is commutative and if we should swap + LIRItem left(x->x(), this); + LIRItem right(x->y(), this); + + if (x->op() == Bytecodes::_ldiv || x->op() == Bytecodes::_lrem) { + + left.load_item(); + + bool need_zero_check = true; + if (right.is_constant()) { + jlong c = right.get_jlong_constant(); + // no need to do div-by-zero check if the divisor is a non-zero constant + if (c != 0) { need_zero_check = false; } + // do not load right if the divisor is a power-of-2 constant + if (c > 0 && is_power_of_2(c)) { + right.dont_load_item(); + } else { + right.load_item(); + } + } else { + right.load_item(); + } + if (need_zero_check) { + CodeEmitInfo* info = state_for(x); + __ cmp(lir_cond_equal, right.result(), LIR_OprFact::longConst(0)); + __ branch(lir_cond_equal, new DivByZeroStub(info)); + } + + rlock_result(x); + switch (x->op()) { + case Bytecodes::_lrem: + __ rem(left.result(), right.result(), x->operand()); + break; + case Bytecodes::_ldiv: + __ div(left.result(), right.result(), x->operand()); + break; + default: + ShouldNotReachHere(); + } + } else { + assert(x->op() == Bytecodes::_lmul || x->op() == Bytecodes::_ladd || x->op() == Bytecodes::_lsub, + "expect lmul, ladd or lsub"); + // add, sub, mul + left.load_item(); + if (!right.is_register()) { + if (x->op() == Bytecodes::_lmul || + !right.is_constant() || + (x->op() == Bytecodes::_ladd && + !Assembler::is_simm12(right.get_jlong_constant())) || + (x->op() == Bytecodes::_lsub && + !Assembler::is_simm12(-right.get_jlong_constant()))) { + right.load_item(); + } else { // add, sub + assert(x->op() == Bytecodes::_ladd || x->op() == Bytecodes::_lsub, "expected ladd or lsub"); + // don't load constants to save register + right.load_nonconstant(); + } + } + rlock_result(x); + arithmetic_op_long(x->op(), x->operand(), left.result(), right.result(), NULL); + } +} + +// for: _iadd, _imul, _isub, _idiv, _irem +void LIRGenerator::do_ArithmeticOp_Int(ArithmeticOp* x) { + + // Test if instr is commutative and if we should swap + LIRItem left(x->x(), this); + LIRItem right(x->y(), this); + LIRItem* left_arg = &left; + LIRItem* right_arg = &right; + if (x->is_commutative() && left.is_stack() && right.is_register()) { + // swap them if left is real stack (or cached) and right is real register(not cached) + left_arg = &right; + right_arg = &left; + } + left_arg->load_item(); + // do not need to load right, as we can handle stack and constants + if (x->op() == Bytecodes::_idiv || x->op() == Bytecodes::_irem) { + + rlock_result(x); + + bool need_zero_check = true; + if (right.is_constant()) { + jint c = right.get_jint_constant(); + // no need to do div-by-zero check if the divisor is a non-zero constant + if (c != 0) { need_zero_check = false; } + // do not load right if the divisor is a power-of-2 constant + if (c > 0 && is_power_of_2(c)) { + right_arg->dont_load_item(); + } else { + right_arg->load_item(); + } + } else { + right_arg->load_item(); + } + if (need_zero_check) { + CodeEmitInfo* info = state_for(x); + __ cmp(lir_cond_equal, right_arg->result(), LIR_OprFact::longConst(0)); + __ branch(lir_cond_equal, new DivByZeroStub(info)); + } + + LIR_Opr ill = LIR_OprFact::illegalOpr; + if (x->op() == Bytecodes::_irem) { + __ irem(left_arg->result(), right_arg->result(), x->operand(), ill, NULL); + } else if (x->op() == Bytecodes::_idiv) { + __ idiv(left_arg->result(), right_arg->result(), x->operand(), ill, NULL); + } + + } else if (x->op() == Bytecodes::_iadd || x->op() == Bytecodes::_isub) { + if (right.is_constant() && + ((x->op() == Bytecodes::_iadd && !Assembler::is_simm12(right.get_jint_constant())) || + (x->op() == Bytecodes::_isub && !Assembler::is_simm12(-right.get_jint_constant())))) { + right.load_nonconstant(); + } else { + right.load_item(); + } + rlock_result(x); + arithmetic_op_int(x->op(), x->operand(), left_arg->result(), right_arg->result(), LIR_OprFact::illegalOpr); + } else { + assert (x->op() == Bytecodes::_imul, "expect imul"); + if (right.is_constant()) { + jint c = right.get_jint_constant(); + if (c > 0 && c < max_jint && (is_power_of_2(c) || is_power_of_2(c - 1) || is_power_of_2(c + 1))) { + right_arg->dont_load_item(); + } else { + // Cannot use constant op. + right_arg->load_item(); + } + } else { + right.load_item(); + } + rlock_result(x); + arithmetic_op_int(x->op(), x->operand(), left_arg->result(), right_arg->result(), new_register(T_INT)); + } +} + +void LIRGenerator::do_ArithmeticOp(ArithmeticOp* x) { + // when an operand with use count 1 is the left operand, then it is + // likely that no move for 2-operand-LIR-form is necessary + if (x->is_commutative() && x->y()->as_Constant() == NULL && x->x()->use_count() > x->y()->use_count()) { + x->swap_operands(); + } + + ValueTag tag = x->type()->tag(); + assert(x->x()->type()->tag() == tag && x->y()->type()->tag() == tag, "wrong parameters"); + switch (tag) { + case floatTag: + case doubleTag: do_ArithmeticOp_FPU(x); return; + case longTag: do_ArithmeticOp_Long(x); return; + case intTag: do_ArithmeticOp_Int(x); return; + default: ShouldNotReachHere(); return; + } +} + +// _ishl, _lshl, _ishr, _lshr, _iushr, _lushr +void LIRGenerator::do_ShiftOp(ShiftOp* x) { + LIRItem value(x->x(), this); + LIRItem count(x->y(), this); + + value.load_item(); + if (count.is_constant()) { + assert(count.type()->as_IntConstant() != NULL || count.type()->as_LongConstant() != NULL , "should be"); + count.dont_load_item(); + } else { + count.load_item(); + } + + LIR_Opr res = rlock_result(x); + shift_op(x->op(), res, value.result(), count.result(), LIR_OprFact::illegalOpr); +} + + +// _iand, _land, _ior, _lor, _ixor, _lxor +void LIRGenerator::do_LogicOp(LogicOp* x) { + + LIRItem left(x->x(), this); + LIRItem right(x->y(), this); + + left.load_item(); + rlock_result(x); + ValueTag tag = right.type()->tag(); + if (right.is_constant() && + ((tag == longTag && Assembler::is_simm12(right.get_jlong_constant())) || + (tag == intTag && Assembler::is_simm12(right.get_jint_constant())))) { + right.dont_load_item(); + } else { + right.load_item(); + } + + switch (x->op()) { + case Bytecodes::_iand: // fall through + case Bytecodes::_land: + __ logical_and(left.result(), right.result(), x->operand()); break; + case Bytecodes::_ior: // fall through + case Bytecodes::_lor: + __ logical_or(left.result(), right.result(), x->operand()); break; + case Bytecodes::_ixor: // fall through + case Bytecodes::_lxor: + __ logical_xor(left.result(), right.result(), x->operand()); break; + default: Unimplemented(); + } +} + +// _lcmp, _fcmpl, _fcmpg, _dcmpl, _dcmpg +void LIRGenerator::do_CompareOp(CompareOp* x) { + LIRItem left(x->x(), this); + LIRItem right(x->y(), this); + ValueTag tag = x->x()->type()->tag(); + if (tag == longTag) { + left.set_destroys_register(); + } + left.load_item(); + right.load_item(); + LIR_Opr reg = rlock_result(x); + + if (x->x()->type()->is_float_kind()) { + Bytecodes::Code code = x->op(); + __ fcmp2int(left.result(), right.result(), reg, (code == Bytecodes::_fcmpl || code == Bytecodes::_dcmpl)); + } else if (x->x()->type()->tag() == longTag) { + __ lcmp2int(left.result(), right.result(), reg); + } else { + Unimplemented(); + } +} + +LIR_Opr LIRGenerator::atomic_cmpxchg(BasicType type, LIR_Opr addr, LIRItem& cmp_value, LIRItem& new_value) { + LIR_Opr ill = LIR_OprFact::illegalOpr; // for convenience + new_value.load_item(); + cmp_value.load_item(); + LIR_Opr result = new_register(T_INT); + if (is_reference_type(type)) { + __ cas_obj(addr, cmp_value.result(), new_value.result(), new_register(T_INT), new_register(T_INT), result); + } else if (type == T_INT) { + __ cas_int(addr->as_address_ptr()->base(), cmp_value.result(), new_value.result(), ill, ill); + } else if (type == T_LONG) { + __ cas_long(addr->as_address_ptr()->base(), cmp_value.result(), new_value.result(), ill, ill); + } else { + ShouldNotReachHere(); + } + __ logical_xor(FrameMap::r5_opr, LIR_OprFact::intConst(1), result); + return result; +} + +LIR_Opr LIRGenerator::atomic_xchg(BasicType type, LIR_Opr addr, LIRItem& value) { + bool is_oop = is_reference_type(type); + LIR_Opr result = new_register(type); + value.load_item(); + assert(type == T_INT || is_oop LP64_ONLY( || type == T_LONG ), "unexpected type"); + LIR_Opr tmp = new_register(T_INT); + __ xchg(addr, value.result(), result, tmp); + return result; +} + +LIR_Opr LIRGenerator::atomic_add(BasicType type, LIR_Opr addr, LIRItem& value) { + LIR_Opr result = new_register(type); + value.load_item(); + assert(type == T_INT LP64_ONLY( || type == T_LONG ), "unexpected type"); + LIR_Opr tmp = new_register(T_INT); + __ xadd(addr, value.result(), result, tmp); + return result; +} + +void LIRGenerator::do_MathIntrinsic(Intrinsic* x) { + assert(x->number_of_arguments() == 1 || (x->number_of_arguments() == 2 && x->id() == vmIntrinsics::_dpow), + "wrong type"); + + switch (x->id()) { + case vmIntrinsics::_dexp: // fall through + case vmIntrinsics::_dlog: // fall through + case vmIntrinsics::_dpow: // fall through + case vmIntrinsics::_dcos: // fall through + case vmIntrinsics::_dsin: // fall through + case vmIntrinsics::_dtan: // fall through + case vmIntrinsics::_dlog10: + do_LibmIntrinsic(x); + break; + case vmIntrinsics::_dabs: // fall through + case vmIntrinsics::_dsqrt: { + assert(x->number_of_arguments() == 1, "wrong type"); + LIRItem value(x->argument_at(0), this); + value.load_item(); + LIR_Opr dst = rlock_result(x); + + switch (x->id()) { + case vmIntrinsics::_dsqrt: { + __ sqrt(value.result(), dst, LIR_OprFact::illegalOpr); + break; + } + case vmIntrinsics::_dabs: { + __ abs(value.result(), dst, LIR_OprFact::illegalOpr); + break; + } + default: + ShouldNotReachHere(); + } + break; + } + default: + ShouldNotReachHere(); + } +} + +void LIRGenerator::do_LibmIntrinsic(Intrinsic* x) { + LIRItem value(x->argument_at(0), this); + value.set_destroys_register(); + + LIR_Opr calc_result = rlock_result(x); + LIR_Opr result_reg = result_register_for(x->type()); + + CallingConvention* cc = NULL; + + if (x->id() == vmIntrinsics::_dpow) { + LIRItem value1(x->argument_at(1), this); + + value1.set_destroys_register(); + + BasicTypeList signature(2); + signature.append(T_DOUBLE); + signature.append(T_DOUBLE); + cc = frame_map()->c_calling_convention(&signature); + value.load_item_force(cc->at(0)); + value1.load_item_force(cc->at(1)); + } else { + BasicTypeList signature(1); + signature.append(T_DOUBLE); + cc = frame_map()->c_calling_convention(&signature); + value.load_item_force(cc->at(0)); + } + + switch (x->id()) { + case vmIntrinsics::_dexp: + if (StubRoutines::dexp() != NULL) { __ call_runtime_leaf(StubRoutines::dexp(), getThreadTemp(), result_reg, cc->args()); } + else { __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dexp), getThreadTemp(), result_reg, cc->args()); } + break; + case vmIntrinsics::_dlog: + if (StubRoutines::dlog() != NULL) { __ call_runtime_leaf(StubRoutines::dlog(), getThreadTemp(), result_reg, cc->args()); } + else { __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dlog), getThreadTemp(), result_reg, cc->args()); } + break; + case vmIntrinsics::_dlog10: + if (StubRoutines::dlog10() != NULL) { __ call_runtime_leaf(StubRoutines::dlog10(), getThreadTemp(), result_reg, cc->args()); } + else { __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dlog10), getThreadTemp(), result_reg, cc->args()); } + break; + case vmIntrinsics::_dsin: + if (StubRoutines::dsin() != NULL) { __ call_runtime_leaf(StubRoutines::dsin(), getThreadTemp(), result_reg, cc->args()); } + else { __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dsin), getThreadTemp(), result_reg, cc->args()); } + break; + case vmIntrinsics::_dcos: + if (StubRoutines::dcos() != NULL) { __ call_runtime_leaf(StubRoutines::dcos(), getThreadTemp(), result_reg, cc->args()); } + else { __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dcos), getThreadTemp(), result_reg, cc->args()); } + break; + case vmIntrinsics::_dtan: + if (StubRoutines::dtan() != NULL) { __ call_runtime_leaf(StubRoutines::dtan(), getThreadTemp(), result_reg, cc->args()); } + else { __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtan), getThreadTemp(), result_reg, cc->args()); } + break; + case vmIntrinsics::_dpow: + if (StubRoutines::dpow() != NULL) { __ call_runtime_leaf(StubRoutines::dpow(), getThreadTemp(), result_reg, cc->args()); } + else { __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dpow), getThreadTemp(), result_reg, cc->args()); } + break; + default: ShouldNotReachHere(); + } + __ move(result_reg, calc_result); +} + + +void LIRGenerator::do_ArrayCopy(Intrinsic* x) { + assert(x->number_of_arguments() == 5, "wrong type"); + + // Make all state_for calls early since they can emit code + CodeEmitInfo* info = state_for(x, x->state()); + + LIRItem src(x->argument_at(0), this); + LIRItem src_pos(x->argument_at(1), this); + LIRItem dst(x->argument_at(2), this); + LIRItem dst_pos(x->argument_at(3), this); + LIRItem length(x->argument_at(4), this); + + // operands for arraycopy must use fixed registers, otherwise + // LinearScan will fail allocation (because arraycopy always needs a + // call) + + // The java calling convention will give us enough registers + // so that on the stub side the args will be perfect already. + // On the other slow/special case side we call C and the arg + // positions are not similar enough to pick one as the best. + // Also because the java calling convention is a "shifted" version + // of the C convention we can process the java args trivially into C + // args without worry of overwriting during the xfer + + src.load_item_force (FrameMap::as_oop_opr(j_rarg0)); + src_pos.load_item_force (FrameMap::as_opr(j_rarg1)); + dst.load_item_force (FrameMap::as_oop_opr(j_rarg2)); + dst_pos.load_item_force (FrameMap::as_opr(j_rarg3)); + length.load_item_force (FrameMap::as_opr(j_rarg4)); + + LIR_Opr tmp = FrameMap::as_opr(j_rarg5); + + set_no_result(x); + + int flags; + ciArrayKlass* expected_type = NULL; + arraycopy_helper(x, &flags, &expected_type); + + __ arraycopy(src.result(), src_pos.result(), dst.result(), dst_pos.result(), length.result(), tmp, + expected_type, flags, info); // does add_safepoint +} + +void LIRGenerator::do_update_CRC32(Intrinsic* x) { + ShouldNotReachHere(); +} + +void LIRGenerator::do_update_CRC32C(Intrinsic* x) { + ShouldNotReachHere(); +} + +void LIRGenerator::do_FmaIntrinsic(Intrinsic* x) { + assert(x->number_of_arguments() == 3, "wrong type"); + assert(UseFMA, "Needs FMA instructions support."); + LIRItem value(x->argument_at(0), this); + LIRItem value1(x->argument_at(1), this); + LIRItem value2(x->argument_at(2), this); + + value.load_item(); + value1.load_item(); + value2.load_item(); + + LIR_Opr calc_input = value.result(); + LIR_Opr calc_input1 = value1.result(); + LIR_Opr calc_input2 = value2.result(); + LIR_Opr calc_result = rlock_result(x); + + switch (x->id()) { + case vmIntrinsics::_fmaD: __ fmad(calc_input, calc_input1, calc_input2, calc_result); break; + case vmIntrinsics::_fmaF: __ fmaf(calc_input, calc_input1, calc_input2, calc_result); break; + default: ShouldNotReachHere(); + } +} + +void LIRGenerator::do_vectorizedMismatch(Intrinsic* x) { + fatal("vectorizedMismatch intrinsic is not implemented on this platform"); +} + +// _i2l, _i2f, _i2d, _l2i, _l2f, _l2d, _f2i, _f2l, _f2d, _d2i, _d2l, _d2f +// _i2b, _i2c, _i2s +void LIRGenerator::do_Convert(Convert* x) { + LIRItem value(x->value(), this); + value.load_item(); + LIR_Opr input = value.result(); + LIR_Opr result = rlock(x); + + // arguments of lir_convert + LIR_Opr conv_input = input; + LIR_Opr conv_result = result; + + __ convert(x->op(), conv_input, conv_result); + + assert(result->is_virtual(), "result must be virtual register"); + set_result(x, result); +} + +void LIRGenerator::do_NewInstance(NewInstance* x) { +#ifndef PRODUCT + if (PrintNotLoaded && !x->klass()->is_loaded()) { + tty->print_cr(" ###class not loaded at new bci %d", x->printable_bci()); + } +#endif + CodeEmitInfo* info = state_for(x, x->state()); + LIR_Opr reg = result_register_for(x->type()); + new_instance(reg, x->klass(), x->is_unresolved(), + FrameMap::r12_oop_opr, + FrameMap::r15_oop_opr, + FrameMap::r14_oop_opr, + LIR_OprFact::illegalOpr, + FrameMap::r13_metadata_opr, + info); + LIR_Opr result = rlock_result(x); + __ move(reg, result); +} + +void LIRGenerator::do_NewTypeArray(NewTypeArray* x) { + CodeEmitInfo* info = state_for(x, x->state()); + + LIRItem length(x->length(), this); + length.load_item_force(FrameMap::r9_opr); + + LIR_Opr reg = result_register_for(x->type()); + LIR_Opr tmp1 = FrameMap::r12_oop_opr; + LIR_Opr tmp2 = FrameMap::r14_oop_opr; + LIR_Opr tmp3 = FrameMap::r15_oop_opr; + LIR_Opr tmp4 = reg; + LIR_Opr klass_reg = FrameMap::r13_metadata_opr; + LIR_Opr len = length.result(); + BasicType elem_type = x->elt_type(); + + __ metadata2reg(ciTypeArrayKlass::make(elem_type)->constant_encoding(), klass_reg); + + CodeStub* slow_path = new NewTypeArrayStub(klass_reg, len, reg, info); + __ allocate_array(reg, len, tmp1, tmp2, tmp3, tmp4, elem_type, klass_reg, slow_path); + + LIR_Opr result = rlock_result(x); + __ move(reg, result); +} + +void LIRGenerator::do_NewObjectArray(NewObjectArray* x) { + LIRItem length(x->length(), this); + // in case of patching (i.e., object class is not yet loaded), we need to reexecute the instruction + // and therefore provide the state before the parameters have been consumed + CodeEmitInfo* patching_info = NULL; + if (!x->klass()->is_loaded() || PatchALot) { + patching_info = state_for(x, x->state_before()); + } + + CodeEmitInfo* info = state_for(x, x->state()); + + LIR_Opr reg = result_register_for(x->type()); + LIR_Opr tmp1 = FrameMap::r12_oop_opr; + LIR_Opr tmp2 = FrameMap::r14_oop_opr; + LIR_Opr tmp3 = FrameMap::r15_oop_opr; + LIR_Opr tmp4 = reg; + LIR_Opr klass_reg = FrameMap::r13_metadata_opr; + + length.load_item_force(FrameMap::r9_opr); + LIR_Opr len = length.result(); + + CodeStub* slow_path = new NewObjectArrayStub(klass_reg, len, reg, info); + ciKlass* obj = (ciKlass*) ciObjArrayKlass::make(x->klass()); + if (obj == ciEnv::unloaded_ciobjarrayklass()) { + BAILOUT("encountered unloaded_ciobjarrayklass due to out of memory error"); + } + klass2reg_with_patching(klass_reg, obj, patching_info); + __ allocate_array(reg, len, tmp1, tmp2, tmp3, tmp4, T_OBJECT, klass_reg, slow_path); + + LIR_Opr result = rlock_result(x); + __ move(reg, result); +} + + +void LIRGenerator::do_NewMultiArray(NewMultiArray* x) { + Values* dims = x->dims(); + int i = dims->length(); + LIRItemList* items = new LIRItemList(i, i, NULL); + while (i-- > 0) { + LIRItem* size = new LIRItem(dims->at(i), this); + items->at_put(i, size); + } + + // Evaluate state_for early since it may emit code. + CodeEmitInfo* patching_info = NULL; + if (!x->klass()->is_loaded() || PatchALot) { + patching_info = state_for(x, x->state_before()); + + // Cannot re-use same xhandlers for multiple CodeEmitInfos, so + // clone all handlers (NOTE: Usually this is handled transparently + // by the CodeEmitInfo cloning logic in CodeStub constructors but + // is done explicitly here because a stub isn't being used). + x->set_exception_handlers(new XHandlers(x->exception_handlers())); + } + CodeEmitInfo* info = state_for(x, x->state()); + + i = dims->length(); + while (i-- > 0) { + LIRItem* size = items->at(i); + size->load_item(); + + store_stack_parameter(size->result(), in_ByteSize(i * BytesPerInt)); + } + + LIR_Opr klass_reg = FrameMap::r10_metadata_opr; + klass2reg_with_patching(klass_reg, x->klass(), patching_info); + + LIR_Opr rank = FrameMap::r9_opr; + __ move(LIR_OprFact::intConst(x->rank()), rank); + LIR_Opr varargs = FrameMap::r12_opr; + __ move(FrameMap::sp_opr, varargs); + LIR_OprList* args = new LIR_OprList(3); + args->append(klass_reg); + args->append(rank); + args->append(varargs); + LIR_Opr reg = result_register_for(x->type()); + __ call_runtime(Runtime1::entry_for(Runtime1::new_multi_array_id), + LIR_OprFact::illegalOpr, + reg, args, info); + + LIR_Opr result = rlock_result(x); + __ move(reg, result); +} + +void LIRGenerator::do_BlockBegin(BlockBegin* x) { + // nothing to do for now +} + +void LIRGenerator::do_CheckCast(CheckCast* x) { + LIRItem obj(x->obj(), this); + + CodeEmitInfo* patching_info = NULL; + if (!x->klass()->is_loaded() || + (PatchALot && !x->is_incompatible_class_change_check() && !x->is_invokespecial_receiver_check())) { + // must do this before locking the destination register as an oop register, + // and before the obj is loaded (the latter is for deoptimization) + patching_info = state_for(x, x->state_before()); + } + obj.load_item(); + + // info for exceptions + CodeEmitInfo* info_for_exception = + (x->needs_exception_state() ? state_for(x) : + state_for(x, x->state_before(), true /*ignore_xhandler*/ )); + + CodeStub* stub = NULL; + if (x->is_incompatible_class_change_check()) { + assert(patching_info == NULL, "can't patch this"); + stub = new SimpleExceptionStub(Runtime1::throw_incompatible_class_change_error_id, LIR_OprFact::illegalOpr, + info_for_exception); + } else if (x->is_invokespecial_receiver_check()) { + assert(patching_info == NULL, "can't patch this"); + stub = new DeoptimizeStub(info_for_exception, + Deoptimization::Reason_class_check, + Deoptimization::Action_none); + } else { + stub = new SimpleExceptionStub(Runtime1::throw_class_cast_exception_id, obj.result(), info_for_exception); + } + LIR_Opr reg = rlock_result(x); + LIR_Opr tmp3 = LIR_OprFact::illegalOpr; + if (!x->klass()->is_loaded() || UseCompressedClassPointers) { + tmp3 = new_register(objectType); + } + __ checkcast(reg, obj.result(), x->klass(), + new_register(objectType), new_register(objectType), tmp3, + x->direct_compare(), info_for_exception, patching_info, stub, + x->profiled_method(), x->profiled_bci()); +} + +void LIRGenerator::do_InstanceOf(InstanceOf* x) { + LIRItem obj(x->obj(), this); + + // result and test object may not be in same register + LIR_Opr reg = rlock_result(x); + CodeEmitInfo* patching_info = NULL; + if ((!x->klass()->is_loaded() || PatchALot)) { + // must do this before locking the destination register as an oop register + patching_info = state_for(x, x->state_before()); + } + obj.load_item(); + LIR_Opr tmp3 = LIR_OprFact::illegalOpr; + if (!x->klass()->is_loaded() || UseCompressedClassPointers) { + tmp3 = new_register(objectType); + } + __ instanceof(reg, obj.result(), x->klass(), + new_register(objectType), new_register(objectType), tmp3, + x->direct_compare(), patching_info, x->profiled_method(), x->profiled_bci()); +} + +void LIRGenerator::do_If(If* x) { + // If should have two successors + assert(x->number_of_sux() == 2, "inconsistency"); + ValueTag tag = x->x()->type()->tag(); + bool is_safepoint = x->is_safepoint(); + + If::Condition cond = x->cond(); + + LIRItem xitem(x->x(), this); + LIRItem yitem(x->y(), this); + LIRItem* xin = &xitem; + LIRItem* yin = &yitem; + + if (tag == longTag) { + // for longs, only conditions "eql", "neq", "lss", "geq" are valid; + // mirror for other conditions + if (cond == If::gtr || cond == If::leq) { + cond = Instruction::mirror(cond); + xin = &yitem; + yin = &xitem; + } + xin->set_destroys_register(); + } + xin->load_item(); + yin->load_item(); + + set_no_result(x); + + LIR_Opr left = xin->result(); + LIR_Opr right = yin->result(); + + // add safepoint before generating condition code so it can be recomputed + if (x->is_safepoint()) { + // increment backedge counter if needed + increment_backedge_counter_conditionally(lir_cond(cond), left, right, state_for(x, x->state_before()), + x->tsux()->bci(), x->fsux()->bci(), x->profiled_bci()); + __ safepoint(LIR_OprFact::illegalOpr, state_for(x, x->state_before())); + } + + // Generate branch profiling. Profiling code doesn't kill flags. + __ cmp(lir_cond(cond), left, right); + profile_branch(x, cond); + move_to_phi(x->state()); + if (x->x()->type()->is_float_kind()) { + __ branch(lir_cond(cond), x->tsux(), x->usux()); + } else { + __ branch(lir_cond(cond), x->tsux()); + } + assert(x->default_sux() == x->fsux(), "wrong destination above"); + __ jump(x->default_sux()); +} + +LIR_Opr LIRGenerator::getThreadPointer() { + return FrameMap::as_pointer_opr(xthread); +} + +void LIRGenerator::trace_block_entry(BlockBegin* block) { Unimplemented(); } + +void LIRGenerator::volatile_field_store(LIR_Opr value, LIR_Address* address, + CodeEmitInfo* info) { + __ volatile_store_mem_reg(value, address, info); +} + +void LIRGenerator::volatile_field_load(LIR_Address* address, LIR_Opr result, + CodeEmitInfo* info) { + __ volatile_load_mem_reg(address, result, info); +} diff --git a/src/hotspot/cpu/riscv/c1_LIR_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIR_riscv.cpp new file mode 100644 index 0000000000000..0317ed9003eea --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_LIR_riscv.cpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/register.hpp" +#include "c1/c1_LIR.hpp" + +FloatRegister LIR_OprDesc::as_float_reg() const { + return as_FloatRegister(fpu_regnr()); +} + +FloatRegister LIR_OprDesc::as_double_reg() const { + return as_FloatRegister(fpu_regnrLo()); +} + +// Reg2 unused. +LIR_Opr LIR_OprFact::double_fpu(int reg1, int reg2) { + assert(as_FloatRegister(reg2) == fnoreg, "Not used on this platform"); + return (LIR_Opr)(intptr_t)((reg1 << LIR_OprDesc::reg1_shift) | + (reg1 << LIR_OprDesc::reg2_shift) | + LIR_OprDesc::double_type | + LIR_OprDesc::fpu_register | + LIR_OprDesc::double_size); +} + +#ifndef PRODUCT +void LIR_Address::verify() const { + assert(base()->is_cpu_register(), "wrong base operand"); + assert(index()->is_illegal() || index()->is_double_cpu() || index()->is_single_cpu(), "wrong index operand"); + assert(base()->type() == T_ADDRESS || base()->type() == T_OBJECT || base()->type() == T_LONG || + base()->type() == T_METADATA, "wrong type for addresses"); +} +#endif // PRODUCT diff --git a/src/hotspot/cpu/riscv/c1_LinearScan_riscv.cpp b/src/hotspot/cpu/riscv/c1_LinearScan_riscv.cpp new file mode 100644 index 0000000000000..78a61128bdd5d --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_LinearScan_riscv.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2005, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "c1/c1_Instruction.hpp" +#include "c1/c1_LinearScan.hpp" +#include "utilities/bitMap.inline.hpp" + +void LinearScan::allocate_fpu_stack() { + // No FPU stack on RISCV +} diff --git a/src/hotspot/cpu/riscv/c1_LinearScan_riscv.hpp b/src/hotspot/cpu/riscv/c1_LinearScan_riscv.hpp new file mode 100644 index 0000000000000..d7ca7b0fd0524 --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_LinearScan_riscv.hpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2005, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_C1_LINEARSCAN_RISCV_HPP +#define CPU_RISCV_C1_LINEARSCAN_RISCV_HPP + +inline bool LinearScan::is_processed_reg_num(int reg_num) +{ + return reg_num <= FrameMap::last_cpu_reg() || reg_num >= pd_nof_cpu_regs_frame_map; +} + +inline int LinearScan::num_physical_regs(BasicType type) { + return 1; +} + +inline bool LinearScan::requires_adjacent_regs(BasicType type) { + return false; +} + +inline bool LinearScan::is_caller_save(int assigned_reg) { + assert(assigned_reg >= 0 && assigned_reg < nof_regs, "should call this only for registers"); + if (assigned_reg < pd_first_callee_saved_reg) { + return true; + } + if (assigned_reg > pd_last_callee_saved_reg && assigned_reg < pd_first_callee_saved_fpu_reg_1) { + return true; + } + if (assigned_reg > pd_last_callee_saved_fpu_reg_1 && assigned_reg < pd_first_callee_saved_fpu_reg_2) { + return true; + } + if (assigned_reg > pd_last_callee_saved_fpu_reg_2 && assigned_reg < pd_last_fpu_reg) { + return true; + } + return false; +} + +inline void LinearScan::pd_add_temps(LIR_Op* op) { + // No special case behaviours yet +} + + +// Implementation of LinearScanWalker + +inline bool LinearScanWalker::pd_init_regs_for_alloc(Interval* cur) +{ + if (allocator()->gen()->is_vreg_flag_set(cur->reg_num(), LIRGenerator::callee_saved)) { + assert(cur->type() != T_FLOAT && cur->type() != T_DOUBLE, "cpu regs only"); + _first_reg = pd_first_callee_saved_reg; + _last_reg = pd_last_callee_saved_reg; + return true; + } else if (cur->type() == T_INT || cur->type() == T_LONG || cur->type() == T_OBJECT || + cur->type() == T_ADDRESS || cur->type() == T_METADATA) { + _first_reg = pd_first_cpu_reg; + _last_reg = pd_last_allocatable_cpu_reg; + return true; + } + return false; +} + +#endif // CPU_RISCV_C1_LINEARSCAN_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp new file mode 100644 index 0000000000000..6a4742ae3dd12 --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp @@ -0,0 +1,451 @@ +/* + * Copyright (c) 1999, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "c1/c1_LIR.hpp" +#include "c1/c1_MacroAssembler.hpp" +#include "c1/c1_Runtime1.hpp" +#include "classfile/systemDictionary.hpp" +#include "gc/shared/barrierSetAssembler.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "interpreter/interpreter.hpp" +#include "oops/arrayOop.hpp" +#include "oops/markWord.hpp" +#include "runtime/basicLock.hpp" +#include "runtime/os.hpp" +#include "runtime/sharedRuntime.hpp" +#include "runtime/stubRoutines.hpp" + +void C1_MacroAssembler::float_cmp(bool is_float, int unordered_result, + FloatRegister freg0, FloatRegister freg1, + Register result) { + if (is_float) { + float_compare(result, freg0, freg1, unordered_result); + } else { + double_compare(result, freg0, freg1, unordered_result); + } +} + +int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register tmp, Label& slow_case) { + const int aligned_mask = BytesPerWord - 1; + const int hdr_offset = oopDesc::mark_offset_in_bytes(); + assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different"); + Label done; + int null_check_offset = -1; + + verify_oop(obj); + + // save object being locked into the BasicObjectLock + sd(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes())); + + null_check_offset = offset(); + + if (DiagnoseSyncOnValueBasedClasses != 0) { + load_klass(hdr, obj); + lwu(hdr, Address(hdr, Klass::access_flags_offset())); + test_bit(t0, hdr, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS)); + bnez(t0, slow_case, true /* is_far */); + } + + if (UseBiasedLocking) { + assert(tmp != noreg, "should have tmp register at this point"); + biased_locking_enter(disp_hdr, obj, hdr, tmp, false, done, &slow_case); + } + + // Load object header + ld(hdr, Address(obj, hdr_offset)); + // and mark it as unlocked + ori(hdr, hdr, markWord::unlocked_value); + // save unlocked object header into the displaced header location on the stack + sd(hdr, Address(disp_hdr, 0)); + // test if object header is still the same (i.e. unlocked), and if so, store the + // displaced header address in the object header - if it is not the same, get the + // object header instead + la(t1, Address(obj, hdr_offset)); + cmpxchgptr(hdr, disp_hdr, t1, t0, done, /*fallthough*/NULL); + // if the object header was the same, we're done + // if the object header was not the same, it is now in the hdr register + // => test if it is a stack pointer into the same stack (recursive locking), i.e.: + // + // 1) (hdr & aligned_mask) == 0 + // 2) sp <= hdr + // 3) hdr <= sp + page_size + // + // these 3 tests can be done by evaluating the following expression: + // + // (hdr -sp) & (aligned_mask - page_size) + // + // assuming both the stack pointer and page_size have their least + // significant 2 bits cleared and page_size is a power of 2 + sub(hdr, hdr, sp); + mv(t0, aligned_mask - os::vm_page_size()); + andr(hdr, hdr, t0); + // for recursive locking, the result is zero => save it in the displaced header + // location (NULL in the displaced hdr location indicates recursive locking) + sd(hdr, Address(disp_hdr, 0)); + // otherwise we don't care about the result and handle locking via runtime call + bnez(hdr, slow_case, /* is_far */ true); + bind(done); + return null_check_offset; +} + +void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) { + const int aligned_mask = BytesPerWord - 1; + const int hdr_offset = oopDesc::mark_offset_in_bytes(); + assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different"); + Label done; + + if (UseBiasedLocking) { + // load object + ld(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes())); + biased_locking_exit(obj, hdr, done); + } + + // load displaced header + ld(hdr, Address(disp_hdr, 0)); + // if the loaded hdr is NULL we had recursive locking + // if we had recursive locking, we are done + beqz(hdr, done); + if (!UseBiasedLocking) { + // load object + ld(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes())); + } + verify_oop(obj); + // test if object header is pointing to the displaced header, and if so, restore + // the displaced header in the object - if the object header is not pointing to + // the displaced header, get the object header instead + // if the object header was not pointing to the displaced header, + // we do unlocking via runtime call + if (hdr_offset) { + la(t0, Address(obj, hdr_offset)); + cmpxchgptr(disp_hdr, hdr, t0, t1, done, &slow_case); + } else { + cmpxchgptr(disp_hdr, hdr, obj, t1, done, &slow_case); + } + bind(done); +} + +// Defines obj, preserves var_size_in_bytes +void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register tmp1, Register tmp2, Label& slow_case) { + if (UseTLAB) { + tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, tmp1, tmp2, slow_case, /* is_far */ true); + } else { + eden_allocate(obj, var_size_in_bytes, con_size_in_bytes, tmp1, slow_case, /* is_far */ true); + } +} + +void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register tmp1, Register tmp2) { + assert_different_registers(obj, klass, len); + if (UseBiasedLocking & !len->is_valid()) { + assert_different_registers(obj, klass, len, tmp1, tmp2); + ld(tmp1, Address(klass, Klass::prototype_header_offset())); + } else { + // This assumes that all prototype bits fitr in an int32_t + mv(tmp1, (int32_t)(intptr_t)markWord::prototype().value()); + } + sd(tmp1, Address(obj, oopDesc::mark_offset_in_bytes())); + + if (UseCompressedClassPointers) { // Take care not to kill klass + encode_klass_not_null(tmp1, klass, tmp2); + sw(tmp1, Address(obj, oopDesc::klass_offset_in_bytes())); + } else { + sd(klass, Address(obj, oopDesc::klass_offset_in_bytes())); + } + + if (len->is_valid()) { + sw(len, Address(obj, arrayOopDesc::length_offset_in_bytes())); + } else if (UseCompressedClassPointers) { + store_klass_gap(obj, zr); + } +} + +// preserves obj, destroys len_in_bytes +void C1_MacroAssembler::initialize_body(Register obj, Register len_in_bytes, int hdr_size_in_bytes, Register tmp) { + assert(hdr_size_in_bytes >= 0, "header size must be positive or 0"); + Label done; + + // len_in_bytes is positive and ptr sized + sub(len_in_bytes, len_in_bytes, hdr_size_in_bytes); + beqz(len_in_bytes, done); + + // Preserve obj + if (hdr_size_in_bytes) { + add(obj, obj, hdr_size_in_bytes); + } + zero_memory(obj, len_in_bytes, tmp); + if (hdr_size_in_bytes) { + sub(obj, obj, hdr_size_in_bytes); + } + + bind(done); +} + +void C1_MacroAssembler::allocate_object(Register obj, Register tmp1, Register tmp2, int header_size, int object_size, Register klass, Label& slow_case) { + assert_different_registers(obj, tmp1, tmp2); + assert(header_size >= 0 && object_size >= header_size, "illegal sizes"); + + try_allocate(obj, noreg, object_size * BytesPerWord, tmp1, tmp2, slow_case); + + initialize_object(obj, klass, noreg, object_size * HeapWordSize, tmp1, tmp2, UseTLAB); +} + +void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register var_size_in_bytes, int con_size_in_bytes, Register tmp1, Register tmp2, bool is_tlab_allocated) { + assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0, + "con_size_in_bytes is not multiple of alignment"); + const int hdr_size_in_bytes = instanceOopDesc::header_size() * HeapWordSize; + + initialize_header(obj, klass, noreg, tmp1, tmp2); + + if (!(UseTLAB && ZeroTLAB && is_tlab_allocated)) { + // clear rest of allocated space + const Register index = tmp2; + // 16: multipler for threshold + const int threshold = 16 * BytesPerWord; // approximate break even point for code size (see comments below) + if (var_size_in_bytes != noreg) { + mv(index, var_size_in_bytes); + initialize_body(obj, index, hdr_size_in_bytes, tmp1); + } else if (con_size_in_bytes <= threshold) { + // use explicit null stores + int i = hdr_size_in_bytes; + if (i < con_size_in_bytes && (con_size_in_bytes % (2 * BytesPerWord))) { // 2: multipler for BytesPerWord + sd(zr, Address(obj, i)); + i += BytesPerWord; + } + for (; i < con_size_in_bytes; i += BytesPerWord) { + sd(zr, Address(obj, i)); + } + } else if (con_size_in_bytes > hdr_size_in_bytes) { + block_comment("zero memory"); + // use loop to null out the fields + int words = (con_size_in_bytes - hdr_size_in_bytes) / BytesPerWord; + mv(index, words / 8); // 8: byte size + + const int unroll = 8; // Number of sd(zr) instructions we'll unroll + int remainder = words % unroll; + la(t0, Address(obj, hdr_size_in_bytes + remainder * BytesPerWord)); + + Label entry_point, loop; + j(entry_point); + + bind(loop); + sub(index, index, 1); + for (int i = -unroll; i < 0; i++) { + if (-i == remainder) { + bind(entry_point); + } + sd(zr, Address(t0, i * wordSize)); + } + if (remainder == 0) { + bind(entry_point); + } + add(t0, t0, unroll * wordSize); + bnez(index, loop); + } + } + + membar(MacroAssembler::StoreStore); + + if (CURRENT_ENV->dtrace_alloc_probes()) { + assert(obj == x10, "must be"); + far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id))); + } + + verify_oop(obj); +} + +void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int header_size, int f, Register klass, Label& slow_case) { + assert_different_registers(obj, len, tmp1, tmp2, klass); + + // determine alignment mask + assert(!(BytesPerWord & 1), "must be multiple of 2 for masking code to work"); + + // check for negative or excessive length + mv(t0, (int32_t)max_array_allocation_length); + bgeu(len, t0, slow_case, /* is_far */ true); + + const Register arr_size = tmp2; // okay to be the same + // align object end + mv(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask); + shadd(arr_size, len, arr_size, t0, f); + andi(arr_size, arr_size, ~(uint)MinObjAlignmentInBytesMask); + + try_allocate(obj, arr_size, 0, tmp1, tmp2, slow_case); + + initialize_header(obj, klass, len, tmp1, tmp2); + + // clear rest of allocated space + const Register len_zero = len; + initialize_body(obj, arr_size, header_size * BytesPerWord, len_zero); + + membar(MacroAssembler::StoreStore); + + if (CURRENT_ENV->dtrace_alloc_probes()) { + assert(obj == x10, "must be"); + far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id))); + } + + verify_oop(obj); +} + +void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache, Label &L) { + verify_oop(receiver); + // explicit NULL check not needed since load from [klass_offset] causes a trap + // check against inline cache + assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check"); + assert_different_registers(receiver, iCache, t0, t2); + cmp_klass(receiver, iCache, t0, t2 /* call-clobbered t2 as a tmp */, L); +} + +void C1_MacroAssembler::build_frame(int framesize, int bang_size_in_bytes) { + assert(bang_size_in_bytes >= framesize, "stack bang size incorrect"); + // Make sure there is enough stack space for this method's activation. + // Note that we do this before creating a frame. + generate_stack_overflow_check(bang_size_in_bytes); + MacroAssembler::build_frame(framesize); + + // Insert nmethod entry barrier into frame. + BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->nmethod_entry_barrier(this); +} + +void C1_MacroAssembler::remove_frame(int framesize) { + MacroAssembler::remove_frame(framesize); +} + + +void C1_MacroAssembler::verified_entry(bool breakAtEntry) { + // If we have to make this method not-entrant we'll overwrite its + // first instruction with a jump. For this action to be legal we + // must ensure that this first instruction is a J, JAL or NOP. + // Make it a NOP. + IncompressibleRegion ir(this); // keep the nop as 4 bytes for patching. + assert_alignment(pc()); + nop(); // 4 bytes +} + +void C1_MacroAssembler::load_parameter(int offset_in_words, Register reg) { + // fp + -2: link + // + -1: return address + // + 0: argument with offset 0 + // + 1: argument with offset 1 + // + 2: ... + ld(reg, Address(fp, offset_in_words * BytesPerWord)); +} + +#ifndef PRODUCT + +void C1_MacroAssembler::verify_stack_oop(int stack_offset) { + if (!VerifyOops) { + return; + } + verify_oop_addr(Address(sp, stack_offset), "oop"); +} + +void C1_MacroAssembler::verify_not_null_oop(Register r) { + if (!VerifyOops) return; + Label not_null; + bnez(r, not_null); + stop("non-null oop required"); + bind(not_null); + verify_oop(r); +} + +void C1_MacroAssembler::invalidate_registers(bool inv_x10, bool inv_x9, bool inv_x12, bool inv_x13, bool inv_x14, bool inv_x15) { +#ifdef ASSERT + static int nn; + if (inv_x10) { mv(x10, 0xDEAD); } + if (inv_x9) { mv(x9, 0xDEAD); } + if (inv_x12) { mv(x12, nn++); } + if (inv_x13) { mv(x13, 0xDEAD); } + if (inv_x14) { mv(x14, 0xDEAD); } + if (inv_x15) { mv(x15, 0xDEAD); } +#endif // ASSERT +} +#endif // ifndef PRODUCT + +typedef void (C1_MacroAssembler::*c1_cond_branch_insn)(Register op1, Register op2, Label& label, bool is_far); +typedef void (C1_MacroAssembler::*c1_float_cond_branch_insn)(FloatRegister op1, FloatRegister op2, + Label& label, bool is_far, bool is_unordered); + +static c1_cond_branch_insn c1_cond_branch[] = +{ + /* SHORT branches */ + (c1_cond_branch_insn)&MacroAssembler::beq, + (c1_cond_branch_insn)&MacroAssembler::bne, + (c1_cond_branch_insn)&MacroAssembler::blt, + (c1_cond_branch_insn)&MacroAssembler::ble, + (c1_cond_branch_insn)&MacroAssembler::bge, + (c1_cond_branch_insn)&MacroAssembler::bgt, + (c1_cond_branch_insn)&MacroAssembler::bleu, // lir_cond_belowEqual + (c1_cond_branch_insn)&MacroAssembler::bgeu // lir_cond_aboveEqual +}; + +static c1_float_cond_branch_insn c1_float_cond_branch[] = +{ + /* FLOAT branches */ + (c1_float_cond_branch_insn)&MacroAssembler::float_beq, + (c1_float_cond_branch_insn)&MacroAssembler::float_bne, + (c1_float_cond_branch_insn)&MacroAssembler::float_blt, + (c1_float_cond_branch_insn)&MacroAssembler::float_ble, + (c1_float_cond_branch_insn)&MacroAssembler::float_bge, + (c1_float_cond_branch_insn)&MacroAssembler::float_bgt, + NULL, // lir_cond_belowEqual + NULL, // lir_cond_aboveEqual + + /* DOUBLE branches */ + (c1_float_cond_branch_insn)&MacroAssembler::double_beq, + (c1_float_cond_branch_insn)&MacroAssembler::double_bne, + (c1_float_cond_branch_insn)&MacroAssembler::double_blt, + (c1_float_cond_branch_insn)&MacroAssembler::double_ble, + (c1_float_cond_branch_insn)&MacroAssembler::double_bge, + (c1_float_cond_branch_insn)&MacroAssembler::double_bgt, + NULL, // lir_cond_belowEqual + NULL // lir_cond_aboveEqual +}; + +void C1_MacroAssembler::c1_cmp_branch(int cmpFlag, Register op1, Register op2, Label& label, + BasicType type, bool is_far) { + if (type == T_OBJECT || type == T_ARRAY) { + assert(cmpFlag == lir_cond_equal || cmpFlag == lir_cond_notEqual, "Should be equal or notEqual"); + if (cmpFlag == lir_cond_equal) { + beq(op1, op2, label, is_far); + } else { + bne(op1, op2, label, is_far); + } + } else { + assert(cmpFlag >= 0 && cmpFlag < (int)(sizeof(c1_cond_branch) / sizeof(c1_cond_branch[0])), + "invalid c1 conditional branch index"); + (this->*c1_cond_branch[cmpFlag])(op1, op2, label, is_far); + } +} + +void C1_MacroAssembler::c1_float_cmp_branch(int cmpFlag, FloatRegister op1, FloatRegister op2, Label& label, + bool is_far, bool is_unordered) { + assert(cmpFlag >= 0 && + cmpFlag < (int)(sizeof(c1_float_cond_branch) / sizeof(c1_float_cond_branch[0])), + "invalid c1 float conditional branch index"); + (this->*c1_float_cond_branch[cmpFlag])(op1, op2, label, is_far, is_unordered); +} diff --git a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.hpp new file mode 100644 index 0000000000000..6f89b8629b0e2 --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.hpp @@ -0,0 +1,121 @@ +/* + * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_C1_MACROASSEMBLER_RISCV_HPP +#define CPU_RISCV_C1_MACROASSEMBLER_RISCV_HPP + +using MacroAssembler::build_frame; +using MacroAssembler::null_check; + +// C1_MacroAssembler contains high-level macros for C1 + + private: + int _rsp_offset; // track rsp changes + // initialization + void pd_init() { _rsp_offset = 0; } + + + public: + void try_allocate( + Register obj, // result: pointer to object after successful allocation + Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise + int con_size_in_bytes, // object size in bytes if known at compile time + Register tmp1, // temp register + Register tmp2, // temp register + Label& slow_case // continuation point if fast allocation fails + ); + + void initialize_header(Register obj, Register klass, Register len, Register tmp1, Register tmp2); + void initialize_body(Register obj, Register len_in_bytes, int hdr_size_in_bytes, Register tmp); + + void float_cmp(bool is_float, int unordered_result, + FloatRegister f0, FloatRegister f1, + Register result); + + // locking + // hdr : must be x10, contents destroyed + // obj : must point to the object to lock, contents preserved + // disp_hdr: must point to the displaced header location, contents preserved + // tmp : temporary register, contents destroyed + // returns code offset at which to add null check debug information + int lock_object (Register swap, Register obj, Register disp_hdr, Register tmp, Label& slow_case); + + // unlocking + // hdr : contents destroyed + // obj : must point to the object to lock, contents preserved + // disp_hdr: must be x10 & must point to the displaced header location, contents destroyed + void unlock_object(Register swap, Register obj, Register lock, Label& slow_case); + + void initialize_object( + Register obj, // result: pointer to object after successful allocation + Register klass, // object klass + Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise + int con_size_in_bytes, // object size in bytes if known at compile time + Register tmp1, // temp register + Register tmp2, // temp register + bool is_tlab_allocated // the object was allocated in a TLAB; relevant for the implementation of ZeroTLAB + ); + + // allocation of fixed-size objects + // (can also be used to allocate fixed-size arrays, by setting + // hdr_size correctly and storing the array length afterwards) + // obj : will contain pointer to allocated object + // t1, t2 : temp registers - contents destroyed + // header_size: size of object header in words + // object_size: total size of object in words + // slow_case : exit to slow case implementation if fast allocation fails + void allocate_object(Register obj, Register tmp1, Register tmp2, int header_size, int object_size, Register klass, Label& slow_case); + + enum { + max_array_allocation_length = 0x00FFFFFF + }; + + // allocation of arrays + // obj : will contain pointer to allocated object + // len : array length in number of elements + // t : temp register - contents destroyed + // header_size: size of object header in words + // f : element scale factor + // slow_case : exit to slow case implementation if fast allocation fails + void allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int header_size, int f, Register klass, Label& slow_case); + + int rsp_offset() const { return _rsp_offset; } + + void invalidate_registers(bool inv_r0, bool inv_r19, bool inv_r2, bool inv_r3, bool inv_r4, bool inv_r5) PRODUCT_RETURN; + + // This platform only uses signal-based null checks. The Label is not needed. + void null_check(Register r, Label *Lnull = NULL) { MacroAssembler::null_check(r); } + + void load_parameter(int offset_in_words, Register reg); + + void inline_cache_check(Register receiver, Register iCache, Label &L); + + static const int c1_double_branch_mask = 1 << 3; // depend on c1_float_cond_branch + void c1_cmp_branch(int cmpFlag, Register op1, Register op2, Label& label, BasicType type, bool is_far); + void c1_float_cmp_branch(int cmpFlag, FloatRegister op1, FloatRegister op2, Label& label, + bool is_far, bool is_unordered = false); + +#endif // CPU_RISCV_C1_MACROASSEMBLER_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/c1_Runtime1_riscv.cpp b/src/hotspot/cpu/riscv/c1_Runtime1_riscv.cpp new file mode 100644 index 0000000000000..2d01a92623fb0 --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_Runtime1_riscv.cpp @@ -0,0 +1,1179 @@ +/* + * Copyright (c) 1999, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/assembler.hpp" +#include "c1/c1_CodeStubs.hpp" +#include "c1/c1_Defs.hpp" +#include "c1/c1_MacroAssembler.hpp" +#include "c1/c1_Runtime1.hpp" +#include "compiler/disassembler.hpp" +#include "compiler/oopMap.hpp" +#include "gc/shared/cardTable.hpp" +#include "gc/shared/cardTableBarrierSet.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "interpreter/interpreter.hpp" +#include "memory/universe.hpp" +#include "nativeInst_riscv.hpp" +#include "oops/compiledICHolder.hpp" +#include "oops/oop.inline.hpp" +#include "prims/jvmtiExport.hpp" +#include "register_riscv.hpp" +#include "runtime/sharedRuntime.hpp" +#include "runtime/signature.hpp" +#include "runtime/stubRoutines.hpp" +#include "runtime/vframe.hpp" +#include "runtime/vframeArray.hpp" +#include "utilities/powerOfTwo.hpp" +#include "vmreg_riscv.inline.hpp" + + +// Implementation of StubAssembler + +int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, int args_size) { + // setup registers + assert(!(oop_result->is_valid() || metadata_result->is_valid()) || oop_result != metadata_result, + "registers must be different"); + assert(oop_result != xthread && metadata_result != xthread, "registers must be different"); + assert(args_size >= 0, "illegal args_size"); + bool align_stack = false; + + mv(c_rarg0, xthread); + set_num_rt_args(0); // Nothing on stack + + Label retaddr; + set_last_Java_frame(sp, fp, retaddr, t0); + + // do the call + RuntimeAddress target(entry); + relocate(target.rspec(), [&] { + int32_t offset; + la_patchable(t0, target, offset); + jalr(x1, t0, offset); + }); + bind(retaddr); + int call_offset = offset(); + // verify callee-saved register +#ifdef ASSERT + push_reg(x10, sp); + { Label L; + get_thread(x10); + beq(xthread, x10, L); + stop("StubAssembler::call_RT: xthread not callee saved?"); + bind(L); + } + pop_reg(x10, sp); +#endif + reset_last_Java_frame(true); + + // check for pending exceptions + { Label L; + // check for pending exceptions (java_thread is set upon return) + ld(t0, Address(xthread, in_bytes(Thread::pending_exception_offset()))); + beqz(t0, L); + // exception pending => remove activation and forward to exception handler + // make sure that the vm_results are cleared + if (oop_result->is_valid()) { + sd(zr, Address(xthread, JavaThread::vm_result_offset())); + } + if (metadata_result->is_valid()) { + sd(zr, Address(xthread, JavaThread::vm_result_2_offset())); + } + if (frame_size() == no_frame_size) { + leave(); + far_jump(RuntimeAddress(StubRoutines::forward_exception_entry())); + } else if (_stub_id == Runtime1::forward_exception_id) { + should_not_reach_here(); + } else { + far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id))); + } + bind(L); + } + // get oop results if there are any and reset the values in the thread + if (oop_result->is_valid()) { + get_vm_result(oop_result, xthread); + } + if (metadata_result->is_valid()) { + get_vm_result_2(metadata_result, xthread); + } + return call_offset; +} + +int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, Register arg1) { + mv(c_rarg1, arg1); + return call_RT(oop_result, metadata_result, entry, 1); +} + +int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, Register arg1, Register arg2) { + const int arg_num = 2; + if (c_rarg1 == arg2) { + if (c_rarg2 == arg1) { + xorr(arg1, arg1, arg2); + xorr(arg2, arg1, arg2); + xorr(arg1, arg1, arg2); + } else { + mv(c_rarg2, arg2); + mv(c_rarg1, arg1); + } + } else { + mv(c_rarg1, arg1); + mv(c_rarg2, arg2); + } + return call_RT(oop_result, metadata_result, entry, arg_num); +} + +int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) { + const int arg_num = 3; + // if there is any conflict use the stack + if (arg1 == c_rarg2 || arg1 == c_rarg3 || + arg2 == c_rarg1 || arg2 == c_rarg3 || + arg3 == c_rarg1 || arg3 == c_rarg2) { + const int arg1_sp_offset = 0; + const int arg2_sp_offset = 1; + const int arg3_sp_offset = 2; + addi(sp, sp, -(arg_num + 1) * wordSize); + sd(arg1, Address(sp, arg1_sp_offset * wordSize)); + sd(arg2, Address(sp, arg2_sp_offset * wordSize)); + sd(arg3, Address(sp, arg3_sp_offset * wordSize)); + + ld(c_rarg1, Address(sp, arg1_sp_offset * wordSize)); + ld(c_rarg2, Address(sp, arg2_sp_offset * wordSize)); + ld(c_rarg3, Address(sp, arg3_sp_offset * wordSize)); + addi(sp, sp, (arg_num + 1) * wordSize); + } else { + mv(c_rarg1, arg1); + mv(c_rarg2, arg2); + mv(c_rarg3, arg3); + } + return call_RT(oop_result, metadata_result, entry, arg_num); +} + +enum return_state_t { + does_not_return, requires_return +}; + +// Implementation of StubFrame + +class StubFrame: public StackObj { + private: + StubAssembler* _sasm; + bool _return_state; + + public: + StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, return_state_t return_state=requires_return); + void load_argument(int offset_in_words, Register reg); + + ~StubFrame(); +};; + +void StubAssembler::prologue(const char* name, bool must_gc_arguments) { + set_info(name, must_gc_arguments); + enter(); +} + +void StubAssembler::epilogue() { + leave(); + ret(); +} + +#define __ _sasm-> + +StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, return_state_t return_state) { + _sasm = sasm; + _return_state = return_state; + __ prologue(name, must_gc_arguments); +} + +// load parameters that were stored with LIR_Assembler::store_parameter +// Note: offsets for store_parameter and load_argument must match +void StubFrame::load_argument(int offset_in_words, Register reg) { + __ load_parameter(offset_in_words, reg); +} + + +StubFrame::~StubFrame() { + if (_return_state == requires_return) { + __ epilogue(); + } else { + __ should_not_reach_here(); + } + _sasm = NULL; +} + +#undef __ + + +// Implementation of Runtime1 + +#define __ sasm-> + +const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2; + +// Stack layout for saving/restoring all the registers needed during a runtime +// call (this includes deoptimization) +// Note: note that users of this frame may well have arguments to some runtime +// while these values are on the stack. These positions neglect those arguments +// but the code in save_live_registers will take the argument count into +// account. +// + +enum reg_save_layout { + reg_save_frame_size = 32 /* float */ + 30 /* integer excluding x3, x4 */ +}; + +// Save off registers which might be killed by calls into the runtime. +// Tries to smart of about FPU registers. In particular we separate +// saving and describing the FPU registers for deoptimization since we +// have to save the FPU registers twice if we describe them. The +// deopt blob is the only thing which needs to describe FPU registers. +// In all other cases it should be sufficient to simply save their +// current value. + +static int cpu_reg_save_offsets[FrameMap::nof_cpu_regs]; +static int fpu_reg_save_offsets[FrameMap::nof_fpu_regs]; + +static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers) { + int frame_size_in_bytes = reg_save_frame_size * BytesPerWord; + sasm->set_frame_size(frame_size_in_bytes / BytesPerWord); + int frame_size_in_slots = frame_size_in_bytes / sizeof(jint); + OopMap* oop_map = new OopMap(frame_size_in_slots, 0); + assert_cond(oop_map != NULL); + + // caller save registers only, see FrameMap::initialize + // in c1_FrameMap_riscv.cpp for detail. + const static Register caller_save_cpu_regs[FrameMap::max_nof_caller_save_cpu_regs] = { + x7, x10, x11, x12, x13, x14, x15, x16, x17, x28, x29, x30, x31 + }; + + for (int i = 0; i < FrameMap::max_nof_caller_save_cpu_regs; i++) { + Register r = caller_save_cpu_regs[i]; + int sp_offset = cpu_reg_save_offsets[r->encoding()]; + oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset), + r->as_VMReg()); + } + + // fpu_regs + if (save_fpu_registers) { + for (int i = 0; i < FrameMap::nof_fpu_regs; i++) { + FloatRegister r = as_FloatRegister(i); + int sp_offset = fpu_reg_save_offsets[i]; + oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset), + r->as_VMReg()); + } + } + return oop_map; +} + +static OopMap* save_live_registers(StubAssembler* sasm, + bool save_fpu_registers = true) { + __ block_comment("save_live_registers"); + + // if the number of pushed regs is odd, one slot will be reserved for alignment + __ push_reg(RegSet::range(x5, x31), sp); // integer registers except ra(x1) & sp(x2) & gp(x3) & tp(x4) + + if (save_fpu_registers) { + // float registers + __ addi(sp, sp, -(FrameMap::nof_fpu_regs * wordSize)); + for (int i = 0; i < FrameMap::nof_fpu_regs; i++) { + __ fsd(as_FloatRegister(i), Address(sp, i * wordSize)); + } + } else { + // we define reg_save_layout = 62 as the fixed frame size, + // we should also sub 32 * wordSize to sp when save_fpu_registers == false + __ addi(sp, sp, -32 * wordSize); + } + + return generate_oop_map(sasm, save_fpu_registers); +} + +static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) { + if (restore_fpu_registers) { + for (int i = 0; i < FrameMap::nof_fpu_regs; i++) { + __ fld(as_FloatRegister(i), Address(sp, i * wordSize)); + } + __ addi(sp, sp, FrameMap::nof_fpu_regs * wordSize); + } else { + // we define reg_save_layout = 64 as the fixed frame size, + // we should also add 32 * wordSize to sp when save_fpu_registers == false + __ addi(sp, sp, 32 * wordSize); + } + + // if the number of popped regs is odd, the reserved slot for alignment will be removed + __ pop_reg(RegSet::range(x5, x31), sp); // integer registers except ra(x1) & sp(x2) & gp(x3) & tp(x4) +} + +static void restore_live_registers_except_r10(StubAssembler* sasm, bool restore_fpu_registers = true) { + if (restore_fpu_registers) { + for (int i = 0; i < FrameMap::nof_fpu_regs; i++) { + __ fld(as_FloatRegister(i), Address(sp, i * wordSize)); + } + __ addi(sp, sp, FrameMap::nof_fpu_regs * wordSize); + } else { + // we define reg_save_layout = 64 as the fixed frame size, + // we should also add 32 * wordSize to sp when save_fpu_registers == false + __ addi(sp, sp, 32 * wordSize); + } + + // pop integer registers except ra(x1) & sp(x2) & gp(x3) & tp(x4) & x10 + // there is one reserved slot for alignment on the stack in save_live_registers(). + __ pop_reg(RegSet::range(x5, x9), sp); // pop x5 ~ x9 with the reserved slot for alignment + __ pop_reg(RegSet::range(x11, x31), sp); // pop x11 ~ x31; x10 will be automatically skipped here +} + +void Runtime1::initialize_pd() { + int i = 0; + int sp_offset = 0; + const int step = 2; // SP offsets are in halfwords + + // all float registers are saved explicitly + for (i = 0; i < FrameMap::nof_fpu_regs; i++) { + fpu_reg_save_offsets[i] = sp_offset; + sp_offset += step; + } + + // a slot reserved for stack 16-byte alignment, see MacroAssembler::push_reg + sp_offset += step; + // we save x5 ~ x31, except x0 ~ x4: loop starts from x5 + for (i = 5; i < FrameMap::nof_cpu_regs; i++) { + cpu_reg_save_offsets[i] = sp_offset; + sp_offset += step; + } +} + +// target: the entry point of the method that creates and posts the exception oop +// has_argument: true if the exception needs arguments (passed in t0 and t1) + +OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) { + // make a frame and preserve the caller's caller-save registers + OopMap* oop_map = save_live_registers(sasm); + assert_cond(oop_map != NULL); + int call_offset = 0; + if (!has_argument) { + call_offset = __ call_RT(noreg, noreg, target); + } else { + __ mv(c_rarg1, t0); + __ mv(c_rarg2, t1); + call_offset = __ call_RT(noreg, noreg, target); + } + OopMapSet* oop_maps = new OopMapSet(); + assert_cond(oop_maps != NULL); + oop_maps->add_gc_map(call_offset, oop_map); + + return oop_maps; +} + +OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) { + __ block_comment("generate_handle_exception"); + + // incoming parameters + const Register exception_oop = x10; + const Register exception_pc = x13; + + OopMapSet* oop_maps = new OopMapSet(); + assert_cond(oop_maps != NULL); + OopMap* oop_map = NULL; + + switch (id) { + case forward_exception_id: + // We're handling an exception in the context of a compiled frame. + // The registers have been saved in the standard places. Perform + // an exception lookup in the caller and dispatch to the handler + // if found. Otherwise unwind and dispatch to the callers + // exception handler. + oop_map = generate_oop_map(sasm, 1 /* thread */); + + // load and clear pending exception oop into x10 + __ ld(exception_oop, Address(xthread, Thread::pending_exception_offset())); + __ sd(zr, Address(xthread, Thread::pending_exception_offset())); + + // load issuing PC (the return address for this stub) into x13 + __ ld(exception_pc, Address(fp, frame::return_addr_offset * BytesPerWord)); + + // make sure that the vm_results are cleared (may be unnecessary) + __ sd(zr, Address(xthread, JavaThread::vm_result_offset())); + __ sd(zr, Address(xthread, JavaThread::vm_result_2_offset())); + break; + case handle_exception_nofpu_id: + case handle_exception_id: + // At this point all registers MAY be live. + oop_map = save_live_registers(sasm, id != handle_exception_nofpu_id); + break; + case handle_exception_from_callee_id: { + // At this point all registers except exception oop (x10) and + // exception pc (ra) are dead. + const int frame_size = 2 /* fp, return address */; + oop_map = new OopMap(frame_size * VMRegImpl::slots_per_word, 0); + sasm->set_frame_size(frame_size); + break; + } + default: ShouldNotReachHere(); + } + + // verify that only x10 and x13 are valid at this time + __ invalidate_registers(false, true, true, false, true, true); + // verify that x10 contains a valid exception + __ verify_not_null_oop(exception_oop); + +#ifdef ASSERT + // check that fields in JavaThread for exception oop and issuing pc are + // empty before writing to them + Label oop_empty; + __ ld(t0, Address(xthread, JavaThread::exception_oop_offset())); + __ beqz(t0, oop_empty); + __ stop("exception oop already set"); + __ bind(oop_empty); + + Label pc_empty; + __ ld(t0, Address(xthread, JavaThread::exception_pc_offset())); + __ beqz(t0, pc_empty); + __ stop("exception pc already set"); + __ bind(pc_empty); +#endif + + // save exception oop and issuing pc into JavaThread + // (exception handler will load it from here) + __ sd(exception_oop, Address(xthread, JavaThread::exception_oop_offset())); + __ sd(exception_pc, Address(xthread, JavaThread::exception_pc_offset())); + + // patch throwing pc into return address (has bci & oop map) + __ sd(exception_pc, Address(fp, frame::return_addr_offset * BytesPerWord)); + + // compute the exception handler. + // the exception oop and the throwing pc are read from the fields in JavaThread + int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc)); + guarantee(oop_map != NULL, "NULL oop_map!"); + oop_maps->add_gc_map(call_offset, oop_map); + + // x10: handler address + // will be the deopt blob if nmethod was deoptimized while we looked up + // handler regardless of whether handler existed in the nmethod. + + // only x10 is valid at this time, all other registers have been destroyed by the runtime call + __ invalidate_registers(false, true, true, true, true, true); + + // patch the return address, this stub will directly return to the exception handler + __ sd(x10, Address(fp, frame::return_addr_offset * BytesPerWord)); + + switch (id) { + case forward_exception_id: + case handle_exception_nofpu_id: + case handle_exception_id: + // Restore the registers that were saved at the beginning. + restore_live_registers(sasm, id != handle_exception_nofpu_id); + break; + case handle_exception_from_callee_id: + break; + default: ShouldNotReachHere(); + } + + return oop_maps; +} + + +void Runtime1::generate_unwind_exception(StubAssembler *sasm) { + // incoming parameters + const Register exception_oop = x10; + // other registers used in this stub + const Register handler_addr = x11; + + // verify that only x10, is valid at this time + __ invalidate_registers(false, true, true, true, true, true); + +#ifdef ASSERT + // check that fields in JavaThread for exception oop and issuing pc are empty + Label oop_empty; + __ ld(t0, Address(xthread, JavaThread::exception_oop_offset())); + __ beqz(t0, oop_empty); + __ stop("exception oop must be empty"); + __ bind(oop_empty); + + Label pc_empty; + __ ld(t0, Address(xthread, JavaThread::exception_pc_offset())); + __ beqz(t0, pc_empty); + __ stop("exception pc must be empty"); + __ bind(pc_empty); +#endif + + // Save our return address because + // exception_handler_for_return_address will destroy it. We also + // save exception_oop + __ addi(sp, sp, -2 * wordSize); + __ sd(exception_oop, Address(sp, wordSize)); + __ sd(ra, Address(sp)); + + // search the exception handler address of the caller (using the return address) + __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), xthread, ra); + // x10: exception handler address of the caller + + // Only x10 is valid at this time; all other registers have been + // destroyed by the call. + __ invalidate_registers(false, true, true, true, false, true); + + // move result of call into correct register + __ mv(handler_addr, x10); + + // get throwing pc (= return address). + // ra has been destroyed by the call + __ ld(ra, Address(sp)); + __ ld(exception_oop, Address(sp, wordSize)); + __ addi(sp, sp, 2 * wordSize); + __ mv(x13, ra); + + __ verify_not_null_oop(exception_oop); + + // continue at exception handler (return address removed) + // note: do *not* remove arguments when unwinding the + // activation since the caller assumes having + // all arguments on the stack when entering the + // runtime to determine the exception handler + // (GC happens at call site with arguments!) + // x10: exception oop + // x13: throwing pc + // x11: exception handler + __ jr(handler_addr); +} + +OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) { + // use the maximum number of runtime-arguments here because it is difficult to + // distinguish each RT-Call. + // Note: This number affects also the RT-Call in generate_handle_exception because + // the oop-map is shared for all calls. + DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob(); + assert(deopt_blob != NULL, "deoptimization blob must have been created"); + + OopMap* oop_map = save_live_registers(sasm); + assert_cond(oop_map != NULL); + + __ mv(c_rarg0, xthread); + Label retaddr; + __ set_last_Java_frame(sp, fp, retaddr, t0); + // do the call + RuntimeAddress addr(target); + __ relocate(addr.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, addr, offset); + __ jalr(x1, t0, offset); + }); + __ bind(retaddr); + OopMapSet* oop_maps = new OopMapSet(); + assert_cond(oop_maps != NULL); + oop_maps->add_gc_map(__ offset(), oop_map); + // verify callee-saved register +#ifdef ASSERT + { Label L; + __ get_thread(t0); + __ beq(xthread, t0, L); + __ stop("StubAssembler::call_RT: xthread not callee saved?"); + __ bind(L); + } +#endif + __ reset_last_Java_frame(true); + +#ifdef ASSERT + // Check that fields in JavaThread for exception oop and issuing pc are empty + Label oop_empty; + __ ld(t0, Address(xthread, Thread::pending_exception_offset())); + __ beqz(t0, oop_empty); + __ stop("exception oop must be empty"); + __ bind(oop_empty); + + Label pc_empty; + __ ld(t0, Address(xthread, JavaThread::exception_pc_offset())); + __ beqz(t0, pc_empty); + __ stop("exception pc must be empty"); + __ bind(pc_empty); +#endif + + // Runtime will return true if the nmethod has been deoptimized, this is the + // expected scenario and anything else is an error. Note that we maintain a + // check on the result purely as a defensive measure. + Label no_deopt; + __ beqz(x10, no_deopt); // Have we deoptimized? + + // Perform a re-execute. The proper return address is already on the stack, + // we just need to restore registers, pop all of our frames but the return + // address and jump to the deopt blob. + + restore_live_registers(sasm); + __ leave(); + __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution())); + + __ bind(no_deopt); + __ stop("deopt not performed"); + + return oop_maps; +} + +OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) { + // for better readability + const bool dont_gc_arguments = false; + + // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu + bool save_fpu_registers = true; + + // stub code & info for the different stubs + OopMapSet* oop_maps = NULL; + switch (id) { + { + case forward_exception_id: + { + oop_maps = generate_handle_exception(id, sasm); + __ leave(); + __ ret(); + } + break; + + case throw_div0_exception_id: + { + StubFrame f(sasm, "throw_div0_exception", dont_gc_arguments, does_not_return); + oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false); + } + break; + + case throw_null_pointer_exception_id: + { StubFrame f(sasm, "throw_null_pointer_exception", dont_gc_arguments, does_not_return); + oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false); + } + break; + + case new_instance_id: + case fast_new_instance_id: + case fast_new_instance_init_check_id: + { + Register klass = x13; // Incoming + Register obj = x10; // Result + + if (id == new_instance_id) { + __ set_info("new_instance", dont_gc_arguments); + } else if (id == fast_new_instance_id) { + __ set_info("fast new_instance", dont_gc_arguments); + } else { + assert(id == fast_new_instance_init_check_id, "bad StubID"); + __ set_info("fast new_instance init check", dont_gc_arguments); + } + + // If TLAB is disabled, see if there is support for inlining contiguous + // allocations. + // Otherwise, just go to the slow path. + if ((id == fast_new_instance_id || id == fast_new_instance_init_check_id) && + !UseTLAB && Universe::heap()->supports_inline_contig_alloc()) { + Label slow_path; + Register obj_size = x12; + Register tmp1 = x9; + Register tmp2 = x14; + assert_different_registers(klass, obj, obj_size, tmp1, tmp2); + + const int sp_offset = 2; + const int x9_offset = 1; + const int zr_offset = 0; + __ addi(sp, sp, -(sp_offset * wordSize)); + __ sd(x9, Address(sp, x9_offset * wordSize)); + __ sd(zr, Address(sp, zr_offset * wordSize)); + + if (id == fast_new_instance_init_check_id) { + // make sure the klass is initialized + __ lbu(t0, Address(klass, InstanceKlass::init_state_offset())); + __ mv(t1, InstanceKlass::fully_initialized); + __ bne(t0, t1, slow_path); + } + +#ifdef ASSERT + // assert object can be fast path allocated + { + Label ok, not_ok; + __ lw(obj_size, Address(klass, Klass::layout_helper_offset())); + // make sure it's an instance. For instances, layout helper is a positive number. + // For arrays, layout helper is a negative number + __ blez(obj_size, not_ok); + __ andi(t0, obj_size, Klass::_lh_instance_slow_path_bit); + __ beqz(t0, ok); + __ bind(not_ok); + __ stop("assert(can be fast path allocated)"); + __ should_not_reach_here(); + __ bind(ok); + } +#endif // ASSERT + + // get the instance size + __ lwu(obj_size, Address(klass, Klass::layout_helper_offset())); + + __ eden_allocate(obj, obj_size, 0, tmp1, slow_path); + + __ initialize_object(obj, klass, obj_size, 0, tmp1, tmp2, /* is_tlab_allocated */ false); + __ verify_oop(obj); + __ ld(x9, Address(sp, x9_offset * wordSize)); + __ ld(zr, Address(sp, zr_offset * wordSize)); + __ addi(sp, sp, sp_offset * wordSize); + __ ret(); + + __ bind(slow_path); + __ ld(x9, Address(sp, x9_offset * wordSize)); + __ ld(zr, Address(sp, zr_offset * wordSize)); + __ addi(sp, sp, sp_offset * wordSize); + } + + __ enter(); + OopMap* map = save_live_registers(sasm); + assert_cond(map != NULL); + int call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass); + oop_maps = new OopMapSet(); + assert_cond(oop_maps != NULL); + oop_maps->add_gc_map(call_offset, map); + restore_live_registers_except_r10(sasm); + __ verify_oop(obj); + __ leave(); + __ ret(); + + // x10: new instance + } + + break; + + case counter_overflow_id: + { + Register bci = x10; + Register method = x11; + __ enter(); + OopMap* map = save_live_registers(sasm); + assert_cond(map != NULL); + + const int bci_off = 0; + const int method_off = 1; + // Retrieve bci + __ lw(bci, Address(fp, bci_off * BytesPerWord)); + // And a pointer to the Method* + __ ld(method, Address(fp, method_off * BytesPerWord)); + int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), bci, method); + oop_maps = new OopMapSet(); + assert_cond(oop_maps != NULL); + oop_maps->add_gc_map(call_offset, map); + restore_live_registers(sasm); + __ leave(); + __ ret(); + } + break; + + case new_type_array_id: + case new_object_array_id: + { + Register length = x9; // Incoming + Register klass = x13; // Incoming + Register obj = x10; // Result + + if (id == new_type_array_id) { + __ set_info("new_type_array", dont_gc_arguments); + } else { + __ set_info("new_object_array", dont_gc_arguments); + } + +#ifdef ASSERT + // assert object type is really an array of the proper kind + { + Label ok; + Register tmp = obj; + __ lwu(tmp, Address(klass, Klass::layout_helper_offset())); + __ sraiw(tmp, tmp, Klass::_lh_array_tag_shift); + int tag = ((id == new_type_array_id) ? Klass::_lh_array_tag_type_value : Klass::_lh_array_tag_obj_value); + __ mv(t0, tag); + __ beq(t0, tmp, ok); + __ stop("assert(is an array klass)"); + __ should_not_reach_here(); + __ bind(ok); + } +#endif // ASSERT + + // If TLAB is disabled, see if there is support for inlining contiguous + // allocations. + // Otherwise, just go to the slow path. + if (!UseTLAB && Universe::heap()->supports_inline_contig_alloc()) { + Register arr_size = x14; + Register tmp1 = x12; + Register tmp2 = x15; + Label slow_path; + assert_different_registers(length, klass, obj, arr_size, tmp1, tmp2); + + // check that array length is small enough for fast path. + __ mv(t0, C1_MacroAssembler::max_array_allocation_length); + __ bgtu(length, t0, slow_path); + + // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F)) + __ lwu(tmp1, Address(klass, Klass::layout_helper_offset())); + __ andi(t0, tmp1, 0x1f); + __ sll(arr_size, length, t0); + int lh_header_size_width = exact_log2(Klass::_lh_header_size_mask + 1); + int lh_header_size_msb = Klass::_lh_header_size_shift + lh_header_size_width; + __ slli(tmp1, tmp1, XLEN - lh_header_size_msb); + __ srli(tmp1, tmp1, XLEN - lh_header_size_width); + __ add(arr_size, arr_size, tmp1); + __ addi(arr_size, arr_size, MinObjAlignmentInBytesMask); // align up + __ andi(arr_size, arr_size, ~(uint)MinObjAlignmentInBytesMask); + + __ eden_allocate(obj, arr_size, 0, tmp1, slow_path); // preserves arr_size + + __ initialize_header(obj, klass, length, tmp1, tmp2); + __ lbu(tmp1, Address(klass, + in_bytes(Klass::layout_helper_offset()) + + (Klass::_lh_header_size_shift / BitsPerByte))); + assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise"); + assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise"); + __ andi(tmp1, tmp1, Klass::_lh_header_size_mask); + __ sub(arr_size, arr_size, tmp1); // body length + __ add(tmp1, tmp1, obj); // body start + __ initialize_body(tmp1, arr_size, 0, tmp2); + __ membar(MacroAssembler::StoreStore); + __ verify_oop(obj); + + __ ret(); + + __ bind(slow_path); + } + + __ enter(); + OopMap* map = save_live_registers(sasm); + assert_cond(map != NULL); + int call_offset = 0; + if (id == new_type_array_id) { + call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length); + } else { + call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length); + } + + oop_maps = new OopMapSet(); + assert_cond(oop_maps != NULL); + oop_maps->add_gc_map(call_offset, map); + restore_live_registers_except_r10(sasm); + + __ verify_oop(obj); + __ leave(); + __ ret(); + + // x10: new array + } + break; + + case new_multi_array_id: + { + StubFrame f(sasm, "new_multi_array", dont_gc_arguments); + // x10: klass + // x9: rank + // x12: address of 1st dimension + OopMap* map = save_live_registers(sasm); + assert_cond(map != NULL); + __ mv(c_rarg1, x10); + __ mv(c_rarg3, x12); + __ mv(c_rarg2, x9); + int call_offset = __ call_RT(x10, noreg, CAST_FROM_FN_PTR(address, new_multi_array), x11, x12, x13); + + oop_maps = new OopMapSet(); + assert_cond(oop_maps != NULL); + oop_maps->add_gc_map(call_offset, map); + restore_live_registers_except_r10(sasm); + + // x10: new multi array + __ verify_oop(x10); + } + break; + + case register_finalizer_id: + { + __ set_info("register_finalizer", dont_gc_arguments); + + // This is called via call_runtime so the arguments + // will be place in C abi locations + __ verify_oop(c_rarg0); + + // load the klass and check the has finalizer flag + Label register_finalizer; + Register t = x15; + __ load_klass(t, x10); + __ lwu(t, Address(t, Klass::access_flags_offset())); + __ test_bit(t0, t, exact_log2(JVM_ACC_HAS_FINALIZER)); + __ bnez(t0, register_finalizer); + __ ret(); + + __ bind(register_finalizer); + __ enter(); + OopMap* oop_map = save_live_registers(sasm); + assert_cond(oop_map != NULL); + int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), x10); + oop_maps = new OopMapSet(); + assert_cond(oop_maps != NULL); + oop_maps->add_gc_map(call_offset, oop_map); + + // Now restore all the live registers + restore_live_registers(sasm); + + __ leave(); + __ ret(); + } + break; + + case throw_class_cast_exception_id: + { + StubFrame f(sasm, "throw_class_cast_exception", dont_gc_arguments, does_not_return); + oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true); + } + break; + + case throw_incompatible_class_change_error_id: + { + StubFrame f(sasm, "throw_incompatible_class_cast_exception", dont_gc_arguments, does_not_return); + oop_maps = generate_exception_throw(sasm, + CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false); + } + break; + + case slow_subtype_check_id: + { + // Typical calling sequence: + // push klass_RInfo (object klass or other subclass) + // push sup_k_RInfo (array element klass or other superclass) + // jump to slow_subtype_check + // Note that the subclass is pushed first, and is therefore deepest. + enum layout { + x10_off, x10_off_hi, + x12_off, x12_off_hi, + x14_off, x14_off_hi, + x15_off, x15_off_hi, + sup_k_off, sup_k_off_hi, + klass_off, klass_off_hi, + framesize, + result_off = sup_k_off + }; + + __ set_info("slow_subtype_check", dont_gc_arguments); + __ push_reg(RegSet::of(x10, x12, x14, x15), sp); + + __ ld(x14, Address(sp, (klass_off) * VMRegImpl::stack_slot_size)); // sub klass + __ ld(x10, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size)); // super klass + + Label miss; + __ check_klass_subtype_slow_path(x14, x10, x12, x15, NULL, &miss); + + // fallthrough on success: + __ mv(t0, 1); + __ sd(t0, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result + __ pop_reg(RegSet::of(x10, x12, x14, x15), sp); + __ ret(); + + __ bind(miss); + __ sd(zr, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result + __ pop_reg(RegSet::of(x10, x12, x14, x15), sp); + __ ret(); + } + break; + + case monitorenter_nofpu_id: + save_fpu_registers = false; + // fall through + case monitorenter_id: + { + StubFrame f(sasm, "monitorenter", dont_gc_arguments); + OopMap* map = save_live_registers(sasm, save_fpu_registers); + assert_cond(map != NULL); + + // Called with store_parameter and not C abi + f.load_argument(1, x10); // x10: object + f.load_argument(0, x11); // x11: lock address + + int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), x10, x11); + + oop_maps = new OopMapSet(); + assert_cond(oop_maps != NULL); + oop_maps->add_gc_map(call_offset, map); + restore_live_registers(sasm, save_fpu_registers); + } + break; + + case monitorexit_nofpu_id: + save_fpu_registers = false; + // fall through + case monitorexit_id: + { + StubFrame f(sasm, "monitorexit", dont_gc_arguments); + OopMap* map = save_live_registers(sasm, save_fpu_registers); + assert_cond(map != NULL); + + // Called with store_parameter and not C abi + f.load_argument(0, x10); // x10: lock address + + // note: really a leaf routine but must setup last java sp + // => use call_RT for now (speed can be improved by + // doing last java sp setup manually) + int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), x10); + + oop_maps = new OopMapSet(); + assert_cond(oop_maps != NULL); + oop_maps->add_gc_map(call_offset, map); + restore_live_registers(sasm, save_fpu_registers); + } + break; + + case deoptimize_id: + { + StubFrame f(sasm, "deoptimize", dont_gc_arguments, does_not_return); + OopMap* oop_map = save_live_registers(sasm); + assert_cond(oop_map != NULL); + f.load_argument(0, c_rarg1); + int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize), c_rarg1); + + oop_maps = new OopMapSet(); + assert_cond(oop_maps != NULL); + oop_maps->add_gc_map(call_offset, oop_map); + restore_live_registers(sasm); + DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob(); + assert(deopt_blob != NULL, "deoptimization blob must have been created"); + __ leave(); + __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution())); + } + break; + + case throw_range_check_failed_id: + { + StubFrame f(sasm, "range_check_failed", dont_gc_arguments, does_not_return); + oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true); + } + break; + + case unwind_exception_id: + { + __ set_info("unwind_exception", dont_gc_arguments); + // note: no stubframe since we are about to leave the current + // activation and we are calling a leaf VM function only. + generate_unwind_exception(sasm); + } + break; + + case access_field_patching_id: + { + StubFrame f(sasm, "access_field_patching", dont_gc_arguments, does_not_return); + // we should set up register map + oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching)); + } + break; + + case load_klass_patching_id: + { + StubFrame f(sasm, "load_klass_patching", dont_gc_arguments, does_not_return); + // we should set up register map + oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching)); + } + break; + + case load_mirror_patching_id: + { + StubFrame f(sasm, "load_mirror_patching", dont_gc_arguments, does_not_return); + // we should set up register map + oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching)); + } + break; + + case load_appendix_patching_id: + { + StubFrame f(sasm, "load_appendix_patching", dont_gc_arguments, does_not_return); + // we should set up register map + oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_appendix_patching)); + } + break; + + case handle_exception_nofpu_id: + case handle_exception_id: + { + StubFrame f(sasm, "handle_exception", dont_gc_arguments); + oop_maps = generate_handle_exception(id, sasm); + } + break; + + case handle_exception_from_callee_id: + { + StubFrame f(sasm, "handle_exception_from_callee", dont_gc_arguments); + oop_maps = generate_handle_exception(id, sasm); + } + break; + + case throw_index_exception_id: + { + StubFrame f(sasm, "index_range_check_failed", dont_gc_arguments, does_not_return); + oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true); + } + break; + + case throw_array_store_exception_id: + { + StubFrame f(sasm, "throw_array_store_exception", dont_gc_arguments, does_not_return); + // tos + 0: link + // + 1: return address + oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true); + } + break; + + case predicate_failed_trap_id: + { + StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments, does_not_return); + + OopMap* map = save_live_registers(sasm); + assert_cond(map != NULL); + + int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap)); + oop_maps = new OopMapSet(); + assert_cond(oop_maps != NULL); + oop_maps->add_gc_map(call_offset, map); + restore_live_registers(sasm); + __ leave(); + DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob(); + assert(deopt_blob != NULL, "deoptimization blob must have been created"); + + __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution())); + } + break; + + case dtrace_object_alloc_id: + { // c_rarg0: object + StubFrame f(sasm, "dtrace_object_alloc", dont_gc_arguments); + save_live_registers(sasm); + + __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), c_rarg0); + + restore_live_registers(sasm); + } + break; + + default: + { + StubFrame f(sasm, "unimplemented entry", dont_gc_arguments, does_not_return); + __ mv(x10, (int)id); + __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), x10); + __ should_not_reach_here(); + } + break; + } + } + return oop_maps; +} + +#undef __ + +const char *Runtime1::pd_name_for_address(address entry) { Unimplemented(); return 0; } diff --git a/src/hotspot/cpu/riscv/c1_globals_riscv.hpp b/src/hotspot/cpu/riscv/c1_globals_riscv.hpp new file mode 100644 index 0000000000000..317107ae91148 --- /dev/null +++ b/src/hotspot/cpu/riscv/c1_globals_riscv.hpp @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_C1_GLOBALS_RISCV_HPP +#define CPU_RISCV_C1_GLOBALS_RISCV_HPP + +#include "utilities/globalDefinitions.hpp" +#include "utilities/macros.hpp" + +// Sets the default values for platform dependent flags used by the client compiler. +// (see c1_globals.hpp) + +#ifndef COMPILER2 +define_pd_global(bool, BackgroundCompilation, true ); +define_pd_global(bool, InlineIntrinsics, true ); +define_pd_global(bool, PreferInterpreterNativeStubs, false); +define_pd_global(bool, ProfileTraps, false); +define_pd_global(bool, UseOnStackReplacement, true ); +define_pd_global(bool, TieredCompilation, false); +define_pd_global(intx, CompileThreshold, 1500 ); + +define_pd_global(intx, OnStackReplacePercentage, 933 ); +define_pd_global(intx, NewSizeThreadIncrease, 4*K ); +define_pd_global(intx, InitialCodeCacheSize, 160*K); +define_pd_global(intx, ReservedCodeCacheSize, 32*M ); +define_pd_global(intx, NonProfiledCodeHeapSize, 13*M ); +define_pd_global(intx, ProfiledCodeHeapSize, 14*M ); +define_pd_global(intx, NonNMethodCodeHeapSize, 5*M ); +define_pd_global(bool, ProfileInterpreter, false); +define_pd_global(intx, CodeCacheExpansionSize, 32*K ); +define_pd_global(uintx, CodeCacheMinBlockLength, 1); +define_pd_global(uintx, CodeCacheMinimumUseSpace, 400*K); +define_pd_global(bool, NeverActAsServerClassMachine, true ); +define_pd_global(uint64_t, MaxRAM, 1ULL*G); +define_pd_global(bool, CICompileOSR, true ); +#endif // !COMPILER2 +define_pd_global(bool, UseTypeProfile, false); + +define_pd_global(bool, OptimizeSinglePrecision, true ); +define_pd_global(bool, CSEArrayLength, false); +define_pd_global(bool, TwoOperandLIRForm, false); + +#endif // CPU_RISCV_C1_GLOBALS_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp new file mode 100644 index 0000000000000..15220c4133e02 --- /dev/null +++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp @@ -0,0 +1,1644 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/assembler.hpp" +#include "asm/assembler.inline.hpp" +#include "opto/c2_MacroAssembler.hpp" +#include "opto/intrinsicnode.hpp" +#include "opto/subnode.hpp" +#include "runtime/stubRoutines.hpp" + +#ifdef PRODUCT +#define BLOCK_COMMENT(str) /* nothing */ +#define STOP(error) stop(error) +#else +#define BLOCK_COMMENT(str) block_comment(str) +#define STOP(error) block_comment(error); stop(error) +#endif + +#define BIND(label) bind(label); BLOCK_COMMENT(#label ":") + +// short string +// StringUTF16.indexOfChar +// StringLatin1.indexOfChar +void C2_MacroAssembler::string_indexof_char_short(Register str1, Register cnt1, + Register ch, Register result, + bool isL) +{ + Register ch1 = t0; + Register index = t1; + + BLOCK_COMMENT("string_indexof_char_short {"); + + Label LOOP, LOOP1, LOOP4, LOOP8; + Label MATCH, MATCH1, MATCH2, MATCH3, + MATCH4, MATCH5, MATCH6, MATCH7, NOMATCH; + + mv(result, -1); + mv(index, zr); + + bind(LOOP); + addi(t0, index, 8); + ble(t0, cnt1, LOOP8); + addi(t0, index, 4); + ble(t0, cnt1, LOOP4); + j(LOOP1); + + bind(LOOP8); + isL ? lbu(ch1, Address(str1, 0)) : lhu(ch1, Address(str1, 0)); + beq(ch, ch1, MATCH); + isL ? lbu(ch1, Address(str1, 1)) : lhu(ch1, Address(str1, 2)); + beq(ch, ch1, MATCH1); + isL ? lbu(ch1, Address(str1, 2)) : lhu(ch1, Address(str1, 4)); + beq(ch, ch1, MATCH2); + isL ? lbu(ch1, Address(str1, 3)) : lhu(ch1, Address(str1, 6)); + beq(ch, ch1, MATCH3); + isL ? lbu(ch1, Address(str1, 4)) : lhu(ch1, Address(str1, 8)); + beq(ch, ch1, MATCH4); + isL ? lbu(ch1, Address(str1, 5)) : lhu(ch1, Address(str1, 10)); + beq(ch, ch1, MATCH5); + isL ? lbu(ch1, Address(str1, 6)) : lhu(ch1, Address(str1, 12)); + beq(ch, ch1, MATCH6); + isL ? lbu(ch1, Address(str1, 7)) : lhu(ch1, Address(str1, 14)); + beq(ch, ch1, MATCH7); + addi(index, index, 8); + addi(str1, str1, isL ? 8 : 16); + blt(index, cnt1, LOOP); + j(NOMATCH); + + bind(LOOP4); + isL ? lbu(ch1, Address(str1, 0)) : lhu(ch1, Address(str1, 0)); + beq(ch, ch1, MATCH); + isL ? lbu(ch1, Address(str1, 1)) : lhu(ch1, Address(str1, 2)); + beq(ch, ch1, MATCH1); + isL ? lbu(ch1, Address(str1, 2)) : lhu(ch1, Address(str1, 4)); + beq(ch, ch1, MATCH2); + isL ? lbu(ch1, Address(str1, 3)) : lhu(ch1, Address(str1, 6)); + beq(ch, ch1, MATCH3); + addi(index, index, 4); + addi(str1, str1, isL ? 4 : 8); + bge(index, cnt1, NOMATCH); + + bind(LOOP1); + isL ? lbu(ch1, Address(str1)) : lhu(ch1, Address(str1)); + beq(ch, ch1, MATCH); + addi(index, index, 1); + addi(str1, str1, isL ? 1 : 2); + blt(index, cnt1, LOOP1); + j(NOMATCH); + + bind(MATCH1); + addi(index, index, 1); + j(MATCH); + + bind(MATCH2); + addi(index, index, 2); + j(MATCH); + + bind(MATCH3); + addi(index, index, 3); + j(MATCH); + + bind(MATCH4); + addi(index, index, 4); + j(MATCH); + + bind(MATCH5); + addi(index, index, 5); + j(MATCH); + + bind(MATCH6); + addi(index, index, 6); + j(MATCH); + + bind(MATCH7); + addi(index, index, 7); + + bind(MATCH); + mv(result, index); + bind(NOMATCH); + BLOCK_COMMENT("} string_indexof_char_short"); +} + +// StringUTF16.indexOfChar +// StringLatin1.indexOfChar +void C2_MacroAssembler::string_indexof_char(Register str1, Register cnt1, + Register ch, Register result, + Register tmp1, Register tmp2, + Register tmp3, Register tmp4, + bool isL) +{ + Label CH1_LOOP, HIT, NOMATCH, DONE, DO_LONG; + Register ch1 = t0; + Register orig_cnt = t1; + Register mask1 = tmp3; + Register mask2 = tmp2; + Register match_mask = tmp1; + Register trailing_char = tmp4; + Register unaligned_elems = tmp4; + + BLOCK_COMMENT("string_indexof_char {"); + beqz(cnt1, NOMATCH); + + addi(t0, cnt1, isL ? -32 : -16); + bgtz(t0, DO_LONG); + string_indexof_char_short(str1, cnt1, ch, result, isL); + j(DONE); + + bind(DO_LONG); + mv(orig_cnt, cnt1); + if (AvoidUnalignedAccesses) { + Label ALIGNED; + andi(unaligned_elems, str1, 0x7); + beqz(unaligned_elems, ALIGNED); + sub(unaligned_elems, unaligned_elems, 8); + neg(unaligned_elems, unaligned_elems); + if (!isL) { + srli(unaligned_elems, unaligned_elems, 1); + } + // do unaligned part per element + string_indexof_char_short(str1, unaligned_elems, ch, result, isL); + bgez(result, DONE); + mv(orig_cnt, cnt1); + sub(cnt1, cnt1, unaligned_elems); + bind(ALIGNED); + } + + // duplicate ch + if (isL) { + slli(ch1, ch, 8); + orr(ch, ch1, ch); + } + slli(ch1, ch, 16); + orr(ch, ch1, ch); + slli(ch1, ch, 32); + orr(ch, ch1, ch); + + if (!isL) { + slli(cnt1, cnt1, 1); + } + + uint64_t mask0101 = UCONST64(0x0101010101010101); + uint64_t mask0001 = UCONST64(0x0001000100010001); + mv(mask1, isL ? mask0101 : mask0001); + uint64_t mask7f7f = UCONST64(0x7f7f7f7f7f7f7f7f); + uint64_t mask7fff = UCONST64(0x7fff7fff7fff7fff); + mv(mask2, isL ? mask7f7f : mask7fff); + + bind(CH1_LOOP); + ld(ch1, Address(str1)); + addi(str1, str1, 8); + addi(cnt1, cnt1, -8); + compute_match_mask(ch1, ch, match_mask, mask1, mask2); + bnez(match_mask, HIT); + bgtz(cnt1, CH1_LOOP); + j(NOMATCH); + + bind(HIT); + ctzc_bit(trailing_char, match_mask, isL, ch1, result); + srli(trailing_char, trailing_char, 3); + addi(cnt1, cnt1, 8); + ble(cnt1, trailing_char, NOMATCH); + // match case + if (!isL) { + srli(cnt1, cnt1, 1); + srli(trailing_char, trailing_char, 1); + } + + sub(result, orig_cnt, cnt1); + add(result, result, trailing_char); + j(DONE); + + bind(NOMATCH); + mv(result, -1); + + bind(DONE); + BLOCK_COMMENT("} string_indexof_char"); +} + +typedef void (MacroAssembler::* load_chr_insn)(Register rd, const Address &adr, Register temp); + +// Search for needle in haystack and return index or -1 +// x10: result +// x11: haystack +// x12: haystack_len +// x13: needle +// x14: needle_len +void C2_MacroAssembler::string_indexof(Register haystack, Register needle, + Register haystack_len, Register needle_len, + Register tmp1, Register tmp2, + Register tmp3, Register tmp4, + Register tmp5, Register tmp6, + Register result, int ae) +{ + assert(ae != StrIntrinsicNode::LU, "Invalid encoding"); + + Label LINEARSEARCH, LINEARSTUB, DONE, NOMATCH; + + Register ch1 = t0; + Register ch2 = t1; + Register nlen_tmp = tmp1; // needle len tmp + Register hlen_tmp = tmp2; // haystack len tmp + Register result_tmp = tmp4; + + bool isLL = ae == StrIntrinsicNode::LL; + + bool needle_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL; + bool haystack_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::LU; + int needle_chr_shift = needle_isL ? 0 : 1; + int haystack_chr_shift = haystack_isL ? 0 : 1; + int needle_chr_size = needle_isL ? 1 : 2; + int haystack_chr_size = haystack_isL ? 1 : 2; + load_chr_insn needle_load_1chr = needle_isL ? (load_chr_insn)&MacroAssembler::lbu : + (load_chr_insn)&MacroAssembler::lhu; + load_chr_insn haystack_load_1chr = haystack_isL ? (load_chr_insn)&MacroAssembler::lbu : + (load_chr_insn)&MacroAssembler::lhu; + + BLOCK_COMMENT("string_indexof {"); + + // Note, inline_string_indexOf() generates checks: + // if (pattern.count > src.count) return -1; + // if (pattern.count == 0) return 0; + + // We have two strings, a source string in haystack, haystack_len and a pattern string + // in needle, needle_len. Find the first occurence of pattern in source or return -1. + + // For larger pattern and source we use a simplified Boyer Moore algorithm. + // With a small pattern and source we use linear scan. + + // needle_len >=8 && needle_len < 256 && needle_len < haystack_len/4, use bmh algorithm. + sub(result_tmp, haystack_len, needle_len); + // needle_len < 8, use linear scan + sub(t0, needle_len, 8); + bltz(t0, LINEARSEARCH); + // needle_len >= 256, use linear scan + sub(t0, needle_len, 256); + bgez(t0, LINEARSTUB); + // needle_len >= haystack_len/4, use linear scan + srli(t0, haystack_len, 2); + bge(needle_len, t0, LINEARSTUB); + + // Boyer-Moore-Horspool introduction: + // The Boyer Moore alogorithm is based on the description here:- + // + // http://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_string_search_algorithm + // + // This describes and algorithm with 2 shift rules. The 'Bad Character' rule + // and the 'Good Suffix' rule. + // + // These rules are essentially heuristics for how far we can shift the + // pattern along the search string. + // + // The implementation here uses the 'Bad Character' rule only because of the + // complexity of initialisation for the 'Good Suffix' rule. + // + // This is also known as the Boyer-Moore-Horspool algorithm: + // + // http://en.wikipedia.org/wiki/Boyer-Moore-Horspool_algorithm + // + // #define ASIZE 256 + // + // int bm(unsigned char *pattern, int m, unsigned char *src, int n) { + // int i, j; + // unsigned c; + // unsigned char bc[ASIZE]; + // + // /* Preprocessing */ + // for (i = 0; i < ASIZE; ++i) + // bc[i] = m; + // for (i = 0; i < m - 1; ) { + // c = pattern[i]; + // ++i; + // // c < 256 for Latin1 string, so, no need for branch + // #ifdef PATTERN_STRING_IS_LATIN1 + // bc[c] = m - i; + // #else + // if (c < ASIZE) bc[c] = m - i; + // #endif + // } + // + // /* Searching */ + // j = 0; + // while (j <= n - m) { + // c = src[i+j]; + // if (pattern[m-1] == c) + // int k; + // for (k = m - 2; k >= 0 && pattern[k] == src[k + j]; --k); + // if (k < 0) return j; + // // c < 256 for Latin1 string, so, no need for branch + // #ifdef SOURCE_STRING_IS_LATIN1_AND_PATTERN_STRING_IS_LATIN1 + // // LL case: (c< 256) always true. Remove branch + // j += bc[pattern[j+m-1]]; + // #endif + // #ifdef SOURCE_STRING_IS_UTF_AND_PATTERN_STRING_IS_UTF + // // UU case: need if (c if not. + // if (c < ASIZE) + // j += bc[pattern[j+m-1]]; + // else + // j += m + // #endif + // } + // return -1; + // } + + // temp register:t0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, result + Label BCLOOP, BCSKIP, BMLOOPSTR2, BMLOOPSTR1, BMSKIP, BMADV, BMMATCH, + BMLOOPSTR1_LASTCMP, BMLOOPSTR1_CMP, BMLOOPSTR1_AFTER_LOAD, BM_INIT_LOOP; + + Register haystack_end = haystack_len; + Register skipch = tmp2; + + // pattern length is >=8, so, we can read at least 1 register for cases when + // UTF->Latin1 conversion is not needed(8 LL or 4UU) and half register for + // UL case. We'll re-read last character in inner pre-loop code to have + // single outer pre-loop load + const int firstStep = isLL ? 7 : 3; + + const int ASIZE = 256; + const int STORE_BYTES = 8; // 8 bytes stored per instruction(sd) + + sub(sp, sp, ASIZE); + + // init BC offset table with default value: needle_len + slli(t0, needle_len, 8); + orr(t0, t0, needle_len); // [63...16][needle_len][needle_len] + slli(tmp1, t0, 16); + orr(t0, tmp1, t0); // [63...32][needle_len][needle_len][needle_len][needle_len] + slli(tmp1, t0, 32); + orr(tmp5, tmp1, t0); // tmp5: 8 elements [needle_len] + + mv(ch1, sp); // ch1 is t0 + mv(tmp6, ASIZE / STORE_BYTES); // loop iterations + + bind(BM_INIT_LOOP); + // for (i = 0; i < ASIZE; ++i) + // bc[i] = m; + for (int i = 0; i < 4; i++) { + sd(tmp5, Address(ch1, i * wordSize)); + } + add(ch1, ch1, 32); + sub(tmp6, tmp6, 4); + bgtz(tmp6, BM_INIT_LOOP); + + sub(nlen_tmp, needle_len, 1); // m - 1, index of the last element in pattern + Register orig_haystack = tmp5; + mv(orig_haystack, haystack); + // result_tmp = tmp4 + shadd(haystack_end, result_tmp, haystack, haystack_end, haystack_chr_shift); + sub(ch2, needle_len, 1); // bc offset init value, ch2 is t1 + mv(tmp3, needle); + + // for (i = 0; i < m - 1; ) { + // c = pattern[i]; + // ++i; + // // c < 256 for Latin1 string, so, no need for branch + // #ifdef PATTERN_STRING_IS_LATIN1 + // bc[c] = m - i; + // #else + // if (c < ASIZE) bc[c] = m - i; + // #endif + // } + bind(BCLOOP); + (this->*needle_load_1chr)(ch1, Address(tmp3), noreg); + add(tmp3, tmp3, needle_chr_size); + if (!needle_isL) { + // ae == StrIntrinsicNode::UU + mv(tmp6, ASIZE); + bgeu(ch1, tmp6, BCSKIP); + } + add(tmp4, sp, ch1); + sb(ch2, Address(tmp4)); // store skip offset to BC offset table + + bind(BCSKIP); + sub(ch2, ch2, 1); // for next pattern element, skip distance -1 + bgtz(ch2, BCLOOP); + + // tmp6: pattern end, address after needle + shadd(tmp6, needle_len, needle, tmp6, needle_chr_shift); + if (needle_isL == haystack_isL) { + // load last 8 bytes (8LL/4UU symbols) + ld(tmp6, Address(tmp6, -wordSize)); + } else { + // UL: from UTF-16(source) search Latin1(pattern) + lwu(tmp6, Address(tmp6, -wordSize / 2)); // load last 4 bytes(4 symbols) + // convert Latin1 to UTF. eg: 0x0000abcd -> 0x0a0b0c0d + // We'll have to wait until load completed, but it's still faster than per-character loads+checks + srli(tmp3, tmp6, BitsPerByte * (wordSize / 2 - needle_chr_size)); // pattern[m-1], eg:0x0000000a + slli(ch2, tmp6, XLEN - 24); + srli(ch2, ch2, XLEN - 8); // pattern[m-2], 0x0000000b + slli(ch1, tmp6, XLEN - 16); + srli(ch1, ch1, XLEN - 8); // pattern[m-3], 0x0000000c + andi(tmp6, tmp6, 0xff); // pattern[m-4], 0x0000000d + slli(ch2, ch2, 16); + orr(ch2, ch2, ch1); // 0x00000b0c + slli(result, tmp3, 48); // use result as temp register + orr(tmp6, tmp6, result); // 0x0a00000d + slli(result, ch2, 16); + orr(tmp6, tmp6, result); // UTF-16:0x0a0b0c0d + } + + // i = m - 1; + // skipch = j + i; + // if (skipch == pattern[m - 1] + // for (k = m - 2; k >= 0 && pattern[k] == src[k + j]; --k); + // else + // move j with bad char offset table + bind(BMLOOPSTR2); + // compare pattern to source string backward + shadd(result, nlen_tmp, haystack, result, haystack_chr_shift); + (this->*haystack_load_1chr)(skipch, Address(result), noreg); + sub(nlen_tmp, nlen_tmp, firstStep); // nlen_tmp is positive here, because needle_len >= 8 + if (needle_isL == haystack_isL) { + // re-init tmp3. It's for free because it's executed in parallel with + // load above. Alternative is to initialize it before loop, but it'll + // affect performance on in-order systems with 2 or more ld/st pipelines + srli(tmp3, tmp6, BitsPerByte * (wordSize - needle_chr_size)); // UU/LL: pattern[m-1] + } + if (!isLL) { // UU/UL case + slli(ch2, nlen_tmp, 1); // offsets in bytes + } + bne(tmp3, skipch, BMSKIP); // if not equal, skipch is bad char + add(result, haystack, isLL ? nlen_tmp : ch2); + ld(ch2, Address(result)); // load 8 bytes from source string + mv(ch1, tmp6); + if (isLL) { + j(BMLOOPSTR1_AFTER_LOAD); + } else { + sub(nlen_tmp, nlen_tmp, 1); // no need to branch for UU/UL case. cnt1 >= 8 + j(BMLOOPSTR1_CMP); + } + + bind(BMLOOPSTR1); + shadd(ch1, nlen_tmp, needle, ch1, needle_chr_shift); + (this->*needle_load_1chr)(ch1, Address(ch1), noreg); + shadd(ch2, nlen_tmp, haystack, ch2, haystack_chr_shift); + (this->*haystack_load_1chr)(ch2, Address(ch2), noreg); + + bind(BMLOOPSTR1_AFTER_LOAD); + sub(nlen_tmp, nlen_tmp, 1); + bltz(nlen_tmp, BMLOOPSTR1_LASTCMP); + + bind(BMLOOPSTR1_CMP); + beq(ch1, ch2, BMLOOPSTR1); + + bind(BMSKIP); + if (!isLL) { + // if we've met UTF symbol while searching Latin1 pattern, then we can + // skip needle_len symbols + if (needle_isL != haystack_isL) { + mv(result_tmp, needle_len); + } else { + mv(result_tmp, 1); + } + mv(t0, ASIZE); + bgeu(skipch, t0, BMADV); + } + add(result_tmp, sp, skipch); + lbu(result_tmp, Address(result_tmp)); // load skip offset + + bind(BMADV); + sub(nlen_tmp, needle_len, 1); + // move haystack after bad char skip offset + shadd(haystack, result_tmp, haystack, result, haystack_chr_shift); + ble(haystack, haystack_end, BMLOOPSTR2); + add(sp, sp, ASIZE); + j(NOMATCH); + + bind(BMLOOPSTR1_LASTCMP); + bne(ch1, ch2, BMSKIP); + + bind(BMMATCH); + sub(result, haystack, orig_haystack); + if (!haystack_isL) { + srli(result, result, 1); + } + add(sp, sp, ASIZE); + j(DONE); + + bind(LINEARSTUB); + sub(t0, needle_len, 16); // small patterns still should be handled by simple algorithm + bltz(t0, LINEARSEARCH); + mv(result, zr); + RuntimeAddress stub = NULL; + if (isLL) { + stub = RuntimeAddress(StubRoutines::riscv::string_indexof_linear_ll()); + assert(stub.target() != NULL, "string_indexof_linear_ll stub has not been generated"); + } else if (needle_isL) { + stub = RuntimeAddress(StubRoutines::riscv::string_indexof_linear_ul()); + assert(stub.target() != NULL, "string_indexof_linear_ul stub has not been generated"); + } else { + stub = RuntimeAddress(StubRoutines::riscv::string_indexof_linear_uu()); + assert(stub.target() != NULL, "string_indexof_linear_uu stub has not been generated"); + } + trampoline_call(stub); + j(DONE); + + bind(NOMATCH); + mv(result, -1); + j(DONE); + + bind(LINEARSEARCH); + string_indexof_linearscan(haystack, needle, haystack_len, needle_len, tmp1, tmp2, tmp3, tmp4, -1, result, ae); + + bind(DONE); + BLOCK_COMMENT("} string_indexof"); +} + +// string_indexof +// result: x10 +// src: x11 +// src_count: x12 +// pattern: x13 +// pattern_count: x14 or 1/2/3/4 +void C2_MacroAssembler::string_indexof_linearscan(Register haystack, Register needle, + Register haystack_len, Register needle_len, + Register tmp1, Register tmp2, + Register tmp3, Register tmp4, + int needle_con_cnt, Register result, int ae) +{ + // Note: + // needle_con_cnt > 0 means needle_len register is invalid, needle length is constant + // for UU/LL: needle_con_cnt[1, 4], UL: needle_con_cnt = 1 + assert(needle_con_cnt <= 4, "Invalid needle constant count"); + assert(ae != StrIntrinsicNode::LU, "Invalid encoding"); + + Register ch1 = t0; + Register ch2 = t1; + Register hlen_neg = haystack_len, nlen_neg = needle_len; + Register nlen_tmp = tmp1, hlen_tmp = tmp2, result_tmp = tmp4; + + bool isLL = ae == StrIntrinsicNode::LL; + + bool needle_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL; + bool haystack_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::LU; + int needle_chr_shift = needle_isL ? 0 : 1; + int haystack_chr_shift = haystack_isL ? 0 : 1; + int needle_chr_size = needle_isL ? 1 : 2; + int haystack_chr_size = haystack_isL ? 1 : 2; + + load_chr_insn needle_load_1chr = needle_isL ? (load_chr_insn)&MacroAssembler::lbu : + (load_chr_insn)&MacroAssembler::lhu; + load_chr_insn haystack_load_1chr = haystack_isL ? (load_chr_insn)&MacroAssembler::lbu : + (load_chr_insn)&MacroAssembler::lhu; + load_chr_insn load_2chr = isLL ? (load_chr_insn)&MacroAssembler::lhu : (load_chr_insn)&MacroAssembler::lwu; + load_chr_insn load_4chr = isLL ? (load_chr_insn)&MacroAssembler::lwu : (load_chr_insn)&MacroAssembler::ld; + + Label DO1, DO2, DO3, MATCH, NOMATCH, DONE; + + Register first = tmp3; + + if (needle_con_cnt == -1) { + Label DOSHORT, FIRST_LOOP, STR2_NEXT, STR1_LOOP, STR1_NEXT; + + sub(t0, needle_len, needle_isL == haystack_isL ? 4 : 2); + bltz(t0, DOSHORT); + + (this->*needle_load_1chr)(first, Address(needle), noreg); + slli(t0, needle_len, needle_chr_shift); + add(needle, needle, t0); + neg(nlen_neg, t0); + slli(t0, result_tmp, haystack_chr_shift); + add(haystack, haystack, t0); + neg(hlen_neg, t0); + + bind(FIRST_LOOP); + add(t0, haystack, hlen_neg); + (this->*haystack_load_1chr)(ch2, Address(t0), noreg); + beq(first, ch2, STR1_LOOP); + + bind(STR2_NEXT); + add(hlen_neg, hlen_neg, haystack_chr_size); + blez(hlen_neg, FIRST_LOOP); + j(NOMATCH); + + bind(STR1_LOOP); + add(nlen_tmp, nlen_neg, needle_chr_size); + add(hlen_tmp, hlen_neg, haystack_chr_size); + bgez(nlen_tmp, MATCH); + + bind(STR1_NEXT); + add(ch1, needle, nlen_tmp); + (this->*needle_load_1chr)(ch1, Address(ch1), noreg); + add(ch2, haystack, hlen_tmp); + (this->*haystack_load_1chr)(ch2, Address(ch2), noreg); + bne(ch1, ch2, STR2_NEXT); + add(nlen_tmp, nlen_tmp, needle_chr_size); + add(hlen_tmp, hlen_tmp, haystack_chr_size); + bltz(nlen_tmp, STR1_NEXT); + j(MATCH); + + bind(DOSHORT); + if (needle_isL == haystack_isL) { + sub(t0, needle_len, 2); + bltz(t0, DO1); + bgtz(t0, DO3); + } + } + + if (needle_con_cnt == 4) { + Label CH1_LOOP; + (this->*load_4chr)(ch1, Address(needle), noreg); + sub(result_tmp, haystack_len, 4); + slli(tmp3, result_tmp, haystack_chr_shift); // result as tmp + add(haystack, haystack, tmp3); + neg(hlen_neg, tmp3); + + bind(CH1_LOOP); + add(ch2, haystack, hlen_neg); + (this->*load_4chr)(ch2, Address(ch2), noreg); + beq(ch1, ch2, MATCH); + add(hlen_neg, hlen_neg, haystack_chr_size); + blez(hlen_neg, CH1_LOOP); + j(NOMATCH); + } + + if ((needle_con_cnt == -1 && needle_isL == haystack_isL) || needle_con_cnt == 2) { + Label CH1_LOOP; + BLOCK_COMMENT("string_indexof DO2 {"); + bind(DO2); + (this->*load_2chr)(ch1, Address(needle), noreg); + if (needle_con_cnt == 2) { + sub(result_tmp, haystack_len, 2); + } + slli(tmp3, result_tmp, haystack_chr_shift); + add(haystack, haystack, tmp3); + neg(hlen_neg, tmp3); + + bind(CH1_LOOP); + add(tmp3, haystack, hlen_neg); + (this->*load_2chr)(ch2, Address(tmp3), noreg); + beq(ch1, ch2, MATCH); + add(hlen_neg, hlen_neg, haystack_chr_size); + blez(hlen_neg, CH1_LOOP); + j(NOMATCH); + BLOCK_COMMENT("} string_indexof DO2"); + } + + if ((needle_con_cnt == -1 && needle_isL == haystack_isL) || needle_con_cnt == 3) { + Label FIRST_LOOP, STR2_NEXT, STR1_LOOP; + BLOCK_COMMENT("string_indexof DO3 {"); + + bind(DO3); + (this->*load_2chr)(first, Address(needle), noreg); + (this->*needle_load_1chr)(ch1, Address(needle, 2 * needle_chr_size), noreg); + if (needle_con_cnt == 3) { + sub(result_tmp, haystack_len, 3); + } + slli(hlen_tmp, result_tmp, haystack_chr_shift); + add(haystack, haystack, hlen_tmp); + neg(hlen_neg, hlen_tmp); + + bind(FIRST_LOOP); + add(ch2, haystack, hlen_neg); + (this->*load_2chr)(ch2, Address(ch2), noreg); + beq(first, ch2, STR1_LOOP); + + bind(STR2_NEXT); + add(hlen_neg, hlen_neg, haystack_chr_size); + blez(hlen_neg, FIRST_LOOP); + j(NOMATCH); + + bind(STR1_LOOP); + add(hlen_tmp, hlen_neg, 2 * haystack_chr_size); + add(ch2, haystack, hlen_tmp); + (this->*haystack_load_1chr)(ch2, Address(ch2), noreg); + bne(ch1, ch2, STR2_NEXT); + j(MATCH); + BLOCK_COMMENT("} string_indexof DO3"); + } + + if (needle_con_cnt == -1 || needle_con_cnt == 1) { + Label DO1_LOOP; + + BLOCK_COMMENT("string_indexof DO1 {"); + bind(DO1); + (this->*needle_load_1chr)(ch1, Address(needle), noreg); + sub(result_tmp, haystack_len, 1); + mv(tmp3, result_tmp); + if (haystack_chr_shift) { + slli(tmp3, result_tmp, haystack_chr_shift); + } + add(haystack, haystack, tmp3); + neg(hlen_neg, tmp3); + + bind(DO1_LOOP); + add(tmp3, haystack, hlen_neg); + (this->*haystack_load_1chr)(ch2, Address(tmp3), noreg); + beq(ch1, ch2, MATCH); + add(hlen_neg, hlen_neg, haystack_chr_size); + blez(hlen_neg, DO1_LOOP); + BLOCK_COMMENT("} string_indexof DO1"); + } + + bind(NOMATCH); + mv(result, -1); + j(DONE); + + bind(MATCH); + srai(t0, hlen_neg, haystack_chr_shift); + add(result, result_tmp, t0); + + bind(DONE); +} + +// Compare strings. +void C2_MacroAssembler::string_compare(Register str1, Register str2, + Register cnt1, Register cnt2, Register result, Register tmp1, Register tmp2, + Register tmp3, int ae) +{ + Label DONE, SHORT_LOOP, SHORT_STRING, SHORT_LAST, TAIL, STUB, + DIFFERENCE, NEXT_WORD, SHORT_LOOP_TAIL, SHORT_LAST2, SHORT_LAST_INIT, + SHORT_LOOP_START, TAIL_CHECK, L; + + const int STUB_THRESHOLD = 64 + 8; + bool isLL = ae == StrIntrinsicNode::LL; + bool isLU = ae == StrIntrinsicNode::LU; + bool isUL = ae == StrIntrinsicNode::UL; + + bool str1_isL = isLL || isLU; + bool str2_isL = isLL || isUL; + + // for L strings, 1 byte for 1 character + // for U strings, 2 bytes for 1 character + int str1_chr_size = str1_isL ? 1 : 2; + int str2_chr_size = str2_isL ? 1 : 2; + int minCharsInWord = isLL ? wordSize : wordSize / 2; + + load_chr_insn str1_load_chr = str1_isL ? (load_chr_insn)&MacroAssembler::lbu : (load_chr_insn)&MacroAssembler::lhu; + load_chr_insn str2_load_chr = str2_isL ? (load_chr_insn)&MacroAssembler::lbu : (load_chr_insn)&MacroAssembler::lhu; + + BLOCK_COMMENT("string_compare {"); + + // Bizzarely, the counts are passed in bytes, regardless of whether they + // are L or U strings, however the result is always in characters. + if (!str1_isL) { + sraiw(cnt1, cnt1, 1); + } + if (!str2_isL) { + sraiw(cnt2, cnt2, 1); + } + + // Compute the minimum of the string lengths and save the difference in result. + sub(result, cnt1, cnt2); + bgt(cnt1, cnt2, L); + mv(cnt2, cnt1); + bind(L); + + // A very short string + mv(t0, minCharsInWord); + ble(cnt2, t0, SHORT_STRING); + + // Compare longwords + // load first parts of strings and finish initialization while loading + { + if (str1_isL == str2_isL) { // LL or UU + // load 8 bytes once to compare + ld(tmp1, Address(str1)); + beq(str1, str2, DONE); + ld(tmp2, Address(str2)); + mv(t0, STUB_THRESHOLD); + bge(cnt2, t0, STUB); + sub(cnt2, cnt2, minCharsInWord); + beqz(cnt2, TAIL_CHECK); + // convert cnt2 from characters to bytes + if (!str1_isL) { + slli(cnt2, cnt2, 1); + } + add(str2, str2, cnt2); + add(str1, str1, cnt2); + sub(cnt2, zr, cnt2); + } else if (isLU) { // LU case + lwu(tmp1, Address(str1)); + ld(tmp2, Address(str2)); + mv(t0, STUB_THRESHOLD); + bge(cnt2, t0, STUB); + addi(cnt2, cnt2, -4); + add(str1, str1, cnt2); + sub(cnt1, zr, cnt2); + slli(cnt2, cnt2, 1); + add(str2, str2, cnt2); + inflate_lo32(tmp3, tmp1); + mv(tmp1, tmp3); + sub(cnt2, zr, cnt2); + addi(cnt1, cnt1, 4); + } else { // UL case + ld(tmp1, Address(str1)); + lwu(tmp2, Address(str2)); + mv(t0, STUB_THRESHOLD); + bge(cnt2, t0, STUB); + addi(cnt2, cnt2, -4); + slli(t0, cnt2, 1); + sub(cnt1, zr, t0); + add(str1, str1, t0); + add(str2, str2, cnt2); + inflate_lo32(tmp3, tmp2); + mv(tmp2, tmp3); + sub(cnt2, zr, cnt2); + addi(cnt1, cnt1, 8); + } + addi(cnt2, cnt2, isUL ? 4 : 8); + bgez(cnt2, TAIL); + xorr(tmp3, tmp1, tmp2); + bnez(tmp3, DIFFERENCE); + + // main loop + bind(NEXT_WORD); + if (str1_isL == str2_isL) { // LL or UU + add(t0, str1, cnt2); + ld(tmp1, Address(t0)); + add(t0, str2, cnt2); + ld(tmp2, Address(t0)); + addi(cnt2, cnt2, 8); + } else if (isLU) { // LU case + add(t0, str1, cnt1); + lwu(tmp1, Address(t0)); + add(t0, str2, cnt2); + ld(tmp2, Address(t0)); + addi(cnt1, cnt1, 4); + inflate_lo32(tmp3, tmp1); + mv(tmp1, tmp3); + addi(cnt2, cnt2, 8); + } else { // UL case + add(t0, str2, cnt2); + lwu(tmp2, Address(t0)); + add(t0, str1, cnt1); + ld(tmp1, Address(t0)); + inflate_lo32(tmp3, tmp2); + mv(tmp2, tmp3); + addi(cnt1, cnt1, 8); + addi(cnt2, cnt2, 4); + } + bgez(cnt2, TAIL); + + xorr(tmp3, tmp1, tmp2); + beqz(tmp3, NEXT_WORD); + j(DIFFERENCE); + bind(TAIL); + xorr(tmp3, tmp1, tmp2); + bnez(tmp3, DIFFERENCE); + // Last longword. In the case where length == 4 we compare the + // same longword twice, but that's still faster than another + // conditional branch. + if (str1_isL == str2_isL) { // LL or UU + ld(tmp1, Address(str1)); + ld(tmp2, Address(str2)); + } else if (isLU) { // LU case + lwu(tmp1, Address(str1)); + ld(tmp2, Address(str2)); + inflate_lo32(tmp3, tmp1); + mv(tmp1, tmp3); + } else { // UL case + lwu(tmp2, Address(str2)); + ld(tmp1, Address(str1)); + inflate_lo32(tmp3, tmp2); + mv(tmp2, tmp3); + } + bind(TAIL_CHECK); + xorr(tmp3, tmp1, tmp2); + beqz(tmp3, DONE); + + // Find the first different characters in the longwords and + // compute their difference. + bind(DIFFERENCE); + ctzc_bit(result, tmp3, isLL); // count zero from lsb to msb + srl(tmp1, tmp1, result); + srl(tmp2, tmp2, result); + if (isLL) { + andi(tmp1, tmp1, 0xFF); + andi(tmp2, tmp2, 0xFF); + } else { + andi(tmp1, tmp1, 0xFFFF); + andi(tmp2, tmp2, 0xFFFF); + } + sub(result, tmp1, tmp2); + j(DONE); + } + + bind(STUB); + RuntimeAddress stub = NULL; + switch (ae) { + case StrIntrinsicNode::LL: + stub = RuntimeAddress(StubRoutines::riscv::compare_long_string_LL()); + break; + case StrIntrinsicNode::UU: + stub = RuntimeAddress(StubRoutines::riscv::compare_long_string_UU()); + break; + case StrIntrinsicNode::LU: + stub = RuntimeAddress(StubRoutines::riscv::compare_long_string_LU()); + break; + case StrIntrinsicNode::UL: + stub = RuntimeAddress(StubRoutines::riscv::compare_long_string_UL()); + break; + default: + ShouldNotReachHere(); + } + assert(stub.target() != NULL, "compare_long_string stub has not been generated"); + trampoline_call(stub); + j(DONE); + + bind(SHORT_STRING); + // Is the minimum length zero? + beqz(cnt2, DONE); + // arrange code to do most branches while loading and loading next characters + // while comparing previous + (this->*str1_load_chr)(tmp1, Address(str1), t0); + addi(str1, str1, str1_chr_size); + addi(cnt2, cnt2, -1); + beqz(cnt2, SHORT_LAST_INIT); + (this->*str2_load_chr)(cnt1, Address(str2), t0); + addi(str2, str2, str2_chr_size); + j(SHORT_LOOP_START); + bind(SHORT_LOOP); + addi(cnt2, cnt2, -1); + beqz(cnt2, SHORT_LAST); + bind(SHORT_LOOP_START); + (this->*str1_load_chr)(tmp2, Address(str1), t0); + addi(str1, str1, str1_chr_size); + (this->*str2_load_chr)(t0, Address(str2), t0); + addi(str2, str2, str2_chr_size); + bne(tmp1, cnt1, SHORT_LOOP_TAIL); + addi(cnt2, cnt2, -1); + beqz(cnt2, SHORT_LAST2); + (this->*str1_load_chr)(tmp1, Address(str1), t0); + addi(str1, str1, str1_chr_size); + (this->*str2_load_chr)(cnt1, Address(str2), t0); + addi(str2, str2, str2_chr_size); + beq(tmp2, t0, SHORT_LOOP); + sub(result, tmp2, t0); + j(DONE); + bind(SHORT_LOOP_TAIL); + sub(result, tmp1, cnt1); + j(DONE); + bind(SHORT_LAST2); + beq(tmp2, t0, DONE); + sub(result, tmp2, t0); + + j(DONE); + bind(SHORT_LAST_INIT); + (this->*str2_load_chr)(cnt1, Address(str2), t0); + addi(str2, str2, str2_chr_size); + bind(SHORT_LAST); + beq(tmp1, cnt1, DONE); + sub(result, tmp1, cnt1); + + bind(DONE); + + BLOCK_COMMENT("} string_compare"); +} + +void C2_MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3, + Register tmp4, Register tmp5, Register tmp6, Register result, + Register cnt1, int elem_size) { + Label DONE, SAME, NEXT_DWORD, SHORT, TAIL, TAIL2, IS_TMP5_ZR; + Register tmp1 = t0; + Register tmp2 = t1; + Register cnt2 = tmp2; // cnt2 only used in array length compare + Register elem_per_word = tmp6; + int log_elem_size = exact_log2(elem_size); + int length_offset = arrayOopDesc::length_offset_in_bytes(); + int base_offset = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE); + + assert(elem_size == 1 || elem_size == 2, "must be char or byte"); + assert_different_registers(a1, a2, result, cnt1, t0, t1, tmp3, tmp4, tmp5, tmp6); + mv(elem_per_word, wordSize / elem_size); + + BLOCK_COMMENT("arrays_equals {"); + + // if (a1 == a2), return true + beq(a1, a2, SAME); + + mv(result, false); + beqz(a1, DONE); + beqz(a2, DONE); + lwu(cnt1, Address(a1, length_offset)); + lwu(cnt2, Address(a2, length_offset)); + bne(cnt2, cnt1, DONE); + beqz(cnt1, SAME); + + slli(tmp5, cnt1, 3 + log_elem_size); + sub(tmp5, zr, tmp5); + add(a1, a1, base_offset); + add(a2, a2, base_offset); + ld(tmp3, Address(a1, 0)); + ld(tmp4, Address(a2, 0)); + ble(cnt1, elem_per_word, SHORT); // short or same + + // Main 16 byte comparison loop with 2 exits + bind(NEXT_DWORD); { + ld(tmp1, Address(a1, wordSize)); + ld(tmp2, Address(a2, wordSize)); + sub(cnt1, cnt1, 2 * wordSize / elem_size); + blez(cnt1, TAIL); + bne(tmp3, tmp4, DONE); + ld(tmp3, Address(a1, 2 * wordSize)); + ld(tmp4, Address(a2, 2 * wordSize)); + add(a1, a1, 2 * wordSize); + add(a2, a2, 2 * wordSize); + ble(cnt1, elem_per_word, TAIL2); + } beq(tmp1, tmp2, NEXT_DWORD); + j(DONE); + + bind(TAIL); + xorr(tmp4, tmp3, tmp4); + xorr(tmp2, tmp1, tmp2); + sll(tmp2, tmp2, tmp5); + orr(tmp5, tmp4, tmp2); + j(IS_TMP5_ZR); + + bind(TAIL2); + bne(tmp1, tmp2, DONE); + + bind(SHORT); + xorr(tmp4, tmp3, tmp4); + sll(tmp5, tmp4, tmp5); + + bind(IS_TMP5_ZR); + bnez(tmp5, DONE); + + bind(SAME); + mv(result, true); + // That's it. + bind(DONE); + + BLOCK_COMMENT("} array_equals"); +} + +// Compare Strings + +// For Strings we're passed the address of the first characters in a1 +// and a2 and the length in cnt1. +// elem_size is the element size in bytes: either 1 or 2. +// There are two implementations. For arrays >= 8 bytes, all +// comparisons (including the final one, which may overlap) are +// performed 8 bytes at a time. For strings < 8 bytes, we compare a +// halfword, then a short, and then a byte. + +void C2_MacroAssembler::string_equals(Register a1, Register a2, + Register result, Register cnt1, int elem_size) +{ + Label SAME, DONE, SHORT, NEXT_WORD; + Register tmp1 = t0; + Register tmp2 = t1; + + assert(elem_size == 1 || elem_size == 2, "must be 2 or 1 byte"); + assert_different_registers(a1, a2, result, cnt1, t0, t1); + + BLOCK_COMMENT("string_equals {"); + + mv(result, false); + + // Check for short strings, i.e. smaller than wordSize. + sub(cnt1, cnt1, wordSize); + bltz(cnt1, SHORT); + + // Main 8 byte comparison loop. + bind(NEXT_WORD); { + ld(tmp1, Address(a1, 0)); + add(a1, a1, wordSize); + ld(tmp2, Address(a2, 0)); + add(a2, a2, wordSize); + sub(cnt1, cnt1, wordSize); + bne(tmp1, tmp2, DONE); + } bgtz(cnt1, NEXT_WORD); + + // Last longword. In the case where length == 4 we compare the + // same longword twice, but that's still faster than another + // conditional branch. + // cnt1 could be 0, -1, -2, -3, -4 for chars; -4 only happens when + // length == 4. + add(tmp1, a1, cnt1); + ld(tmp1, Address(tmp1, 0)); + add(tmp2, a2, cnt1); + ld(tmp2, Address(tmp2, 0)); + bne(tmp1, tmp2, DONE); + j(SAME); + + bind(SHORT); + Label TAIL03, TAIL01; + + // 0-7 bytes left. + test_bit(t0, cnt1, 2); + beqz(t0, TAIL03); + { + lwu(tmp1, Address(a1, 0)); + add(a1, a1, 4); + lwu(tmp2, Address(a2, 0)); + add(a2, a2, 4); + bne(tmp1, tmp2, DONE); + } + + bind(TAIL03); + // 0-3 bytes left. + test_bit(t0, cnt1, 1); + beqz(t0, TAIL01); + { + lhu(tmp1, Address(a1, 0)); + add(a1, a1, 2); + lhu(tmp2, Address(a2, 0)); + add(a2, a2, 2); + bne(tmp1, tmp2, DONE); + } + + bind(TAIL01); + if (elem_size == 1) { // Only needed when comparing 1-byte elements + // 0-1 bytes left. + test_bit(t0, cnt1, 0); + beqz(t0, SAME); + { + lbu(tmp1, Address(a1, 0)); + lbu(tmp2, Address(a2, 0)); + bne(tmp1, tmp2, DONE); + } + } + + // Arrays are equal. + bind(SAME); + mv(result, true); + + // That's it. + bind(DONE); + BLOCK_COMMENT("} string_equals"); +} + +typedef void (Assembler::*conditional_branch_insn)(Register op1, Register op2, Label& label, bool is_far); +typedef void (MacroAssembler::*float_conditional_branch_insn)(FloatRegister op1, FloatRegister op2, Label& label, + bool is_far, bool is_unordered); + +static conditional_branch_insn conditional_branches[] = +{ + /* SHORT branches */ + (conditional_branch_insn)&MacroAssembler::beq, + (conditional_branch_insn)&MacroAssembler::bgt, + NULL, // BoolTest::overflow + (conditional_branch_insn)&MacroAssembler::blt, + (conditional_branch_insn)&MacroAssembler::bne, + (conditional_branch_insn)&MacroAssembler::ble, + NULL, // BoolTest::no_overflow + (conditional_branch_insn)&MacroAssembler::bge, + + /* UNSIGNED branches */ + (conditional_branch_insn)&MacroAssembler::beq, + (conditional_branch_insn)&MacroAssembler::bgtu, + NULL, + (conditional_branch_insn)&MacroAssembler::bltu, + (conditional_branch_insn)&MacroAssembler::bne, + (conditional_branch_insn)&MacroAssembler::bleu, + NULL, + (conditional_branch_insn)&MacroAssembler::bgeu +}; + +static float_conditional_branch_insn float_conditional_branches[] = +{ + /* FLOAT SHORT branches */ + (float_conditional_branch_insn)&MacroAssembler::float_beq, + (float_conditional_branch_insn)&MacroAssembler::float_bgt, + NULL, // BoolTest::overflow + (float_conditional_branch_insn)&MacroAssembler::float_blt, + (float_conditional_branch_insn)&MacroAssembler::float_bne, + (float_conditional_branch_insn)&MacroAssembler::float_ble, + NULL, // BoolTest::no_overflow + (float_conditional_branch_insn)&MacroAssembler::float_bge, + + /* DOUBLE SHORT branches */ + (float_conditional_branch_insn)&MacroAssembler::double_beq, + (float_conditional_branch_insn)&MacroAssembler::double_bgt, + NULL, + (float_conditional_branch_insn)&MacroAssembler::double_blt, + (float_conditional_branch_insn)&MacroAssembler::double_bne, + (float_conditional_branch_insn)&MacroAssembler::double_ble, + NULL, + (float_conditional_branch_insn)&MacroAssembler::double_bge +}; + +void C2_MacroAssembler::cmp_branch(int cmpFlag, Register op1, Register op2, Label& label, bool is_far) { + assert(cmpFlag >= 0 && cmpFlag < (int)(sizeof(conditional_branches) / sizeof(conditional_branches[0])), + "invalid conditional branch index"); + (this->*conditional_branches[cmpFlag])(op1, op2, label, is_far); +} + +// This is a function should only be used by C2. Flip the unordered when unordered-greater, C2 would use +// unordered-lesser instead of unordered-greater. Finally, commute the result bits at function do_one_bytecode(). +void C2_MacroAssembler::float_cmp_branch(int cmpFlag, FloatRegister op1, FloatRegister op2, Label& label, bool is_far) { + assert(cmpFlag >= 0 && cmpFlag < (int)(sizeof(float_conditional_branches) / sizeof(float_conditional_branches[0])), + "invalid float conditional branch index"); + int booltest_flag = cmpFlag & ~(C2_MacroAssembler::double_branch_mask); + (this->*float_conditional_branches[cmpFlag])(op1, op2, label, is_far, + (booltest_flag == (BoolTest::ge) || booltest_flag == (BoolTest::gt)) ? false : true); +} + +void C2_MacroAssembler::enc_cmpUEqNeLeGt_imm0_branch(int cmpFlag, Register op1, Label& L, bool is_far) { + switch (cmpFlag) { + case BoolTest::eq: + case BoolTest::le: + beqz(op1, L, is_far); + break; + case BoolTest::ne: + case BoolTest::gt: + bnez(op1, L, is_far); + break; + default: + ShouldNotReachHere(); + } +} + +void C2_MacroAssembler::enc_cmpEqNe_imm0_branch(int cmpFlag, Register op1, Label& L, bool is_far) { + switch (cmpFlag) { + case BoolTest::eq: + beqz(op1, L, is_far); + break; + case BoolTest::ne: + bnez(op1, L, is_far); + break; + default: + ShouldNotReachHere(); + } +} + +void C2_MacroAssembler::enc_cmove(int cmpFlag, Register op1, Register op2, Register dst, Register src) { + Label L; + cmp_branch(cmpFlag ^ (1 << neg_cond_bits), op1, op2, L); + mv(dst, src); + bind(L); +} + +// Set dst to NaN if any NaN input. +void C2_MacroAssembler::minmax_FD(FloatRegister dst, FloatRegister src1, FloatRegister src2, + bool is_double, bool is_min) { + assert_different_registers(dst, src1, src2); + + Label Done, Compare; + + is_double ? fclass_d(t0, src1) + : fclass_s(t0, src1); + is_double ? fclass_d(t1, src2) + : fclass_s(t1, src2); + orr(t0, t0, t1); + andi(t0, t0, 0b1100000000); //if src1 or src2 is quiet or signaling NaN then return NaN + beqz(t0, Compare); + is_double ? fadd_d(dst, src1, src2) + : fadd_s(dst, src1, src2); + j(Done); + + bind(Compare); + if (is_double) { + is_min ? fmin_d(dst, src1, src2) + : fmax_d(dst, src1, src2); + } else { + is_min ? fmin_s(dst, src1, src2) + : fmax_s(dst, src1, src2); + } + + bind(Done); +} + +void C2_MacroAssembler::element_compare(Register a1, Register a2, Register result, Register cnt, Register tmp1, Register tmp2, + VectorRegister vr1, VectorRegister vr2, VectorRegister vrs, bool islatin, Label &DONE) { + Label loop; + Assembler::SEW sew = islatin ? Assembler::e8 : Assembler::e16; + + bind(loop); + vsetvli(tmp1, cnt, sew, Assembler::m2); + vlex_v(vr1, a1, sew); + vlex_v(vr2, a2, sew); + vmsne_vv(vrs, vr1, vr2); + vfirst_m(tmp2, vrs); + bgez(tmp2, DONE); + sub(cnt, cnt, tmp1); + if (!islatin) { + slli(tmp1, tmp1, 1); // get byte counts + } + add(a1, a1, tmp1); + add(a2, a2, tmp1); + bnez(cnt, loop); + + mv(result, true); +} + +void C2_MacroAssembler::string_equals_v(Register a1, Register a2, Register result, Register cnt, int elem_size) { + Label DONE; + Register tmp1 = t0; + Register tmp2 = t1; + + BLOCK_COMMENT("string_equals_v {"); + + mv(result, false); + + if (elem_size == 2) { + srli(cnt, cnt, 1); + } + + element_compare(a1, a2, result, cnt, tmp1, tmp2, v0, v2, v0, elem_size == 1, DONE); + + bind(DONE); + BLOCK_COMMENT("} string_equals_v"); +} + +// used by C2 ClearArray patterns. +// base: Address of a buffer to be zeroed +// cnt: Count in HeapWords +// +// base, cnt, v0, v1 and t0 are clobbered. +void C2_MacroAssembler::clear_array_v(Register base, Register cnt) { + Label loop; + + // making zero words + vsetvli(t0, cnt, Assembler::e64, Assembler::m4); + vxor_vv(v0, v0, v0); + + bind(loop); + vsetvli(t0, cnt, Assembler::e64, Assembler::m4); + vse64_v(v0, base); + sub(cnt, cnt, t0); + shadd(base, t0, base, t0, 3); + bnez(cnt, loop); +} + +void C2_MacroAssembler::arrays_equals_v(Register a1, Register a2, Register result, + Register cnt1, int elem_size) { + Label DONE; + Register tmp1 = t0; + Register tmp2 = t1; + Register cnt2 = tmp2; + int length_offset = arrayOopDesc::length_offset_in_bytes(); + int base_offset = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE); + + BLOCK_COMMENT("arrays_equals_v {"); + + // if (a1 == a2), return true + mv(result, true); + beq(a1, a2, DONE); + + mv(result, false); + // if a1 == null or a2 == null, return false + beqz(a1, DONE); + beqz(a2, DONE); + // if (a1.length != a2.length), return false + lwu(cnt1, Address(a1, length_offset)); + lwu(cnt2, Address(a2, length_offset)); + bne(cnt1, cnt2, DONE); + + la(a1, Address(a1, base_offset)); + la(a2, Address(a2, base_offset)); + + element_compare(a1, a2, result, cnt1, tmp1, tmp2, v0, v2, v0, elem_size == 1, DONE); + + bind(DONE); + + BLOCK_COMMENT("} arrays_equals_v"); +} + +void C2_MacroAssembler::string_compare_v(Register str1, Register str2, Register cnt1, Register cnt2, + Register result, Register tmp1, Register tmp2, int encForm) { + Label DIFFERENCE, DONE, L, loop; + bool encLL = encForm == StrIntrinsicNode::LL; + bool encLU = encForm == StrIntrinsicNode::LU; + bool encUL = encForm == StrIntrinsicNode::UL; + + bool str1_isL = encLL || encLU; + bool str2_isL = encLL || encUL; + + int minCharsInWord = encLL ? wordSize : wordSize / 2; + + BLOCK_COMMENT("string_compare {"); + + // for Lating strings, 1 byte for 1 character + // for UTF16 strings, 2 bytes for 1 character + if (!str1_isL) + sraiw(cnt1, cnt1, 1); + if (!str2_isL) + sraiw(cnt2, cnt2, 1); + + // if str1 == str2, return the difference + // save the minimum of the string lengths in cnt2. + sub(result, cnt1, cnt2); + bgt(cnt1, cnt2, L); + mv(cnt2, cnt1); + bind(L); + + if (str1_isL == str2_isL) { // LL or UU + element_compare(str1, str2, zr, cnt2, tmp1, tmp2, v2, v4, v1, encLL, DIFFERENCE); + j(DONE); + } else { // LU or UL + Register strL = encLU ? str1 : str2; + Register strU = encLU ? str2 : str1; + VectorRegister vstr1 = encLU ? v4 : v0; + VectorRegister vstr2 = encLU ? v0 : v4; + + bind(loop); + vsetvli(tmp1, cnt2, Assembler::e8, Assembler::m2); + vle8_v(vstr1, strL); + vsetvli(tmp1, cnt2, Assembler::e16, Assembler::m4); + vzext_vf2(vstr2, vstr1); + vle16_v(vstr1, strU); + vmsne_vv(v0, vstr2, vstr1); + vfirst_m(tmp2, v0); + bgez(tmp2, DIFFERENCE); + sub(cnt2, cnt2, tmp1); + add(strL, strL, tmp1); + shadd(strU, tmp1, strU, tmp1, 1); + bnez(cnt2, loop); + j(DONE); + } + bind(DIFFERENCE); + slli(tmp1, tmp2, 1); + add(str1, str1, str1_isL ? tmp2 : tmp1); + add(str2, str2, str2_isL ? tmp2 : tmp1); + str1_isL ? lbu(tmp1, Address(str1, 0)) : lhu(tmp1, Address(str1, 0)); + str2_isL ? lbu(tmp2, Address(str2, 0)) : lhu(tmp2, Address(str2, 0)); + sub(result, tmp1, tmp2); + + bind(DONE); +} + +void C2_MacroAssembler::byte_array_inflate_v(Register src, Register dst, Register len, Register tmp) { + Label loop; + assert_different_registers(src, dst, len, tmp, t0); + + BLOCK_COMMENT("byte_array_inflate_v {"); + bind(loop); + vsetvli(tmp, len, Assembler::e8, Assembler::m2); + vle8_v(v2, src); + vsetvli(t0, len, Assembler::e16, Assembler::m4); + vzext_vf2(v0, v2); + vse16_v(v0, dst); + sub(len, len, tmp); + add(src, src, tmp); + shadd(dst, tmp, dst, tmp, 1); + bnez(len, loop); + BLOCK_COMMENT("} byte_array_inflate_v"); +} + +// Compress char[] array to byte[]. +// result: the array length if every element in array can be encoded; 0, otherwise. +void C2_MacroAssembler::char_array_compress_v(Register src, Register dst, Register len, Register result, Register tmp) { + Label done; + encode_iso_array_v(src, dst, len, result, tmp); + beqz(len, done); + mv(result, zr); + bind(done); +} + +// result: the number of elements had been encoded. +void C2_MacroAssembler::encode_iso_array_v(Register src, Register dst, Register len, Register result, Register tmp) { + Label loop, DIFFERENCE, DONE; + + BLOCK_COMMENT("encode_iso_array_v {"); + mv(result, 0); + + bind(loop); + mv(tmp, 0xff); + vsetvli(t0, len, Assembler::e16, Assembler::m2); + vle16_v(v2, src); + // if element > 0xff, stop + vmsgtu_vx(v1, v2, tmp); + vfirst_m(tmp, v1); + vmsbf_m(v0, v1); + // compress char to byte + vsetvli(t0, len, Assembler::e8); + vncvt_x_x_w(v1, v2, Assembler::v0_t); + vse8_v(v1, dst, Assembler::v0_t); + + bgez(tmp, DIFFERENCE); + add(result, result, t0); + add(dst, dst, t0); + sub(len, len, t0); + shadd(src, t0, src, t0, 1); + bnez(len, loop); + j(DONE); + + bind(DIFFERENCE); + add(result, result, tmp); + + bind(DONE); + BLOCK_COMMENT("} encode_iso_array_v"); +} + +void C2_MacroAssembler::has_negatives_v(Register ary, Register len, + Register result, Register tmp) { + Label LOOP, DONE; + + BLOCK_COMMENT("has_negatives_v {"); + assert_different_registers(ary, len, result, tmp); + + mv(result, true); + + bind(LOOP); + vsetvli(t0, len, Assembler::e8, Assembler::m4); + vle8_v(v0, ary); + vmslt_vx(v0, v0, zr); + vfirst_m(tmp, v0); + bgez(tmp, DONE); + + sub(len, len, t0); + add(ary, ary, t0); + bnez(len, LOOP); + mv(result, false); + + bind(DONE); + BLOCK_COMMENT("} has_negatives_v"); +} + +void C2_MacroAssembler::string_indexof_char_v(Register str1, Register cnt1, + Register ch, Register result, + Register tmp1, Register tmp2, + bool isL) { + mv(result, zr); + + Label loop, MATCH, DONE; + Assembler::SEW sew = isL ? Assembler::e8 : Assembler::e16; + bind(loop); + vsetvli(tmp1, cnt1, sew, Assembler::m4); + vlex_v(v0, str1, sew); + vmseq_vx(v0, v0, ch); + vfirst_m(tmp2, v0); + bgez(tmp2, MATCH); // if equal, return index + + add(result, result, tmp1); + sub(cnt1, cnt1, tmp1); + if (!isL) slli(tmp1, tmp1, 1); + add(str1, str1, tmp1); + bnez(cnt1, loop); + + mv(result, -1); + j(DONE); + + bind(MATCH); + add(result, result, tmp2); + + bind(DONE); +} + +// Set dst to NaN if any NaN input. +void C2_MacroAssembler::minmax_FD_v(VectorRegister dst, VectorRegister src1, VectorRegister src2, + bool is_double, bool is_min) { + assert_different_registers(dst, src1, src2); + + vsetvli(t0, x0, is_double ? Assembler::e64 : Assembler::e32); + + is_min ? vfmin_vv(dst, src1, src2) + : vfmax_vv(dst, src1, src2); + + vmfne_vv(v0, src1, src1); + vfadd_vv(dst, src1, src1, Assembler::v0_t); + vmfne_vv(v0, src2, src2); + vfadd_vv(dst, src2, src2, Assembler::v0_t); +} + +// Set dst to NaN if any NaN input. +void C2_MacroAssembler::reduce_minmax_FD_v(FloatRegister dst, + FloatRegister src1, VectorRegister src2, + VectorRegister tmp1, VectorRegister tmp2, + bool is_double, bool is_min) { + assert_different_registers(src2, tmp1, tmp2); + + Label L_done, L_NaN; + vsetvli(t0, x0, is_double ? Assembler::e64 : Assembler::e32); + vfmv_s_f(tmp2, src1); + + is_min ? vfredmin_vs(tmp1, src2, tmp2) + : vfredmax_vs(tmp1, src2, tmp2); + + fsflags(zr); + // Checking NaNs + vmflt_vf(tmp2, src2, src1); + frflags(t0); + bnez(t0, L_NaN); + j(L_done); + + bind(L_NaN); + vfmv_s_f(tmp2, src1); + vfredusum_vs(tmp1, src2, tmp2); + + bind(L_done); + vfmv_f_s(dst, tmp1); +} diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp new file mode 100644 index 0000000000000..e904249283a18 --- /dev/null +++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_C2_MACROASSEMBLER_RISCV_HPP +#define CPU_RISCV_C2_MACROASSEMBLER_RISCV_HPP + +// C2_MacroAssembler contains high-level macros for C2 + + private: + void element_compare(Register r1, Register r2, + Register result, Register cnt, + Register tmp1, Register tmp2, + VectorRegister vr1, VectorRegister vr2, + VectorRegister vrs, + bool is_latin, Label& DONE); + public: + + void string_compare(Register str1, Register str2, + Register cnt1, Register cnt2, Register result, + Register tmp1, Register tmp2, Register tmp3, + int ae); + + void string_indexof_char_short(Register str1, Register cnt1, + Register ch, Register result, + bool isL); + + void string_indexof_char(Register str1, Register cnt1, + Register ch, Register result, + Register tmp1, Register tmp2, + Register tmp3, Register tmp4, + bool isL); + + void string_indexof(Register str1, Register str2, + Register cnt1, Register cnt2, + Register tmp1, Register tmp2, + Register tmp3, Register tmp4, + Register tmp5, Register tmp6, + Register result, int ae); + + void string_indexof_linearscan(Register haystack, Register needle, + Register haystack_len, Register needle_len, + Register tmp1, Register tmp2, + Register tmp3, Register tmp4, + int needle_con_cnt, Register result, int ae); + + void arrays_equals(Register r1, Register r2, + Register tmp3, Register tmp4, + Register tmp5, Register tmp6, + Register result, Register cnt1, + int elem_size); + + void string_equals(Register r1, Register r2, + Register result, Register cnt1, + int elem_size); + + // refer to conditional_branches and float_conditional_branches + static const int bool_test_bits = 3; + static const int neg_cond_bits = 2; + static const int unsigned_branch_mask = 1 << bool_test_bits; + static const int double_branch_mask = 1 << bool_test_bits; + + // cmp + void cmp_branch(int cmpFlag, + Register op1, Register op2, + Label& label, bool is_far = false); + + void float_cmp_branch(int cmpFlag, + FloatRegister op1, FloatRegister op2, + Label& label, bool is_far = false); + + void enc_cmpUEqNeLeGt_imm0_branch(int cmpFlag, Register op, + Label& L, bool is_far = false); + + void enc_cmpEqNe_imm0_branch(int cmpFlag, Register op, + Label& L, bool is_far = false); + + void enc_cmove(int cmpFlag, + Register op1, Register op2, + Register dst, Register src); + + void spill(Register r, bool is64, int offset) { + is64 ? sd(r, Address(sp, offset)) + : sw(r, Address(sp, offset)); + } + + void spill(FloatRegister f, bool is64, int offset) { + is64 ? fsd(f, Address(sp, offset)) + : fsw(f, Address(sp, offset)); + } + + void spill(VectorRegister v, int offset) { + add(t0, sp, offset); + vs1r_v(v, t0); + } + + void unspill(Register r, bool is64, int offset) { + is64 ? ld(r, Address(sp, offset)) + : lw(r, Address(sp, offset)); + } + + void unspillu(Register r, bool is64, int offset) { + is64 ? ld(r, Address(sp, offset)) + : lwu(r, Address(sp, offset)); + } + + void unspill(FloatRegister f, bool is64, int offset) { + is64 ? fld(f, Address(sp, offset)) + : flw(f, Address(sp, offset)); + } + + void unspill(VectorRegister v, int offset) { + add(t0, sp, offset); + vl1re8_v(v, t0); + } + + void spill_copy_vector_stack_to_stack(int src_offset, int dst_offset, int vec_reg_size_in_bytes) { + assert(vec_reg_size_in_bytes % 16 == 0, "unexpected vector reg size"); + unspill(v0, src_offset); + spill(v0, dst_offset); + } + + void minmax_FD(FloatRegister dst, + FloatRegister src1, FloatRegister src2, + bool is_double, bool is_min); + + // intrinsic methods implemented by rvv instructions + void string_equals_v(Register r1, Register r2, + Register result, Register cnt1, + int elem_size); + + void arrays_equals_v(Register r1, Register r2, + Register result, Register cnt1, + int elem_size); + + void string_compare_v(Register str1, Register str2, + Register cnt1, Register cnt2, + Register result, + Register tmp1, Register tmp2, + int encForm); + + void clear_array_v(Register base, Register cnt); + + void byte_array_inflate_v(Register src, Register dst, + Register len, Register tmp); + + void char_array_compress_v(Register src, Register dst, + Register len, Register result, + Register tmp); + + void encode_iso_array_v(Register src, Register dst, + Register len, Register result, + Register tmp); + + void has_negatives_v(Register ary, Register len, + Register result, Register tmp); + + void string_indexof_char_v(Register str1, Register cnt1, + Register ch, Register result, + Register tmp1, Register tmp2, + bool isL); + + void minmax_FD_v(VectorRegister dst, + VectorRegister src1, VectorRegister src2, + bool is_double, bool is_min); + + void reduce_minmax_FD_v(FloatRegister dst, + FloatRegister src1, VectorRegister src2, + VectorRegister tmp1, VectorRegister tmp2, + bool is_double, bool is_min); + +#endif // CPU_RISCV_C2_MACROASSEMBLER_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/c2_globals_riscv.hpp b/src/hotspot/cpu/riscv/c2_globals_riscv.hpp new file mode 100644 index 0000000000000..c63d2c0ffb9ce --- /dev/null +++ b/src/hotspot/cpu/riscv/c2_globals_riscv.hpp @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_C2_GLOBALS_RISCV_HPP +#define CPU_RISCV_C2_GLOBALS_RISCV_HPP + +#include "utilities/globalDefinitions.hpp" +#include "utilities/macros.hpp" + +// Sets the default values for platform dependent flags used by the server compiler. +// (see c2_globals.hpp). Alpha-sorted. + +define_pd_global(bool, BackgroundCompilation, true); +define_pd_global(bool, CICompileOSR, true); +define_pd_global(bool, InlineIntrinsics, true); +define_pd_global(bool, PreferInterpreterNativeStubs, false); +define_pd_global(bool, ProfileTraps, true); +define_pd_global(bool, UseOnStackReplacement, true); +define_pd_global(bool, ProfileInterpreter, true); +define_pd_global(bool, TieredCompilation, COMPILER1_PRESENT(true) NOT_COMPILER1(false)); +define_pd_global(intx, CompileThreshold, 10000); + +define_pd_global(intx, OnStackReplacePercentage, 140); +define_pd_global(intx, ConditionalMoveLimit, 0); +define_pd_global(intx, FLOATPRESSURE, 32); +define_pd_global(intx, FreqInlineSize, 325); +define_pd_global(intx, MinJumpTableSize, 10); +define_pd_global(intx, INTPRESSURE, 24); +define_pd_global(intx, InteriorEntryAlignment, 16); +define_pd_global(intx, NewSizeThreadIncrease, ScaleForWordSize(4*K)); +define_pd_global(intx, LoopUnrollLimit, 60); +define_pd_global(intx, LoopPercentProfileLimit, 10); +// InitialCodeCacheSize derived from specjbb2000 run. +define_pd_global(intx, InitialCodeCacheSize, 2496*K); // Integral multiple of CodeCacheExpansionSize +define_pd_global(intx, CodeCacheExpansionSize, 64*K); + +// Ergonomics related flags +define_pd_global(uint64_t,MaxRAM, 128ULL*G); +define_pd_global(intx, RegisterCostAreaRatio, 16000); + +// Peephole and CISC spilling both break the graph, and so makes the +// scheduler sick. +define_pd_global(bool, OptoPeephole, false); +define_pd_global(bool, UseCISCSpill, false); +define_pd_global(bool, OptoScheduling, true); +define_pd_global(bool, OptoBundling, false); +define_pd_global(bool, OptoRegScheduling, false); +define_pd_global(bool, SuperWordLoopUnrollAnalysis, true); +define_pd_global(bool, IdealizeClearArrayNode, true); + +define_pd_global(intx, ReservedCodeCacheSize, 48*M); +define_pd_global(intx, NonProfiledCodeHeapSize, 21*M); +define_pd_global(intx, ProfiledCodeHeapSize, 22*M); +define_pd_global(intx, NonNMethodCodeHeapSize, 5*M ); +define_pd_global(uintx, CodeCacheMinBlockLength, 6); +define_pd_global(uintx, CodeCacheMinimumUseSpace, 400*K); + +// Ergonomics related flags +define_pd_global(bool, NeverActAsServerClassMachine, false); + +define_pd_global(bool, TrapBasedRangeChecks, false); // Not needed. + +#endif // CPU_RISCV_C2_GLOBALS_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/c2_init_riscv.cpp b/src/hotspot/cpu/riscv/c2_init_riscv.cpp new file mode 100644 index 0000000000000..cdbd69807bee1 --- /dev/null +++ b/src/hotspot/cpu/riscv/c2_init_riscv.cpp @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2019, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "opto/compile.hpp" +#include "opto/node.hpp" + +// processor dependent initialization for riscv + +extern void reg_mask_init(); + +void Compile::pd_compiler2_init() { + guarantee(CodeEntryAlignment >= InteriorEntryAlignment, "" ); + reg_mask_init(); +} diff --git a/src/hotspot/cpu/riscv/c2_safepointPollStubTable_riscv.cpp b/src/hotspot/cpu/riscv/c2_safepointPollStubTable_riscv.cpp new file mode 100644 index 0000000000000..d22a6a758432c --- /dev/null +++ b/src/hotspot/cpu/riscv/c2_safepointPollStubTable_riscv.cpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.hpp" +#include "opto/compile.hpp" +#include "opto/node.hpp" +#include "opto/output.hpp" +#include "runtime/sharedRuntime.hpp" + +#define __ masm. +void C2SafepointPollStubTable::emit_stub_impl(MacroAssembler& masm, C2SafepointPollStub* entry) const { + assert(SharedRuntime::polling_page_return_handler_blob() != NULL, + "polling page return stub not created yet"); + address stub = SharedRuntime::polling_page_return_handler_blob()->entry_point(); + RuntimeAddress callback_addr(stub); + + __ bind(entry->_stub_label); + InternalAddress safepoint_pc(__ pc() - __ offset() + entry->_safepoint_offset); + __ relocate(safepoint_pc.rspec(), [&] { + __ la(t0, safepoint_pc.target()); + }); + __ sd(t0, Address(xthread, JavaThread::saved_exception_pc_offset())); + __ far_jump(callback_addr); +} +#undef __ diff --git a/test/hotspot/jtreg/gc/TestMemoryInitialization.java b/src/hotspot/cpu/riscv/codeBuffer_riscv.hpp similarity index 65% rename from test/hotspot/jtreg/gc/TestMemoryInitialization.java rename to src/hotspot/cpu/riscv/codeBuffer_riscv.hpp index e81c8a3d4c42b..14a68b45026da 100644 --- a/test/hotspot/jtreg/gc/TestMemoryInitialization.java +++ b/src/hotspot/cpu/riscv/codeBuffer_riscv.hpp @@ -1,6 +1,7 @@ - /* * Copyright (c) 2002, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -20,29 +21,16 @@ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. + * */ -package gc; - -/* - * test TestMemoryInitialization - * bug 4668531 - * Simple test for -XX:+CheckMemoryInitialization doesn't crash VM - */ - -public class TestMemoryInitialization { - final static int LOOP_LENGTH = 10; - final static int CHUNK_SIZE = 1500000; +#ifndef CPU_RISCV_CODEBUFFER_RISCV_HPP +#define CPU_RISCV_CODEBUFFER_RISCV_HPP - public static byte[] buffer; +private: + void pd_initialize() {} - public static void main(String args[]) { +public: + void flush_bundle(bool start_new_bundle) {} - for (int i = 0; i < LOOP_LENGTH; i++) { - for (int j = 0; j < LOOP_LENGTH; j++) { - buffer = new byte[CHUNK_SIZE]; - buffer = null; - } - } - } -} +#endif // CPU_RISCV_CODEBUFFER_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/compiledIC_riscv.cpp b/src/hotspot/cpu/riscv/compiledIC_riscv.cpp new file mode 100644 index 0000000000000..d202db92a4cbd --- /dev/null +++ b/src/hotspot/cpu/riscv/compiledIC_riscv.cpp @@ -0,0 +1,149 @@ +/* + * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2018, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "code/compiledIC.hpp" +#include "code/icBuffer.hpp" +#include "code/nmethod.hpp" +#include "memory/resourceArea.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/safepoint.hpp" + +// ---------------------------------------------------------------------------- + +#define __ _masm. +address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) { + precond(cbuf.stubs()->start() != badAddress); + precond(cbuf.stubs()->end() != badAddress); + // Stub is fixed up when the corresponding call is converted from + // calling compiled code to calling interpreted code. + // mv xmethod, 0 + // jalr -4 # to self + + if (mark == NULL) { + mark = cbuf.insts_mark(); // Get mark within main instrs section. + } + + // Note that the code buffer's insts_mark is always relative to insts. + // That's why we must use the macroassembler to generate a stub. + MacroAssembler _masm(&cbuf); + + address base = __ start_a_stub(to_interp_stub_size()); + int offset = __ offset(); + if (base == NULL) { + return NULL; // CodeBuffer::expand failed + } + // static stub relocation stores the instruction address of the call + __ relocate(static_stub_Relocation::spec(mark)); + + __ emit_static_call_stub(); + + assert((__ offset() - offset) <= (int)to_interp_stub_size(), "stub too big"); + __ end_a_stub(); + return base; +} +#undef __ + +int CompiledStaticCall::to_interp_stub_size() { + // (lui, addi, slli, addi, slli, addi) + (lui, addi, slli, addi, slli) + jalr + return 12 * NativeInstruction::instruction_size; +} + +int CompiledStaticCall::to_trampoline_stub_size() { + // Somewhat pessimistically, we count 4 instructions here (although + // there are only 3) because we sometimes emit an alignment nop. + // Trampoline stubs are always word aligned. + return NativeInstruction::instruction_size + NativeCallTrampolineStub::instruction_size; +} + +// Relocation entries for call stub, compiled java to interpreter. +int CompiledStaticCall::reloc_to_interp_stub() { + return 4; // 3 in emit_to_interp_stub + 1 in emit_call +} + +void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, address entry) { + address stub = find_stub(); + guarantee(stub != NULL, "stub not found"); + + if (TraceICs) { + ResourceMark rm; + tty->print_cr("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s", + p2i(instruction_address()), + callee->name_and_sig_as_C_string()); + } + + // Creation also verifies the object. + NativeMovConstReg* method_holder + = nativeMovConstReg_at(stub); +#ifdef ASSERT + NativeGeneralJump* jump = nativeGeneralJump_at(method_holder->next_instruction_address()); + + verify_mt_safe(callee, entry, method_holder, jump); +#endif + // Update stub. + method_holder->set_data((intptr_t)callee()); + NativeGeneralJump::insert_unconditional(method_holder->next_instruction_address(), entry); + ICache::invalidate_range(stub, to_interp_stub_size()); + // Update jump to call. + set_destination_mt_safe(stub); +} + +void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_stub) { + // Reset stub. + address stub = static_stub->addr(); + assert(stub != NULL, "stub not found"); + assert(CompiledICLocker::is_safe(stub), "mt unsafe call"); + // Creation also verifies the object. + NativeMovConstReg* method_holder + = nativeMovConstReg_at(stub); + method_holder->set_data(0); + NativeJump* jump = nativeJump_at(method_holder->next_instruction_address()); + jump->set_jump_destination((address)-1); +} + +//----------------------------------------------------------------------------- +// Non-product mode code +#ifndef PRODUCT + +void CompiledDirectStaticCall::verify() { + // Verify call. + _call->verify(); + _call->verify_alignment(); + + // Verify stub. + address stub = find_stub(); + assert(stub != NULL, "no stub found for static call"); + // Creation also verifies the object. + NativeMovConstReg* method_holder + = nativeMovConstReg_at(stub); + NativeJump* jump = nativeJump_at(method_holder->next_instruction_address()); + + // Verify state. + assert(is_clean() || is_call_to_compiled() || is_call_to_interpreted(), "sanity check"); +} + +#endif // !PRODUCT diff --git a/src/hotspot/cpu/riscv/copy_riscv.hpp b/src/hotspot/cpu/riscv/copy_riscv.hpp new file mode 100644 index 0000000000000..098a9620a83cf --- /dev/null +++ b/src/hotspot/cpu/riscv/copy_riscv.hpp @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_COPY_RISCV_HPP +#define CPU_RISCV_COPY_RISCV_HPP + +#include OS_CPU_HEADER(copy) + +static void pd_fill_to_words(HeapWord* tohw, size_t count, juint value) { + julong* to = (julong*) tohw; + julong v = ((julong) value << 32) | value; + while (count-- > 0) { + *to++ = v; + } +} + +static void pd_fill_to_aligned_words(HeapWord* tohw, size_t count, juint value) { + pd_fill_to_words(tohw, count, value); +} + +static void pd_fill_to_bytes(void* to, size_t count, jubyte value) { + (void)memset(to, value, count); +} + +static void pd_zero_to_words(HeapWord* tohw, size_t count) { + pd_fill_to_words(tohw, count, 0); +} + +static void pd_zero_to_bytes(void* to, size_t count) { + (void)memset(to, 0, count); +} + +static void pd_conjoint_words(const HeapWord* from, HeapWord* to, size_t count) { + (void)memmove(to, from, count * HeapWordSize); +} + +static inline void pd_disjoint_words_helper(const HeapWord* from, HeapWord* to, size_t count, bool is_atomic) { + switch (count) { + case 8: to[7] = from[7]; // fall through + case 7: to[6] = from[6]; // fall through + case 6: to[5] = from[5]; // fall through + case 5: to[4] = from[4]; // fall through + case 4: to[3] = from[3]; // fall through + case 3: to[2] = from[2]; // fall through + case 2: to[1] = from[1]; // fall through + case 1: to[0] = from[0]; // fall through + case 0: break; + default: + if (is_atomic) { + while (count-- > 0) { *to++ = *from++; } + } else { + memcpy(to, from, count * HeapWordSize); + } + } +} + +static void pd_disjoint_words(const HeapWord* from, HeapWord* to, size_t count) { + pd_disjoint_words_helper(from, to, count, false); +} + +static void pd_disjoint_words_atomic(const HeapWord* from, HeapWord* to, size_t count) { + pd_disjoint_words_helper(from, to, count, true); +} + +static void pd_aligned_conjoint_words(const HeapWord* from, HeapWord* to, size_t count) { + pd_conjoint_words(from, to, count); +} + +static void pd_aligned_disjoint_words(const HeapWord* from, HeapWord* to, size_t count) { + pd_disjoint_words(from, to, count); +} + +static void pd_conjoint_bytes(const void* from, void* to, size_t count) { + (void)memmove(to, from, count); +} + +static void pd_conjoint_bytes_atomic(const void* from, void* to, size_t count) { + pd_conjoint_bytes(from, to, count); +} + +static void pd_conjoint_jshorts_atomic(const jshort* from, jshort* to, size_t count) { + _Copy_conjoint_jshorts_atomic(from, to, count); +} + +static void pd_conjoint_jints_atomic(const jint* from, jint* to, size_t count) { + _Copy_conjoint_jints_atomic(from, to, count); +} + +static void pd_conjoint_jlongs_atomic(const jlong* from, jlong* to, size_t count) { + _Copy_conjoint_jlongs_atomic(from, to, count); +} + +static void pd_conjoint_oops_atomic(const oop* from, oop* to, size_t count) { + assert(BytesPerLong == BytesPerOop, "jlongs and oops must be the same size."); + _Copy_conjoint_jlongs_atomic((const jlong*)from, (jlong*)to, count); +} + +static void pd_arrayof_conjoint_bytes(const HeapWord* from, HeapWord* to, size_t count) { + _Copy_arrayof_conjoint_bytes(from, to, count); +} + +static void pd_arrayof_conjoint_jshorts(const HeapWord* from, HeapWord* to, size_t count) { + _Copy_arrayof_conjoint_jshorts(from, to, count); +} + +static void pd_arrayof_conjoint_jints(const HeapWord* from, HeapWord* to, size_t count) { + _Copy_arrayof_conjoint_jints(from, to, count); +} + +static void pd_arrayof_conjoint_jlongs(const HeapWord* from, HeapWord* to, size_t count) { + _Copy_arrayof_conjoint_jlongs(from, to, count); +} + +static void pd_arrayof_conjoint_oops(const HeapWord* from, HeapWord* to, size_t count) { + assert(!UseCompressedOops, "foo!"); + assert(BytesPerLong == BytesPerOop, "jlongs and oops must be the same size"); + _Copy_arrayof_conjoint_jlongs(from, to, count); +} + +#endif // CPU_RISCV_COPY_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/disassembler_riscv.hpp b/src/hotspot/cpu/riscv/disassembler_riscv.hpp new file mode 100644 index 0000000000000..2c7ecc39d4045 --- /dev/null +++ b/src/hotspot/cpu/riscv/disassembler_riscv.hpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_DISASSEMBLER_RISCV_HPP +#define CPU_RISCV_DISASSEMBLER_RISCV_HPP + +static int pd_instruction_alignment() { + return 1; +} + +static const char* pd_cpu_opts() { + return ""; +} + +// special-case instruction decoding. +// There may be cases where the binutils disassembler doesn't do +// the perfect job. In those cases, decode_instruction0 may kick in +// and do it right. +// If nothing had to be done, just return "here", otherwise return "here + instr_len(here)" +static address decode_instruction0(address here, outputStream* st, address virtual_begin = NULL) { + return here; +} + +// platform-specific instruction annotations (like value of loaded constants) +static void annotate(address pc, outputStream* st) {} + +#endif // CPU_RISCV_DISASSEMBLER_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/foreign_globals_riscv.cpp b/src/hotspot/cpu/riscv/foreign_globals_riscv.cpp new file mode 100644 index 0000000000000..5c700be9c91cc --- /dev/null +++ b/src/hotspot/cpu/riscv/foreign_globals_riscv.cpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "prims/foreign_globals.hpp" +#include "utilities/debug.hpp" + +// Stubbed out, implement later +const ABIDescriptor ForeignGlobals::parse_abi_descriptor_impl(jobject jabi) const { + Unimplemented(); + return {}; +} + +const BufferLayout ForeignGlobals::parse_buffer_layout_impl(jobject jlayout) const { + Unimplemented(); + return {}; +} + +const CallRegs ForeignGlobals::parse_call_regs_impl(jobject jconv) const { + ShouldNotCallThis(); + return {}; +} diff --git a/src/jdk.internal.le/windows/classes/module-info.java.extra b/src/hotspot/cpu/riscv/foreign_globals_riscv.hpp similarity index 71% rename from src/jdk.internal.le/windows/classes/module-info.java.extra rename to src/hotspot/cpu/riscv/foreign_globals_riscv.hpp index 3e791de0b3637..3ac89752c27ee 100644 --- a/src/jdk.internal.le/windows/classes/module-info.java.extra +++ b/src/hotspot/cpu/riscv/foreign_globals_riscv.hpp @@ -1,12 +1,11 @@ /* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. + * published by the Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or @@ -21,7 +20,13 @@ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. + * */ +#ifndef CPU_RISCV_FOREIGN_GLOBALS_RISCV_HPP +#define CPU_RISCV_FOREIGN_GLOBALS_RISCV_HPP + +class ABIDescriptor {}; +class BufferLayout {}; -provides jdk.internal.org.jline.terminal.spi.JnaSupport with jdk.internal.org.jline.terminal.impl.jna.JnaSupportImpl; +#endif // CPU_RISCV_FOREIGN_GLOBALS_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/frame_riscv.cpp b/src/hotspot/cpu/riscv/frame_riscv.cpp new file mode 100644 index 0000000000000..8123317a60958 --- /dev/null +++ b/src/hotspot/cpu/riscv/frame_riscv.cpp @@ -0,0 +1,688 @@ +/* + * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "compiler/oopMap.hpp" +#include "interpreter/interpreter.hpp" +#include "memory/resourceArea.hpp" +#include "memory/universe.hpp" +#include "oops/markWord.hpp" +#include "oops/method.hpp" +#include "oops/oop.inline.hpp" +#include "prims/methodHandles.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/javaCalls.hpp" +#include "runtime/monitorChunk.hpp" +#include "runtime/os.inline.hpp" +#include "runtime/signature.hpp" +#include "runtime/stackWatermarkSet.hpp" +#include "runtime/stubCodeGenerator.hpp" +#include "runtime/stubRoutines.hpp" +#include "vmreg_riscv.inline.hpp" +#ifdef COMPILER1 +#include "c1/c1_Runtime1.hpp" +#include "runtime/vframeArray.hpp" +#endif + +#ifdef ASSERT +void RegisterMap::check_location_valid() { +} +#endif + + +// Profiling/safepoint support + +bool frame::safe_for_sender(JavaThread *thread) { + address addr_sp = (address)_sp; + address addr_fp = (address)_fp; + address unextended_sp = (address)_unextended_sp; + + // consider stack guards when trying to determine "safe" stack pointers + // sp must be within the usable part of the stack (not in guards) + if (!thread->is_in_usable_stack(addr_sp)) { + return false; + } + + // When we are running interpreted code the machine stack pointer, SP, is + // set low enough so that the Java expression stack can grow and shrink + // without ever exceeding the machine stack bounds. So, ESP >= SP. + + // When we call out of an interpreted method, SP is incremented so that + // the space between SP and ESP is removed. The SP saved in the callee's + // frame is the SP *before* this increment. So, when we walk a stack of + // interpreter frames the sender's SP saved in a frame might be less than + // the SP at the point of call. + + // So unextended sp must be within the stack but we need not to check + // that unextended sp >= sp + + if (!thread->is_in_full_stack_checked(unextended_sp)) { + return false; + } + + // an fp must be within the stack and above (but not equal) sp + // second evaluation on fp+ is added to handle situation where fp is -1 + bool fp_safe = thread->is_in_stack_range_excl(addr_fp, addr_sp) && + thread->is_in_full_stack_checked(addr_fp + (return_addr_offset * sizeof(void*))); + + // We know sp/unextended_sp are safe only fp is questionable here + + // If the current frame is known to the code cache then we can attempt to + // to construct the sender and do some validation of it. This goes a long way + // toward eliminating issues when we get in frame construction code + + if (_cb != NULL) { + + // First check if frame is complete and tester is reliable + // Unfortunately we can only check frame complete for runtime stubs and nmethod + // other generic buffer blobs are more problematic so we just assume they are + // ok. adapter blobs never have a frame complete and are never ok. + + if (!_cb->is_frame_complete_at(_pc)) { + if (_cb->is_nmethod() || _cb->is_adapter_blob() || _cb->is_runtime_stub()) { + return false; + } + } + + // Could just be some random pointer within the codeBlob + if (!_cb->code_contains(_pc)) { + return false; + } + + // Entry frame checks + if (is_entry_frame()) { + // an entry frame must have a valid fp. + return fp_safe && is_entry_frame_valid(thread); + } + + intptr_t* sender_sp = NULL; + intptr_t* sender_unextended_sp = NULL; + address sender_pc = NULL; + intptr_t* saved_fp = NULL; + + if (is_interpreted_frame()) { + // fp must be safe + if (!fp_safe) { + return false; + } + + sender_pc = (address)this->fp()[return_addr_offset]; + // for interpreted frames, the value below is the sender "raw" sp, + // which can be different from the sender unextended sp (the sp seen + // by the sender) because of current frame local variables + sender_sp = (intptr_t*) addr_at(sender_sp_offset); + sender_unextended_sp = (intptr_t*) this->fp()[interpreter_frame_sender_sp_offset]; + saved_fp = (intptr_t*) this->fp()[link_offset]; + } else { + // must be some sort of compiled/runtime frame + // fp does not have to be safe (although it could be check for c1?) + + // check for a valid frame_size, otherwise we are unlikely to get a valid sender_pc + if (_cb->frame_size() <= 0) { + return false; + } + + sender_sp = _unextended_sp + _cb->frame_size(); + // Is sender_sp safe? + if (!thread->is_in_full_stack_checked((address)sender_sp)) { + return false; + } + + sender_unextended_sp = sender_sp; + sender_pc = (address) *(sender_sp - 1); + saved_fp = (intptr_t*) *(sender_sp - 2); + } + + + // If the potential sender is the interpreter then we can do some more checking + if (Interpreter::contains(sender_pc)) { + + // fp is always saved in a recognizable place in any code we generate. However + // only if the sender is interpreted/call_stub (c1 too?) are we certain that the saved fp + // is really a frame pointer. + if (!thread->is_in_stack_range_excl((address)saved_fp, (address)sender_sp)) { + return false; + } + + // construct the potential sender + frame sender(sender_sp, sender_unextended_sp, saved_fp, sender_pc); + + return sender.is_interpreted_frame_valid(thread); + } + + // We must always be able to find a recognizable pc + CodeBlob* sender_blob = CodeCache::find_blob_unsafe(sender_pc); + if (sender_pc == NULL || sender_blob == NULL) { + return false; + } + + // Could be a zombie method + if (sender_blob->is_zombie() || sender_blob->is_unloaded()) { + return false; + } + + // Could just be some random pointer within the codeBlob + if (!sender_blob->code_contains(sender_pc)) { + return false; + } + + // We should never be able to see an adapter if the current frame is something from code cache + if (sender_blob->is_adapter_blob()) { + return false; + } + + // Could be the call_stub + if (StubRoutines::returns_to_call_stub(sender_pc)) { + if (!thread->is_in_stack_range_excl((address)saved_fp, (address)sender_sp)) { + return false; + } + + // construct the potential sender + frame sender(sender_sp, sender_unextended_sp, saved_fp, sender_pc); + + // Validate the JavaCallWrapper an entry frame must have + address jcw = (address)sender.entry_frame_call_wrapper(); + + bool jcw_safe = (jcw < thread->stack_base()) && (jcw > (address)sender.fp()); + + return jcw_safe; + } + + CompiledMethod* nm = sender_blob->as_compiled_method_or_null(); + if (nm != NULL) { + if (nm->is_deopt_mh_entry(sender_pc) || nm->is_deopt_entry(sender_pc) || + nm->method()->is_method_handle_intrinsic()) { + return false; + } + } + + // If the frame size is 0 something (or less) is bad because every nmethod has a non-zero frame size + // because the return address counts against the callee's frame. + if (sender_blob->frame_size() <= 0) { + assert(!sender_blob->is_compiled(), "should count return address at least"); + return false; + } + + // We should never be able to see anything here except an nmethod. If something in the + // code cache (current frame) is called by an entity within the code cache that entity + // should not be anything but the call stub (already covered), the interpreter (already covered) + // or an nmethod. + if (!sender_blob->is_compiled()) { + return false; + } + + // Could put some more validation for the potential non-interpreted sender + // frame we'd create by calling sender if I could think of any. Wait for next crash in forte... + + // One idea is seeing if the sender_pc we have is one that we'd expect to call to current cb + + // We've validated the potential sender that would be created + return true; + } + + // Must be native-compiled frame. Since sender will try and use fp to find + // linkages it must be safe + if (!fp_safe) { + return false; + } + + // Will the pc we fetch be non-zero (which we'll find at the oldest frame) + if ((address)this->fp()[return_addr_offset] == NULL) { return false; } + + return true; +} + +void frame::patch_pc(Thread* thread, address pc) { + assert(_cb == CodeCache::find_blob(pc), "unexpected pc"); + address* pc_addr = &(((address*) sp())[-1]); + if (TracePcPatching) { + tty->print_cr("patch_pc at address " INTPTR_FORMAT " [" INTPTR_FORMAT " -> " INTPTR_FORMAT "]", + p2i(pc_addr), p2i(*pc_addr), p2i(pc)); + } + // Either the return address is the original one or we are going to + // patch in the same address that's already there. + assert(_pc == *pc_addr || pc == *pc_addr, "must be"); + *pc_addr = pc; + address original_pc = CompiledMethod::get_deopt_original_pc(this); + if (original_pc != NULL) { + assert(original_pc == _pc, "expected original PC to be stored before patching"); + _deopt_state = is_deoptimized; + // leave _pc as is + } else { + _deopt_state = not_deoptimized; + _pc = pc; + } +} + +bool frame::is_interpreted_frame() const { + return Interpreter::contains(pc()); +} + +int frame::frame_size(RegisterMap* map) const { + frame sender = this->sender(map); + return sender.sp() - sp(); +} + +intptr_t* frame::entry_frame_argument_at(int offset) const { + // convert offset to index to deal with tsi + int index = (Interpreter::expr_offset_in_bytes(offset)/wordSize); + // Entry frame's arguments are always in relation to unextended_sp() + return &unextended_sp()[index]; +} + +// sender_sp +intptr_t* frame::interpreter_frame_sender_sp() const { + assert(is_interpreted_frame(), "interpreted frame expected"); + return (intptr_t*) at(interpreter_frame_sender_sp_offset); +} + +void frame::set_interpreter_frame_sender_sp(intptr_t* sender_sp) { + assert(is_interpreted_frame(), "interpreted frame expected"); + ptr_at_put(interpreter_frame_sender_sp_offset, (intptr_t) sender_sp); +} + + +// monitor elements + +BasicObjectLock* frame::interpreter_frame_monitor_begin() const { + return (BasicObjectLock*) addr_at(interpreter_frame_monitor_block_bottom_offset); +} + +BasicObjectLock* frame::interpreter_frame_monitor_end() const { + BasicObjectLock* result = (BasicObjectLock*) *addr_at(interpreter_frame_monitor_block_top_offset); + // make sure the pointer points inside the frame + assert(sp() <= (intptr_t*) result, "monitor end should be above the stack pointer"); + assert((intptr_t*) result < fp(), "monitor end should be strictly below the frame pointer"); + return result; +} + +void frame::interpreter_frame_set_monitor_end(BasicObjectLock* value) { + *((BasicObjectLock**)addr_at(interpreter_frame_monitor_block_top_offset)) = value; +} + +// Used by template based interpreter deoptimization +void frame::interpreter_frame_set_last_sp(intptr_t* last_sp) { + *((intptr_t**)addr_at(interpreter_frame_last_sp_offset)) = last_sp; +} + +frame frame::sender_for_entry_frame(RegisterMap* map) const { + assert(map != NULL, "map must be set"); + // Java frame called from C; skip all C frames and return top C + // frame of that chunk as the sender + JavaFrameAnchor* jfa = entry_frame_call_wrapper()->anchor(); + assert(!entry_frame_is_first(), "next Java fp must be non zero"); + assert(jfa->last_Java_sp() > sp(), "must be above this frame on stack"); + // Since we are walking the stack now this nested anchor is obviously walkable + // even if it wasn't when it was stacked. + jfa->make_walkable(); + map->clear(); + assert(map->include_argument_oops(), "should be set by clear"); + vmassert(jfa->last_Java_pc() != NULL, "not walkable"); + frame fr(jfa->last_Java_sp(), jfa->last_Java_fp(), jfa->last_Java_pc()); + return fr; +} + +OptimizedEntryBlob::FrameData* OptimizedEntryBlob::frame_data_for_frame(const frame& frame) const { + ShouldNotCallThis(); + return nullptr; +} + +bool frame::optimized_entry_frame_is_first() const { + ShouldNotCallThis(); + return false; +} + +frame frame::sender_for_optimized_entry_frame(RegisterMap* map) const { + ShouldNotCallThis(); + return {}; +} + +//------------------------------------------------------------------------------ +// frame::verify_deopt_original_pc +// +// Verifies the calculated original PC of a deoptimization PC for the +// given unextended SP. +#ifdef ASSERT +void frame::verify_deopt_original_pc(CompiledMethod* nm, intptr_t* unextended_sp) { + frame fr; + + // This is ugly but it's better than to change {get,set}_original_pc + // to take an SP value as argument. And it's only a debugging + // method anyway. + fr._unextended_sp = unextended_sp; + + assert_cond(nm != NULL); + address original_pc = nm->get_original_pc(&fr); + assert(nm->insts_contains_inclusive(original_pc), + "original PC must be in the main code section of the the compiled method (or must be immediately following it)"); +} +#endif + +//------------------------------------------------------------------------------ +// frame::adjust_unextended_sp +void frame::adjust_unextended_sp() { + // On riscv, sites calling method handle intrinsics and lambda forms are treated + // as any other call site. Therefore, no special action is needed when we are + // returning to any of these call sites. + + if (_cb != NULL) { + CompiledMethod* sender_cm = _cb->as_compiled_method_or_null(); + if (sender_cm != NULL) { + // If the sender PC is a deoptimization point, get the original PC. + if (sender_cm->is_deopt_entry(_pc) || + sender_cm->is_deopt_mh_entry(_pc)) { + DEBUG_ONLY(verify_deopt_original_pc(sender_cm, _unextended_sp)); + } + } + } +} + +//------------------------------------------------------------------------------ +// frame::update_map_with_saved_link +void frame::update_map_with_saved_link(RegisterMap* map, intptr_t** link_addr) { + // The interpreter and compiler(s) always save fp in a known + // location on entry. We must record where that location is + // so that if fp was live on callout from c2 we can find + // the saved copy no matter what it called. + + // Since the interpreter always saves fp if we record where it is then + // we don't have to always save fp on entry and exit to c2 compiled + // code, on entry will be enough. + assert(map != NULL, "map must be set"); + map->set_location(::fp->as_VMReg(), (address) link_addr); + // this is weird "H" ought to be at a higher address however the + // oopMaps seems to have the "H" regs at the same address and the + // vanilla register. + map->set_location(::fp->as_VMReg()->next(), (address) link_addr); +} + + +//------------------------------------------------------------------------------ +// frame::sender_for_interpreter_frame +frame frame::sender_for_interpreter_frame(RegisterMap* map) const { + // SP is the raw SP from the sender after adapter or interpreter + // extension. + intptr_t* sender_sp = this->sender_sp(); + + // This is the sp before any possible extension (adapter/locals). + intptr_t* unextended_sp = interpreter_frame_sender_sp(); + +#ifdef COMPILER2 + assert(map != NULL, "map must be set"); + if (map->update_map()) { + update_map_with_saved_link(map, (intptr_t**) addr_at(link_offset)); + } +#endif // COMPILER2 + + return frame(sender_sp, unextended_sp, link(), sender_pc()); +} + + +//------------------------------------------------------------------------------ +// frame::sender_for_compiled_frame +frame frame::sender_for_compiled_frame(RegisterMap* map) const { + // we cannot rely upon the last fp having been saved to the thread + // in C2 code but it will have been pushed onto the stack. so we + // have to find it relative to the unextended sp + + assert(_cb->frame_size() >= 0, "must have non-zero frame size"); + intptr_t* l_sender_sp = unextended_sp() + _cb->frame_size(); + intptr_t* unextended_sp = l_sender_sp; + + // the return_address is always the word on the stack + address sender_pc = (address) *(l_sender_sp + frame::return_addr_offset); + + intptr_t** saved_fp_addr = (intptr_t**) (l_sender_sp + frame::link_offset); + + assert(map != NULL, "map must be set"); + if (map->update_map()) { + // Tell GC to use argument oopmaps for some runtime stubs that need it. + // For C1, the runtime stub might not have oop maps, so set this flag + // outside of update_register_map. + map->set_include_argument_oops(_cb->caller_must_gc_arguments(map->thread())); + if (_cb->oop_maps() != NULL) { + OopMapSet::update_register_map(this, map); + } + + // Since the prolog does the save and restore of FP there is no + // oopmap for it so we must fill in its location as if there was + // an oopmap entry since if our caller was compiled code there + // could be live jvm state in it. + update_map_with_saved_link(map, saved_fp_addr); + } + + return frame(l_sender_sp, unextended_sp, *saved_fp_addr, sender_pc); +} + +//------------------------------------------------------------------------------ +// frame::sender_raw +frame frame::sender_raw(RegisterMap* map) const { + // Default is we done have to follow them. The sender_for_xxx will + // update it accordingly + assert(map != NULL, "map must be set"); + map->set_include_argument_oops(false); + + if (is_entry_frame()) { + return sender_for_entry_frame(map); + } + if (is_interpreted_frame()) { + return sender_for_interpreter_frame(map); + } + assert(_cb == CodeCache::find_blob(pc()),"Must be the same"); + + // This test looks odd: why is it not is_compiled_frame() ? That's + // because stubs also have OOP maps. + if (_cb != NULL) { + return sender_for_compiled_frame(map); + } + + // Must be native-compiled frame, i.e. the marshaling code for native + // methods that exists in the core system. + return frame(sender_sp(), link(), sender_pc()); +} + +frame frame::sender(RegisterMap* map) const { + frame result = sender_raw(map); + + if (map->process_frames()) { + StackWatermarkSet::on_iteration(map->thread(), result); + } + + return result; +} + +bool frame::is_interpreted_frame_valid(JavaThread* thread) const { + assert(is_interpreted_frame(), "Not an interpreted frame"); + // These are reasonable sanity checks + if (fp() == NULL || (intptr_t(fp()) & (wordSize-1)) != 0) { + return false; + } + if (sp() == NULL || (intptr_t(sp()) & (wordSize-1)) != 0) { + return false; + } + if (fp() + interpreter_frame_initial_sp_offset < sp()) { + return false; + } + // These are hacks to keep us out of trouble. + // The problem with these is that they mask other problems + if (fp() <= sp()) { // this attempts to deal with unsigned comparison above + return false; + } + + // do some validation of frame elements + + // first the method + Method* m = *interpreter_frame_method_addr(); + // validate the method we'd find in this potential sender + if (!Method::is_valid_method(m)) { + return false; + } + + // stack frames shouldn't be much larger than max_stack elements + // this test requires the use of unextended_sp which is the sp as seen by + // the current frame, and not sp which is the "raw" pc which could point + // further because of local variables of the callee method inserted after + // method arguments + if (fp() - unextended_sp() > 1024 + m->max_stack()*Interpreter::stackElementSize) { + return false; + } + + // validate bci/bcx + address bcp = interpreter_frame_bcp(); + if (m->validate_bci_from_bcp(bcp) < 0) { + return false; + } + + // validate constantPoolCache* + ConstantPoolCache* cp = *interpreter_frame_cache_addr(); + if (MetaspaceObj::is_valid(cp) == false) { + return false; + } + + // validate locals + address locals = (address) *interpreter_frame_locals_addr(); + if (locals > thread->stack_base() || locals < (address) fp()) { + return false; + } + + // We'd have to be pretty unlucky to be mislead at this point + return true; +} + +BasicType frame::interpreter_frame_result(oop* oop_result, jvalue* value_result) { + assert(is_interpreted_frame(), "interpreted frame expected"); + Method* method = interpreter_frame_method(); + BasicType type = method->result_type(); + + intptr_t* tos_addr = NULL; + if (method->is_native()) { + tos_addr = (intptr_t*)sp(); + if (type == T_FLOAT || type == T_DOUBLE) { + // This is because we do a push(ltos) after push(dtos) in generate_native_entry. + tos_addr += 2 * Interpreter::stackElementWords; + } + } else { + tos_addr = (intptr_t*)interpreter_frame_tos_address(); + } + + switch (type) { + case T_OBJECT : + case T_ARRAY : { + oop obj; + if (method->is_native()) { + obj = cast_to_oop(at(interpreter_frame_oop_temp_offset)); + } else { + oop* obj_p = (oop*)tos_addr; + obj = (obj_p == NULL) ? (oop)NULL : *obj_p; + } + assert(Universe::is_in_heap_or_null(obj), "sanity check"); + *oop_result = obj; + break; + } + case T_BOOLEAN : value_result->z = *(jboolean*)tos_addr; break; + case T_BYTE : value_result->b = *(jbyte*)tos_addr; break; + case T_CHAR : value_result->c = *(jchar*)tos_addr; break; + case T_SHORT : value_result->s = *(jshort*)tos_addr; break; + case T_INT : value_result->i = *(jint*)tos_addr; break; + case T_LONG : value_result->j = *(jlong*)tos_addr; break; + case T_FLOAT : { + value_result->f = *(jfloat*)tos_addr; + break; + } + case T_DOUBLE : value_result->d = *(jdouble*)tos_addr; break; + case T_VOID : /* Nothing to do */ break; + default : ShouldNotReachHere(); + } + + return type; +} + + +intptr_t* frame::interpreter_frame_tos_at(jint offset) const { + int index = (Interpreter::expr_offset_in_bytes(offset)/wordSize); + return &interpreter_frame_tos_address()[index]; +} + +#ifndef PRODUCT + +#define DESCRIBE_FP_OFFSET(name) \ + values.describe(frame_no, fp() + frame::name##_offset, #name) + +void frame::describe_pd(FrameValues& values, int frame_no) { + if (is_interpreted_frame()) { + DESCRIBE_FP_OFFSET(interpreter_frame_sender_sp); + DESCRIBE_FP_OFFSET(interpreter_frame_last_sp); + DESCRIBE_FP_OFFSET(interpreter_frame_method); + DESCRIBE_FP_OFFSET(interpreter_frame_mdp); + DESCRIBE_FP_OFFSET(interpreter_frame_mirror); + DESCRIBE_FP_OFFSET(interpreter_frame_cache); + DESCRIBE_FP_OFFSET(interpreter_frame_locals); + DESCRIBE_FP_OFFSET(interpreter_frame_bcp); + DESCRIBE_FP_OFFSET(interpreter_frame_initial_sp); + } +} +#endif + +intptr_t *frame::initial_deoptimization_info() { + // Not used on riscv, but we must return something. + return NULL; +} + +intptr_t* frame::real_fp() const { + if (_cb != NULL) { + // use the frame size if valid + int size = _cb->frame_size(); + if (size > 0) { + return unextended_sp() + size; + } + } + // else rely on fp() + assert(!is_compiled_frame(), "unknown compiled frame size"); + return fp(); +} + +#undef DESCRIBE_FP_OFFSET + +#ifndef PRODUCT +// This is a generic constructor which is only used by pns() in debug.cpp. +frame::frame(void* ptr_sp, void* ptr_fp, void* pc) { + init((intptr_t*)ptr_sp, (intptr_t*)ptr_fp, (address)pc); +} + +void frame::pd_ps() {} +#endif + +void JavaFrameAnchor::make_walkable() { + // last frame set? + if (last_Java_sp() == NULL) { return; } + // already walkable? + if (walkable()) { return; } + vmassert(last_Java_sp() != NULL, "not called from Java code?"); + vmassert(last_Java_pc() == NULL, "already walkable"); + _last_Java_pc = (address)_last_Java_sp[-1]; + vmassert(walkable(), "something went wrong"); +} diff --git a/src/hotspot/cpu/riscv/frame_riscv.hpp b/src/hotspot/cpu/riscv/frame_riscv.hpp new file mode 100644 index 0000000000000..d71ee908aa518 --- /dev/null +++ b/src/hotspot/cpu/riscv/frame_riscv.hpp @@ -0,0 +1,204 @@ +/* + * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_FRAME_RISCV_HPP +#define CPU_RISCV_FRAME_RISCV_HPP + +#include "runtime/synchronizer.hpp" + +// A frame represents a physical stack frame (an activation). Frames can be +// C or Java frames, and the Java frames can be interpreted or compiled. +// In contrast, vframes represent source-level activations, so that one physical frame +// can correspond to multiple source level frames because of inlining. +// A frame is comprised of {pc, fp, sp} +// ------------------------------ Asm interpreter ---------------------------------------- +// Layout of asm interpreter frame: +// [expression stack ] * <- sp + +// [monitors[0] ] \ +// ... | monitor block size = k +// [monitors[k-1] ] / +// [frame initial esp ] ( == &monitors[0], initially here) initial_sp_offset +// [byte code index/pointr] = bcx() bcx_offset + +// [pointer to locals ] = locals() locals_offset +// [constant pool cache ] = cache() cache_offset + +// [klass of method ] = mirror() mirror_offset +// [padding ] + +// [methodData ] = mdp() mdx_offset +// [Method ] = method() method_offset + +// [last esp ] = last_sp() last_sp_offset +// [old stack pointer ] (sender_sp) sender_sp_offset + +// [old frame pointer ] +// [return pc ] + +// [last sp ] <- fp = link() +// [oop temp ] (only for native calls) + +// [padding ] (to preserve machine SP alignment) +// [locals and parameters ] +// <- sender sp +// ------------------------------ Asm interpreter ---------------------------------------- + +// ------------------------------ C Frame ------------------------------------------------ +// Stack: gcc with -fno-omit-frame-pointer +// . +// . +// +-> . +// | +-----------------+ | +// | | return address | | +// | | previous fp ------+ +// | | saved registers | +// | | local variables | +// | | ... | <-+ +// | +-----------------+ | +// | | return address | | +// +------ previous fp | | +// | saved registers | | +// | local variables | | +// +-> | ... | | +// | +-----------------+ | +// | | return address | | +// | | previous fp ------+ +// | | saved registers | +// | | local variables | +// | | ... | <-+ +// | +-----------------+ | +// | | return address | | +// +------ previous fp | | +// | saved registers | | +// | local variables | | +// $fp --> | ... | | +// +-----------------+ | +// | return address | | +// | previous fp ------+ +// | saved registers | +// $sp --> | local variables | +// +-----------------+ +// ------------------------------ C Frame ------------------------------------------------ + + public: + enum { + pc_return_offset = 0, + + // All frames + link_offset = -2, + return_addr_offset = -1, + sender_sp_offset = 0, + + // Interpreter frames + interpreter_frame_oop_temp_offset = 1, // for native calls only + + interpreter_frame_sender_sp_offset = -3, + // outgoing sp before a call to an invoked method + interpreter_frame_last_sp_offset = interpreter_frame_sender_sp_offset - 1, + interpreter_frame_method_offset = interpreter_frame_last_sp_offset - 1, + interpreter_frame_mdp_offset = interpreter_frame_method_offset - 1, + interpreter_frame_padding_offset = interpreter_frame_mdp_offset - 1, + interpreter_frame_mirror_offset = interpreter_frame_padding_offset - 1, + interpreter_frame_cache_offset = interpreter_frame_mirror_offset - 1, + interpreter_frame_locals_offset = interpreter_frame_cache_offset - 1, + interpreter_frame_bcp_offset = interpreter_frame_locals_offset - 1, + interpreter_frame_initial_sp_offset = interpreter_frame_bcp_offset - 1, + + interpreter_frame_monitor_block_top_offset = interpreter_frame_initial_sp_offset, + interpreter_frame_monitor_block_bottom_offset = interpreter_frame_initial_sp_offset, + + // Entry frames + // n.b. these values are determined by the layout defined in + // stubGenerator for the Java call stub + entry_frame_after_call_words = 34, + entry_frame_call_wrapper_offset = -10, + + // we don't need a save area + arg_reg_save_area_bytes = 0 + }; + + intptr_t ptr_at(int offset) const { + return *ptr_at_addr(offset); + } + + void ptr_at_put(int offset, intptr_t value) { + *ptr_at_addr(offset) = value; + } + + private: + // an additional field beyond _sp and _pc: + intptr_t* _fp; // frame pointer + // The interpreter and adapters will extend the frame of the caller. + // Since oopMaps are based on the sp of the caller before extension + // we need to know that value. However in order to compute the address + // of the return address we need the real "raw" sp. Since sparc already + // uses sp() to mean "raw" sp and unextended_sp() to mean the caller's + // original sp we use that convention. + + intptr_t* _unextended_sp; + void adjust_unextended_sp(); + + intptr_t* ptr_at_addr(int offset) const { + return (intptr_t*) addr_at(offset); + } + +#ifdef ASSERT + // Used in frame::sender_for_{interpreter,compiled}_frame + static void verify_deopt_original_pc( CompiledMethod* nm, intptr_t* unextended_sp); +#endif + + public: + // Constructors + + frame(intptr_t* ptr_sp, intptr_t* ptr_fp, address pc); + + frame(intptr_t* ptr_sp, intptr_t* unextended_sp, intptr_t* ptr_fp, address pc); + + frame(intptr_t* ptr_sp, intptr_t* ptr_fp); + + void init(intptr_t* ptr_sp, intptr_t* ptr_fp, address pc); + + // accessors for the instance variables + // Note: not necessarily the real 'frame pointer' (see real_fp) + intptr_t* fp() const { return _fp; } + + inline address* sender_pc_addr() const; + + // expression stack tos if we are nested in a java call + intptr_t* interpreter_frame_last_sp() const; + + // helper to update a map with callee-saved RBP + static void update_map_with_saved_link(RegisterMap* map, intptr_t** link_addr); + + // deoptimization support + void interpreter_frame_set_last_sp(intptr_t* last_sp); + + static jint interpreter_frame_expression_stack_direction() { return -1; } + + // returns the sending frame, without applying any barriers + frame sender_raw(RegisterMap* map) const; + +#endif // CPU_RISCV_FRAME_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/frame_riscv.inline.hpp b/src/hotspot/cpu/riscv/frame_riscv.inline.hpp new file mode 100644 index 0000000000000..0536340e369ce --- /dev/null +++ b/src/hotspot/cpu/riscv/frame_riscv.inline.hpp @@ -0,0 +1,246 @@ +/* + * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_FRAME_RISCV_INLINE_HPP +#define CPU_RISCV_FRAME_RISCV_INLINE_HPP + +#include "code/codeCache.hpp" +#include "code/vmreg.inline.hpp" + +// Inline functions for RISCV frames: + +// Constructors: + +inline frame::frame() { + _pc = NULL; + _sp = NULL; + _unextended_sp = NULL; + _fp = NULL; + _cb = NULL; + _deopt_state = unknown; +} + +static int spin; + +inline void frame::init(intptr_t* ptr_sp, intptr_t* ptr_fp, address pc) { + intptr_t a = intptr_t(ptr_sp); + intptr_t b = intptr_t(ptr_fp); + _sp = ptr_sp; + _unextended_sp = ptr_sp; + _fp = ptr_fp; + _pc = pc; + assert(pc != NULL, "no pc?"); + _cb = CodeCache::find_blob(pc); + adjust_unextended_sp(); + + address original_pc = CompiledMethod::get_deopt_original_pc(this); + if (original_pc != NULL) { + _pc = original_pc; + _deopt_state = is_deoptimized; + } else { + _deopt_state = not_deoptimized; + } +} + +inline frame::frame(intptr_t* ptr_sp, intptr_t* ptr_fp, address pc) { + init(ptr_sp, ptr_fp, pc); +} + +inline frame::frame(intptr_t* ptr_sp, intptr_t* unextended_sp, intptr_t* ptr_fp, address pc) { + intptr_t a = intptr_t(ptr_sp); + intptr_t b = intptr_t(ptr_fp); + _sp = ptr_sp; + _unextended_sp = unextended_sp; + _fp = ptr_fp; + _pc = pc; + assert(pc != NULL, "no pc?"); + _cb = CodeCache::find_blob(pc); + adjust_unextended_sp(); + + address original_pc = CompiledMethod::get_deopt_original_pc(this); + if (original_pc != NULL) { + _pc = original_pc; + assert(_cb->as_compiled_method()->insts_contains_inclusive(_pc), + "original PC must be in the main code section of the the compiled method (or must be immediately following it)"); + _deopt_state = is_deoptimized; + } else { + _deopt_state = not_deoptimized; + } +} + +inline frame::frame(intptr_t* ptr_sp, intptr_t* ptr_fp) { + intptr_t a = intptr_t(ptr_sp); + intptr_t b = intptr_t(ptr_fp); + _sp = ptr_sp; + _unextended_sp = ptr_sp; + _fp = ptr_fp; + _pc = (address)(ptr_sp[-1]); + + // Here's a sticky one. This constructor can be called via AsyncGetCallTrace + // when last_Java_sp is non-null but the pc fetched is junk. If we are truly + // unlucky the junk value could be to a zombied method and we'll die on the + // find_blob call. This is also why we can have no asserts on the validity + // of the pc we find here. AsyncGetCallTrace -> pd_get_top_frame_for_signal_handler + // -> pd_last_frame should use a specialized version of pd_last_frame which could + // call a specilaized frame constructor instead of this one. + // Then we could use the assert below. However this assert is of somewhat dubious + // value. + + _cb = CodeCache::find_blob(_pc); + adjust_unextended_sp(); + + address original_pc = CompiledMethod::get_deopt_original_pc(this); + if (original_pc != NULL) { + _pc = original_pc; + _deopt_state = is_deoptimized; + } else { + _deopt_state = not_deoptimized; + } +} + +// Accessors + +inline bool frame::equal(frame other) const { + bool ret = sp() == other.sp() && + unextended_sp() == other.unextended_sp() && + fp() == other.fp() && + pc() == other.pc(); + assert(!ret || ret && cb() == other.cb() && _deopt_state == other._deopt_state, "inconsistent construction"); + return ret; +} + +// Return unique id for this frame. The id must have a value where we can distinguish +// identity and younger/older relationship. NULL represents an invalid (incomparable) +// frame. +inline intptr_t* frame::id(void) const { return unextended_sp(); } + +// Return true if the frame is older (less recent activation) than the frame represented by id +inline bool frame::is_older(intptr_t* id) const { assert(this->id() != NULL && id != NULL, "NULL frame id"); + return this->id() > id ; } + +inline intptr_t* frame::link() const { return (intptr_t*) *(intptr_t **)addr_at(link_offset); } + +inline intptr_t* frame::link_or_null() const { + intptr_t** ptr = (intptr_t **)addr_at(link_offset); + return os::is_readable_pointer(ptr) ? *ptr : NULL; +} + +inline intptr_t* frame::unextended_sp() const { return _unextended_sp; } + +// Return address +inline address* frame::sender_pc_addr() const { return (address*) addr_at(return_addr_offset); } +inline address frame::sender_pc() const { return *sender_pc_addr(); } +inline intptr_t* frame::sender_sp() const { return addr_at(sender_sp_offset); } + +inline intptr_t** frame::interpreter_frame_locals_addr() const { + return (intptr_t**)addr_at(interpreter_frame_locals_offset); +} + +inline intptr_t* frame::interpreter_frame_last_sp() const { + return *(intptr_t**)addr_at(interpreter_frame_last_sp_offset); +} + +inline intptr_t* frame::interpreter_frame_bcp_addr() const { + return (intptr_t*)addr_at(interpreter_frame_bcp_offset); +} + +inline intptr_t* frame::interpreter_frame_mdp_addr() const { + return (intptr_t*)addr_at(interpreter_frame_mdp_offset); +} + + +// Constant pool cache + +inline ConstantPoolCache** frame::interpreter_frame_cache_addr() const { + return (ConstantPoolCache**)addr_at(interpreter_frame_cache_offset); +} + +// Method + +inline Method** frame::interpreter_frame_method_addr() const { + return (Method**)addr_at(interpreter_frame_method_offset); +} + +// Mirror + +inline oop* frame::interpreter_frame_mirror_addr() const { + return (oop*)addr_at(interpreter_frame_mirror_offset); +} + +// top of expression stack +inline intptr_t* frame::interpreter_frame_tos_address() const { + intptr_t* last_sp = interpreter_frame_last_sp(); + if (last_sp == NULL) { + return sp(); + } else { + // sp() may have been extended or shrunk by an adapter. At least + // check that we don't fall behind the legal region. + // For top deoptimized frame last_sp == interpreter_frame_monitor_end. + assert(last_sp <= (intptr_t*) interpreter_frame_monitor_end(), "bad tos"); + return last_sp; + } +} + +inline oop* frame::interpreter_frame_temp_oop_addr() const { + return (oop *)(fp() + interpreter_frame_oop_temp_offset); +} + +inline int frame::interpreter_frame_monitor_size() { + return BasicObjectLock::size(); +} + + +// expression stack +// (the max_stack arguments are used by the GC; see class FrameClosure) + +inline intptr_t* frame::interpreter_frame_expression_stack() const { + intptr_t* monitor_end = (intptr_t*) interpreter_frame_monitor_end(); + return monitor_end-1; +} + + +// Entry frames + +inline JavaCallWrapper** frame::entry_frame_call_wrapper_addr() const { + return (JavaCallWrapper**)addr_at(entry_frame_call_wrapper_offset); +} + + +// Compiled frames + +inline oop frame::saved_oop_result(RegisterMap* map) const { + oop* result_adr = (oop *)map->location(x10->as_VMReg()); + guarantee(result_adr != NULL, "bad register save location"); + return (*result_adr); +} + +inline void frame::set_saved_oop_result(RegisterMap* map, oop obj) { + oop* result_adr = (oop *)map->location(x10->as_VMReg()); + guarantee(result_adr != NULL, "bad register save location"); + *result_adr = obj; +} + +#endif // CPU_RISCV_FRAME_RISCV_INLINE_HPP diff --git a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp new file mode 100644 index 0000000000000..2a2ccd0ee6dc5 --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp @@ -0,0 +1,478 @@ +/* + * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "gc/g1/g1BarrierSet.hpp" +#include "gc/g1/g1BarrierSetAssembler.hpp" +#include "gc/g1/g1BarrierSetRuntime.hpp" +#include "gc/g1/g1CardTable.hpp" +#include "gc/g1/g1ThreadLocalData.hpp" +#include "gc/g1/heapRegion.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "interpreter/interp_masm.hpp" +#include "runtime/sharedRuntime.hpp" +#include "runtime/thread.hpp" +#ifdef COMPILER1 +#include "c1/c1_LIRAssembler.hpp" +#include "c1/c1_MacroAssembler.hpp" +#include "gc/g1/c1/g1BarrierSetC1.hpp" +#endif + +#define __ masm-> + +void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register addr, Register count, RegSet saved_regs) { + bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; + if (!dest_uninitialized) { + Label done; + Address in_progress(xthread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); + + // Is marking active? + if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { + __ lwu(t0, in_progress); + } else { + assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); + __ lbu(t0, in_progress); + } + __ beqz(t0, done); + + __ push_reg(saved_regs, sp); + if (count == c_rarg0) { + if (addr == c_rarg1) { + // exactly backwards!! + __ mv(t0, c_rarg0); + __ mv(c_rarg0, c_rarg1); + __ mv(c_rarg1, t0); + } else { + __ mv(c_rarg1, count); + __ mv(c_rarg0, addr); + } + } else { + __ mv(c_rarg0, addr); + __ mv(c_rarg1, count); + } + if (UseCompressedOops) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), 2); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), 2); + } + __ pop_reg(saved_regs, sp); + + __ bind(done); + } +} + +void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register start, Register count, Register tmp, RegSet saved_regs) { + __ push_reg(saved_regs, sp); + assert_different_registers(start, count, tmp); + assert_different_registers(c_rarg0, count); + __ mv(c_rarg0, start); + __ mv(c_rarg1, count); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2); + __ pop_reg(saved_regs, sp); +} + +void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp, + bool tosca_live, + bool expand_call) { + // If expand_call is true then we expand the call_VM_leaf macro + // directly to skip generating the check by + // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp. + + assert(thread == xthread, "must be"); + + Label done; + Label runtime; + + assert_different_registers(obj, pre_val, tmp, t0); + assert(pre_val != noreg && tmp != noreg, "expecting a register"); + + Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); + Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset())); + Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset())); + + // Is marking active? + if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { // 4-byte width + __ lwu(tmp, in_progress); + } else { + assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); + __ lbu(tmp, in_progress); + } + __ beqz(tmp, done); + + // Do we need to load the previous value? + if (obj != noreg) { + __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); + } + + // Is the previous value null? + __ beqz(pre_val, done); + + // Can we store original value in the thread's buffer? + // Is index == 0? + // (The index field is typed as size_t.) + + __ ld(tmp, index); // tmp := *index_adr + __ beqz(tmp, runtime); // tmp == 0? + // If yes, goto runtime + + __ sub(tmp, tmp, wordSize); // tmp := tmp - wordSize + __ sd(tmp, index); // *index_adr := tmp + __ ld(t0, buffer); + __ add(tmp, tmp, t0); // tmp := tmp + *buffer_adr + + // Record the previous value + __ sd(pre_val, Address(tmp, 0)); + __ j(done); + + __ bind(runtime); + // save the live input values + RegSet saved = RegSet::of(pre_val); + if (tosca_live) { saved += RegSet::of(x10); } + if (obj != noreg) { saved += RegSet::of(obj); } + + __ push_reg(saved, sp); + + if (expand_call) { + assert(pre_val != c_rarg1, "smashed arg"); + __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread); + } + + __ pop_reg(saved, sp); + + __ bind(done); + +} + +void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, + Register store_addr, + Register new_val, + Register thread, + Register tmp, + Register tmp2) { + assert(thread == xthread, "must be"); + assert_different_registers(store_addr, new_val, thread, tmp, tmp2, + t0); + assert(store_addr != noreg && new_val != noreg && tmp != noreg && + tmp2 != noreg, "expecting a register"); + + Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset())); + Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset())); + + BarrierSet* bs = BarrierSet::barrier_set(); + CardTableBarrierSet* ctbs = barrier_set_cast(bs); + CardTable* ct = ctbs->card_table(); + + Label done; + Label runtime; + + // Does store cross heap regions? + + __ xorr(tmp, store_addr, new_val); + __ srli(tmp, tmp, HeapRegion::LogOfHRGrainBytes); + __ beqz(tmp, done); + + // crosses regions, storing NULL? + + __ beqz(new_val, done); + + // storing region crossing non-NULL, is card already dirty? + + ExternalAddress cardtable((address) ct->byte_map_base()); + const Register card_addr = tmp; + + __ srli(card_addr, store_addr, CardTable::card_shift); + + // get the address of the card + __ load_byte_map_base(tmp2); + __ add(card_addr, card_addr, tmp2); + __ lbu(tmp2, Address(card_addr)); + __ mv(t0, (int)G1CardTable::g1_young_card_val()); + __ beq(tmp2, t0, done); + + assert((int)CardTable::dirty_card_val() == 0, "must be 0"); + + __ membar(MacroAssembler::StoreLoad); + + __ lbu(tmp2, Address(card_addr)); + __ beqz(tmp2, done); + + // storing a region crossing, non-NULL oop, card is clean. + // dirty card and log. + + __ sb(zr, Address(card_addr)); + + __ ld(t0, queue_index); + __ beqz(t0, runtime); + __ sub(t0, t0, wordSize); + __ sd(t0, queue_index); + + __ ld(tmp2, buffer); + __ add(t0, tmp2, t0); + __ sd(card_addr, Address(t0, 0)); + __ j(done); + + __ bind(runtime); + // save the live input values + RegSet saved = RegSet::of(store_addr); + __ push_reg(saved, sp); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread); + __ pop_reg(saved, sp); + + __ bind(done); +} + +void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register dst, Address src, Register tmp1, Register tmp_thread) { + bool on_oop = is_reference_type(type); + bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; + bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; + bool on_reference = on_weak || on_phantom; + ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); + if (on_oop && on_reference) { + // RA is live. It must be saved around calls. + __ enter(); // barrier may call runtime + // Generate the G1 pre-barrier code to log the value of + // the referent field in an SATB buffer. + g1_write_barrier_pre(masm /* masm */, + noreg /* obj */, + dst /* pre_val */, + xthread /* thread */, + tmp1 /* tmp */, + true /* tosca_live */, + true /* expand_call */); + __ leave(); + } +} + +void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2) { + // flatten object address if needed + if (dst.offset() == 0) { + if (dst.base() != x13) { + __ mv(x13, dst.base()); + } + } else { + __ la(x13, dst); + } + + g1_write_barrier_pre(masm, + x13 /* obj */, + tmp2 /* pre_val */, + xthread /* thread */, + tmp1 /* tmp */, + val != noreg /* tosca_live */, + false /* expand_call */); + + if (val == noreg) { + BarrierSetAssembler::store_at(masm, decorators, type, Address(x13, 0), noreg, noreg, noreg); + } else { + // G1 barrier needs uncompressed oop for region cross check. + Register new_val = val; + if (UseCompressedOops) { + new_val = t1; + __ mv(new_val, val); + } + BarrierSetAssembler::store_at(masm, decorators, type, Address(x13, 0), val, noreg, noreg); + g1_write_barrier_post(masm, + x13 /* store_adr */, + new_val /* new_val */, + xthread /* thread */, + tmp1 /* tmp */, + tmp2 /* tmp2 */); + } +} + +#ifdef COMPILER1 + +#undef __ +#define __ ce->masm()-> + +void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) { + G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); + + // At this point we know that marking is in progress. + // If do_load() is true then we have to emit the + // load of the previous value; otherwise it has already + // been loaded into _pre_val. + __ bind(*stub->entry()); + + assert(stub->pre_val()->is_register(), "Precondition."); + + Register pre_val_reg = stub->pre_val()->as_register(); + + if (stub->do_load()) { + ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /* wide */, false /* unaligned */); + } + __ beqz(pre_val_reg, *stub->continuation(), /* is_far */ true); + ce->store_parameter(stub->pre_val()->as_register(), 0); + __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); + __ j(*stub->continuation()); +} + +void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) { + G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); + __ bind(*stub->entry()); + assert(stub->addr()->is_register(), "Precondition"); + assert(stub->new_val()->is_register(), "Precondition"); + Register new_val_reg = stub->new_val()->as_register(); + __ beqz(new_val_reg, *stub->continuation(), /* is_far */ true); + ce->store_parameter(stub->addr()->as_pointer_register(), 0); + __ far_call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin())); + __ j(*stub->continuation()); +} + +#undef __ + +#define __ sasm-> + +void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { + __ prologue("g1_pre_barrier", false); + + BarrierSet* bs = BarrierSet::barrier_set(); + + // arg0 : previous value of memory + const Register pre_val = x10; + const Register thread = xthread; + const Register tmp = t0; + + Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); + Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset())); + Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset())); + + Label done; + Label runtime; + + // Is marking still active? + if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { // 4-byte width + __ lwu(tmp, in_progress); + } else { + assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); + __ lbu(tmp, in_progress); + } + __ beqz(tmp, done); + + // Can we store original value in the thread's buffer? + __ ld(tmp, queue_index); + __ beqz(tmp, runtime); + + __ sub(tmp, tmp, wordSize); + __ sd(tmp, queue_index); + __ ld(t1, buffer); + __ add(tmp, tmp, t1); + __ load_parameter(0, t1); + __ sd(t1, Address(tmp, 0)); + __ j(done); + + __ bind(runtime); + __ push_call_clobbered_registers(); + __ load_parameter(0, pre_val); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread); + __ pop_call_clobbered_registers(); + __ bind(done); + + __ epilogue(); +} + +void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) { + __ prologue("g1_post_barrier", false); + + // arg0 : store_address + Address store_addr(fp, 2 * BytesPerWord); // 2 BytesPerWord from fp + + BarrierSet* bs = BarrierSet::barrier_set(); + CardTableBarrierSet* ctbs = barrier_set_cast(bs); + CardTable* ct = ctbs->card_table(); + + Label done; + Label runtime; + + // At this point we know new_value is non-NULL and the new_value crosses regions. + // Must check to see if card is already dirty + const Register thread = xthread; + + Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset())); + Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset())); + + const Register card_offset = t1; + // RA is free here, so we can use it to hold the byte_map_base. + const Register byte_map_base = ra; + + assert_different_registers(card_offset, byte_map_base, t0); + + __ load_parameter(0, card_offset); + __ srli(card_offset, card_offset, CardTable::card_shift); + __ load_byte_map_base(byte_map_base); + + // Convert card offset into an address in card_addr + Register card_addr = card_offset; + __ add(card_addr, byte_map_base, card_addr); + + __ lbu(t0, Address(card_addr, 0)); + __ sub(t0, t0, (int)G1CardTable::g1_young_card_val()); + __ beqz(t0, done); + + assert((int)CardTable::dirty_card_val() == 0, "must be 0"); + + __ membar(MacroAssembler::StoreLoad); + __ lbu(t0, Address(card_addr, 0)); + __ beqz(t0, done); + + // storing region crossing non-NULL, card is clean. + // dirty card and log. + __ sb(zr, Address(card_addr, 0)); + + __ ld(t0, queue_index); + __ beqz(t0, runtime); + __ sub(t0, t0, wordSize); + __ sd(t0, queue_index); + + // Reuse RA to hold buffer_addr + const Register buffer_addr = ra; + + __ ld(buffer_addr, buffer); + __ add(t0, buffer_addr, t0); + __ sd(card_addr, Address(t0, 0)); + __ j(done); + + __ bind(runtime); + __ push_call_clobbered_registers(); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread); + __ pop_call_clobbered_registers(); + __ bind(done); + __ epilogue(); +} + +#undef __ + +#endif // COMPILER1 diff --git a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp new file mode 100644 index 0000000000000..37bc183f39c76 --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_GC_G1_G1BARRIERSETASSEMBLER_RISCV_HPP +#define CPU_RISCV_GC_G1_G1BARRIERSETASSEMBLER_RISCV_HPP + +#include "asm/macroAssembler.hpp" +#include "gc/shared/modRefBarrierSetAssembler.hpp" +#include "utilities/macros.hpp" + +#ifdef COMPILER1 +class LIR_Assembler; +#endif +class StubAssembler; +class G1PreBarrierStub; +class G1PostBarrierStub; + +class G1BarrierSetAssembler: public ModRefBarrierSetAssembler { +protected: + void gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register addr, Register count, RegSet saved_regs); + void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register start, Register count, Register tmp, RegSet saved_regs); + + void g1_write_barrier_pre(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp, + bool tosca_live, + bool expand_call); + + void g1_write_barrier_post(MacroAssembler* masm, + Register store_addr, + Register new_val, + Register thread, + Register tmp, + Register tmp2); + + virtual void oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2); + +public: +#ifdef COMPILER1 + void gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub); + void gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub); + + void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm); + void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm); +#endif + + void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register dst, Address src, Register tmp1, Register tmp_thread); +}; + +#endif // CPU_RISCV_GC_G1_G1BARRIERSETASSEMBLER_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/gc/g1/g1Globals_riscv.hpp b/src/hotspot/cpu/riscv/gc/g1/g1Globals_riscv.hpp new file mode 100644 index 0000000000000..8735fd014ffff --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/g1/g1Globals_riscv.hpp @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_GC_G1_G1GLOBALS_RISCV_HPP +#define CPU_RISCV_GC_G1_G1GLOBALS_RISCV_HPP + +const size_t G1MergeHeapRootsPrefetchCacheSize = 16; + +#endif // CPU_RISCV_GC_G1_G1GLOBALS_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp new file mode 100644 index 0000000000000..22a8e2d7dba8d --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp @@ -0,0 +1,298 @@ +/* + * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/classLoaderData.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/barrierSetAssembler.hpp" +#include "gc/shared/barrierSetNMethod.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "interpreter/interp_masm.hpp" +#include "memory/universe.hpp" +#include "runtime/jniHandles.hpp" +#include "runtime/sharedRuntime.hpp" +#include "runtime/stubRoutines.hpp" +#include "runtime/thread.hpp" + +#define __ masm-> + +void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register dst, Address src, Register tmp1, Register tmp_thread) { + // RA is live. It must be saved around calls. + + bool in_heap = (decorators & IN_HEAP) != 0; + bool in_native = (decorators & IN_NATIVE) != 0; + bool is_not_null = (decorators & IS_NOT_NULL) != 0; + switch (type) { + case T_OBJECT: // fall through + case T_ARRAY: { + if (in_heap) { + if (UseCompressedOops) { + __ lwu(dst, src); + if (is_not_null) { + __ decode_heap_oop_not_null(dst); + } else { + __ decode_heap_oop(dst); + } + } else { + __ ld(dst, src); + } + } else { + assert(in_native, "why else?"); + __ ld(dst, src); + } + break; + } + case T_BOOLEAN: __ load_unsigned_byte (dst, src); break; + case T_BYTE: __ load_signed_byte (dst, src); break; + case T_CHAR: __ load_unsigned_short(dst, src); break; + case T_SHORT: __ load_signed_short (dst, src); break; + case T_INT: __ lw (dst, src); break; + case T_LONG: __ ld (dst, src); break; + case T_ADDRESS: __ ld (dst, src); break; + case T_FLOAT: __ flw (f10, src); break; + case T_DOUBLE: __ fld (f10, src); break; + default: Unimplemented(); + } +} + +void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2) { + bool in_heap = (decorators & IN_HEAP) != 0; + bool in_native = (decorators & IN_NATIVE) != 0; + switch (type) { + case T_OBJECT: // fall through + case T_ARRAY: { + val = val == noreg ? zr : val; + if (in_heap) { + if (UseCompressedOops) { + assert(!dst.uses(val), "not enough registers"); + if (val != zr) { + __ encode_heap_oop(val); + } + __ sw(val, dst); + } else { + __ sd(val, dst); + } + } else { + assert(in_native, "why else?"); + __ sd(val, dst); + } + break; + } + case T_BOOLEAN: + __ andi(val, val, 0x1); // boolean is true if LSB is 1 + __ sb(val, dst); + break; + case T_BYTE: __ sb(val, dst); break; + case T_CHAR: __ sh(val, dst); break; + case T_SHORT: __ sh(val, dst); break; + case T_INT: __ sw(val, dst); break; + case T_LONG: __ sd(val, dst); break; + case T_ADDRESS: __ sd(val, dst); break; + case T_FLOAT: __ fsw(f10, dst); break; + case T_DOUBLE: __ fsd(f10, dst); break; + default: Unimplemented(); + } + +} + +void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env, + Register obj, Register tmp, Label& slowpath) { + // If mask changes we need to ensure that the inverse is still encodable as an immediate + STATIC_ASSERT(JNIHandles::weak_tag_mask == 1); + __ andi(obj, obj, ~JNIHandles::weak_tag_mask); + __ ld(obj, Address(obj, 0)); // *obj +} + +// Defines obj, preserves var_size_in_bytes, okay for tmp2 == var_size_in_bytes. +void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj, + Register var_size_in_bytes, + int con_size_in_bytes, + Register tmp1, + Register tmp2, + Label& slow_case, + bool is_far) { + assert_different_registers(obj, tmp2); + assert_different_registers(obj, var_size_in_bytes); + Register end = tmp2; + + __ ld(obj, Address(xthread, JavaThread::tlab_top_offset())); + if (var_size_in_bytes == noreg) { + __ la(end, Address(obj, con_size_in_bytes)); + } else { + __ add(end, obj, var_size_in_bytes); + } + __ ld(t0, Address(xthread, JavaThread::tlab_end_offset())); + __ bgtu(end, t0, slow_case, is_far); + + // update the tlab top pointer + __ sd(end, Address(xthread, JavaThread::tlab_top_offset())); + + // recover var_size_in_bytes if necessary + if (var_size_in_bytes == end) { + __ sub(var_size_in_bytes, var_size_in_bytes, obj); + } +} + +// Defines obj, preserves var_size_in_bytes +void BarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register obj, + Register var_size_in_bytes, + int con_size_in_bytes, + Register tmp1, + Label& slow_case, + bool is_far) { + assert_cond(masm != NULL); + assert_different_registers(obj, var_size_in_bytes, tmp1); + if (!Universe::heap()->supports_inline_contig_alloc()) { + __ j(slow_case); + } else { + Register end = tmp1; + Label retry; + __ bind(retry); + + // Get the current end of the heap + ExternalAddress address_end((address) Universe::heap()->end_addr()); + { + int32_t offset; + __ la_patchable(t1, address_end, offset); + __ ld(t1, Address(t1, offset)); + } + + // Get the current top of the heap + ExternalAddress address_top((address) Universe::heap()->top_addr()); + { + int32_t offset; + __ la_patchable(t0, address_top, offset); + __ addi(t0, t0, offset); + __ lr_d(obj, t0, Assembler::aqrl); + } + + // Adjust it my the size of our new object + if (var_size_in_bytes == noreg) { + __ la(end, Address(obj, con_size_in_bytes)); + } else { + __ add(end, obj, var_size_in_bytes); + } + + // if end < obj then we wrapped around high memory + __ bltu(end, obj, slow_case, is_far); + + __ bgtu(end, t1, slow_case, is_far); + + // If heap_top hasn't been changed by some other thread, update it. + __ sc_d(t1, end, t0, Assembler::rl); + __ bnez(t1, retry); + + incr_allocated_bytes(masm, var_size_in_bytes, con_size_in_bytes, tmp1); + } +} + +void BarrierSetAssembler::incr_allocated_bytes(MacroAssembler* masm, + Register var_size_in_bytes, + int con_size_in_bytes, + Register tmp1) { + assert(tmp1->is_valid(), "need temp reg"); + + __ ld(tmp1, Address(xthread, in_bytes(JavaThread::allocated_bytes_offset()))); + if (var_size_in_bytes->is_valid()) { + __ add(tmp1, tmp1, var_size_in_bytes); + } else { + __ add(tmp1, tmp1, con_size_in_bytes); + } + __ sd(tmp1, Address(xthread, in_bytes(JavaThread::allocated_bytes_offset()))); +} + +void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) { + BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); + + if (bs_nm == NULL) { + return; + } + + Assembler::IncompressibleRegion ir(masm); // Fixed length: see entry_barrier_offset() + + // RISCV atomic operations require that the memory address be naturally aligned. + __ align(4); + + Label skip, guard; + Address thread_disarmed_addr(xthread, in_bytes(bs_nm->thread_disarmed_offset())); + + __ lwu(t0, guard); + + // Subsequent loads of oops must occur after load of guard value. + // BarrierSetNMethod::disarm sets guard with release semantics. + __ membar(MacroAssembler::LoadLoad); + __ lwu(t1, thread_disarmed_addr); + __ beq(t0, t1, skip); + + int32_t offset = 0; + __ movptr(t0, StubRoutines::riscv::method_entry_barrier(), offset); + __ jalr(ra, t0, offset); + __ j(skip); + + __ bind(guard); + + MacroAssembler::assert_alignment(__ pc()); + __ emit_int32(0); // nmethod guard value. Skipped over in common case. + + __ bind(skip); +} + +void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) { + BarrierSetNMethod* bs = BarrierSet::barrier_set()->barrier_set_nmethod(); + if (bs == NULL) { + return; + } + + Label bad_call; + __ beqz(xmethod, bad_call); + + // Pointer chase to the method holder to find out if the method is concurrently unloading. + Label method_live; + __ load_method_holder_cld(t0, xmethod); + + // Is it a strong CLD? + __ lwu(t1, Address(t0, ClassLoaderData::keep_alive_offset())); + __ bnez(t1, method_live); + + // Is it a weak but alive CLD? + __ push_reg(RegSet::of(x28, x29), sp); + + __ ld(x28, Address(t0, ClassLoaderData::holder_offset())); + + // Uses x28 & x29, so we must pass new temporaries. + __ resolve_weak_handle(x28, x29); + __ mv(t0, x28); + + __ pop_reg(RegSet::of(x28, x29), sp); + + __ bnez(t0, method_live); + + __ bind(bad_call); + + __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); + __ bind(method_live); +} diff --git a/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.hpp new file mode 100644 index 0000000000000..b85f7f5582b2a --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.hpp @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_GC_SHARED_BARRIERSETASSEMBLER_RISCV_HPP +#define CPU_RISCV_GC_SHARED_BARRIERSETASSEMBLER_RISCV_HPP + +#include "asm/macroAssembler.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/barrierSetNMethod.hpp" +#include "memory/allocation.hpp" +#include "oops/access.hpp" + +class BarrierSetAssembler: public CHeapObj { +private: + void incr_allocated_bytes(MacroAssembler* masm, + Register var_size_in_bytes, int con_size_in_bytes, + Register t1 = noreg); + +public: + virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, + Register src, Register dst, Register count, RegSet saved_regs) {} + virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, + Register start, Register end, Register tmp, RegSet saved_regs) {} + virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register dst, Address src, Register tmp1, Register tmp_thread); + virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2); + + virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env, + Register obj, Register tmp, Label& slowpath); + + virtual void tlab_allocate(MacroAssembler* masm, + Register obj, // result: pointer to object after successful allocation + Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise + int con_size_in_bytes, // object size in bytes if known at compile time + Register tmp1, // temp register + Register tmp2, // temp register + Label& slow_case, // continuation point if fast allocation fails + bool is_far = false + ); + + void eden_allocate(MacroAssembler* masm, + Register obj, // result: pointer to object after successful allocation + Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise + int con_size_in_bytes, // object size in bytes if known at compile time + Register tmp1, // temp register + Label& slow_case, // continuation point if fast allocation fails + bool is_far = false + ); + virtual void barrier_stubs_init() {} + + virtual void nmethod_entry_barrier(MacroAssembler* masm); + virtual void c2i_entry_barrier(MacroAssembler* masm); + virtual ~BarrierSetAssembler() {} +}; + +#endif // CPU_RISCV_GC_SHARED_BARRIERSETASSEMBLER_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/gc/shared/barrierSetNMethod_riscv.cpp b/src/hotspot/cpu/riscv/gc/shared/barrierSetNMethod_riscv.cpp new file mode 100644 index 0000000000000..39ffbea03e8e4 --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/shared/barrierSetNMethod_riscv.cpp @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "code/codeCache.hpp" +#include "code/nativeInst.hpp" +#include "gc/shared/barrierSetNMethod.hpp" +#include "logging/log.hpp" +#include "memory/resourceArea.hpp" +#include "runtime/sharedRuntime.hpp" +#include "runtime/registerMap.hpp" +#include "runtime/thread.hpp" +#include "utilities/align.hpp" +#include "utilities/debug.hpp" + +class NativeNMethodBarrier: public NativeInstruction { + address instruction_address() const { return addr_at(0); } + + int *guard_addr() { + /* auipc + lwu + fence + lwu + beq + lui + addi + slli + addi + slli + jalr + j */ + return reinterpret_cast(instruction_address() + 12 * 4); + } + +public: + int get_value() { + return Atomic::load_acquire(guard_addr()); + } + + void set_value(int value) { + Atomic::release_store(guard_addr(), value); + } + + void verify() const; +}; + +// Store the instruction bitmask, bits and name for checking the barrier. +struct CheckInsn { + uint32_t mask; + uint32_t bits; + const char *name; +}; + +static const struct CheckInsn barrierInsn[] = { + { 0x00000fff, 0x00000297, "auipc t0, 0 "}, + { 0x000fffff, 0x0002e283, "lwu t0, 48(t0) "}, + { 0xffffffff, 0x0aa0000f, "fence ir, ir "}, + { 0x000fffff, 0x000be303, "lwu t1, 112(xthread)"}, + { 0x01fff07f, 0x00628063, "beq t0, t1, skip "}, + { 0x00000fff, 0x000002b7, "lui t0, imm0 "}, + { 0x000fffff, 0x00028293, "addi t0, t0, imm1 "}, + { 0xffffffff, 0x00b29293, "slli t0, t0, 11 "}, + { 0x000fffff, 0x00028293, "addi t0, t0, imm2 "}, + { 0xffffffff, 0x00629293, "slli t0, t0, 6 "}, + { 0x000fffff, 0x000280e7, "jalr ra, imm3(t0) "}, + { 0x00000fff, 0x0000006f, "j skip "} + /* guard: */ + /* 32bit nmethod guard value */ + /* skip: */ +}; + +// The encodings must match the instructions emitted by +// BarrierSetAssembler::nmethod_entry_barrier. The matching ignores the specific +// register numbers and immediate values in the encoding. +void NativeNMethodBarrier::verify() const { + intptr_t addr = (intptr_t) instruction_address(); + for(unsigned int i = 0; i < sizeof(barrierInsn)/sizeof(struct CheckInsn); i++ ) { + uint32_t inst = *((uint32_t*) addr); + if ((inst & barrierInsn[i].mask) != barrierInsn[i].bits) { + tty->print_cr("Addr: " INTPTR_FORMAT " Code: 0x%x", addr, inst); + fatal("not an %s instruction.", barrierInsn[i].name); + } + addr += 4; + } +} + + +/* We're called from an nmethod when we need to deoptimize it. We do + this by throwing away the nmethod's frame and jumping to the + ic_miss stub. This looks like there has been an IC miss at the + entry of the nmethod, so we resolve the call, which will fall back + to the interpreter if the nmethod has been unloaded. */ +void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) { + + typedef struct { + intptr_t *sp; intptr_t *fp; address ra; address pc; + } frame_pointers_t; + + frame_pointers_t *new_frame = (frame_pointers_t *)(return_address_ptr - 5); + + JavaThread *thread = JavaThread::current(); + RegisterMap reg_map(thread, false); + frame frame = thread->last_frame(); + + assert(frame.is_compiled_frame() || frame.is_native_frame(), "must be"); + assert(frame.cb() == nm, "must be"); + frame = frame.sender(®_map); + + LogTarget(Trace, nmethod, barrier) out; + if (out.is_enabled()) { + ResourceMark mark; + log_trace(nmethod, barrier)("deoptimize(nmethod: %s(%p), return_addr: %p, osr: %d, thread: %p(%s), making rsp: %p) -> %p", + nm->method()->name_and_sig_as_C_string(), + nm, *(address *) return_address_ptr, nm->is_osr_method(), thread, + thread->name(), frame.sp(), nm->verified_entry_point()); + } + + new_frame->sp = frame.sp(); + new_frame->fp = frame.fp(); + new_frame->ra = frame.pc(); + new_frame->pc = SharedRuntime::get_handle_wrong_method_stub(); +} + +// This is the offset of the entry barrier from where the frame is completed. +// If any code changes between the end of the verified entry where the entry +// barrier resides, and the completion of the frame, then +// NativeNMethodCmpBarrier::verify() will immediately complain when it does +// not find the expected native instruction at this offset, which needs updating. +// Note that this offset is invariant of PreserveFramePointer. + +// see BarrierSetAssembler::nmethod_entry_barrier +// auipc + lwu + fence + lwu + beq + movptr(5 instructions) + jalr + j + int32 +static const int entry_barrier_offset = -4 * 13; + +static NativeNMethodBarrier* native_nmethod_barrier(nmethod* nm) { + address barrier_address = nm->code_begin() + nm->frame_complete_offset() + entry_barrier_offset; + NativeNMethodBarrier* barrier = reinterpret_cast(barrier_address); + debug_only(barrier->verify()); + return barrier; +} + +void BarrierSetNMethod::disarm(nmethod* nm) { + if (!supports_entry_barrier(nm)) { + return; + } + + // Disarms the nmethod guard emitted by BarrierSetAssembler::nmethod_entry_barrier. + NativeNMethodBarrier* barrier = native_nmethod_barrier(nm); + + barrier->set_value(disarmed_value()); +} + +bool BarrierSetNMethod::is_armed(nmethod* nm) { + if (!supports_entry_barrier(nm)) { + return false; + } + + NativeNMethodBarrier* barrier = native_nmethod_barrier(nm); + return barrier->get_value() != disarmed_value(); +} diff --git a/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.cpp new file mode 100644 index 0000000000000..640e1ccf47c89 --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.cpp @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/cardTable.hpp" +#include "gc/shared/cardTableBarrierSet.hpp" +#include "gc/shared/cardTableBarrierSetAssembler.hpp" +#include "gc/shared/gc_globals.hpp" +#include "interpreter/interp_masm.hpp" + +#define __ masm-> + + +void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Register tmp) { + assert_different_registers(obj, tmp); + BarrierSet* bs = BarrierSet::barrier_set(); + assert(bs->kind() == BarrierSet::CardTableBarrierSet, "Wrong barrier set kind"); + + __ srli(obj, obj, CardTable::card_shift); + + assert(CardTable::dirty_card_val() == 0, "must be"); + + __ load_byte_map_base(tmp); + __ add(tmp, obj, tmp); + + if (UseCondCardMark) { + Label L_already_dirty; + __ lbu(t1, Address(tmp)); + __ beqz(t1, L_already_dirty); + __ sb(zr, Address(tmp)); + __ bind(L_already_dirty); + } else { + __ sb(zr, Address(tmp)); + } +} + +void CardTableBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register start, Register count, Register tmp, RegSet saved_regs) { + assert_different_registers(start, tmp); + assert_different_registers(count, tmp); + + Label L_loop, L_done; + const Register end = count; + + __ beqz(count, L_done); // zero count - nothing to do + // end = start + count << LogBytesPerHeapOop + __ shadd(end, count, start, count, LogBytesPerHeapOop); + __ sub(end, end, BytesPerHeapOop); // last element address to make inclusive + + __ srli(start, start, CardTable::card_shift); + __ srli(end, end, CardTable::card_shift); + __ sub(count, end, start); // number of bytes to copy + + __ load_byte_map_base(tmp); + __ add(start, start, tmp); + + __ bind(L_loop); + __ add(tmp, start, count); + __ sb(zr, Address(tmp)); + __ sub(count, count, 1); + __ bgez(count, L_loop); + __ bind(L_done); +} + +void CardTableBarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2) { + bool in_heap = (decorators & IN_HEAP) != 0; + bool is_array = (decorators & IS_ARRAY) != 0; + bool on_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0; + bool precise = is_array || on_anonymous; + + bool needs_post_barrier = val != noreg && in_heap; + BarrierSetAssembler::store_at(masm, decorators, type, dst, val, noreg, noreg); + if (needs_post_barrier) { + // flatten object address if needed + if (!precise || dst.offset() == 0) { + store_check(masm, dst.base(), x13); + } else { + __ la(x13, dst); + store_check(masm, x13, t0); + } + } +} diff --git a/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.hpp new file mode 100644 index 0000000000000..686fe8fa47861 --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.hpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_GC_SHARED_CARDTABLEBARRIERSETASSEMBLER_RISCV_HPP +#define CPU_RISCV_GC_SHARED_CARDTABLEBARRIERSETASSEMBLER_RISCV_HPP + +#include "asm/macroAssembler.hpp" +#include "gc/shared/modRefBarrierSetAssembler.hpp" + +class CardTableBarrierSetAssembler: public ModRefBarrierSetAssembler { +protected: + void store_check(MacroAssembler* masm, Register obj, Register tmp); + + virtual void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register start, Register count, Register tmp, RegSet saved_regs); + virtual void oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2); +}; + +#endif // #ifndef CPU_RISCV_GC_SHARED_CARDTABLEBARRIERSETASSEMBLER_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/gc/shared/modRefBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/shared/modRefBarrierSetAssembler_riscv.cpp new file mode 100644 index 0000000000000..4b7982eb213a7 --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/shared/modRefBarrierSetAssembler_riscv.cpp @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "gc/shared/modRefBarrierSetAssembler.hpp" + +#define __ masm-> + +void ModRefBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, + Register src, Register dst, Register count, RegSet saved_regs) { + if (is_oop) { + gen_write_ref_array_pre_barrier(masm, decorators, dst, count, saved_regs); + } +} + +void ModRefBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, + Register start, Register count, Register tmp, + RegSet saved_regs) { + if (is_oop) { + gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp, saved_regs); + } +} + +void ModRefBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2) { + if (is_reference_type(type)) { + oop_store_at(masm, decorators, type, dst, val, tmp1, tmp2); + } else { + BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); + } +} diff --git a/src/hotspot/cpu/riscv/gc/shared/modRefBarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/shared/modRefBarrierSetAssembler_riscv.hpp new file mode 100644 index 0000000000000..00419c3163c2d --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/shared/modRefBarrierSetAssembler_riscv.hpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_GC_SHARED_MODREFBARRIERSETASSEMBLER_RISCV_HPP +#define CPU_RISCV_GC_SHARED_MODREFBARRIERSETASSEMBLER_RISCV_HPP + +#include "asm/macroAssembler.hpp" +#include "gc/shared/barrierSetAssembler.hpp" + +// The ModRefBarrierSetAssembler filters away accesses on BasicTypes other +// than T_OBJECT/T_ARRAY (oops). The oop accesses call one of the protected +// accesses, which are overridden in the concrete BarrierSetAssembler. + +class ModRefBarrierSetAssembler: public BarrierSetAssembler { +protected: + virtual void gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register addr, Register count, RegSet saved_regs) {} + virtual void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register start, Register count, Register tmp, RegSet saved_regs) {} + + virtual void oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2) = 0; + +public: + virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, + Register src, Register dst, Register count, RegSet saved_regs); + virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, + Register start, Register count, Register tmp, RegSet saved_regs); + virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2); +}; + +#endif // CPU_RISCV_GC_SHARED_MODREFBARRIERSETASSEMBLER_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp b/src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp new file mode 100644 index 0000000000000..cd568cc723fe9 --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "c1/c1_LIRAssembler.hpp" +#include "c1/c1_MacroAssembler.hpp" +#include "gc/shared/gc_globals.hpp" +#include "gc/shenandoah/shenandoahBarrierSet.hpp" +#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" +#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp" + +#define __ masm->masm()-> + +void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler* masm) { + Register addr = _addr->as_register_lo(); + Register newval = _new_value->as_register(); + Register cmpval = _cmp_value->as_register(); + Register tmp1 = _tmp1->as_register(); + Register tmp2 = _tmp2->as_register(); + Register result = result_opr()->as_register(); + + ShenandoahBarrierSet::assembler()->iu_barrier(masm->masm(), newval, t1); + + if (UseCompressedOops) { + __ encode_heap_oop(tmp1, cmpval); + cmpval = tmp1; + __ encode_heap_oop(tmp2, newval); + newval = tmp2; + } + + ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), addr, cmpval, newval, /* acquire */ Assembler::aq, + /* release */ Assembler::rl, /* is_cae */ false, result); +} + +#undef __ + +#ifdef ASSERT +#define __ gen->lir(__FILE__, __LINE__)-> +#else +#define __ gen->lir()-> +#endif + +LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) { + BasicType bt = access.type(); + if (access.is_oop()) { + LIRGenerator *gen = access.gen(); + if (ShenandoahSATBBarrier) { + pre_barrier(gen, access.access_emit_info(), access.decorators(), access.resolved_addr(), + LIR_OprFact::illegalOpr /* pre_val */); + } + if (ShenandoahCASBarrier) { + cmp_value.load_item(); + new_value.load_item(); + + LIR_Opr tmp1 = gen->new_register(T_OBJECT); + LIR_Opr tmp2 = gen->new_register(T_OBJECT); + LIR_Opr addr = access.resolved_addr()->as_address_ptr()->base(); + LIR_Opr result = gen->new_register(T_INT); + + __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), tmp1, tmp2, result)); + return result; + } + } + return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value); +} + +LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) { + LIRGenerator* gen = access.gen(); + BasicType type = access.type(); + + LIR_Opr result = gen->new_register(type); + value.load_item(); + LIR_Opr value_opr = value.result(); + + if (access.is_oop()) { + value_opr = iu_barrier(access.gen(), value_opr, access.access_emit_info(), access.decorators()); + } + + assert(type == T_INT || is_reference_type(type) LP64_ONLY( || type == T_LONG ), "unexpected type"); + LIR_Opr tmp = gen->new_register(T_INT); + __ xchg(access.resolved_addr(), value_opr, result, tmp); + + if (access.is_oop()) { + result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), access.decorators()); + LIR_Opr tmp_opr = gen->new_register(type); + __ move(result, tmp_opr); + result = tmp_opr; + if (ShenandoahSATBBarrier) { + pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr, + result /* pre_val */); + } + } + + return result; +} diff --git a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp new file mode 100644 index 0000000000000..76caa33b61f83 --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp @@ -0,0 +1,714 @@ +/* + * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shenandoah/shenandoahBarrierSet.hpp" +#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" +#include "gc/shenandoah/shenandoahForwarding.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.hpp" +#include "gc/shenandoah/shenandoahRuntime.hpp" +#include "gc/shenandoah/shenandoahThreadLocalData.hpp" +#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" +#include "interpreter/interpreter.hpp" +#include "interpreter/interp_masm.hpp" +#include "runtime/sharedRuntime.hpp" +#include "runtime/thread.hpp" +#ifdef COMPILER1 +#include "c1/c1_LIRAssembler.hpp" +#include "c1/c1_MacroAssembler.hpp" +#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp" +#endif + +#define __ masm-> + +void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, + Register src, Register dst, Register count, RegSet saved_regs) { + if (is_oop) { + bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; + if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) { + + Label done; + + // Avoid calling runtime if count == 0 + __ beqz(count, done); + + // Is GC active? + Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); + assert_different_registers(src, dst, count, t0); + + __ lbu(t0, gc_state); + if (ShenandoahSATBBarrier && dest_uninitialized) { + __ test_bit(t0, t0, ShenandoahHeap::HAS_FORWARDED_BITPOS); + __ beqz(t0, done); + } else { + __ andi(t0, t0, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING); + __ beqz(t0, done); + } + + __ push_reg(saved_regs, sp); + if (UseCompressedOops) { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry), + src, dst, count); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry), src, dst, count); + } + __ pop_reg(saved_regs, sp); + __ bind(done); + } + } +} + +void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp, + bool tosca_live, + bool expand_call) { + if (ShenandoahSATBBarrier) { + satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call); + } +} + +void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp, + bool tosca_live, + bool expand_call) { + // If expand_call is true then we expand the call_VM_leaf macro + // directly to skip generating the check by + // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp. + assert(thread == xthread, "must be"); + + Label done; + Label runtime; + + assert_different_registers(obj, pre_val, tmp, t0); + assert(pre_val != noreg && tmp != noreg, "expecting a register"); + + Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset())); + Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset())); + Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); + + // Is marking active? + if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { + __ lwu(tmp, in_progress); + } else { + assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); + __ lbu(tmp, in_progress); + } + __ beqz(tmp, done); + + // Do we need to load the previous value? + if (obj != noreg) { + __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); + } + + // Is the previous value null? + __ beqz(pre_val, done); + + // Can we store original value in the thread's buffer? + // Is index == 0? + // (The index field is typed as size_t.) + __ ld(tmp, index); // tmp := *index_adr + __ beqz(tmp, runtime); // tmp == 0? If yes, goto runtime + + __ sub(tmp, tmp, wordSize); // tmp := tmp - wordSize + __ sd(tmp, index); // *index_adr := tmp + __ ld(t0, buffer); + __ add(tmp, tmp, t0); // tmp := tmp + *buffer_adr + + // Record the previous value + __ sd(pre_val, Address(tmp, 0)); + __ j(done); + + __ bind(runtime); + // save the live input values + RegSet saved = RegSet::of(pre_val); + if (tosca_live) saved += RegSet::of(x10); + if (obj != noreg) saved += RegSet::of(obj); + + __ push_reg(saved, sp); + + // Calling the runtime using the regular call_VM_leaf mechanism generates + // code (generated by InterpreterMacroAssember::call_VM_leaf_base) + // that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL. + // + // If we care generating the pre-barrier without a frame (e.g. in the + // intrinsified Reference.get() routine) then ebp might be pointing to + // the caller frame and so this check will most likely fail at runtime. + // + // Expanding the call directly bypasses the generation of the check. + // So when we do not have have a full interpreter frame on the stack + // expand_call should be passed true. + if (expand_call) { + assert(pre_val != c_rarg1, "smashed arg"); + __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread); + } else { + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread); + } + + __ pop_reg(saved, sp); + + __ bind(done); +} + +void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) { + assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled"); + + Label is_null; + __ beqz(dst, is_null); + resolve_forward_pointer_not_null(masm, dst, tmp); + __ bind(is_null); +} + +// IMPORTANT: This must preserve all registers, even t0 and t1, except those explicitely +// passed in. +void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) { + assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled"); + // The below loads the mark word, checks if the lowest two bits are + // set, and if so, clear the lowest two bits and copy the result + // to dst. Otherwise it leaves dst alone. + // Implementing this is surprisingly awkward. I do it here by: + // - Inverting the mark word + // - Test lowest two bits == 0 + // - If so, set the lowest two bits + // - Invert the result back, and copy to dst + RegSet saved_regs = RegSet::of(t2); + bool borrow_reg = (tmp == noreg); + if (borrow_reg) { + // No free registers available. Make one useful. + tmp = t0; + if (tmp == dst) { + tmp = t1; + } + saved_regs += RegSet::of(tmp); + } + + assert_different_registers(tmp, dst, t2); + __ push_reg(saved_regs, sp); + + Label done; + __ ld(tmp, Address(dst, oopDesc::mark_offset_in_bytes())); + __ xori(tmp, tmp, -1); // eon with 0 is equivalent to XOR with -1 + __ andi(t2, tmp, markWord::lock_mask_in_place); + __ bnez(t2, done); + __ ori(tmp, tmp, markWord::marked_value); + __ xori(dst, tmp, -1); // eon with 0 is equivalent to XOR with -1 + __ bind(done); + + __ pop_reg(saved_regs, sp); +} + +void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, + Register dst, + Address load_addr, + DecoratorSet decorators) { + assert(ShenandoahLoadRefBarrier, "Should be enabled"); + assert(dst != t1 && load_addr.base() != t1, "need t1"); + assert_different_registers(load_addr.base(), t0, t1); + + bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); + bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); + bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); + bool is_native = ShenandoahBarrierSet::is_native_access(decorators); + bool is_narrow = UseCompressedOops && !is_native; + + Label heap_stable, not_cset; + __ enter(); + Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); + __ lbu(t1, gc_state); + + // Check for heap stability + if (is_strong) { + __ test_bit(t1, t1, ShenandoahHeap::HAS_FORWARDED_BITPOS); + __ beqz(t1, heap_stable); + } else { + Label lrb; + __ test_bit(t0, t1, ShenandoahHeap::WEAK_ROOTS_BITPOS); + __ bnez(t0, lrb); + __ test_bit(t0, t1, ShenandoahHeap::HAS_FORWARDED_BITPOS); + __ beqz(t0, heap_stable); + __ bind(lrb); + } + + // use x11 for load address + Register result_dst = dst; + if (dst == x11) { + __ mv(t1, dst); + dst = t1; + } + + // Save x10 and x11, unless it is an output register + RegSet saved_regs = RegSet::of(x10, x11) - result_dst; + __ push_reg(saved_regs, sp); + __ la(x11, load_addr); + __ mv(x10, dst); + + // Test for in-cset + if (is_strong) { + __ mv(t1, ShenandoahHeap::in_cset_fast_test_addr()); + __ srli(t0, x10, ShenandoahHeapRegion::region_size_bytes_shift_jint()); + __ add(t1, t1, t0); + __ lbu(t1, Address(t1)); + __ test_bit(t0, t1, 0); + __ beqz(t0, not_cset); + } + + __ push_call_clobbered_registers(); + address target = NULL; + if (is_strong) { + if (is_narrow) { + target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow); + } else { + target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong); + } + } else if (is_weak) { + if (is_narrow) { + target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow); + } else { + target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak); + } + } else { + assert(is_phantom, "only remaining strength"); + assert(!is_narrow, "phantom access cannot be narrow"); + target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak); + } + __ call(target); + __ mv(t0, x10); + __ pop_call_clobbered_registers(); + __ mv(x10, t0); + __ bind(not_cset); + __ mv(result_dst, x10); + __ pop_reg(saved_regs, sp); + + __ bind(heap_stable); + __ leave(); +} + +void ShenandoahBarrierSetAssembler::iu_barrier(MacroAssembler* masm, Register dst, Register tmp) { + if (ShenandoahIUBarrier) { + __ push_call_clobbered_registers(); + + satb_write_barrier_pre(masm, noreg, dst, xthread, tmp, true, false); + + __ pop_call_clobbered_registers(); + } +} + +// +// Arguments: +// +// Inputs: +// src: oop location to load from, might be clobbered +// +// Output: +// dst: oop loaded from src location +// +// Kill: +// x30 (tmp reg) +// +// Alias: +// dst: x30 (might use x30 as temporary output register to avoid clobbering src) +// +void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, + DecoratorSet decorators, + BasicType type, + Register dst, + Address src, + Register tmp1, + Register tmp_thread) { + // 1: non-reference load, no additional barrier is needed + if (!is_reference_type(type)) { + BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); + return; + } + + // 2: load a reference from src location and apply LRB if needed + if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) { + Register result_dst = dst; + + // Preserve src location for LRB + RegSet saved_regs; + if (dst == src.base()) { + dst = (src.base() == x28) ? x29 : x28; + saved_regs = RegSet::of(dst); + __ push_reg(saved_regs, sp); + } + assert_different_registers(dst, src.base()); + + BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); + + load_reference_barrier(masm, dst, src, decorators); + + if (dst != result_dst) { + __ mv(result_dst, dst); + dst = result_dst; + } + + if (saved_regs.bits() != 0) { + __ pop_reg(saved_regs, sp); + } + } else { + BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); + } + + // 3: apply keep-alive barrier if needed + if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) { + __ enter(); + __ push_call_clobbered_registers(); + satb_write_barrier_pre(masm /* masm */, + noreg /* obj */, + dst /* pre_val */, + xthread /* thread */, + tmp1 /* tmp */, + true /* tosca_live */, + true /* expand_call */); + __ pop_call_clobbered_registers(); + __ leave(); + } +} + +void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2) { + bool on_oop = is_reference_type(type); + if (!on_oop) { + BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); + return; + } + + // flatten object address if needed + if (dst.offset() == 0) { + if (dst.base() != x13) { + __ mv(x13, dst.base()); + } + } else { + __ la(x13, dst); + } + + shenandoah_write_barrier_pre(masm, + x13 /* obj */, + tmp2 /* pre_val */, + xthread /* thread */, + tmp1 /* tmp */, + val != noreg /* tosca_live */, + false /* expand_call */); + + if (val == noreg) { + BarrierSetAssembler::store_at(masm, decorators, type, Address(x13, 0), noreg, noreg, noreg); + } else { + iu_barrier(masm, val, tmp1); + // G1 barrier needs uncompressed oop for region cross check. + Register new_val = val; + if (UseCompressedOops) { + new_val = t1; + __ mv(new_val, val); + } + BarrierSetAssembler::store_at(masm, decorators, type, Address(x13, 0), val, noreg, noreg); + } +} + +void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env, + Register obj, Register tmp, Label& slowpath) { + Label done; + // Resolve jobject + BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath); + + // Check for null. + __ beqz(obj, done); + + assert(obj != t1, "need t1"); + Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset()); + __ lbu(t1, gc_state); + + // Check for heap in evacuation phase + __ test_bit(t0, t1, ShenandoahHeap::EVACUATION_BITPOS); + __ bnez(t0, slowpath); + + __ bind(done); +} + +// Special Shenandoah CAS implementation that handles false negatives due +// to concurrent evacuation. The service is more complex than a +// traditional CAS operation because the CAS operation is intended to +// succeed if the reference at addr exactly matches expected or if the +// reference at addr holds a pointer to a from-space object that has +// been relocated to the location named by expected. There are two +// races that must be addressed: +// a) A parallel thread may mutate the contents of addr so that it points +// to a different object. In this case, the CAS operation should fail. +// b) A parallel thread may heal the contents of addr, replacing a +// from-space pointer held in addr with the to-space pointer +// representing the new location of the object. +// Upon entry to cmpxchg_oop, it is assured that new_val equals NULL +// or it refers to an object that is not being evacuated out of +// from-space, or it refers to the to-space version of an object that +// is being evacuated out of from-space. +// +// By default the value held in the result register following execution +// of the generated code sequence is 0 to indicate failure of CAS, +// non-zero to indicate success. If is_cae, the result is the value most +// recently fetched from addr rather than a boolean success indicator. +// +// Clobbers t0, t1 +void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, + Register addr, + Register expected, + Register new_val, + Assembler::Aqrl acquire, + Assembler::Aqrl release, + bool is_cae, + Register result) { + bool is_narrow = UseCompressedOops; + Assembler::operand_size size = is_narrow ? Assembler::uint32 : Assembler::int64; + + assert_different_registers(addr, expected, t0, t1); + assert_different_registers(addr, new_val, t0, t1); + + Label retry, success, fail, done; + + __ bind(retry); + + // Step1: Try to CAS. + __ cmpxchg(addr, expected, new_val, size, acquire, release, /* result */ t1); + + // If success, then we are done. + __ beq(expected, t1, success); + + // Step2: CAS failed, check the forwared pointer. + __ mv(t0, t1); + + if (is_narrow) { + __ decode_heap_oop(t0, t0); + } + resolve_forward_pointer(masm, t0); + + __ encode_heap_oop(t0, t0); + + // Report failure when the forwarded oop was not expected. + __ bne(t0, expected, fail); + + // Step 3: CAS again using the forwarded oop. + __ cmpxchg(addr, t1, new_val, size, acquire, release, /* result */ t0); + + // Retry when failed. + __ bne(t0, t1, retry); + + __ bind(success); + if (is_cae) { + __ mv(result, expected); + } else { + __ mv(result, 1); + } + __ j(done); + + __ bind(fail); + if (is_cae) { + __ mv(result, t0); + } else { + __ mv(result, zr); + } + + __ bind(done); +} + +#undef __ + +#ifdef COMPILER1 + +#define __ ce->masm()-> + +void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) { + ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); + // At this point we know that marking is in progress. + // If do_load() is true then we have to emit the + // load of the previous value; otherwise it has already + // been loaded into _pre_val. + __ bind(*stub->entry()); + + assert(stub->pre_val()->is_register(), "Precondition."); + + Register pre_val_reg = stub->pre_val()->as_register(); + + if (stub->do_load()) { + ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /* wide */, false /* unaligned */); + } + __ beqz(pre_val_reg, *stub->continuation(), /* is_far */ true); + ce->store_parameter(stub->pre_val()->as_register(), 0); + __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); + __ j(*stub->continuation()); +} + +void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, + ShenandoahLoadReferenceBarrierStub* stub) { + ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); + __ bind(*stub->entry()); + + DecoratorSet decorators = stub->decorators(); + bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); + bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); + bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); + bool is_native = ShenandoahBarrierSet::is_native_access(decorators); + + Register obj = stub->obj()->as_register(); + Register res = stub->result()->as_register(); + Register addr = stub->addr()->as_pointer_register(); + Register tmp1 = stub->tmp1()->as_register(); + Register tmp2 = stub->tmp2()->as_register(); + + assert(res == x10, "result must arrive in x10"); + assert_different_registers(tmp1, tmp2, t0); + + if (res != obj) { + __ mv(res, obj); + } + + if (is_strong) { + // Check for object in cset. + __ mv(tmp2, ShenandoahHeap::in_cset_fast_test_addr()); + __ srli(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint()); + __ add(tmp2, tmp2, tmp1); + __ lbu(tmp2, Address(tmp2)); + __ beqz(tmp2, *stub->continuation(), true /* is_far */); + } + + ce->store_parameter(res, 0); + ce->store_parameter(addr, 1); + + if (is_strong) { + if (is_native) { + __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin())); + } else { + __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_rt_code_blob()->code_begin())); + } + } else if (is_weak) { + __ far_call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin())); + } else { + assert(is_phantom, "only remaining strength"); + __ far_call(RuntimeAddress(bs->load_reference_barrier_phantom_rt_code_blob()->code_begin())); + } + + __ j(*stub->continuation()); +} + +#undef __ + +#define __ sasm-> + +void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { + __ prologue("shenandoah_pre_barrier", false); + + // arg0 : previous value of memory + + BarrierSet* bs = BarrierSet::barrier_set(); + + const Register pre_val = x10; + const Register thread = xthread; + const Register tmp = t0; + + Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset())); + Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); + + Label done; + Label runtime; + + // Is marking still active? + Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); + __ lb(tmp, gc_state); + __ test_bit(tmp, tmp, ShenandoahHeap::MARKING_BITPOS); + __ beqz(tmp, done); + + // Can we store original value in the thread's buffer? + __ ld(tmp, queue_index); + __ beqz(tmp, runtime); + + __ sub(tmp, tmp, wordSize); + __ sd(tmp, queue_index); + __ ld(t1, buffer); + __ add(tmp, tmp, t1); + __ load_parameter(0, t1); + __ sd(t1, Address(tmp, 0)); + __ j(done); + + __ bind(runtime); + __ push_call_clobbered_registers(); + __ load_parameter(0, pre_val); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread); + __ pop_call_clobbered_registers(); + __ bind(done); + + __ epilogue(); +} + +void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, + DecoratorSet decorators) { + __ prologue("shenandoah_load_reference_barrier", false); + // arg0 : object to be resolved + + __ push_call_clobbered_registers(); + __ load_parameter(0, x10); + __ load_parameter(1, x11); + + bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); + bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators); + bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators); + bool is_native = ShenandoahBarrierSet::is_native_access(decorators); + address target = NULL; + if (is_strong) { + if (is_native) { + target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong); + } else { + if (UseCompressedOops) { + target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow); + } else { + target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong); + } + } + } else if (is_weak) { + assert(!is_native, "weak must not be called off-heap"); + if (UseCompressedOops) { + target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow); + } else { + target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak); + } + } else { + assert(is_phantom, "only remaining strength"); + assert(is_native, "phantom must only be called off-heap"); + target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom); + } + __ call(target); + __ mv(t0, x10); + __ pop_call_clobbered_registers(); + __ mv(x10, t0); + + __ epilogue(); +} + +#undef __ + +#endif // COMPILER1 diff --git a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp new file mode 100644 index 0000000000000..a705f49766777 --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_RISCV_HPP +#define CPU_RISCV_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_RISCV_HPP + +#include "asm/macroAssembler.hpp" +#include "gc/shared/barrierSetAssembler.hpp" +#include "gc/shenandoah/shenandoahBarrierSet.hpp" +#ifdef COMPILER1 +class LIR_Assembler; +class ShenandoahPreBarrierStub; +class ShenandoahLoadReferenceBarrierStub; +class StubAssembler; +#endif +class StubCodeGenerator; + +class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { +private: + + void satb_write_barrier_pre(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp, + bool tosca_live, + bool expand_call); + void shenandoah_write_barrier_pre(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp, + bool tosca_live, + bool expand_call); + + void resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp = noreg); + void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp = noreg); + void load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, DecoratorSet decorators); + +public: + + void iu_barrier(MacroAssembler* masm, Register dst, Register tmp); + +#ifdef COMPILER1 + void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub); + void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub); + void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm); + void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators); +#endif + + virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, + Register src, Register dst, Register count, RegSet saved_regs); + + virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register dst, Address src, Register tmp1, Register tmp_thread); + virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Address dst, Register val, Register tmp1, Register tmp2); + + virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env, + Register obj, Register tmp, Label& slowpath); + + void cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val, + Assembler::Aqrl acquire, Assembler::Aqrl release, bool is_cae, Register result); +}; + +#endif // CPU_RISCV_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoah_riscv64.ad b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoah_riscv64.ad new file mode 100644 index 0000000000000..6c855f23c2af1 --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoah_riscv64.ad @@ -0,0 +1,285 @@ +// +// Copyright (c) 2018, Red Hat, Inc. All rights reserved. +// Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. +// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +// +// This code is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License version 2 only, as +// published by the Free Software Foundation. +// +// This code is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// version 2 for more details (a copy is included in the LICENSE file that +// accompanied this code). +// +// You should have received a copy of the GNU General Public License version +// 2 along with this work; if not, write to the Free Software Foundation, +// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +// +// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +// or visit www.oracle.com if you need additional information or have any +// questions. +// +// + +source_hpp %{ +#include "gc/shenandoah/shenandoahBarrierSet.hpp" +#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" +%} + +instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{ + match(Set res (ShenandoahCompareAndSwapP mem (Binary oldval newval))); + ins_cost(10 * DEFAULT_COST); + + effect(TEMP tmp, KILL cr); + + format %{ + "cmpxchg_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp, #@compareAndSwapP_shenandoah" + %} + + ins_encode %{ + Register tmp = $tmp$$Register; + __ mv(tmp, $oldval$$Register); // Must not clobber oldval. + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, + Assembler::relaxed /* acquire */, Assembler::rl /* release */, + false /* is_cae */, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{ + match(Set res (ShenandoahCompareAndSwapN mem (Binary oldval newval))); + ins_cost(10 * DEFAULT_COST); + + effect(TEMP tmp, KILL cr); + + format %{ + "cmpxchgw_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp, #@compareAndSwapN_shenandoah" + %} + + ins_encode %{ + Register tmp = $tmp$$Register; + __ mv(tmp, $oldval$$Register); // Must not clobber oldval. + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, + Assembler::relaxed /* acquire */, Assembler::rl /* release */, + false /* is_cae */, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{ + predicate(needs_acquiring_load_reserved(n)); + match(Set res (ShenandoahCompareAndSwapP mem (Binary oldval newval))); + ins_cost(10 * DEFAULT_COST); + + effect(TEMP tmp, KILL cr); + + format %{ + "cmpxchg_acq_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp, #@compareAndSwapPAcq_shenandoah" + %} + + ins_encode %{ + Register tmp = $tmp$$Register; + __ mv(tmp, $oldval$$Register); // Must not clobber oldval. + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, + Assembler::aq /* acquire */, Assembler::rl /* release */, + false /* is_cae */, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{ + predicate(needs_acquiring_load_reserved(n)); + match(Set res (ShenandoahCompareAndSwapN mem (Binary oldval newval))); + ins_cost(10 * DEFAULT_COST); + + effect(TEMP tmp, KILL cr); + + format %{ + "cmpxchgw_acq_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp, #@compareAndSwapNAcq_shenandoah" + %} + + ins_encode %{ + Register tmp = $tmp$$Register; + __ mv(tmp, $oldval$$Register); // Must not clobber oldval. + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, + Assembler::aq /* acquire */, Assembler::rl /* release */, + false /* is_cae */, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeN_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{ + match(Set res (ShenandoahCompareAndExchangeN mem (Binary oldval newval))); + ins_cost(10 * DEFAULT_COST); + effect(TEMP_DEF res, TEMP tmp, KILL cr); + + format %{ + "cmpxchgw_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeN_shenandoah" + %} + + ins_encode %{ + Register tmp = $tmp$$Register; + __ mv(tmp, $oldval$$Register); // Must not clobber oldval. + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, + Assembler::relaxed /* acquire */, Assembler::rl /* release */, + true /* is_cae */, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeP_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{ + match(Set res (ShenandoahCompareAndExchangeP mem (Binary oldval newval))); + ins_cost(10 * DEFAULT_COST); + + effect(TEMP_DEF res, TEMP tmp, KILL cr); + format %{ + "cmpxchg_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp, #@compareAndExchangeP_shenandoah" + %} + + ins_encode %{ + Register tmp = $tmp$$Register; + __ mv(tmp, $oldval$$Register); // Must not clobber oldval. + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, + Assembler::relaxed /* acquire */, Assembler::rl /* release */, + true /* is_cae */, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{ + match(Set res (ShenandoahWeakCompareAndSwapN mem (Binary oldval newval))); + ins_cost(10 * DEFAULT_COST); + + effect(TEMP tmp, KILL cr); + format %{ + "cmpxchgw_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval, #@weakCompareAndSwapN_shenandoah" + "mv $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + + ins_encode %{ + Register tmp = $tmp$$Register; + __ mv(tmp, $oldval$$Register); // Must not clobber oldval. + // Weak is not current supported by ShenandoahBarrierSet::cmpxchg_oop + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, + Assembler::relaxed /* acquire */, Assembler::rl /* release */, + false /* is_cae */, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeNAcq_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{ + predicate(needs_acquiring_load_reserved(n)); + match(Set res (ShenandoahCompareAndExchangeN mem (Binary oldval newval))); + ins_cost(10 * DEFAULT_COST); + + effect(TEMP_DEF res, TEMP tmp, KILL cr); + format %{ + "cmpxchgw_acq_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeNAcq_shenandoah" + %} + + ins_encode %{ + Register tmp = $tmp$$Register; + __ mv(tmp, $oldval$$Register); + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, + Assembler::aq /* acquire */, Assembler::rl /* release */, + true /* is_cae */, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangePAcq_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{ + predicate(needs_acquiring_load_reserved(n)); + match(Set res (ShenandoahCompareAndExchangeP mem (Binary oldval newval))); + ins_cost(10 * DEFAULT_COST); + + effect(TEMP_DEF res, TEMP tmp, KILL cr); + format %{ + "cmpxchg_acq_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangePAcq_shenandoah" + %} + + ins_encode %{ + Register tmp = $tmp$$Register; + __ mv(tmp, $oldval$$Register); + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, + Assembler::aq /* acquire */, Assembler::rl /* release */, + true /* is_cae */, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{ + match(Set res (ShenandoahWeakCompareAndSwapP mem (Binary oldval newval))); + ins_cost(10 * DEFAULT_COST); + + effect(TEMP tmp, KILL cr); + format %{ + "cmpxchg_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval, #@weakCompareAndSwapP_shenandoah" + %} + + ins_encode %{ + Register tmp = $tmp$$Register; + __ mv(tmp, $oldval$$Register); // Must not clobber oldval. + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, + Assembler::relaxed /* acquire */, Assembler::rl /* release */, + false /* is_cae */, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{ + predicate(needs_acquiring_load_reserved(n)); + match(Set res (ShenandoahWeakCompareAndSwapN mem (Binary oldval newval))); + ins_cost(10 * DEFAULT_COST); + + effect(TEMP tmp, KILL cr); + format %{ + "cmpxchgw_acq_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval, #@weakCompareAndSwapNAcq_shenandoah" + "mv $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + + ins_encode %{ + Register tmp = $tmp$$Register; + __ mv(tmp, $oldval$$Register); // Must not clobber oldval. + // Weak is not current supported by ShenandoahBarrierSet::cmpxchg_oop + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, + Assembler::aq /* acquire */, Assembler::rl /* release */, + false /* is_cae */, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{ + predicate(needs_acquiring_load_reserved(n)); + match(Set res (ShenandoahWeakCompareAndSwapP mem (Binary oldval newval))); + ins_cost(10 * DEFAULT_COST); + + effect(TEMP tmp, KILL cr); + format %{ + "cmpxchg_acq_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval, #@weakCompareAndSwapPAcq_shenandoah" + "mv $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + + ins_encode %{ + Register tmp = $tmp$$Register; + __ mv(tmp, $oldval$$Register); // Must not clobber oldval. + // Weak is not current supported by ShenandoahBarrierSet::cmpxchg_oop + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, + Assembler::aq /* acquire */, Assembler::rl /* release */, + false /* is_cae */, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} diff --git a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp new file mode 100644 index 0000000000000..f166bf27f42ab --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp @@ -0,0 +1,445 @@ +/* + * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "code/codeBlob.hpp" +#include "code/vmreg.inline.hpp" +#include "gc/z/zBarrier.inline.hpp" +#include "gc/z/zBarrierSet.hpp" +#include "gc/z/zBarrierSetAssembler.hpp" +#include "gc/z/zBarrierSetRuntime.hpp" +#include "gc/z/zThreadLocalData.hpp" +#include "memory/resourceArea.hpp" +#include "runtime/sharedRuntime.hpp" +#include "utilities/macros.hpp" +#ifdef COMPILER1 +#include "c1/c1_LIRAssembler.hpp" +#include "c1/c1_MacroAssembler.hpp" +#include "gc/z/c1/zBarrierSetC1.hpp" +#endif // COMPILER1 +#ifdef COMPILER2 +#include "gc/z/c2/zBarrierSetC2.hpp" +#endif // COMPILER2 + +#ifdef PRODUCT +#define BLOCK_COMMENT(str) /* nothing */ +#else +#define BLOCK_COMMENT(str) __ block_comment(str) +#endif + +#undef __ +#define __ masm-> + +void ZBarrierSetAssembler::load_at(MacroAssembler* masm, + DecoratorSet decorators, + BasicType type, + Register dst, + Address src, + Register tmp1, + Register tmp_thread) { + if (!ZBarrierSet::barrier_needed(decorators, type)) { + // Barrier not needed + BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); + return; + } + + assert_different_registers(t1, src.base()); + assert_different_registers(t0, t1, dst); + + Label done; + + // Load bad mask into temp register. + __ la(t0, src); + __ ld(t1, address_bad_mask_from_thread(xthread)); + __ ld(dst, Address(t0)); + + // Test reference against bad mask. If mask bad, then we need to fix it up. + __ andr(t1, dst, t1); + __ beqz(t1, done); + + __ enter(); + + __ push_call_clobbered_registers_except(RegSet::of(dst)); + + if (c_rarg0 != dst) { + __ mv(c_rarg0, dst); + } + + __ mv(c_rarg1, t0); + + __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2); + + // Make sure dst has the return value. + if (dst != x10) { + __ mv(dst, x10); + } + + __ pop_call_clobbered_registers_except(RegSet::of(dst)); + __ leave(); + + __ bind(done); +} + +#ifdef ASSERT + +void ZBarrierSetAssembler::store_at(MacroAssembler* masm, + DecoratorSet decorators, + BasicType type, + Address dst, + Register val, + Register tmp1, + Register tmp2) { + // Verify value + if (is_reference_type(type)) { + // Note that src could be noreg, which means we + // are storing null and can skip verification. + if (val != noreg) { + Label done; + + // tmp1 and tmp2 are often set to noreg. + RegSet savedRegs = RegSet::of(t0); + __ push_reg(savedRegs, sp); + + __ ld(t0, address_bad_mask_from_thread(xthread)); + __ andr(t0, val, t0); + __ beqz(t0, done); + __ stop("Verify oop store failed"); + __ should_not_reach_here(); + __ bind(done); + __ pop_reg(savedRegs, sp); + } + } + + // Store value + BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); +} + +#endif // ASSERT + +void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, + DecoratorSet decorators, + bool is_oop, + Register src, + Register dst, + Register count, + RegSet saved_regs) { + if (!is_oop) { + // Barrier not needed + return; + } + + BLOCK_COMMENT("ZBarrierSetAssembler::arraycopy_prologue {"); + + assert_different_registers(src, count, t0); + + __ push_reg(saved_regs, sp); + + if (count == c_rarg0 && src == c_rarg1) { + // exactly backwards!! + __ xorr(c_rarg0, c_rarg0, c_rarg1); + __ xorr(c_rarg1, c_rarg0, c_rarg1); + __ xorr(c_rarg0, c_rarg0, c_rarg1); + } else { + __ mv(c_rarg0, src); + __ mv(c_rarg1, count); + } + + __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_array_addr(), 2); + + __ pop_reg(saved_regs, sp); + + BLOCK_COMMENT("} ZBarrierSetAssembler::arraycopy_prologue"); +} + +void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, + Register jni_env, + Register robj, + Register tmp, + Label& slowpath) { + BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_jobject_in_native {"); + + assert_different_registers(jni_env, robj, tmp); + + // Resolve jobject + BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, robj, tmp, slowpath); + + // Compute the offset of address bad mask from the field of jni_environment + long int bad_mask_relative_offset = (long int) (in_bytes(ZThreadLocalData::address_bad_mask_offset()) - + in_bytes(JavaThread::jni_environment_offset())); + + // Load the address bad mask + __ ld(tmp, Address(jni_env, bad_mask_relative_offset)); + + // Check address bad mask + __ andr(tmp, robj, tmp); + __ bnez(tmp, slowpath); + + BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native"); +} + +#ifdef COMPILER2 + +OptoReg::Name ZBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) { + if (!OptoReg::is_reg(opto_reg)) { + return OptoReg::Bad; + } + + const VMReg vm_reg = OptoReg::as_VMReg(opto_reg); + if (vm_reg->is_FloatRegister()) { + return opto_reg & ~1; + } + + return opto_reg; +} + +#undef __ +#define __ _masm-> + +class ZSaveLiveRegisters { +private: + MacroAssembler* const _masm; + RegSet _gp_regs; + FloatRegSet _fp_regs; + VectorRegSet _vp_regs; + +public: + void initialize(ZLoadBarrierStubC2* stub) { + // Record registers that needs to be saved/restored + RegMaskIterator rmi(stub->live()); + while (rmi.has_next()) { + const OptoReg::Name opto_reg = rmi.next(); + if (OptoReg::is_reg(opto_reg)) { + const VMReg vm_reg = OptoReg::as_VMReg(opto_reg); + if (vm_reg->is_Register()) { + _gp_regs += RegSet::of(vm_reg->as_Register()); + } else if (vm_reg->is_FloatRegister()) { + _fp_regs += FloatRegSet::of(vm_reg->as_FloatRegister()); + } else if (vm_reg->is_VectorRegister()) { + const VMReg vm_reg_base = OptoReg::as_VMReg(opto_reg & ~(VectorRegisterImpl::max_slots_per_register - 1)); + _vp_regs += VectorRegSet::of(vm_reg_base->as_VectorRegister()); + } else { + fatal("Unknown register type"); + } + } + } + + // Remove C-ABI SOE registers, tmp regs and _ref register that will be updated + _gp_regs -= RegSet::range(x18, x27) + RegSet::of(x2) + RegSet::of(x8, x9) + RegSet::of(x5, stub->ref()); + } + + ZSaveLiveRegisters(MacroAssembler* masm, ZLoadBarrierStubC2* stub) : + _masm(masm), + _gp_regs(), + _fp_regs(), + _vp_regs() { + // Figure out what registers to save/restore + initialize(stub); + + // Save registers + __ push_reg(_gp_regs, sp); + __ push_fp(_fp_regs, sp); + __ push_v(_vp_regs, sp); + } + + ~ZSaveLiveRegisters() { + // Restore registers + __ pop_v(_vp_regs, sp); + __ pop_fp(_fp_regs, sp); + __ pop_reg(_gp_regs, sp); + } +}; + +class ZSetupArguments { +private: + MacroAssembler* const _masm; + const Register _ref; + const Address _ref_addr; + +public: + ZSetupArguments(MacroAssembler* masm, ZLoadBarrierStubC2* stub) : + _masm(masm), + _ref(stub->ref()), + _ref_addr(stub->ref_addr()) { + + // Setup arguments + if (_ref_addr.base() == noreg) { + // No self healing + if (_ref != c_rarg0) { + __ mv(c_rarg0, _ref); + } + __ mv(c_rarg1, zr); + } else { + // Self healing + if (_ref == c_rarg0) { + // _ref is already at correct place + __ la(c_rarg1, _ref_addr); + } else if (_ref != c_rarg1) { + // _ref is in wrong place, but not in c_rarg1, so fix it first + __ la(c_rarg1, _ref_addr); + __ mv(c_rarg0, _ref); + } else if (_ref_addr.base() != c_rarg0) { + assert(_ref == c_rarg1, "Mov ref first, vacating c_rarg0"); + __ mv(c_rarg0, _ref); + __ la(c_rarg1, _ref_addr); + } else { + assert(_ref == c_rarg1, "Need to vacate c_rarg1 and _ref_addr is using c_rarg0"); + if (_ref_addr.base() == c_rarg0) { + __ mv(t1, c_rarg1); + __ la(c_rarg1, _ref_addr); + __ mv(c_rarg0, t1); + } else { + ShouldNotReachHere(); + } + } + } + } + + ~ZSetupArguments() { + // Transfer result + if (_ref != x10) { + __ mv(_ref, x10); + } + } +}; + +#undef __ +#define __ masm-> + +void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const { + BLOCK_COMMENT("ZLoadBarrierStubC2"); + + // Stub entry + __ bind(*stub->entry()); + + { + ZSaveLiveRegisters save_live_registers(masm, stub); + ZSetupArguments setup_arguments(masm, stub); + + Address target(stub->slow_path()); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ jalr(x1, t0, offset); + }); + } + + // Stub exit + __ j(*stub->continuation()); +} + +#undef __ + +#endif // COMPILER2 + +#ifdef COMPILER1 +#undef __ +#define __ ce->masm()-> + +void ZBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce, + LIR_Opr ref) const { + assert_different_registers(xthread, ref->as_register(), t1); + __ ld(t1, address_bad_mask_from_thread(xthread)); + __ andr(t1, t1, ref->as_register()); +} + +void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce, + ZLoadBarrierStubC1* stub) const { + // Stub entry + __ bind(*stub->entry()); + + Register ref = stub->ref()->as_register(); + Register ref_addr = noreg; + Register tmp = noreg; + + if (stub->tmp()->is_valid()) { + // Load address into tmp register + ce->leal(stub->ref_addr(), stub->tmp()); + ref_addr = tmp = stub->tmp()->as_pointer_register(); + } else { + // Address already in register + ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register(); + } + + assert_different_registers(ref, ref_addr, noreg); + + // Save x10 unless it is the result or tmp register + // Set up SP to accomodate parameters and maybe x10. + if (ref != x10 && tmp != x10) { + __ sub(sp, sp, 32); + __ sd(x10, Address(sp, 16)); + } else { + __ sub(sp, sp, 16); + } + + // Setup arguments and call runtime stub + ce->store_parameter(ref_addr, 1); + ce->store_parameter(ref, 0); + + __ far_call(stub->runtime_stub()); + + // Verify result + __ verify_oop(x10, "Bad oop"); + + + // Move result into place + if (ref != x10) { + __ mv(ref, x10); + } + + // Restore x10 unless it is the result or tmp register + if (ref != x10 && tmp != x10) { + __ ld(x10, Address(sp, 16)); + __ add(sp, sp, 32); + } else { + __ add(sp, sp, 16); + } + + // Stub exit + __ j(*stub->continuation()); +} + +#undef __ +#define __ sasm-> + +void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm, + DecoratorSet decorators) const { + __ prologue("zgc_load_barrier stub", false); + + __ push_call_clobbered_registers_except(RegSet::of(x10)); + + // Setup arguments + __ load_parameter(0, c_rarg0); + __ load_parameter(1, c_rarg1); + + __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2); + + __ pop_call_clobbered_registers_except(RegSet::of(x10)); + + __ epilogue(); +} + +#undef __ +#endif // COMPILER1 diff --git a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp new file mode 100644 index 0000000000000..9169b98acb612 --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_GC_Z_ZBARRIERSETASSEMBLER_RISCV_HPP +#define CPU_RISCV_GC_Z_ZBARRIERSETASSEMBLER_RISCV_HPP + +#include "code/vmreg.hpp" +#include "oops/accessDecorators.hpp" +#ifdef COMPILER2 +#include "opto/optoreg.hpp" +#endif // COMPILER2 + +#ifdef COMPILER1 +class LIR_Assembler; +class LIR_OprDesc; +typedef LIR_OprDesc* LIR_Opr; +class StubAssembler; +class ZLoadBarrierStubC1; +#endif // COMPILER1 + +#ifdef COMPILER2 +class Node; +class ZLoadBarrierStubC2; +#endif // COMPILER2 + +class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase { +public: + virtual void load_at(MacroAssembler* masm, + DecoratorSet decorators, + BasicType type, + Register dst, + Address src, + Register tmp1, + Register tmp_thread); + +#ifdef ASSERT + virtual void store_at(MacroAssembler* masm, + DecoratorSet decorators, + BasicType type, + Address dst, + Register val, + Register tmp1, + Register tmp2); +#endif // ASSERT + + virtual void arraycopy_prologue(MacroAssembler* masm, + DecoratorSet decorators, + bool is_oop, + Register src, + Register dst, + Register count, + RegSet saved_regs); + + virtual void try_resolve_jobject_in_native(MacroAssembler* masm, + Register jni_env, + Register robj, + Register tmp, + Label& slowpath); + +#ifdef COMPILER1 + void generate_c1_load_barrier_test(LIR_Assembler* ce, + LIR_Opr ref) const; + + void generate_c1_load_barrier_stub(LIR_Assembler* ce, + ZLoadBarrierStubC1* stub) const; + + void generate_c1_load_barrier_runtime_stub(StubAssembler* sasm, + DecoratorSet decorators) const; +#endif // COMPILER1 + +#ifdef COMPILER2 + OptoReg::Name refine_register(const Node* node, + OptoReg::Name opto_reg); + + void generate_c2_load_barrier_stub(MacroAssembler* masm, + ZLoadBarrierStubC2* stub) const; +#endif // COMPILER2 +}; + +#endif // CPU_RISCV_GC_Z_ZBARRIERSETASSEMBLER_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/gc/z/zGlobals_riscv.cpp b/src/hotspot/cpu/riscv/gc/z/zGlobals_riscv.cpp new file mode 100644 index 0000000000000..d14997790afaa --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/z/zGlobals_riscv.cpp @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/gcLogPrecious.hpp" +#include "gc/shared/gc_globals.hpp" +#include "gc/z/zGlobals.hpp" +#include "runtime/globals.hpp" +#include "runtime/os.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/powerOfTwo.hpp" + +#ifdef LINUX +#include +#endif // LINUX + +// +// The heap can have three different layouts, depending on the max heap size. +// +// Address Space & Pointer Layout 1 +// -------------------------------- +// +// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB) +// . . +// . . +// . . +// +--------------------------------+ 0x0000014000000000 (20TB) +// | Remapped View | +// +--------------------------------+ 0x0000010000000000 (16TB) +// . . +// +--------------------------------+ 0x00000c0000000000 (12TB) +// | Marked1 View | +// +--------------------------------+ 0x0000080000000000 (8TB) +// | Marked0 View | +// +--------------------------------+ 0x0000040000000000 (4TB) +// . . +// +--------------------------------+ 0x0000000000000000 +// +// 6 4 4 4 4 +// 3 6 5 2 1 0 +// +--------------------+----+-----------------------------------------------+ +// |00000000 00000000 00|1111|11 11111111 11111111 11111111 11111111 11111111| +// +--------------------+----+-----------------------------------------------+ +// | | | +// | | * 41-0 Object Offset (42-bits, 4TB address space) +// | | +// | * 45-42 Metadata Bits (4-bits) 0001 = Marked0 (Address view 4-8TB) +// | 0010 = Marked1 (Address view 8-12TB) +// | 0100 = Remapped (Address view 16-20TB) +// | 1000 = Finalizable (Address view N/A) +// | +// * 63-46 Fixed (18-bits, always zero) +// +// +// Address Space & Pointer Layout 2 +// -------------------------------- +// +// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB) +// . . +// . . +// . . +// +--------------------------------+ 0x0000280000000000 (40TB) +// | Remapped View | +// +--------------------------------+ 0x0000200000000000 (32TB) +// . . +// +--------------------------------+ 0x0000180000000000 (24TB) +// | Marked1 View | +// +--------------------------------+ 0x0000100000000000 (16TB) +// | Marked0 View | +// +--------------------------------+ 0x0000080000000000 (8TB) +// . . +// +--------------------------------+ 0x0000000000000000 +// +// 6 4 4 4 4 +// 3 7 6 3 2 0 +// +------------------+-----+------------------------------------------------+ +// |00000000 00000000 0|1111|111 11111111 11111111 11111111 11111111 11111111| +// +-------------------+----+------------------------------------------------+ +// | | | +// | | * 42-0 Object Offset (43-bits, 8TB address space) +// | | +// | * 46-43 Metadata Bits (4-bits) 0001 = Marked0 (Address view 8-16TB) +// | 0010 = Marked1 (Address view 16-24TB) +// | 0100 = Remapped (Address view 32-40TB) +// | 1000 = Finalizable (Address view N/A) +// | +// * 63-47 Fixed (17-bits, always zero) +// +// +// Address Space & Pointer Layout 3 +// -------------------------------- +// +// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB) +// . . +// . . +// . . +// +--------------------------------+ 0x0000500000000000 (80TB) +// | Remapped View | +// +--------------------------------+ 0x0000400000000000 (64TB) +// . . +// +--------------------------------+ 0x0000300000000000 (48TB) +// | Marked1 View | +// +--------------------------------+ 0x0000200000000000 (32TB) +// | Marked0 View | +// +--------------------------------+ 0x0000100000000000 (16TB) +// . . +// +--------------------------------+ 0x0000000000000000 +// +// 6 4 4 4 4 +// 3 8 7 4 3 0 +// +------------------+----+-------------------------------------------------+ +// |00000000 00000000 |1111|1111 11111111 11111111 11111111 11111111 11111111| +// +------------------+----+-------------------------------------------------+ +// | | | +// | | * 43-0 Object Offset (44-bits, 16TB address space) +// | | +// | * 47-44 Metadata Bits (4-bits) 0001 = Marked0 (Address view 16-32TB) +// | 0010 = Marked1 (Address view 32-48TB) +// | 0100 = Remapped (Address view 64-80TB) +// | 1000 = Finalizable (Address view N/A) +// | +// * 63-48 Fixed (16-bits, always zero) +// + +// Default value if probing is not implemented for a certain platform: 128TB +static const size_t DEFAULT_MAX_ADDRESS_BIT = 47; +// Minimum value returned, if probing fails: 64GB +static const size_t MINIMUM_MAX_ADDRESS_BIT = 36; + +static size_t probe_valid_max_address_bit() { +#ifdef LINUX + size_t max_address_bit = 0; + const size_t page_size = os::vm_page_size(); + for (size_t i = DEFAULT_MAX_ADDRESS_BIT; i > MINIMUM_MAX_ADDRESS_BIT; --i) { + const uintptr_t base_addr = ((uintptr_t) 1U) << i; + if (msync((void*)base_addr, page_size, MS_ASYNC) == 0) { + // msync suceeded, the address is valid, and maybe even already mapped. + max_address_bit = i; + break; + } + if (errno != ENOMEM) { + // Some error occured. This should never happen, but msync + // has some undefined behavior, hence ignore this bit. +#ifdef ASSERT + fatal("Received '%s' while probing the address space for the highest valid bit", os::errno_name(errno)); +#else // ASSERT + log_warning_p(gc)("Received '%s' while probing the address space for the highest valid bit", os::errno_name(errno)); +#endif // ASSERT + continue; + } + // Since msync failed with ENOMEM, the page might not be mapped. + // Try to map it, to see if the address is valid. + void* const result_addr = mmap((void*) base_addr, page_size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0); + if (result_addr != MAP_FAILED) { + munmap(result_addr, page_size); + } + if ((uintptr_t) result_addr == base_addr) { + // address is valid + max_address_bit = i; + break; + } + } + if (max_address_bit == 0) { + // probing failed, allocate a very high page and take that bit as the maximum + const uintptr_t high_addr = ((uintptr_t) 1U) << DEFAULT_MAX_ADDRESS_BIT; + void* const result_addr = mmap((void*) high_addr, page_size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0); + if (result_addr != MAP_FAILED) { + max_address_bit = BitsPerSize_t - count_leading_zeros((size_t) result_addr) - 1; + munmap(result_addr, page_size); + } + } + log_info_p(gc, init)("Probing address space for the highest valid bit: " SIZE_FORMAT, max_address_bit); + return MAX2(max_address_bit, MINIMUM_MAX_ADDRESS_BIT); +#else // LINUX + return DEFAULT_MAX_ADDRESS_BIT; +#endif // LINUX +} + +size_t ZPlatformAddressOffsetBits() { + const static size_t valid_max_address_offset_bits = probe_valid_max_address_bit() + 1; + const size_t max_address_offset_bits = valid_max_address_offset_bits - 3; + const size_t min_address_offset_bits = max_address_offset_bits - 2; + const size_t address_offset = round_up_power_of_2(MaxHeapSize * ZVirtualToPhysicalRatio); + const size_t address_offset_bits = log2i_exact(address_offset); + return clamp(address_offset_bits, min_address_offset_bits, max_address_offset_bits); +} + +size_t ZPlatformAddressMetadataShift() { + return ZPlatformAddressOffsetBits(); +} diff --git a/src/hotspot/cpu/riscv/gc/z/zGlobals_riscv.hpp b/src/hotspot/cpu/riscv/gc/z/zGlobals_riscv.hpp new file mode 100644 index 0000000000000..f20ecd9b073c0 --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/z/zGlobals_riscv.hpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_GC_Z_ZGLOBALS_RISCV_HPP +#define CPU_RISCV_GC_Z_ZGLOBALS_RISCV_HPP + +const size_t ZPlatformGranuleSizeShift = 21; // 2MB +const size_t ZPlatformHeapViews = 3; +const size_t ZPlatformCacheLineSize = 64; + +size_t ZPlatformAddressOffsetBits(); +size_t ZPlatformAddressMetadataShift(); + +#endif // CPU_RISCV_GC_Z_ZGLOBALS_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/gc/z/z_riscv64.ad b/src/hotspot/cpu/riscv/gc/z/z_riscv64.ad new file mode 100644 index 0000000000000..6b6f87814a56e --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/z/z_riscv64.ad @@ -0,0 +1,233 @@ +// +// Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. +// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +// +// This code is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License version 2 only, as +// published by the Free Software Foundation. +// +// This code is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// version 2 for more details (a copy is included in the LICENSE file that +// accompanied this code). +// +// You should have received a copy of the GNU General Public License version +// 2 along with this work; if not, write to the Free Software Foundation, +// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +// +// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +// or visit www.oracle.com if you need additional information or have any +// questions. +// + +source_hpp %{ + +#include "gc/shared/gc_globals.hpp" +#include "gc/z/c2/zBarrierSetC2.hpp" +#include "gc/z/zThreadLocalData.hpp" + +%} + +source %{ + +static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, int barrier_data) { + if (barrier_data == ZLoadBarrierElided) { + return; + } + ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data); + __ ld(tmp, Address(xthread, ZThreadLocalData::address_bad_mask_offset())); + __ andr(tmp, tmp, ref); + __ bnez(tmp, *stub->entry(), true /* far */); + __ bind(*stub->continuation()); +} + +static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) { + ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, ZLoadBarrierStrong); + __ j(*stub->entry()); + __ bind(*stub->continuation()); +} + +%} + +// Load Pointer +instruct zLoadP(iRegPNoSp dst, memory mem) +%{ + match(Set dst (LoadP mem)); + predicate(UseZGC && (n->as_Load()->barrier_data() != 0)); + effect(TEMP dst); + + ins_cost(4 * DEFAULT_COST); + + format %{ "ld $dst, $mem, #@zLoadP" %} + + ins_encode %{ + const Address ref_addr (as_Register($mem$$base), $mem$$disp); + __ ld($dst$$Register, ref_addr); + z_load_barrier(_masm, this, ref_addr, $dst$$Register, t0 /* tmp */, barrier_data()); + %} + + ins_pipe(iload_reg_mem); +%} + +instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + match(Set res (CompareAndSwapP mem (Binary oldval newval))); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + predicate(UseZGC && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); + effect(KILL cr, TEMP_DEF res); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ "cmpxchg $mem, $oldval, $newval, #@zCompareAndSwapP\n\t" + "mv $res, $res == $oldval" %} + + ins_encode %{ + Label failed; + guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64, + Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register, + true /* result_as_bool */); + __ beqz($res$$Register, failed); + __ mv(t0, $oldval$$Register); + __ bind(failed); + if (barrier_data() != ZLoadBarrierElided) { + Label good; + __ ld(t1, Address(xthread, ZThreadLocalData::address_bad_mask_offset()), t1 /* tmp */); + __ andr(t1, t1, t0); + __ beqz(t1, good); + z_load_barrier_slow_path(_masm, this, Address($mem$$Register), t0 /* ref */, t1 /* tmp */); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64, + Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register, + true /* result_as_bool */); + __ bind(good); + } + %} + + ins_pipe(pipe_slow); +%} + +instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + match(Set res (CompareAndSwapP mem (Binary oldval newval))); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + predicate(UseZGC && needs_acquiring_load_reserved(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); + effect(KILL cr, TEMP_DEF res); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ "cmpxchg $mem, $oldval, $newval, #@zCompareAndSwapPAcq\n\t" + "mv $res, $res == $oldval" %} + + ins_encode %{ + Label failed; + guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64, + Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register, + true /* result_as_bool */); + __ beqz($res$$Register, failed); + __ mv(t0, $oldval$$Register); + __ bind(failed); + if (barrier_data() != ZLoadBarrierElided) { + Label good; + __ ld(t1, Address(xthread, ZThreadLocalData::address_bad_mask_offset()), t1 /* tmp */); + __ andr(t1, t1, t0); + __ beqz(t1, good); + z_load_barrier_slow_path(_masm, this, Address($mem$$Register), t0 /* ref */, t1 /* tmp */); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64, + Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register, + true /* result_as_bool */); + __ bind(good); + } + %} + + ins_pipe(pipe_slow); +%} + +instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval) %{ + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + predicate(UseZGC && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); + effect(TEMP_DEF res); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ "cmpxchg $res = $mem, $oldval, $newval, #@zCompareAndExchangeP" %} + + ins_encode %{ + guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64, + Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register); + if (barrier_data() != ZLoadBarrierElided) { + Label good; + __ ld(t0, Address(xthread, ZThreadLocalData::address_bad_mask_offset())); + __ andr(t0, t0, $res$$Register); + __ beqz(t0, good); + z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, t0 /* tmp */); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64, + Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register); + __ bind(good); + } + %} + + ins_pipe(pipe_slow); +%} + +instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval) %{ + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + predicate(UseZGC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); + effect(TEMP_DEF res); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ "cmpxchg $res = $mem, $oldval, $newval, #@zCompareAndExchangePAcq" %} + + ins_encode %{ + guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64, + Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register); + if (barrier_data() != ZLoadBarrierElided) { + Label good; + __ ld(t0, Address(xthread, ZThreadLocalData::address_bad_mask_offset())); + __ andr(t0, t0, $res$$Register); + __ beqz(t0, good); + z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, t0 /* tmp */); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64, + Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register); + __ bind(good); + } + %} + + ins_pipe(pipe_slow); +%} + +instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ + match(Set prev (GetAndSetP mem newv)); + predicate(UseZGC && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0); + effect(TEMP_DEF prev, KILL cr); + + ins_cost(2 * VOLATILE_REF_COST); + + format %{ "atomic_xchg $prev, $newv, [$mem], #@zGetAndSetP" %} + + ins_encode %{ + __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base)); + z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, t0 /* tmp */, barrier_data()); + %} + + ins_pipe(pipe_serial); +%} + +instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ + match(Set prev (GetAndSetP mem newv)); + predicate(UseZGC && needs_acquiring_load_reserved(n) && (n->as_LoadStore()->barrier_data() != 0)); + effect(TEMP_DEF prev, KILL cr); + + ins_cost(VOLATILE_REF_COST); + + format %{ "atomic_xchg_acq $prev, $newv, [$mem], #@zGetAndSetPAcq" %} + + ins_encode %{ + __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base)); + z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, t0 /* tmp */, barrier_data()); + %} + ins_pipe(pipe_serial); +%} diff --git a/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp b/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp new file mode 100644 index 0000000000000..2936837d95183 --- /dev/null +++ b/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp @@ -0,0 +1,52 @@ +/* + * Copyright (c) 1999, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_GLOBALDEFINITIONS_RISCV_HPP +#define CPU_RISCV_GLOBALDEFINITIONS_RISCV_HPP + +const int StackAlignmentInBytes = 16; + +// Indicates whether the C calling conventions require that +// 32-bit integer argument values are extended to 64 bits. +const bool CCallingConventionRequiresIntsAsLongs = false; + +// RISCV has adopted a multicopy atomic model closely following +// that of ARMv8. +#define CPU_MULTI_COPY_ATOMIC + +// To be safe, we deoptimize when we come across an access that needs +// patching. This is similar to what is done on aarch64. +#define DEOPTIMIZE_WHEN_PATCHING + +#define SUPPORTS_NATIVE_CX8 + +#define SUPPORT_RESERVED_STACK_AREA + +#define COMPRESSED_CLASS_POINTERS_DEPENDS_ON_COMPRESSED_OOPS false + +#define USE_POINTERS_TO_REGISTER_IMPL_ARRAY + +#endif // CPU_RISCV_GLOBALDEFINITIONS_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/globals_riscv.hpp b/src/hotspot/cpu/riscv/globals_riscv.hpp new file mode 100644 index 0000000000000..fa322407f85ef --- /dev/null +++ b/src/hotspot/cpu/riscv/globals_riscv.hpp @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_GLOBALS_RISCV_HPP +#define CPU_RISCV_GLOBALS_RISCV_HPP + +#include "utilities/globalDefinitions.hpp" +#include "utilities/macros.hpp" + +// Sets the default values for platform dependent flags used by the runtime system. +// (see globals.hpp) + +define_pd_global(bool, ImplicitNullChecks, true); // Generate code for implicit null checks +define_pd_global(bool, TrapBasedNullChecks, false); +define_pd_global(bool, UncommonNullCast, true); // Uncommon-trap NULLs past to check cast + +define_pd_global(uintx, CodeCacheSegmentSize, 64 COMPILER1_AND_COMPILER2_PRESENT(+64)); // Tiered compilation has large code-entry alignment. +define_pd_global(intx, CodeEntryAlignment, 64); +define_pd_global(intx, OptoLoopAlignment, 16); +define_pd_global(intx, InlineFrequencyCount, 100); + +#define DEFAULT_STACK_YELLOW_PAGES (2) +#define DEFAULT_STACK_RED_PAGES (1) +// Java_java_net_SocketOutputStream_socketWrite0() uses a 64k buffer on the +// stack if compiled for unix and LP64. To pass stack overflow tests we need +// 20 shadow pages. +#define DEFAULT_STACK_SHADOW_PAGES (20 DEBUG_ONLY(+5)) +#define DEFAULT_STACK_RESERVED_PAGES (1) + +#define MIN_STACK_YELLOW_PAGES DEFAULT_STACK_YELLOW_PAGES +#define MIN_STACK_RED_PAGES DEFAULT_STACK_RED_PAGES +#define MIN_STACK_SHADOW_PAGES DEFAULT_STACK_SHADOW_PAGES +#define MIN_STACK_RESERVED_PAGES (0) + +define_pd_global(intx, StackYellowPages, DEFAULT_STACK_YELLOW_PAGES); +define_pd_global(intx, StackRedPages, DEFAULT_STACK_RED_PAGES); +define_pd_global(intx, StackShadowPages, DEFAULT_STACK_SHADOW_PAGES); +define_pd_global(intx, StackReservedPages, DEFAULT_STACK_RESERVED_PAGES); + +define_pd_global(bool, RewriteBytecodes, true); +define_pd_global(bool, RewriteFrequentPairs, true); + +define_pd_global(bool, PreserveFramePointer, false); + +define_pd_global(uintx, TypeProfileLevel, 111); + +define_pd_global(bool, CompactStrings, true); + +// Clear short arrays bigger than one word in an arch-specific way +define_pd_global(intx, InitArrayShortSize, BytesPerLong); + +define_pd_global(intx, InlineSmallCode, 1000); + +#define ARCH_FLAGS(develop, \ + product, \ + notproduct, \ + range, \ + constraint) \ + \ + product(bool, NearCpool, true, \ + "constant pool is close to instructions") \ + product(intx, BlockZeroingLowLimit, 256, \ + "Minimum size in bytes when block zeroing will be used") \ + range(1, max_jint) \ + product(bool, TraceTraps, false, "Trace all traps the signal handler") \ + /* For now we're going to be safe and add the I/O bits to userspace fences. */ \ + product(bool, UseConservativeFence, true, \ + "Extend i for r and o for w in the pred/succ flags of fence") \ + product(bool, AvoidUnalignedAccesses, true, \ + "Avoid generating unaligned memory accesses") \ + product(bool, UseRVC, true, "Use RVC instructions") \ + product(bool, UseRVV, false, EXPERIMENTAL, "Use RVV instructions") \ + product(bool, UseZba, false, EXPERIMENTAL, "Use Zba instructions") \ + product(bool, UseZbb, false, EXPERIMENTAL, "Use Zbb instructions") \ + product(bool, UseZbs, false, EXPERIMENTAL, "Use Zbs instructions") \ + product(bool, UseRVVForBigIntegerShiftIntrinsics, true, \ + "Use RVV instructions for left/right shift of BigInteger") + +#endif // CPU_RISCV_GLOBALS_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/icBuffer_riscv.cpp b/src/hotspot/cpu/riscv/icBuffer_riscv.cpp new file mode 100644 index 0000000000000..cc93103dc5567 --- /dev/null +++ b/src/hotspot/cpu/riscv/icBuffer_riscv.cpp @@ -0,0 +1,79 @@ +/* + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "code/icBuffer.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "interpreter/bytecodes.hpp" +#include "memory/resourceArea.hpp" +#include "nativeInst_riscv.hpp" +#include "oops/oop.inline.hpp" + +int InlineCacheBuffer::ic_stub_code_size() { + // 6: auipc + ld + auipc + jalr + address(2 * instruction_size) + // 5: auipc + ld + j + address(2 * instruction_size) + return (MacroAssembler::far_branches() ? 6 : 5) * NativeInstruction::instruction_size; +} + +#define __ masm-> + +void InlineCacheBuffer::assemble_ic_buffer_code(address code_begin, void* cached_value, address entry_point) { + assert_cond(code_begin != NULL && entry_point != NULL); + ResourceMark rm; + CodeBuffer code(code_begin, ic_stub_code_size()); + MacroAssembler* masm = new MacroAssembler(&code); + // Note: even though the code contains an embedded value, we do not need reloc info + // because + // (1) the value is old (i.e., doesn't matter for scavenges) + // (2) these ICStubs are removed *before* a GC happens, so the roots disappear + + address start = __ pc(); + Label l; + __ ld(t1, l); + __ far_jump(ExternalAddress(entry_point)); + __ align(wordSize); + __ bind(l); + __ emit_int64((intptr_t)cached_value); + // Only need to invalidate the 1st two instructions - not the whole ic stub + ICache::invalidate_range(code_begin, InlineCacheBuffer::ic_stub_code_size()); + assert(__ pc() - start == ic_stub_code_size(), "must be"); +} + +address InlineCacheBuffer::ic_buffer_entry_point(address code_begin) { + NativeMovConstReg* move = nativeMovConstReg_at(code_begin); // creation also verifies the object + NativeJump* jump = nativeJump_at(move->next_instruction_address()); + return jump->jump_destination(); +} + + +void* InlineCacheBuffer::ic_buffer_cached_value(address code_begin) { + // The word containing the cached value is at the end of this IC buffer + uintptr_t *p = (uintptr_t *)(code_begin + ic_stub_code_size() - wordSize); + void* o = (void*)*p; + return o; +} diff --git a/src/hotspot/cpu/riscv/icache_riscv.cpp b/src/hotspot/cpu/riscv/icache_riscv.cpp new file mode 100644 index 0000000000000..e6c2586e5bde1 --- /dev/null +++ b/src/hotspot/cpu/riscv/icache_riscv.cpp @@ -0,0 +1,59 @@ +/* + * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.hpp" +#include "runtime/icache.hpp" + +#define __ _masm-> + +static int icache_flush(address addr, int lines, int magic) { + // To make a store to instruction memory visible to all RISC-V harts, + // the writing hart has to execute a data FENCE before requesting that + // all remote RISC-V harts execute a FENCE.I. + // + // No sush assurance is defined at the interface level of the builtin + // method, and so we should make sure it works. + __asm__ volatile("fence rw, rw" : : : "memory"); + + __builtin___clear_cache(addr, addr + (lines << ICache::log2_line_size)); + return magic; +} + +void ICacheStubGenerator::generate_icache_flush(ICache::flush_icache_stub_t* flush_icache_stub) { + address start = (address)icache_flush; + *flush_icache_stub = (ICache::flush_icache_stub_t)start; + + // ICache::invalidate_range() contains explicit condition that the first + // call is invoked on the generated icache flush stub code range. + ICache::invalidate_range(start, 0); + + { + StubCodeMark mark(this, "ICache", "fake_stub_for_inlined_icache_flush"); + __ ret(); + } +} + +#undef __ diff --git a/src/hotspot/cpu/riscv/icache_riscv.hpp b/src/hotspot/cpu/riscv/icache_riscv.hpp new file mode 100644 index 0000000000000..5bf40ca820485 --- /dev/null +++ b/src/hotspot/cpu/riscv/icache_riscv.hpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_ICACHE_RISCV_HPP +#define CPU_RISCV_ICACHE_RISCV_HPP + +// Interface for updating the instruction cache. Whenever the VM +// modifies code, part of the processor instruction cache potentially +// has to be flushed. + +class ICache : public AbstractICache { +public: + enum { + stub_size = 16, // Size of the icache flush stub in bytes + line_size = BytesPerWord, // conservative + log2_line_size = LogBytesPerWord // log2(line_size) + }; +}; + +#endif // CPU_RISCV_ICACHE_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp new file mode 100644 index 0000000000000..92c128fe93a9b --- /dev/null +++ b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp @@ -0,0 +1,1950 @@ +/* + * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/barrierSetAssembler.hpp" +#include "interp_masm_riscv.hpp" +#include "interpreter/interpreter.hpp" +#include "interpreter/interpreterRuntime.hpp" +#include "logging/log.hpp" +#include "oops/arrayOop.hpp" +#include "oops/markWord.hpp" +#include "oops/method.hpp" +#include "oops/methodData.hpp" +#include "prims/jvmtiExport.hpp" +#include "prims/jvmtiThreadState.hpp" +#include "runtime/basicLock.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/safepointMechanism.hpp" +#include "runtime/sharedRuntime.hpp" +#include "runtime/thread.inline.hpp" +#include "utilities/powerOfTwo.hpp" + +void InterpreterMacroAssembler::narrow(Register result) { + // Get method->_constMethod->_result_type + ld(t0, Address(fp, frame::interpreter_frame_method_offset * wordSize)); + ld(t0, Address(t0, Method::const_offset())); + lbu(t0, Address(t0, ConstMethod::result_type_offset())); + + Label done, notBool, notByte, notChar; + + // common case first + mv(t1, T_INT); + beq(t0, t1, done); + + // mask integer result to narrower return type. + mv(t1, T_BOOLEAN); + bne(t0, t1, notBool); + + andi(result, result, 0x1); + j(done); + + bind(notBool); + mv(t1, T_BYTE); + bne(t0, t1, notByte); + sign_extend(result, result, 8); + j(done); + + bind(notByte); + mv(t1, T_CHAR); + bne(t0, t1, notChar); + zero_extend(result, result, 16); + j(done); + + bind(notChar); + sign_extend(result, result, 16); + + bind(done); + sign_extend(result, result, 32); +} + +void InterpreterMacroAssembler::jump_to_entry(address entry) { + assert(entry != NULL, "Entry must have been generated by now"); + j(entry); +} + +void InterpreterMacroAssembler::check_and_handle_popframe(Register java_thread) { + if (JvmtiExport::can_pop_frame()) { + Label L; + // Initiate popframe handling only if it is not already being + // processed. If the flag has the popframe_processing bit set, + // it means that this code is called *during* popframe handling - we + // don't want to reenter. + // This method is only called just after the call into the vm in + // call_VM_base, so the arg registers are available. + lwu(t1, Address(xthread, JavaThread::popframe_condition_offset())); + test_bit(t0, t1, exact_log2(JavaThread::popframe_pending_bit)); + beqz(t0, L); + test_bit(t0, t1, exact_log2(JavaThread::popframe_processing_bit)); + bnez(t0, L); + // Call Interpreter::remove_activation_preserving_args_entry() to get the + // address of the same-named entrypoint in the generated interpreter code. + call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry)); + jr(x10); + bind(L); + } +} + + +void InterpreterMacroAssembler::load_earlyret_value(TosState state) { + ld(x12, Address(xthread, JavaThread::jvmti_thread_state_offset())); + const Address tos_addr(x12, JvmtiThreadState::earlyret_tos_offset()); + const Address oop_addr(x12, JvmtiThreadState::earlyret_oop_offset()); + const Address val_addr(x12, JvmtiThreadState::earlyret_value_offset()); + switch (state) { + case atos: + ld(x10, oop_addr); + sd(zr, oop_addr); + verify_oop(x10); + break; + case ltos: + ld(x10, val_addr); + break; + case btos: // fall through + case ztos: // fall through + case ctos: // fall through + case stos: // fall through + case itos: + lwu(x10, val_addr); + break; + case ftos: + flw(f10, val_addr); + break; + case dtos: + fld(f10, val_addr); + break; + case vtos: + /* nothing to do */ + break; + default: + ShouldNotReachHere(); + } + // Clean up tos value in the thread object + mv(t0, (int)ilgl); + sw(t0, tos_addr); + sw(zr, val_addr); +} + + +void InterpreterMacroAssembler::check_and_handle_earlyret(Register java_thread) { + if (JvmtiExport::can_force_early_return()) { + Label L; + ld(t0, Address(xthread, JavaThread::jvmti_thread_state_offset())); + beqz(t0, L); // if [thread->jvmti_thread_state() == NULL] then exit + + // Initiate earlyret handling only if it is not already being processed. + // If the flag has the earlyret_processing bit set, it means that this code + // is called *during* earlyret handling - we don't want to reenter. + lwu(t0, Address(t0, JvmtiThreadState::earlyret_state_offset())); + mv(t1, JvmtiThreadState::earlyret_pending); + bne(t0, t1, L); + + // Call Interpreter::remove_activation_early_entry() to get the address of the + // same-named entrypoint in the generated interpreter code. + ld(t0, Address(xthread, JavaThread::jvmti_thread_state_offset())); + lwu(t0, Address(t0, JvmtiThreadState::earlyret_tos_offset())); + call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), t0); + jr(x10); + bind(L); + } +} + +void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(Register reg, int bcp_offset) { + assert(bcp_offset >= 0, "bcp is still pointing to start of bytecode"); + lhu(reg, Address(xbcp, bcp_offset)); + revb_h(reg, reg); +} + +void InterpreterMacroAssembler::get_dispatch() { + ExternalAddress target((address)Interpreter::dispatch_table()); + relocate(target.rspec(), [&] { + int32_t offset; + la_patchable(xdispatch, target, offset); + addi(xdispatch, xdispatch, offset); + }); +} + +void InterpreterMacroAssembler::get_cache_index_at_bcp(Register index, + int bcp_offset, + size_t index_size) { + assert(bcp_offset > 0, "bcp is still pointing to start of bytecode"); + if (index_size == sizeof(u2)) { + load_unsigned_short(index, Address(xbcp, bcp_offset)); + } else if (index_size == sizeof(u4)) { + lwu(index, Address(xbcp, bcp_offset)); + // Check if the secondary index definition is still ~x, otherwise + // we have to change the following assembler code to calculate the + // plain index. + assert(ConstantPool::decode_invokedynamic_index(~123) == 123, "else change next line"); + xori(index, index, -1); + sign_extend(index, index, 32); + } else if (index_size == sizeof(u1)) { + load_unsigned_byte(index, Address(xbcp, bcp_offset)); + } else { + ShouldNotReachHere(); + } +} + +// Return +// Rindex: index into constant pool +// Rcache: address of cache entry - ConstantPoolCache::base_offset() +// +// A caller must add ConstantPoolCache::base_offset() to Rcache to get +// the true address of the cache entry. +// +void InterpreterMacroAssembler::get_cache_and_index_at_bcp(Register cache, + Register index, + int bcp_offset, + size_t index_size) { + assert_different_registers(cache, index); + assert_different_registers(cache, xcpool); + get_cache_index_at_bcp(index, bcp_offset, index_size); + assert(sizeof(ConstantPoolCacheEntry) == 4 * wordSize, "adjust code below"); + // Convert from field index to ConstantPoolCacheEntry + // riscv already has the cache in xcpool so there is no need to + // install it in cache. Instead we pre-add the indexed offset to + // xcpool and return it in cache. All clients of this method need to + // be modified accordingly. + shadd(cache, index, xcpool, cache, 5); +} + + +void InterpreterMacroAssembler::get_cache_and_index_and_bytecode_at_bcp(Register cache, + Register index, + Register bytecode, + int byte_no, + int bcp_offset, + size_t index_size) { + get_cache_and_index_at_bcp(cache, index, bcp_offset, index_size); + // We use a 32-bit load here since the layout of 64-bit words on + // little-endian machines allow us that. + // n.b. unlike x86 cache already includes the index offset + la(bytecode, Address(cache, + ConstantPoolCache::base_offset() + + ConstantPoolCacheEntry::indices_offset())); + membar(MacroAssembler::AnyAny); + lwu(bytecode, bytecode); + membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); + const int shift_count = (1 + byte_no) * BitsPerByte; + slli(bytecode, bytecode, XLEN - (shift_count + BitsPerByte)); + srli(bytecode, bytecode, XLEN - BitsPerByte); +} + +void InterpreterMacroAssembler::get_cache_entry_pointer_at_bcp(Register cache, + Register tmp, + int bcp_offset, + size_t index_size) { + assert_different_registers(cache, tmp); + get_cache_index_at_bcp(tmp, bcp_offset, index_size); + assert(sizeof(ConstantPoolCacheEntry) == 4 * wordSize, "adjust code below"); + // Convert from field index to ConstantPoolCacheEntry index + // and from word offset to byte offset + assert(exact_log2(in_bytes(ConstantPoolCacheEntry::size_in_bytes())) == 2 + LogBytesPerWord, + "else change next line"); + ld(cache, Address(fp, frame::interpreter_frame_cache_offset * wordSize)); + // skip past the header + add(cache, cache, in_bytes(ConstantPoolCache::base_offset())); + // construct pointer to cache entry + shadd(cache, tmp, cache, tmp, 2 + LogBytesPerWord); +} + +// Load object from cpool->resolved_references(index) +void InterpreterMacroAssembler::load_resolved_reference_at_index( + Register result, Register index, Register tmp) { + assert_different_registers(result, index); + + get_constant_pool(result); + // Load pointer for resolved_references[] objArray + ld(result, Address(result, ConstantPool::cache_offset_in_bytes())); + ld(result, Address(result, ConstantPoolCache::resolved_references_offset_in_bytes())); + resolve_oop_handle(result, tmp); + // Add in the index + addi(index, index, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop); + shadd(result, index, result, index, LogBytesPerHeapOop); + load_heap_oop(result, Address(result, 0)); +} + +void InterpreterMacroAssembler::load_resolved_klass_at_offset( + Register cpool, Register index, Register klass, Register temp) { + shadd(temp, index, cpool, temp, LogBytesPerWord); + lhu(temp, Address(temp, sizeof(ConstantPool))); // temp = resolved_klass_index + ld(klass, Address(cpool, ConstantPool::resolved_klasses_offset_in_bytes())); // klass = cpool->_resolved_klasses + shadd(klass, temp, klass, temp, LogBytesPerWord); + ld(klass, Address(klass, Array::base_offset_in_bytes())); +} + +void InterpreterMacroAssembler::load_resolved_method_at_index(int byte_no, + Register method, + Register cache) { + const int method_offset = in_bytes( + ConstantPoolCache::base_offset() + + ((byte_no == TemplateTable::f2_byte) + ? ConstantPoolCacheEntry::f2_offset() + : ConstantPoolCacheEntry::f1_offset())); + + ld(method, Address(cache, method_offset)); // get f1 Method* +} + +// Generate a subtype check: branch to ok_is_subtype if sub_klass is a +// subtype of super_klass. +// +// Args: +// x10: superklass +// Rsub_klass: subklass +// +// Kills: +// x12, x15 +void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass, + Label& ok_is_subtype) { + assert(Rsub_klass != x10, "x10 holds superklass"); + assert(Rsub_klass != x12, "x12 holds 2ndary super array length"); + assert(Rsub_klass != x15, "x15 holds 2ndary super array scan ptr"); + + // Profile the not-null value's klass. + profile_typecheck(x12, Rsub_klass, x15); // blows x12, reloads x15 + + // Do the check. + check_klass_subtype(Rsub_klass, x10, x12, ok_is_subtype); // blows x12 + + // Profile the failure of the check. + profile_typecheck_failed(x12); // blows x12 +} + +// Java Expression Stack + +void InterpreterMacroAssembler::pop_ptr(Register r) { + ld(r, Address(esp, 0)); + addi(esp, esp, wordSize); +} + +void InterpreterMacroAssembler::pop_i(Register r) { + lw(r, Address(esp, 0)); // lw do signed extended + addi(esp, esp, wordSize); +} + +void InterpreterMacroAssembler::pop_l(Register r) { + ld(r, Address(esp, 0)); + addi(esp, esp, 2 * Interpreter::stackElementSize); +} + +void InterpreterMacroAssembler::push_ptr(Register r) { + addi(esp, esp, -wordSize); + sd(r, Address(esp, 0)); +} + +void InterpreterMacroAssembler::push_i(Register r) { + addi(esp, esp, -wordSize); + sign_extend(r, r, 32); + sd(r, Address(esp, 0)); +} + +void InterpreterMacroAssembler::push_l(Register r) { + addi(esp, esp, -2 * wordSize); + sd(zr, Address(esp, wordSize)); + sd(r, Address(esp)); +} + +void InterpreterMacroAssembler::pop_f(FloatRegister r) { + flw(r, Address(esp, 0)); + addi(esp, esp, wordSize); +} + +void InterpreterMacroAssembler::pop_d(FloatRegister r) { + fld(r, Address(esp, 0)); + addi(esp, esp, 2 * Interpreter::stackElementSize); +} + +void InterpreterMacroAssembler::push_f(FloatRegister r) { + addi(esp, esp, -wordSize); + fsw(r, Address(esp, 0)); +} + +void InterpreterMacroAssembler::push_d(FloatRegister r) { + addi(esp, esp, -2 * wordSize); + fsd(r, Address(esp, 0)); +} + +void InterpreterMacroAssembler::pop(TosState state) { + switch (state) { + case atos: + pop_ptr(); + verify_oop(x10); + break; + case btos: // fall through + case ztos: // fall through + case ctos: // fall through + case stos: // fall through + case itos: + pop_i(); + break; + case ltos: + pop_l(); + break; + case ftos: + pop_f(); + break; + case dtos: + pop_d(); + break; + case vtos: + /* nothing to do */ + break; + default: + ShouldNotReachHere(); + } +} + +void InterpreterMacroAssembler::push(TosState state) { + switch (state) { + case atos: + verify_oop(x10); + push_ptr(); + break; + case btos: // fall through + case ztos: // fall through + case ctos: // fall through + case stos: // fall through + case itos: + push_i(); + break; + case ltos: + push_l(); + break; + case ftos: + push_f(); + break; + case dtos: + push_d(); + break; + case vtos: + /* nothing to do */ + break; + default: + ShouldNotReachHere(); + } +} + +// Helpers for swap and dup +void InterpreterMacroAssembler::load_ptr(int n, Register val) { + ld(val, Address(esp, Interpreter::expr_offset_in_bytes(n))); +} + +void InterpreterMacroAssembler::store_ptr(int n, Register val) { + sd(val, Address(esp, Interpreter::expr_offset_in_bytes(n))); +} + +void InterpreterMacroAssembler::load_float(Address src) { + flw(f10, src); +} + +void InterpreterMacroAssembler::load_double(Address src) { + fld(f10, src); +} + +void InterpreterMacroAssembler::prepare_to_jump_from_interpreted() { + // set sender sp + mv(x30, sp); + // record last_sp + sd(esp, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize)); +} + +// Jump to from_interpreted entry of a call unless single stepping is possible +// in this thread in which case we must call the i2i entry +void InterpreterMacroAssembler::jump_from_interpreted(Register method) { + prepare_to_jump_from_interpreted(); + if (JvmtiExport::can_post_interpreter_events()) { + Label run_compiled_code; + // JVMTI events, such as single-stepping, are implemented partly by avoiding running + // compiled code in threads for which the event is enabled. Check here for + // interp_only_mode if these events CAN be enabled. + lwu(t0, Address(xthread, JavaThread::interp_only_mode_offset())); + beqz(t0, run_compiled_code); + ld(t0, Address(method, Method::interpreter_entry_offset())); + jr(t0); + bind(run_compiled_code); + } + + ld(t0, Address(method, Method::from_interpreted_offset())); + jr(t0); +} + +// The following two routines provide a hook so that an implementation +// can schedule the dispatch in two parts. amd64 does not do this. +void InterpreterMacroAssembler::dispatch_prolog(TosState state, int step) { +} + +void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) { + dispatch_next(state, step); +} + +void InterpreterMacroAssembler::dispatch_base(TosState state, + address* table, + bool verifyoop, + bool generate_poll, + Register Rs) { + // Pay attention to the argument Rs, which is acquiesce in t0. + if (VerifyActivationFrameSize) { + Unimplemented(); + } + if (verifyoop && state == atos) { + verify_oop(x10); + } + + Label safepoint; + address* const safepoint_table = Interpreter::safept_table(state); + bool needs_thread_local_poll = generate_poll && table != safepoint_table; + + if (needs_thread_local_poll) { + NOT_PRODUCT(block_comment("Thread-local Safepoint poll")); + ld(t1, Address(xthread, JavaThread::polling_word_offset())); + test_bit(t1, t1, exact_log2(SafepointMechanism::poll_bit())); + bnez(t1, safepoint); + } + if (table == Interpreter::dispatch_table(state)) { + mv(t1, Interpreter::distance_from_dispatch_table(state)); + add(t1, Rs, t1); + shadd(t1, t1, xdispatch, t1, 3); + } else { + mv(t1, (address)table); + shadd(t1, Rs, t1, Rs, 3); + } + ld(t1, Address(t1)); + jr(t1); + + if (needs_thread_local_poll) { + bind(safepoint); + la(t1, ExternalAddress((address)safepoint_table)); + shadd(t1, Rs, t1, Rs, 3); + ld(t1, Address(t1)); + jr(t1); + } +} + +void InterpreterMacroAssembler::dispatch_only(TosState state, bool generate_poll, Register Rs) { + dispatch_base(state, Interpreter::dispatch_table(state), true, generate_poll, Rs); +} + +void InterpreterMacroAssembler::dispatch_only_normal(TosState state, Register Rs) { + dispatch_base(state, Interpreter::normal_table(state), Rs); +} + +void InterpreterMacroAssembler::dispatch_only_noverify(TosState state, Register Rs) { + dispatch_base(state, Interpreter::normal_table(state), false, Rs); +} + +void InterpreterMacroAssembler::dispatch_next(TosState state, int step, bool generate_poll) { + // load next bytecode + load_unsigned_byte(t0, Address(xbcp, step)); + add(xbcp, xbcp, step); + dispatch_base(state, Interpreter::dispatch_table(state), true, generate_poll); +} + +void InterpreterMacroAssembler::dispatch_via(TosState state, address* table) { + // load current bytecode + lbu(t0, Address(xbcp, 0)); + dispatch_base(state, table); +} + +// remove activation +// +// Apply stack watermark barrier. +// Unlock the receiver if this is a synchronized method. +// Unlock any Java monitors from syncronized blocks. +// Remove the activation from the stack. +// +// If there are locked Java monitors +// If throw_monitor_exception +// throws IllegalMonitorStateException +// Else if install_monitor_exception +// installs IllegalMonitorStateException +// Else +// no error processing +void InterpreterMacroAssembler::remove_activation( + TosState state, + bool throw_monitor_exception, + bool install_monitor_exception, + bool notify_jvmdi) { + // Note: Registers x13 may be in use for the + // result check if synchronized method + Label unlocked, unlock, no_unlock; + + // The below poll is for the stack watermark barrier. It allows fixing up frames lazily, + // that would normally not be safe to use. Such bad returns into unsafe territory of + // the stack, will call InterpreterRuntime::at_unwind. + Label slow_path; + Label fast_path; + safepoint_poll(slow_path, true /* at_return */, false /* acquire */, false /* in_nmethod */); + j(fast_path); + + bind(slow_path); + push(state); + set_last_Java_frame(esp, fp, (address)pc(), t0); + super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind), xthread); + reset_last_Java_frame(true); + pop(state); + + bind(fast_path); + + // get the value of _do_not_unlock_if_synchronized into x13 + const Address do_not_unlock_if_synchronized(xthread, + in_bytes(JavaThread::do_not_unlock_if_synchronized_offset())); + lbu(x13, do_not_unlock_if_synchronized); + sb(zr, do_not_unlock_if_synchronized); // reset the flag + + // get method access flags + ld(x11, Address(fp, frame::interpreter_frame_method_offset * wordSize)); + ld(x12, Address(x11, Method::access_flags_offset())); + test_bit(t0, x12, exact_log2(JVM_ACC_SYNCHRONIZED)); + beqz(t0, unlocked); + + // Don't unlock anything if the _do_not_unlock_if_synchronized flag + // is set. + bnez(x13, no_unlock); + + // unlock monitor + push(state); // save result + + // BasicObjectLock will be first in list, since this is a + // synchronized method. However, need to check that the object has + // not been unlocked by an explicit monitorexit bytecode. + const Address monitor(fp, frame::interpreter_frame_initial_sp_offset * + wordSize - (int) sizeof(BasicObjectLock)); + // We use c_rarg1 so that if we go slow path it will be the correct + // register for unlock_object to pass to VM directly + la(c_rarg1, monitor); // address of first monitor + + ld(x10, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes())); + bnez(x10, unlock); + + pop(state); + if (throw_monitor_exception) { + // Entry already unlocked, need to throw exception + call_VM(noreg, CAST_FROM_FN_PTR(address, + InterpreterRuntime::throw_illegal_monitor_state_exception)); + should_not_reach_here(); + } else { + // Monitor already unlocked during a stack unroll. If requested, + // install an illegal_monitor_state_exception. Continue with + // stack unrolling. + if (install_monitor_exception) { + call_VM(noreg, CAST_FROM_FN_PTR(address, + InterpreterRuntime::new_illegal_monitor_state_exception)); + } + j(unlocked); + } + + bind(unlock); + unlock_object(c_rarg1); + pop(state); + + // Check that for block-structured locking (i.e., that all locked + // objects has been unlocked) + bind(unlocked); + + // x10: Might contain return value + + // Check that all monitors are unlocked + { + Label loop, exception, entry, restart; + const int entry_size = frame::interpreter_frame_monitor_size() * wordSize; + const Address monitor_block_top( + fp, frame::interpreter_frame_monitor_block_top_offset * wordSize); + const Address monitor_block_bot( + fp, frame::interpreter_frame_initial_sp_offset * wordSize); + + bind(restart); + // We use c_rarg1 so that if we go slow path it will be the correct + // register for unlock_object to pass to VM directly + ld(c_rarg1, monitor_block_top); // points to current entry, starting + // with top-most entry + la(x9, monitor_block_bot); // points to word before bottom of + // monitor block + + j(entry); + + // Entry already locked, need to throw exception + bind(exception); + + if (throw_monitor_exception) { + // Throw exception + MacroAssembler::call_VM(noreg, + CAST_FROM_FN_PTR(address, InterpreterRuntime:: + throw_illegal_monitor_state_exception)); + + should_not_reach_here(); + } else { + // Stack unrolling. Unlock object and install illegal_monitor_exception. + // Unlock does not block, so don't have to worry about the frame. + // We don't have to preserve c_rarg1 since we are going to throw an exception. + + push(state); + unlock_object(c_rarg1); + pop(state); + + if (install_monitor_exception) { + call_VM(noreg, CAST_FROM_FN_PTR(address, + InterpreterRuntime:: + new_illegal_monitor_state_exception)); + } + + j(restart); + } + + bind(loop); + // check if current entry is used + add(t0, c_rarg1, BasicObjectLock::obj_offset_in_bytes()); + ld(t0, Address(t0, 0)); + bnez(t0, exception); + + add(c_rarg1, c_rarg1, entry_size); // otherwise advance to next entry + bind(entry); + bne(c_rarg1, x9, loop); // check if bottom reached if not at bottom then check this entry + } + + bind(no_unlock); + + // jvmti support + if (notify_jvmdi) { + notify_method_exit(state, NotifyJVMTI); // preserve TOSCA + + } else { + notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA + } + + // remove activation + // get sender esp + ld(t1, + Address(fp, frame::interpreter_frame_sender_sp_offset * wordSize)); + if (StackReservedPages > 0) { + // testing if reserved zone needs to be re-enabled + Label no_reserved_zone_enabling; + + ld(t0, Address(xthread, JavaThread::reserved_stack_activation_offset())); + ble(t1, t0, no_reserved_zone_enabling); + + call_VM_leaf( + CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), xthread); + call_VM(noreg, CAST_FROM_FN_PTR(address, + InterpreterRuntime::throw_delayed_StackOverflowError)); + should_not_reach_here(); + + bind(no_reserved_zone_enabling); + } + + // restore sender esp + mv(esp, t1); + + // remove frame anchor + leave(); + // If we're returning to interpreted code we will shortly be + // adjusting SP to allow some space for ESP. If we're returning to + // compiled code the saved sender SP was saved in sender_sp, so this + // restores it. + andi(sp, esp, -16); +} + +// Lock object +// +// Args: +// c_rarg1: BasicObjectLock to be used for locking +// +// Kills: +// x10 +// c_rarg0, c_rarg1, c_rarg2, c_rarg3, .. (param regs) +// t0, t1 (temp regs) +void InterpreterMacroAssembler::lock_object(Register lock_reg) +{ + assert(lock_reg == c_rarg1, "The argument is only for looks. It must be c_rarg1"); + if (UseHeavyMonitors) { + call_VM(noreg, + CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), + lock_reg); + } else { + Label done; + + const Register swap_reg = x10; + const Register tmp = c_rarg2; + const Register obj_reg = c_rarg3; // Will contain the oop + + const int obj_offset = BasicObjectLock::obj_offset_in_bytes(); + const int lock_offset = BasicObjectLock::lock_offset_in_bytes (); + const int mark_offset = lock_offset + + BasicLock::displaced_header_offset_in_bytes(); + + Label slow_case; + + // Load object pointer into obj_reg c_rarg3 + ld(obj_reg, Address(lock_reg, obj_offset)); + + if (DiagnoseSyncOnValueBasedClasses != 0) { + load_klass(tmp, obj_reg); + lwu(tmp, Address(tmp, Klass::access_flags_offset())); + test_bit(tmp, tmp, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS)); + bnez(tmp, slow_case); + } + + if (UseBiasedLocking) { + biased_locking_enter(lock_reg, obj_reg, swap_reg, tmp, false, done, &slow_case); + } + + // Load (object->mark() | 1) into swap_reg + ld(t0, Address(obj_reg, oopDesc::mark_offset_in_bytes())); + ori(swap_reg, t0, 1); + + // Save (object->mark() | 1) into BasicLock's displaced header + sd(swap_reg, Address(lock_reg, mark_offset)); + + assert(lock_offset == 0, + "displached header must be first word in BasicObjectLock"); + + cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, t0, done, /*fallthrough*/NULL); + + // Test if the oopMark is an obvious stack pointer, i.e., + // 1) (mark & 7) == 0, and + // 2) sp <= mark < mark + os::pagesize() + // + // These 3 tests can be done by evaluating the following + // expression: ((mark - sp) & (7 - os::vm_page_size())), + // assuming both stack pointer and pagesize have their + // least significant 3 bits clear. + // NOTE: the oopMark is in swap_reg x10 as the result of cmpxchg + sub(swap_reg, swap_reg, sp); + mv(t0, (int64_t)(7 - os::vm_page_size())); + andr(swap_reg, swap_reg, t0); + + // Save the test result, for recursive case, the result is zero + sd(swap_reg, Address(lock_reg, mark_offset)); + beqz(swap_reg, done); + + bind(slow_case); + + // Call the runtime routine for slow case + call_VM(noreg, + CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), + lock_reg); + + bind(done); + } +} + + +// Unlocks an object. Used in monitorexit bytecode and +// remove_activation. Throws an IllegalMonitorException if object is +// not locked by current thread. +// +// Args: +// c_rarg1: BasicObjectLock for lock +// +// Kills: +// x10 +// c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs) +// t0, t1 (temp regs) +void InterpreterMacroAssembler::unlock_object(Register lock_reg) +{ + assert(lock_reg == c_rarg1, "The argument is only for looks. It must be rarg1"); + + if (UseHeavyMonitors) { + call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg); + } else { + Label done; + + const Register swap_reg = x10; + const Register header_reg = c_rarg2; // Will contain the old oopMark + const Register obj_reg = c_rarg3; // Will contain the oop + + save_bcp(); // Save in case of exception + + // Convert from BasicObjectLock structure to object and BasicLock + // structure Store the BasicLock address into x10 + la(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes())); + + // Load oop into obj_reg(c_rarg3) + ld(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes())); + + // Free entry + sd(zr, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes())); + + if (UseBiasedLocking) { + biased_locking_exit(obj_reg, header_reg, done); + } + + // Load the old header from BasicLock structure + ld(header_reg, Address(swap_reg, + BasicLock::displaced_header_offset_in_bytes())); + + // Test for recursion + beqz(header_reg, done); + + // Atomic swap back the old header + cmpxchg_obj_header(swap_reg, header_reg, obj_reg, t0, done, /*fallthrough*/NULL); + + // Call the runtime routine for slow case. + sd(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes())); // restore obj + call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg); + + bind(done); + + restore_bcp(); + } +} + + +void InterpreterMacroAssembler::test_method_data_pointer(Register mdp, + Label& zero_continue) { + assert(ProfileInterpreter, "must be profiling interpreter"); + ld(mdp, Address(fp, frame::interpreter_frame_mdp_offset * wordSize)); + beqz(mdp, zero_continue); +} + +// Set the method data pointer for the current bcp. +void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() { + assert(ProfileInterpreter, "must be profiling interpreter"); + Label set_mdp; + push_reg(RegSet::of(x10, x11), sp); // save x10, x11 + + // Test MDO to avoid the call if it is NULL. + ld(x10, Address(xmethod, in_bytes(Method::method_data_offset()))); + beqz(x10, set_mdp); + call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), xmethod, xbcp); + // x10: mdi + // mdo is guaranteed to be non-zero here, we checked for it before the call. + ld(x11, Address(xmethod, in_bytes(Method::method_data_offset()))); + la(x11, Address(x11, in_bytes(MethodData::data_offset()))); + add(x10, x11, x10); + sd(x10, Address(fp, frame::interpreter_frame_mdp_offset * wordSize)); + bind(set_mdp); + pop_reg(RegSet::of(x10, x11), sp); +} + +void InterpreterMacroAssembler::verify_method_data_pointer() { + assert(ProfileInterpreter, "must be profiling interpreter"); +#ifdef ASSERT + Label verify_continue; + add(sp, sp, -4 * wordSize); + sd(x10, Address(sp, 0)); + sd(x11, Address(sp, wordSize)); + sd(x12, Address(sp, 2 * wordSize)); + sd(x13, Address(sp, 3 * wordSize)); + test_method_data_pointer(x13, verify_continue); // If mdp is zero, continue + get_method(x11); + + // If the mdp is valid, it will point to a DataLayout header which is + // consistent with the bcp. The converse is highly probable also. + lh(x12, Address(x13, in_bytes(DataLayout::bci_offset()))); + ld(t0, Address(x11, Method::const_offset())); + add(x12, x12, t0); + la(x12, Address(x12, ConstMethod::codes_offset())); + beq(x12, xbcp, verify_continue); + // x10: method + // xbcp: bcp // xbcp == 22 + // x13: mdp + call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp), + x11, xbcp, x13); + bind(verify_continue); + ld(x10, Address(sp, 0)); + ld(x11, Address(sp, wordSize)); + ld(x12, Address(sp, 2 * wordSize)); + ld(x13, Address(sp, 3 * wordSize)); + add(sp, sp, 4 * wordSize); +#endif // ASSERT +} + + +void InterpreterMacroAssembler::set_mdp_data_at(Register mdp_in, + int constant, + Register value) { + assert(ProfileInterpreter, "must be profiling interpreter"); + Address data(mdp_in, constant); + sd(value, data); +} + + +void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in, + int constant, + bool decrement) { + increment_mdp_data_at(mdp_in, noreg, constant, decrement); +} + +void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in, + Register reg, + int constant, + bool decrement) { + assert(ProfileInterpreter, "must be profiling interpreter"); + // %%% this does 64bit counters at best it is wasting space + // at worst it is a rare bug when counters overflow + + assert_different_registers(t1, t0, mdp_in, reg); + + Address addr1(mdp_in, constant); + Address addr2(t1, 0); + Address &addr = addr1; + if (reg != noreg) { + la(t1, addr1); + add(t1, t1, reg); + addr = addr2; + } + + if (decrement) { + ld(t0, addr); + addi(t0, t0, -DataLayout::counter_increment); + Label L; + bltz(t0, L); // skip store if counter underflow + sd(t0, addr); + bind(L); + } else { + assert(DataLayout::counter_increment == 1, + "flow-free idiom only works with 1"); + ld(t0, addr); + addi(t0, t0, DataLayout::counter_increment); + Label L; + blez(t0, L); // skip store if counter overflow + sd(t0, addr); + bind(L); + } +} + +void InterpreterMacroAssembler::set_mdp_flag_at(Register mdp_in, + int flag_byte_constant) { + assert(ProfileInterpreter, "must be profiling interpreter"); + int flags_offset = in_bytes(DataLayout::flags_offset()); + // Set the flag + lbu(t1, Address(mdp_in, flags_offset)); + ori(t1, t1, flag_byte_constant); + sb(t1, Address(mdp_in, flags_offset)); +} + + +void InterpreterMacroAssembler::test_mdp_data_at(Register mdp_in, + int offset, + Register value, + Register test_value_out, + Label& not_equal_continue) { + assert(ProfileInterpreter, "must be profiling interpreter"); + if (test_value_out == noreg) { + ld(t1, Address(mdp_in, offset)); + bne(value, t1, not_equal_continue); + } else { + // Put the test value into a register, so caller can use it: + ld(test_value_out, Address(mdp_in, offset)); + bne(value, test_value_out, not_equal_continue); + } +} + + +void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, + int offset_of_disp) { + assert(ProfileInterpreter, "must be profiling interpreter"); + ld(t1, Address(mdp_in, offset_of_disp)); + add(mdp_in, mdp_in, t1); + sd(mdp_in, Address(fp, frame::interpreter_frame_mdp_offset * wordSize)); +} + +void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, + Register reg, + int offset_of_disp) { + assert(ProfileInterpreter, "must be profiling interpreter"); + add(t1, mdp_in, reg); + ld(t1, Address(t1, offset_of_disp)); + add(mdp_in, mdp_in, t1); + sd(mdp_in, Address(fp, frame::interpreter_frame_mdp_offset * wordSize)); +} + + +void InterpreterMacroAssembler::update_mdp_by_constant(Register mdp_in, + int constant) { + assert(ProfileInterpreter, "must be profiling interpreter"); + addi(mdp_in, mdp_in, (unsigned)constant); + sd(mdp_in, Address(fp, frame::interpreter_frame_mdp_offset * wordSize)); +} + + +void InterpreterMacroAssembler::update_mdp_for_ret(Register return_bci) { + assert(ProfileInterpreter, "must be profiling interpreter"); + + // save/restore across call_VM + addi(sp, sp, -2 * wordSize); + sd(zr, Address(sp, 0)); + sd(return_bci, Address(sp, wordSize)); + call_VM(noreg, + CAST_FROM_FN_PTR(address, InterpreterRuntime::update_mdp_for_ret), + return_bci); + ld(zr, Address(sp, 0)); + ld(return_bci, Address(sp, wordSize)); + addi(sp, sp, 2 * wordSize); +} + +void InterpreterMacroAssembler::profile_taken_branch(Register mdp, + Register bumped_count) { + if (ProfileInterpreter) { + Label profile_continue; + + // If no method data exists, go to profile_continue. + // Otherwise, assign to mdp + test_method_data_pointer(mdp, profile_continue); + + // We are taking a branch. Increment the taken count. + Address data(mdp, in_bytes(JumpData::taken_offset())); + ld(bumped_count, data); + assert(DataLayout::counter_increment == 1, + "flow-free idiom only works with 1"); + addi(bumped_count, bumped_count, DataLayout::counter_increment); + Label L; + // eg: bumped_count=0x7fff ffff ffff ffff + 1 < 0. so we use <= 0; + blez(bumped_count, L); // skip store if counter overflow, + sd(bumped_count, data); + bind(L); + // The method data pointer needs to be updated to reflect the new target. + update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset())); + bind(profile_continue); + } +} + +void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) { + if (ProfileInterpreter) { + Label profile_continue; + + // If no method data exists, go to profile_continue. + test_method_data_pointer(mdp, profile_continue); + + // We are taking a branch. Increment the not taken count. + increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset())); + + // The method data pointer needs to be updated to correspond to + // the next bytecode + update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size())); + bind(profile_continue); + } +} + +void InterpreterMacroAssembler::profile_call(Register mdp) { + if (ProfileInterpreter) { + Label profile_continue; + + // If no method data exists, go to profile_continue. + test_method_data_pointer(mdp, profile_continue); + + // We are making a call. Increment the count. + increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset())); + + // The method data pointer needs to be updated to reflect the new target. + update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size())); + bind(profile_continue); + } +} + +void InterpreterMacroAssembler::profile_final_call(Register mdp) { + if (ProfileInterpreter) { + Label profile_continue; + + // If no method data exists, go to profile_continue. + test_method_data_pointer(mdp, profile_continue); + + // We are making a call. Increment the count. + increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset())); + + // The method data pointer needs to be updated to reflect the new target. + update_mdp_by_constant(mdp, + in_bytes(VirtualCallData:: + virtual_call_data_size())); + bind(profile_continue); + } +} + + +void InterpreterMacroAssembler::profile_virtual_call(Register receiver, + Register mdp, + Register reg2, + bool receiver_can_be_null) { + if (ProfileInterpreter) { + Label profile_continue; + + // If no method data exists, go to profile_continue. + test_method_data_pointer(mdp, profile_continue); + + Label skip_receiver_profile; + if (receiver_can_be_null) { + Label not_null; + // We are making a call. Increment the count for null receiver. + increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset())); + j(skip_receiver_profile); + bind(not_null); + } + + // Record the receiver type. + record_klass_in_profile(receiver, mdp, reg2, true); + bind(skip_receiver_profile); + + // The method data pointer needs to be updated to reflect the new target. + + update_mdp_by_constant(mdp, + in_bytes(VirtualCallData:: + virtual_call_data_size())); + bind(profile_continue); + } +} + +// This routine creates a state machine for updating the multi-row +// type profile at a virtual call site (or other type-sensitive bytecode). +// The machine visits each row (of receiver/count) until the receiver type +// is found, or until it runs out of rows. At the same time, it remembers +// the location of the first empty row. (An empty row records null for its +// receiver, and can be allocated for a newly-observed receiver type.) +// Because there are two degrees of freedom in the state, a simple linear +// search will not work; it must be a decision tree. Hence this helper +// function is recursive, to generate the required tree structured code. +// It's the interpreter, so we are trading off code space for speed. +// See below for example code. +void InterpreterMacroAssembler::record_klass_in_profile_helper( + Register receiver, Register mdp, + Register reg2, + Label& done, bool is_virtual_call) { + if (TypeProfileWidth == 0) { + if (is_virtual_call) { + increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset())); + } + + } else { + int non_profiled_offset = -1; + if (is_virtual_call) { + non_profiled_offset = in_bytes(CounterData::count_offset()); + } + + record_item_in_profile_helper(receiver, mdp, reg2, 0, done, TypeProfileWidth, + &VirtualCallData::receiver_offset, &VirtualCallData::receiver_count_offset, non_profiled_offset); + } +} + +void InterpreterMacroAssembler::record_item_in_profile_helper( + Register item, Register mdp, Register reg2, int start_row, Label& done, int total_rows, + OffsetFunction item_offset_fn, OffsetFunction item_count_offset_fn, int non_profiled_offset) { + int last_row = total_rows - 1; + assert(start_row <= last_row, "must be work left to do"); + // Test this row for both the item and for null. + // Take any of three different outcomes: + // 1. found item => increment count and goto done + // 2. found null => keep looking for case 1, maybe allocate this cell + // 3. found something else => keep looking for cases 1 and 2 + // Case 3 is handled by a recursive call. + for (int row = start_row; row <= last_row; row++) { + Label next_test; + bool test_for_null_also = (row == start_row); + + // See if the item is item[n]. + int item_offset = in_bytes(item_offset_fn(row)); + test_mdp_data_at(mdp, item_offset, item, + (test_for_null_also ? reg2 : noreg), + next_test); + // (Reg2 now contains the item from the CallData.) + + // The item is item[n]. Increment count[n]. + int count_offset = in_bytes(item_count_offset_fn(row)); + increment_mdp_data_at(mdp, count_offset); + j(done); + bind(next_test); + + if (test_for_null_also) { + Label found_null; + // Failed the equality check on item[n]... Test for null. + if (start_row == last_row) { + // The only thing left to do is handle the null case. + if (non_profiled_offset >= 0) { + beqz(reg2, found_null); + // Item did not match any saved item and there is no empty row for it. + // Increment total counter to indicate polymorphic case. + increment_mdp_data_at(mdp, non_profiled_offset); + j(done); + bind(found_null); + } else { + bnez(reg2, done); + } + break; + } + // Since null is rare, make it be the branch-taken case. + beqz(reg2, found_null); + + // Put all the "Case 3" tests here. + record_item_in_profile_helper(item, mdp, reg2, start_row + 1, done, total_rows, + item_offset_fn, item_count_offset_fn, non_profiled_offset); + + // Found a null. Keep searching for a matching item, + // but remember that this is an empty (unused) slot. + bind(found_null); + } + } + + // In the fall-through case, we found no matching item, but we + // observed the item[start_row] is NULL. + // Fill in the item field and increment the count. + int item_offset = in_bytes(item_offset_fn(start_row)); + set_mdp_data_at(mdp, item_offset, item); + int count_offset = in_bytes(item_count_offset_fn(start_row)); + mv(reg2, DataLayout::counter_increment); + set_mdp_data_at(mdp, count_offset, reg2); + if (start_row > 0) { + j(done); + } +} + +// Example state machine code for three profile rows: +// # main copy of decision tree, rooted at row[1] +// if (row[0].rec == rec) then [ +// row[0].incr() +// goto done +// ] +// if (row[0].rec != NULL) then [ +// # inner copy of decision tree, rooted at row[1] +// if (row[1].rec == rec) then [ +// row[1].incr() +// goto done +// ] +// if (row[1].rec != NULL) then [ +// # degenerate decision tree, rooted at row[2] +// if (row[2].rec == rec) then [ +// row[2].incr() +// goto done +// ] +// if (row[2].rec != NULL) then [ +// count.incr() +// goto done +// ] # overflow +// row[2].init(rec) +// goto done +// ] else [ +// # remember row[1] is empty +// if (row[2].rec == rec) then [ +// row[2].incr() +// goto done +// ] +// row[1].init(rec) +// goto done +// ] +// else [ +// # remember row[0] is empty +// if (row[1].rec == rec) then [ +// row[1].incr() +// goto done +// ] +// if (row[2].rec == rec) then [ +// row[2].incr() +// goto done +// ] +// row[0].init(rec) +// goto done +// ] +// done: + +void InterpreterMacroAssembler::record_klass_in_profile(Register receiver, + Register mdp, Register reg2, + bool is_virtual_call) { + assert(ProfileInterpreter, "must be profiling"); + Label done; + + record_klass_in_profile_helper(receiver, mdp, reg2, done, is_virtual_call); + + bind(done); +} + +void InterpreterMacroAssembler::profile_ret(Register return_bci, Register mdp) { + if (ProfileInterpreter) { + Label profile_continue; + + // If no method data exists, go to profile_continue. + test_method_data_pointer(mdp, profile_continue); + + // Update the total ret count. + increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset())); + + for (uint row = 0; row < RetData::row_limit(); row++) { + Label next_test; + + // See if return_bci is equal to bci[n]: + test_mdp_data_at(mdp, + in_bytes(RetData::bci_offset(row)), + return_bci, noreg, + next_test); + + // return_bci is equal to bci[n]. Increment the count. + increment_mdp_data_at(mdp, in_bytes(RetData::bci_count_offset(row))); + + // The method data pointer needs to be updated to reflect the new target. + update_mdp_by_offset(mdp, + in_bytes(RetData::bci_displacement_offset(row))); + j(profile_continue); + bind(next_test); + } + + update_mdp_for_ret(return_bci); + + bind(profile_continue); + } +} + +void InterpreterMacroAssembler::profile_null_seen(Register mdp) { + if (ProfileInterpreter) { + Label profile_continue; + + // If no method data exists, go to profile_continue. + test_method_data_pointer(mdp, profile_continue); + + set_mdp_flag_at(mdp, BitData::null_seen_byte_constant()); + + // The method data pointer needs to be updated. + int mdp_delta = in_bytes(BitData::bit_data_size()); + if (TypeProfileCasts) { + mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); + } + update_mdp_by_constant(mdp, mdp_delta); + + bind(profile_continue); + } +} + +void InterpreterMacroAssembler::profile_typecheck_failed(Register mdp) { + if (ProfileInterpreter && TypeProfileCasts) { + Label profile_continue; + + // If no method data exists, go to profile_continue. + test_method_data_pointer(mdp, profile_continue); + + int count_offset = in_bytes(CounterData::count_offset()); + // Back up the address, since we have already bumped the mdp. + count_offset -= in_bytes(VirtualCallData::virtual_call_data_size()); + + // *Decrement* the counter. We expect to see zero or small negatives. + increment_mdp_data_at(mdp, count_offset, true); + + bind (profile_continue); + } +} + +void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass, Register reg2) { + if (ProfileInterpreter) { + Label profile_continue; + + // If no method data exists, go to profile_continue. + test_method_data_pointer(mdp, profile_continue); + + // The method data pointer needs to be updated. + int mdp_delta = in_bytes(BitData::bit_data_size()); + if (TypeProfileCasts) { + mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); + + // Record the object type. + record_klass_in_profile(klass, mdp, reg2, false); + } + update_mdp_by_constant(mdp, mdp_delta); + + bind(profile_continue); + } +} + +void InterpreterMacroAssembler::profile_switch_default(Register mdp) { + if (ProfileInterpreter) { + Label profile_continue; + + // If no method data exists, go to profile_continue. + test_method_data_pointer(mdp, profile_continue); + + // Update the default case count + increment_mdp_data_at(mdp, + in_bytes(MultiBranchData::default_count_offset())); + + // The method data pointer needs to be updated. + update_mdp_by_offset(mdp, + in_bytes(MultiBranchData:: + default_displacement_offset())); + + bind(profile_continue); + } +} + +void InterpreterMacroAssembler::profile_switch_case(Register index, + Register mdp, + Register reg2) { + if (ProfileInterpreter) { + Label profile_continue; + + // If no method data exists, go to profile_continue. + test_method_data_pointer(mdp, profile_continue); + + // Build the base (index * per_case_size_in_bytes()) + + // case_array_offset_in_bytes() + mv(reg2, in_bytes(MultiBranchData::per_case_size())); + mv(t0, in_bytes(MultiBranchData::case_array_offset())); + Assembler::mul(index, index, reg2); + Assembler::add(index, index, t0); + + // Update the case count + increment_mdp_data_at(mdp, + index, + in_bytes(MultiBranchData::relative_count_offset())); + + // The method data pointer need to be updated. + update_mdp_by_offset(mdp, + index, + in_bytes(MultiBranchData:: + relative_displacement_offset())); + + bind(profile_continue); + } +} + +void InterpreterMacroAssembler::verify_FPU(int stack_depth, TosState state) { ; } + +void InterpreterMacroAssembler::notify_method_entry() { + // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to + // track stack depth. If it is possible to enter interp_only_mode we add + // the code to check if the event should be sent. + if (JvmtiExport::can_post_interpreter_events()) { + Label L; + lwu(x13, Address(xthread, JavaThread::interp_only_mode_offset())); + beqz(x13, L); + call_VM(noreg, CAST_FROM_FN_PTR(address, + InterpreterRuntime::post_method_entry)); + bind(L); + } + + { + SkipIfEqual skip(this, &DTraceMethodProbes, false); + get_method(c_rarg1); + call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry), + xthread, c_rarg1); + } + + // RedefineClasses() tracing support for obsolete method entry + if (log_is_enabled(Trace, redefine, class, obsolete)) { + get_method(c_rarg1); + call_VM_leaf( + CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry), + xthread, c_rarg1); + } +} + + +void InterpreterMacroAssembler::notify_method_exit( + TosState state, NotifyMethodExitMode mode) { + // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to + // track stack depth. If it is possible to enter interp_only_mode we add + // the code to check if the event should be sent. + if (mode == NotifyJVMTI && JvmtiExport::can_post_interpreter_events()) { + Label L; + // Note: frame::interpreter_frame_result has a dependency on how the + // method result is saved across the call to post_method_exit. If this + // is changed then the interpreter_frame_result implementation will + // need to be updated too. + + // template interpreter will leave the result on the top of the stack. + push(state); + lwu(x13, Address(xthread, JavaThread::interp_only_mode_offset())); + beqz(x13, L); + call_VM(noreg, + CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit)); + bind(L); + pop(state); + } + + { + SkipIfEqual skip(this, &DTraceMethodProbes, false); + push(state); + get_method(c_rarg1); + call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), + xthread, c_rarg1); + pop(state); + } +} + + +// Jump if ((*counter_addr += increment) & mask) satisfies the condition. +void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr, + int increment, Address mask, + Register tmp1, Register tmp2, + bool preloaded, Label* where) { + Label done; + if (!preloaded) { + lwu(tmp1, counter_addr); + } + add(tmp1, tmp1, increment); + sw(tmp1, counter_addr); + lwu(tmp2, mask); + andr(tmp1, tmp1, tmp2); + bnez(tmp1, done); + j(*where); // offset is too large so we have to use j instead of beqz here + bind(done); +} + +void InterpreterMacroAssembler::call_VM_leaf_base(address entry_point, + int number_of_arguments) { + // interpreter specific + // + // Note: No need to save/restore rbcp & rlocals pointer since these + // are callee saved registers and no blocking/ GC can happen + // in leaf calls. +#ifdef ASSERT + { + Label L; + ld(t0, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize)); + beqz(t0, L); + stop("InterpreterMacroAssembler::call_VM_leaf_base:" + " last_sp != NULL"); + bind(L); + } +#endif /* ASSERT */ + // super call + MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments); +} + +void InterpreterMacroAssembler::call_VM_base(Register oop_result, + Register java_thread, + Register last_java_sp, + address entry_point, + int number_of_arguments, + bool check_exceptions) { + // interpreter specific + // + // Note: Could avoid restoring locals ptr (callee saved) - however doesn't + // really make a difference for these runtime calls, since they are + // slow anyway. Btw., bcp must be saved/restored since it may change + // due to GC. + save_bcp(); +#ifdef ASSERT + { + Label L; + ld(t0, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize)); + beqz(t0, L); + stop("InterpreterMacroAssembler::call_VM_base:" + " last_sp != NULL"); + bind(L); + } +#endif /* ASSERT */ + // super call + MacroAssembler::call_VM_base(oop_result, noreg, last_java_sp, + entry_point, number_of_arguments, + check_exceptions); +// interpreter specific + restore_bcp(); + restore_locals(); +} + +void InterpreterMacroAssembler::profile_obj_type(Register obj, const Address& mdo_addr, Register tmp) { + assert_different_registers(obj, tmp, t0, mdo_addr.base()); + Label update, next, none; + + verify_oop(obj); + + bnez(obj, update); + orptr(mdo_addr, TypeEntries::null_seen, t0, tmp); + j(next); + + bind(update); + load_klass(obj, obj); + + ld(t0, mdo_addr); + xorr(obj, obj, t0); + andi(t0, obj, TypeEntries::type_klass_mask); + beqz(t0, next); // klass seen before, nothing to + // do. The unknown bit may have been + // set already but no need to check. + + test_bit(t0, obj, exact_log2(TypeEntries::type_unknown)); + bnez(t0, next); + // already unknown. Nothing to do anymore. + + ld(t0, mdo_addr); + beqz(t0, none); + mv(tmp, (u1)TypeEntries::null_seen); + beq(t0, tmp, none); + // There is a chance that the checks above (re-reading profiling + // data from memory) fail if another thread has just set the + // profiling to this obj's klass + ld(t0, mdo_addr); + xorr(obj, obj, t0); + andi(t0, obj, TypeEntries::type_klass_mask); + beqz(t0, next); + + // different than before. Cannot keep accurate profile. + orptr(mdo_addr, TypeEntries::type_unknown, t0, tmp); + j(next); + + bind(none); + // first time here. Set profile type. + sd(obj, mdo_addr); + + bind(next); +} + +void InterpreterMacroAssembler::profile_arguments_type(Register mdp, Register callee, Register tmp, bool is_virtual) { + if (!ProfileInterpreter) { + return; + } + + if (MethodData::profile_arguments() || MethodData::profile_return()) { + Label profile_continue; + + test_method_data_pointer(mdp, profile_continue); + + int off_to_start = is_virtual ? in_bytes(VirtualCallData::virtual_call_data_size()) : in_bytes(CounterData::counter_data_size()); + + lbu(t0, Address(mdp, in_bytes(DataLayout::tag_offset()) - off_to_start)); + if (is_virtual) { + mv(tmp, (u1)DataLayout::virtual_call_type_data_tag); + bne(t0, tmp, profile_continue); + } else { + mv(tmp, (u1)DataLayout::call_type_data_tag); + bne(t0, tmp, profile_continue); + } + + // calculate slot step + static int stack_slot_offset0 = in_bytes(TypeEntriesAtCall::stack_slot_offset(0)); + static int slot_step = in_bytes(TypeEntriesAtCall::stack_slot_offset(1)) - stack_slot_offset0; + + // calculate type step + static int argument_type_offset0 = in_bytes(TypeEntriesAtCall::argument_type_offset(0)); + static int type_step = in_bytes(TypeEntriesAtCall::argument_type_offset(1)) - argument_type_offset0; + + if (MethodData::profile_arguments()) { + Label done, loop, loopEnd, profileArgument, profileReturnType; + RegSet pushed_registers; + pushed_registers += x15; + pushed_registers += x16; + pushed_registers += x17; + Register mdo_addr = x15; + Register index = x16; + Register off_to_args = x17; + push_reg(pushed_registers, sp); + + mv(off_to_args, in_bytes(TypeEntriesAtCall::args_data_offset())); + mv(t0, TypeProfileArgsLimit); + beqz(t0, loopEnd); + + mv(index, zr); // index < TypeProfileArgsLimit + bind(loop); + bgtz(index, profileReturnType); + mv(t0, (int)MethodData::profile_return()); + beqz(t0, profileArgument); // (index > 0 || MethodData::profile_return()) == false + bind(profileReturnType); + // If return value type is profiled we may have no argument to profile + ld(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset()))); + mv(t1, - TypeStackSlotEntries::per_arg_count()); + mul(t1, index, t1); + add(tmp, tmp, t1); + mv(t1, TypeStackSlotEntries::per_arg_count()); + add(t0, mdp, off_to_args); + blt(tmp, t1, done); + + bind(profileArgument); + + ld(tmp, Address(callee, Method::const_offset())); + load_unsigned_short(tmp, Address(tmp, ConstMethod::size_of_parameters_offset())); + // stack offset o (zero based) from the start of the argument + // list, for n arguments translates into offset n - o - 1 from + // the end of the argument list + mv(t0, stack_slot_offset0); + mv(t1, slot_step); + mul(t1, index, t1); + add(t0, t0, t1); + add(t0, mdp, t0); + ld(t0, Address(t0)); + sub(tmp, tmp, t0); + addi(tmp, tmp, -1); + Address arg_addr = argument_address(tmp); + ld(tmp, arg_addr); + + mv(t0, argument_type_offset0); + mv(t1, type_step); + mul(t1, index, t1); + add(t0, t0, t1); + add(mdo_addr, mdp, t0); + Address mdo_arg_addr(mdo_addr, 0); + profile_obj_type(tmp, mdo_arg_addr, t1); + + int to_add = in_bytes(TypeStackSlotEntries::per_arg_size()); + addi(off_to_args, off_to_args, to_add); + + // increment index by 1 + addi(index, index, 1); + mv(t1, TypeProfileArgsLimit); + blt(index, t1, loop); + bind(loopEnd); + + if (MethodData::profile_return()) { + ld(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset()))); + addi(tmp, tmp, -TypeProfileArgsLimit*TypeStackSlotEntries::per_arg_count()); + } + + add(t0, mdp, off_to_args); + bind(done); + mv(mdp, t0); + + // unspill the clobbered registers + pop_reg(pushed_registers, sp); + + if (MethodData::profile_return()) { + // We're right after the type profile for the last + // argument. tmp is the number of cells left in the + // CallTypeData/VirtualCallTypeData to reach its end. Non null + // if there's a return to profile. + assert(ReturnTypeEntry::static_cell_count() < TypeStackSlotEntries::per_arg_count(), "can't move past ret type"); + shadd(mdp, tmp, mdp, tmp, exact_log2(DataLayout::cell_size)); + } + sd(mdp, Address(fp, frame::interpreter_frame_mdp_offset * wordSize)); + } else { + assert(MethodData::profile_return(), "either profile call args or call ret"); + update_mdp_by_constant(mdp, in_bytes(TypeEntriesAtCall::return_only_size())); + } + + // mdp points right after the end of the + // CallTypeData/VirtualCallTypeData, right after the cells for the + // return value type if there's one + + bind(profile_continue); + } +} + +void InterpreterMacroAssembler::profile_return_type(Register mdp, Register ret, Register tmp) { + assert_different_registers(mdp, ret, tmp, xbcp, t0, t1); + if (ProfileInterpreter && MethodData::profile_return()) { + Label profile_continue, done; + + test_method_data_pointer(mdp, profile_continue); + + if (MethodData::profile_return_jsr292_only()) { + assert(Method::intrinsic_id_size_in_bytes() == 2, "assuming Method::_intrinsic_id is u2"); + + // If we don't profile all invoke bytecodes we must make sure + // it's a bytecode we indeed profile. We can't go back to the + // begining of the ProfileData we intend to update to check its + // type because we're right after it and we don't known its + // length + Label do_profile; + lbu(t0, Address(xbcp, 0)); + mv(tmp, (u1)Bytecodes::_invokedynamic); + beq(t0, tmp, do_profile); + mv(tmp, (u1)Bytecodes::_invokehandle); + beq(t0, tmp, do_profile); + get_method(tmp); + lhu(t0, Address(tmp, Method::intrinsic_id_offset_in_bytes())); + mv(t1, static_cast(vmIntrinsics::_compiledLambdaForm)); + bne(t0, t1, profile_continue); + bind(do_profile); + } + + Address mdo_ret_addr(mdp, -in_bytes(ReturnTypeEntry::size())); + mv(tmp, ret); + profile_obj_type(tmp, mdo_ret_addr, t1); + + bind(profile_continue); + } +} + +void InterpreterMacroAssembler::profile_parameters_type(Register mdp, Register tmp1, Register tmp2, Register tmp3) { + assert_different_registers(t0, t1, mdp, tmp1, tmp2, tmp3); + if (ProfileInterpreter && MethodData::profile_parameters()) { + Label profile_continue, done; + + test_method_data_pointer(mdp, profile_continue); + + // Load the offset of the area within the MDO used for + // parameters. If it's negative we're not profiling any parameters + lwu(tmp1, Address(mdp, in_bytes(MethodData::parameters_type_data_di_offset()) - in_bytes(MethodData::data_offset()))); + srli(tmp2, tmp1, 31); + bnez(tmp2, profile_continue); // i.e. sign bit set + + // Compute a pointer to the area for parameters from the offset + // and move the pointer to the slot for the last + // parameters. Collect profiling from last parameter down. + // mdo start + parameters offset + array length - 1 + add(mdp, mdp, tmp1); + ld(tmp1, Address(mdp, ArrayData::array_len_offset())); + add(tmp1, tmp1, - TypeStackSlotEntries::per_arg_count()); + + Label loop; + bind(loop); + + int off_base = in_bytes(ParametersTypeData::stack_slot_offset(0)); + int type_base = in_bytes(ParametersTypeData::type_offset(0)); + int per_arg_scale = exact_log2(DataLayout::cell_size); + add(t0, mdp, off_base); + add(t1, mdp, type_base); + + shadd(tmp2, tmp1, t0, tmp2, per_arg_scale); + // load offset on the stack from the slot for this parameter + ld(tmp2, Address(tmp2, 0)); + neg(tmp2, tmp2); + + // read the parameter from the local area + shadd(tmp2, tmp2, xlocals, tmp2, Interpreter::logStackElementSize); + ld(tmp2, Address(tmp2, 0)); + + // profile the parameter + shadd(t1, tmp1, t1, t0, per_arg_scale); + Address arg_type(t1, 0); + profile_obj_type(tmp2, arg_type, tmp3); + + // go to next parameter + add(tmp1, tmp1, - TypeStackSlotEntries::per_arg_count()); + bgez(tmp1, loop); + + bind(profile_continue); + } +} + +void InterpreterMacroAssembler::get_method_counters(Register method, + Register mcs, Label& skip) { + Label has_counters; + ld(mcs, Address(method, Method::method_counters_offset())); + bnez(mcs, has_counters); + call_VM(noreg, CAST_FROM_FN_PTR(address, + InterpreterRuntime::build_method_counters), method); + ld(mcs, Address(method, Method::method_counters_offset())); + beqz(mcs, skip); // No MethodCounters allocated, OutOfMemory + bind(has_counters); +} + +#ifdef ASSERT +void InterpreterMacroAssembler::verify_access_flags(Register access_flags, uint32_t flag, + const char* msg, bool stop_by_hit) { + Label L; + test_bit(t0, access_flags, exact_log2(flag)); + if (stop_by_hit) { + beqz(t0, L); + } else { + bnez(t0, L); + } + stop(msg); + bind(L); +} + +void InterpreterMacroAssembler::verify_frame_setup() { + Label L; + const Address monitor_block_top(fp, frame::interpreter_frame_monitor_block_top_offset * wordSize); + ld(t0, monitor_block_top); + beq(esp, t0, L); + stop("broken stack frame setup in interpreter"); + bind(L); +} +#endif diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.hpp b/src/hotspot/cpu/riscv/interp_masm_riscv.hpp new file mode 100644 index 0000000000000..c820291e5fe07 --- /dev/null +++ b/src/hotspot/cpu/riscv/interp_masm_riscv.hpp @@ -0,0 +1,285 @@ +/* + * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_INTERP_MASM_RISCV_HPP +#define CPU_RISCV_INTERP_MASM_RISCV_HPP + +#include "asm/macroAssembler.hpp" +#include "interpreter/invocationCounter.hpp" +#include "runtime/frame.hpp" + +// This file specializes the assember with interpreter-specific macros + +typedef ByteSize (*OffsetFunction)(uint); + +class InterpreterMacroAssembler: public MacroAssembler { + protected: + // Interpreter specific version of call_VM_base + using MacroAssembler::call_VM_leaf_base; + + virtual void call_VM_leaf_base(address entry_point, + int number_of_arguments); + + virtual void call_VM_base(Register oop_result, + Register java_thread, + Register last_java_sp, + address entry_point, + int number_of_arguments, + bool check_exceptions); + + // base routine for all dispatches + void dispatch_base(TosState state, address* table, bool verifyoop = true, + bool generate_poll = false, Register Rs = t0); + + public: + InterpreterMacroAssembler(CodeBuffer* code) : MacroAssembler(code) {} + virtual ~InterpreterMacroAssembler() {} + + void load_earlyret_value(TosState state); + + void jump_to_entry(address entry); + + virtual void check_and_handle_popframe(Register java_thread); + virtual void check_and_handle_earlyret(Register java_thread); + + // Interpreter-specific registers + void save_bcp() { + sd(xbcp, Address(fp, frame::interpreter_frame_bcp_offset * wordSize)); + } + + void restore_bcp() { + ld(xbcp, Address(fp, frame::interpreter_frame_bcp_offset * wordSize)); + } + + void restore_locals() { + ld(xlocals, Address(fp, frame::interpreter_frame_locals_offset * wordSize)); + } + + void restore_constant_pool_cache() { + ld(xcpool, Address(fp, frame::interpreter_frame_cache_offset * wordSize)); + } + + void get_dispatch(); + + // Helpers for runtime call arguments/results + void get_method(Register reg) { + ld(reg, Address(fp, frame::interpreter_frame_method_offset * wordSize)); + } + + void get_const(Register reg) { + get_method(reg); + ld(reg, Address(reg, in_bytes(Method::const_offset()))); + } + + void get_constant_pool(Register reg) { + get_const(reg); + ld(reg, Address(reg, in_bytes(ConstMethod::constants_offset()))); + } + + void get_constant_pool_cache(Register reg) { + get_constant_pool(reg); + ld(reg, Address(reg, ConstantPool::cache_offset_in_bytes())); + } + + void get_cpool_and_tags(Register cpool, Register tags) { + get_constant_pool(cpool); + ld(tags, Address(cpool, ConstantPool::tags_offset_in_bytes())); + } + + void get_unsigned_2_byte_index_at_bcp(Register reg, int bcp_offset); + void get_cache_and_index_at_bcp(Register cache, Register index, int bcp_offset, size_t index_size = sizeof(u2)); + void get_cache_and_index_and_bytecode_at_bcp(Register cache, Register index, Register bytecode, int byte_no, int bcp_offset, size_t index_size = sizeof(u2)); + void get_cache_entry_pointer_at_bcp(Register cache, Register tmp, int bcp_offset, size_t index_size = sizeof(u2)); + void get_cache_index_at_bcp(Register index, int bcp_offset, size_t index_size = sizeof(u2)); + void get_method_counters(Register method, Register mcs, Label& skip); + + // Load cpool->resolved_references(index). + void load_resolved_reference_at_index(Register result, Register index, Register tmp = x15); + + // Load cpool->resolved_klass_at(index). + void load_resolved_klass_at_offset(Register cpool, Register index, Register klass, Register temp); + + void load_resolved_method_at_index(int byte_no, Register method, Register cache); + + void pop_ptr(Register r = x10); + void pop_i(Register r = x10); + void pop_l(Register r = x10); + void pop_f(FloatRegister r = f10); + void pop_d(FloatRegister r = f10); + void push_ptr(Register r = x10); + void push_i(Register r = x10); + void push_l(Register r = x10); + void push_f(FloatRegister r = f10); + void push_d(FloatRegister r = f10); + + void pop(TosState state); // transition vtos -> state + void push(TosState state); // transition state -> vtos + + void empty_expression_stack() { + ld(esp, Address(fp, frame::interpreter_frame_monitor_block_top_offset * wordSize)); + // NULL last_sp until next java call + sd(zr, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize)); + } + + // Helpers for swap and dup + void load_ptr(int n, Register val); + void store_ptr(int n, Register val); + + // Load float value from 'address'. The value is loaded onto the fp register f10. + void load_float(Address src); + void load_double(Address src); + + // Generate a subtype check: branch to ok_is_subtype if sub_klass is + // a subtype of super_klass. + void gen_subtype_check( Register sub_klass, Label &ok_is_subtype ); + + // Dispatching + void dispatch_prolog(TosState state, int step = 0); + void dispatch_epilog(TosState state, int step = 0); + // dispatch via t0 + void dispatch_only(TosState state, bool generate_poll = false, Register Rs = t0); + // dispatch normal table via t0 (assume t0 is loaded already) + void dispatch_only_normal(TosState state, Register Rs = t0); + void dispatch_only_noverify(TosState state, Register Rs = t0); + // load t0 from [xbcp + step] and dispatch via t0 + void dispatch_next(TosState state, int step = 0, bool generate_poll = false); + // load t0 from [xbcp] and dispatch via t0 and table + void dispatch_via (TosState state, address* table); + + // jump to an invoked target + void prepare_to_jump_from_interpreted(); + void jump_from_interpreted(Register method); + + + // Returning from interpreted functions + // + // Removes the current activation (incl. unlocking of monitors) + // and sets up the return address. This code is also used for + // exception unwindwing. In that case, we do not want to throw + // IllegalMonitorStateExceptions, since that might get us into an + // infinite rethrow exception loop. + // Additionally this code is used for popFrame and earlyReturn. + // In popFrame case we want to skip throwing an exception, + // installing an exception, and notifying jvmdi. + // In earlyReturn case we only want to skip throwing an exception + // and installing an exception. + void remove_activation(TosState state, + bool throw_monitor_exception = true, + bool install_monitor_exception = true, + bool notify_jvmdi = true); + + // FIXME: Give us a valid frame at a null check. + virtual void null_check(Register reg, int offset = -1) { + MacroAssembler::null_check(reg, offset); + } + + // Object locking + void lock_object (Register lock_reg); + void unlock_object(Register lock_reg); + + // Interpreter profiling operations + void set_method_data_pointer_for_bcp(); + void test_method_data_pointer(Register mdp, Label& zero_continue); + void verify_method_data_pointer(); + + void set_mdp_data_at(Register mdp_in, int constant, Register value); + void increment_mdp_data_at(Address data, bool decrement = false); + void increment_mdp_data_at(Register mdp_in, int constant, + bool decrement = false); + void increment_mdp_data_at(Register mdp_in, Register reg, int constant, + bool decrement = false); + void increment_mask_and_jump(Address counter_addr, + int increment, Address mask, + Register tmp1, Register tmp2, + bool preloaded, Label* where); + + void set_mdp_flag_at(Register mdp_in, int flag_constant); + void test_mdp_data_at(Register mdp_in, int offset, Register value, + Register test_value_out, + Label& not_equal_continue); + + void record_klass_in_profile(Register receiver, Register mdp, + Register reg2, bool is_virtual_call); + void record_klass_in_profile_helper(Register receiver, Register mdp, + Register reg2, + Label& done, bool is_virtual_call); + void record_item_in_profile_helper(Register item, Register mdp, + Register reg2, int start_row, Label& done, int total_rows, + OffsetFunction item_offset_fn, OffsetFunction item_count_offset_fn, + int non_profiled_offset); + + void update_mdp_by_offset(Register mdp_in, int offset_of_offset); + void update_mdp_by_offset(Register mdp_in, Register reg, int offset_of_disp); + void update_mdp_by_constant(Register mdp_in, int constant); + void update_mdp_for_ret(Register return_bci); + + // narrow int return value + void narrow(Register result); + + void profile_taken_branch(Register mdp, Register bumped_count); + void profile_not_taken_branch(Register mdp); + void profile_call(Register mdp); + void profile_final_call(Register mdp); + void profile_virtual_call(Register receiver, Register mdp, + Register t1, + bool receiver_can_be_null = false); + void profile_ret(Register return_bci, Register mdp); + void profile_null_seen(Register mdp); + void profile_typecheck(Register mdp, Register klass, Register temp); + void profile_typecheck_failed(Register mdp); + void profile_switch_default(Register mdp); + void profile_switch_case(Register index_in_scratch, Register mdp, + Register temp); + + void profile_obj_type(Register obj, const Address& mdo_addr, Register tmp); + void profile_arguments_type(Register mdp, Register callee, Register tmp, bool is_virtual); + void profile_return_type(Register mdp, Register ret, Register tmp); + void profile_parameters_type(Register mdp, Register tmp1, Register tmp2, Register tmp3); + + // Debugging + // only if +VerifyFPU && (state == ftos || state == dtos) + void verify_FPU(int stack_depth, TosState state = ftos); + + typedef enum { NotifyJVMTI, SkipNotifyJVMTI } NotifyMethodExitMode; + + // support for jvmti/dtrace + void notify_method_entry(); + void notify_method_exit(TosState state, NotifyMethodExitMode mode); + + virtual void _call_Unimplemented(address call_site) { + save_bcp(); + set_last_Java_frame(esp, fp, (address) pc(), t0); + MacroAssembler::_call_Unimplemented(call_site); + } + +#ifdef ASSERT + void verify_access_flags(Register access_flags, uint32_t flag, + const char* msg, bool stop_by_hit = true); + void verify_frame_setup(); +#endif +}; + +#endif // CPU_RISCV_INTERP_MASM_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/interpreterRT_riscv.cpp b/src/hotspot/cpu/riscv/interpreterRT_riscv.cpp new file mode 100644 index 0000000000000..d93530d856401 --- /dev/null +++ b/src/hotspot/cpu/riscv/interpreterRT_riscv.cpp @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "interpreter/interp_masm.hpp" +#include "interpreter/interpreter.hpp" +#include "interpreter/interpreterRuntime.hpp" +#include "memory/allocation.inline.hpp" +#include "memory/universe.hpp" +#include "oops/method.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/icache.hpp" +#include "runtime/interfaceSupport.inline.hpp" +#include "runtime/signature.hpp" + +#define __ _masm-> + +// Implementation of SignatureHandlerGenerator +Register InterpreterRuntime::SignatureHandlerGenerator::from() { return xlocals; } +Register InterpreterRuntime::SignatureHandlerGenerator::to() { return sp; } +Register InterpreterRuntime::SignatureHandlerGenerator::temp() { return t0; } + +Register InterpreterRuntime::SignatureHandlerGenerator::next_gpr() { + if (_num_reg_int_args < Argument::n_int_register_parameters_c - 1) { + return g_INTArgReg[++_num_reg_int_args]; + } + return noreg; +} + +FloatRegister InterpreterRuntime::SignatureHandlerGenerator::next_fpr() { + if (_num_reg_fp_args < Argument::n_float_register_parameters_c) { + return g_FPArgReg[_num_reg_fp_args++]; + } else { + return fnoreg; + } +} + +int InterpreterRuntime::SignatureHandlerGenerator::next_stack_offset() { + int ret = _stack_offset; + _stack_offset += wordSize; + return ret; +} + +InterpreterRuntime::SignatureHandlerGenerator::SignatureHandlerGenerator( + const methodHandle& method, CodeBuffer* buffer) : NativeSignatureIterator(method) { + _masm = new MacroAssembler(buffer); // allocate on resourse area by default + _num_reg_int_args = (method->is_static() ? 1 : 0); + _num_reg_fp_args = 0; + _stack_offset = 0; +} + +void InterpreterRuntime::SignatureHandlerGenerator::pass_int() { + const Address src(from(), Interpreter::local_offset_in_bytes(offset())); + + Register reg = next_gpr(); + if (reg != noreg) { + __ lw(reg, src); + } else { + __ lw(x10, src); + __ sw(x10, Address(to(), next_stack_offset())); + } +} + +void InterpreterRuntime::SignatureHandlerGenerator::pass_long() { + const Address src(from(), Interpreter::local_offset_in_bytes(offset() + 1)); + + Register reg = next_gpr(); + if (reg != noreg) { + __ ld(reg, src); + } else { + __ ld(x10, src); + __ sd(x10, Address(to(), next_stack_offset())); + } +} + +void InterpreterRuntime::SignatureHandlerGenerator::pass_float() { + const Address src(from(), Interpreter::local_offset_in_bytes(offset())); + + FloatRegister reg = next_fpr(); + if (reg != fnoreg) { + __ flw(reg, src); + } else { + // a floating-point argument is passed according to the integer calling + // convention if no floating-point argument register available + pass_int(); + } +} + +void InterpreterRuntime::SignatureHandlerGenerator::pass_double() { + const Address src(from(), Interpreter::local_offset_in_bytes(offset() + 1)); + + FloatRegister reg = next_fpr(); + if (reg != fnoreg) { + __ fld(reg, src); + } else { + // a floating-point argument is passed according to the integer calling + // convention if no floating-point argument register available + pass_long(); + } +} + +void InterpreterRuntime::SignatureHandlerGenerator::pass_object() { + Register reg = next_gpr(); + if (reg == c_rarg1) { + assert(offset() == 0, "argument register 1 can only be (non-null) receiver"); + __ addi(c_rarg1, from(), Interpreter::local_offset_in_bytes(offset())); + } else if (reg != noreg) { + // c_rarg2-c_rarg7 + __ addi(x10, from(), Interpreter::local_offset_in_bytes(offset())); + __ mv(reg, zr); //_num_reg_int_args:c_rarg -> 1:c_rarg2, 2:c_rarg3... + __ ld(temp(), x10); + Label L; + __ beqz(temp(), L); + __ mv(reg, x10); + __ bind(L); + } else { + //to stack + __ addi(x10, from(), Interpreter::local_offset_in_bytes(offset())); + __ ld(temp(), x10); + Label L; + __ bnez(temp(), L); + __ mv(x10, zr); + __ bind(L); + assert(sizeof(jobject) == wordSize, ""); + __ sd(x10, Address(to(), next_stack_offset())); + } +} + +void InterpreterRuntime::SignatureHandlerGenerator::generate(uint64_t fingerprint) { + // generate code to handle arguments + iterate(fingerprint); + + // return result handler + __ la(x10, ExternalAddress(Interpreter::result_handler(method()->result_type()))); + __ ret(); + + __ flush(); +} + + +// Implementation of SignatureHandlerLibrary + +void SignatureHandlerLibrary::pd_set_handler(address handler) {} + + +class SlowSignatureHandler + : public NativeSignatureIterator { + private: + address _from; + intptr_t* _to; + intptr_t* _int_args; + intptr_t* _fp_args; + intptr_t* _fp_identifiers; + unsigned int _num_reg_int_args; + unsigned int _num_reg_fp_args; + + intptr_t* single_slot_addr() { + intptr_t* from_addr = (intptr_t*)(_from + Interpreter::local_offset_in_bytes(0)); + _from -= Interpreter::stackElementSize; + return from_addr; + } + + intptr_t* double_slot_addr() { + intptr_t* from_addr = (intptr_t*)(_from + Interpreter::local_offset_in_bytes(1)); + _from -= 2 * Interpreter::stackElementSize; + return from_addr; + } + + int pass_gpr(intptr_t value) { + if (_num_reg_int_args < Argument::n_int_register_parameters_c - 1) { + *_int_args++ = value; + return _num_reg_int_args++; + } + return -1; + } + + int pass_fpr(intptr_t value) { + if (_num_reg_fp_args < Argument::n_float_register_parameters_c) { + *_fp_args++ = value; + return _num_reg_fp_args++; + } + return -1; + } + + void pass_stack(intptr_t value) { + *_to++ = value; + } + + virtual void pass_int() { + jint value = *(jint*)single_slot_addr(); + if (pass_gpr(value) < 0) { + pass_stack(value); + } + } + + virtual void pass_long() { + intptr_t value = *double_slot_addr(); + if (pass_gpr(value) < 0) { + pass_stack(value); + } + } + + virtual void pass_object() { + intptr_t* addr = single_slot_addr(); + intptr_t value = *addr == 0 ? NULL : (intptr_t)addr; + if (pass_gpr(value) < 0) { + pass_stack(value); + } + } + + virtual void pass_float() { + jint value = *(jint*) single_slot_addr(); + // a floating-point argument is passed according to the integer calling + // convention if no floating-point argument register available + if (pass_fpr(value) < 0 && pass_gpr(value) < 0) { + pass_stack(value); + } + } + + virtual void pass_double() { + intptr_t value = *double_slot_addr(); + int arg = pass_fpr(value); + if (0 <= arg) { + *_fp_identifiers |= (1ull << arg); // mark as double + } else if (pass_gpr(value) < 0) { // no need to mark if passing by integer registers or stack + pass_stack(value); + } + } + + public: + SlowSignatureHandler(const methodHandle& method, address from, intptr_t* to) + : NativeSignatureIterator(method) + { + _from = from; + _to = to; + + _int_args = to - (method->is_static() ? 16 : 17); + _fp_args = to - 8; + _fp_identifiers = to - 9; + *(int*) _fp_identifiers = 0; + _num_reg_int_args = (method->is_static() ? 1 : 0); + _num_reg_fp_args = 0; + } + + ~SlowSignatureHandler() + { + _from = NULL; + _to = NULL; + _int_args = NULL; + _fp_args = NULL; + _fp_identifiers = NULL; + } +}; + + +JRT_ENTRY(address, + InterpreterRuntime::slow_signature_handler(JavaThread* current, + Method* method, + intptr_t* from, + intptr_t* to)) + methodHandle m(current, (Method*)method); + assert(m->is_native(), "sanity check"); + + // handle arguments + SlowSignatureHandler ssh(m, (address)from, to); + ssh.iterate(UCONST64(-1)); + + // return result handler + return Interpreter::result_handler(m->result_type()); +JRT_END diff --git a/src/hotspot/cpu/riscv/interpreterRT_riscv.hpp b/src/hotspot/cpu/riscv/interpreterRT_riscv.hpp new file mode 100644 index 0000000000000..05df63ba2ae5f --- /dev/null +++ b/src/hotspot/cpu/riscv/interpreterRT_riscv.hpp @@ -0,0 +1,68 @@ +/* + * Copyright (c) 1998, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_INTERPRETERRT_RISCV_HPP +#define CPU_RISCV_INTERPRETERRT_RISCV_HPP + +// This is included in the middle of class Interpreter. +// Do not include files here. + +// native method calls + +class SignatureHandlerGenerator: public NativeSignatureIterator { + private: + MacroAssembler* _masm; + unsigned int _num_reg_fp_args; + unsigned int _num_reg_int_args; + int _stack_offset; + + void pass_int(); + void pass_long(); + void pass_float(); + void pass_double(); + void pass_object(); + + Register next_gpr(); + FloatRegister next_fpr(); + int next_stack_offset(); + + public: + // Creation + SignatureHandlerGenerator(const methodHandle& method, CodeBuffer* buffer); + virtual ~SignatureHandlerGenerator() { + _masm = NULL; + } + + // Code generation + void generate(uint64_t fingerprint); + + // Code generation support + static Register from(); + static Register to(); + static Register temp(); +}; + +#endif // CPU_RISCV_INTERPRETERRT_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/javaFrameAnchor_riscv.hpp b/src/hotspot/cpu/riscv/javaFrameAnchor_riscv.hpp new file mode 100644 index 0000000000000..a0cec4820f223 --- /dev/null +++ b/src/hotspot/cpu/riscv/javaFrameAnchor_riscv.hpp @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2002, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_JAVAFRAMEANCHOR_RISCV_HPP +#define CPU_RISCV_JAVAFRAMEANCHOR_RISCV_HPP + +private: + + // FP value associated with _last_Java_sp: + intptr_t* volatile _last_Java_fp; // pointer is volatile not what it points to + +public: + // Each arch must define reset, save, restore + // These are used by objects that only care about: + // 1 - initializing a new state (thread creation, javaCalls) + // 2 - saving a current state (javaCalls) + // 3 - restoring an old state (javaCalls) + + void clear(void) { + // clearing _last_Java_sp must be first + _last_Java_sp = NULL; + OrderAccess::release(); + _last_Java_fp = NULL; + _last_Java_pc = NULL; + } + + void copy(JavaFrameAnchor* src) { + // In order to make sure the transition state is valid for "this" + // We must clear _last_Java_sp before copying the rest of the new data + // + // Hack Alert: Temporary bugfix for 4717480/4721647 + // To act like previous version (pd_cache_state) don't NULL _last_Java_sp + // unless the value is changing + // + assert(src != NULL, "Src should not be NULL."); + if (_last_Java_sp != src->_last_Java_sp) { + _last_Java_sp = NULL; + OrderAccess::release(); + } + _last_Java_fp = src->_last_Java_fp; + _last_Java_pc = src->_last_Java_pc; + // Must be last so profiler will always see valid frame if has_last_frame() is true + _last_Java_sp = src->_last_Java_sp; + } + + bool walkable(void) { return _last_Java_sp != NULL && _last_Java_pc != NULL; } + + void make_walkable(); + + intptr_t* last_Java_sp(void) const { return _last_Java_sp; } + + const address last_Java_pc(void) { return _last_Java_pc; } + +private: + + static ByteSize last_Java_fp_offset() { return byte_offset_of(JavaFrameAnchor, _last_Java_fp); } + +public: + + void set_last_Java_sp(intptr_t* java_sp) { _last_Java_sp = java_sp; OrderAccess::release(); } + + intptr_t* last_Java_fp(void) { return _last_Java_fp; } + +#endif // CPU_RISCV_JAVAFRAMEANCHOR_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/jniFastGetField_riscv.cpp b/src/hotspot/cpu/riscv/jniFastGetField_riscv.cpp new file mode 100644 index 0000000000000..8a7a1d8378a43 --- /dev/null +++ b/src/hotspot/cpu/riscv/jniFastGetField_riscv.cpp @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2004, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/barrierSetAssembler.hpp" +#include "memory/resourceArea.hpp" +#include "prims/jniFastGetField.hpp" +#include "prims/jvm_misc.hpp" +#include "prims/jvmtiExport.hpp" +#include "runtime/safepoint.hpp" + +#define __ masm-> + +#define BUFFER_SIZE 30*wordSize + +// Instead of issuing a LoadLoad barrier we create an address +// dependency between loads; this might be more efficient. + +// Common register usage: +// x10/f10: result +// c_rarg0: jni env +// c_rarg1: obj +// c_rarg2: jfield id + +static const Register robj = x13; +static const Register rcounter = x14; +static const Register roffset = x15; +static const Register rcounter_addr = x16; +static const Register result = x17; + +address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) { + const char *name; + switch (type) { + case T_BOOLEAN: name = "jni_fast_GetBooleanField"; break; + case T_BYTE: name = "jni_fast_GetByteField"; break; + case T_CHAR: name = "jni_fast_GetCharField"; break; + case T_SHORT: name = "jni_fast_GetShortField"; break; + case T_INT: name = "jni_fast_GetIntField"; break; + case T_LONG: name = "jni_fast_GetLongField"; break; + case T_FLOAT: name = "jni_fast_GetFloatField"; break; + case T_DOUBLE: name = "jni_fast_GetDoubleField"; break; + default: ShouldNotReachHere(); + name = NULL; // unreachable + } + ResourceMark rm; + BufferBlob* blob = BufferBlob::create(name, BUFFER_SIZE); + CodeBuffer cbuf(blob); + MacroAssembler* masm = new MacroAssembler(&cbuf); + address fast_entry = __ pc(); + + Address target(SafepointSynchronize::safepoint_counter_addr()); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(rcounter_addr, target, offset); + __ addi(rcounter_addr, rcounter_addr, offset); + }); + + Label slow; + Address safepoint_counter_addr(rcounter_addr, 0); + __ lwu(rcounter, safepoint_counter_addr); + // An even value means there are no ongoing safepoint operations + __ test_bit(t0, rcounter, 0); + __ bnez(t0, slow); + + if (JvmtiExport::can_post_field_access()) { + // Using barrier to order wrt. JVMTI check and load of result. + __ membar(MacroAssembler::LoadLoad); + + // Check to see if a field access watch has been set before we + // take the fast path. + ExternalAddress target((address) JvmtiExport::get_field_access_count_addr()); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(result, target, offset); + __ lwu(result, Address(result, offset)); + }); + __ bnez(result, slow); + + __ mv(robj, c_rarg1); + } else { + // Using address dependency to order wrt. load of result. + __ xorr(robj, c_rarg1, rcounter); + __ xorr(robj, robj, rcounter); // obj, since + // robj ^ rcounter ^ rcounter == robj + // robj is address dependent on rcounter. + } + + // Both robj and t0 are clobbered by try_resolve_jobject_in_native. + BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); + assert_cond(bs != NULL); + bs->try_resolve_jobject_in_native(masm, c_rarg0, robj, t0, slow); + + __ srli(roffset, c_rarg2, 2); // offset + + assert(count < LIST_CAPACITY, "LIST_CAPACITY too small"); + speculative_load_pclist[count] = __ pc(); // Used by the segfault handler + __ add(roffset, robj, roffset); + + switch (type) { + case T_BOOLEAN: __ lbu(result, Address(roffset, 0)); break; + case T_BYTE: __ lb(result, Address(roffset, 0)); break; + case T_CHAR: __ lhu(result, Address(roffset, 0)); break; + case T_SHORT: __ lh(result, Address(roffset, 0)); break; + case T_INT: __ lw(result, Address(roffset, 0)); break; + case T_LONG: __ ld(result, Address(roffset, 0)); break; + case T_FLOAT: { + __ flw(f28, Address(roffset, 0)); // f28 as temporaries + __ fmv_x_w(result, f28); // f{31--0}-->x + break; + } + case T_DOUBLE: { + __ fld(f28, Address(roffset, 0)); // f28 as temporaries + __ fmv_x_d(result, f28); // d{63--0}-->x + break; + } + default: ShouldNotReachHere(); + } + + // Using acquire: Order JVMTI check and load of result wrt. succeeding check + // (LoadStore for volatile field). + __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); + + __ lw(t0, safepoint_counter_addr); + __ bne(rcounter, t0, slow); + + switch (type) { + case T_FLOAT: __ fmv_w_x(f10, result); break; + case T_DOUBLE: __ fmv_d_x(f10, result); break; + default: __ mv(x10, result); break; + } + __ ret(); + + slowcase_entry_pclist[count++] = __ pc(); + __ bind(slow); + address slow_case_addr; + switch (type) { + case T_BOOLEAN: slow_case_addr = jni_GetBooleanField_addr(); break; + case T_BYTE: slow_case_addr = jni_GetByteField_addr(); break; + case T_CHAR: slow_case_addr = jni_GetCharField_addr(); break; + case T_SHORT: slow_case_addr = jni_GetShortField_addr(); break; + case T_INT: slow_case_addr = jni_GetIntField_addr(); break; + case T_LONG: slow_case_addr = jni_GetLongField_addr(); break; + case T_FLOAT: slow_case_addr = jni_GetFloatField_addr(); break; + case T_DOUBLE: slow_case_addr = jni_GetDoubleField_addr(); break; + default: ShouldNotReachHere(); + slow_case_addr = NULL; // unreachable + } + + { + __ enter(); + ExternalAddress target(slow_case_addr); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ jalr(x1, t0, offset); + }); + __ leave(); + __ ret(); + } + __ flush(); + + return fast_entry; +} + + +address JNI_FastGetField::generate_fast_get_boolean_field() { + return generate_fast_get_int_field0(T_BOOLEAN); +} + +address JNI_FastGetField::generate_fast_get_byte_field() { + return generate_fast_get_int_field0(T_BYTE); +} + +address JNI_FastGetField::generate_fast_get_char_field() { + return generate_fast_get_int_field0(T_CHAR); +} + +address JNI_FastGetField::generate_fast_get_short_field() { + return generate_fast_get_int_field0(T_SHORT); +} + +address JNI_FastGetField::generate_fast_get_int_field() { + return generate_fast_get_int_field0(T_INT); +} + +address JNI_FastGetField::generate_fast_get_long_field() { + return generate_fast_get_int_field0(T_LONG); +} + +address JNI_FastGetField::generate_fast_get_float_field() { + return generate_fast_get_int_field0(T_FLOAT); +} + +address JNI_FastGetField::generate_fast_get_double_field() { + return generate_fast_get_int_field0(T_DOUBLE); +} diff --git a/src/hotspot/cpu/riscv/jniTypes_riscv.hpp b/src/hotspot/cpu/riscv/jniTypes_riscv.hpp new file mode 100644 index 0000000000000..83ffcc55d8355 --- /dev/null +++ b/src/hotspot/cpu/riscv/jniTypes_riscv.hpp @@ -0,0 +1,106 @@ +/* + * Copyright (c) 1998, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_JNITYPES_RISCV_HPP +#define CPU_RISCV_JNITYPES_RISCV_HPP + +#include "jni.h" +#include "memory/allStatic.hpp" +#include "oops/oop.hpp" + +// This file holds platform-dependent routines used to write primitive jni +// types to the array of arguments passed into JavaCalls::call + +class JNITypes : private AllStatic { + // These functions write a java primitive type (in native format) + // to a java stack slot array to be passed as an argument to JavaCalls:calls. + // I.e., they are functionally 'push' operations if they have a 'pos' + // formal parameter. Note that jlong's and jdouble's are written + // _in reverse_ of the order in which they appear in the interpreter + // stack. This is because call stubs (see stubGenerator_sparc.cpp) + // reverse the argument list constructed by JavaCallArguments (see + // javaCalls.hpp). + +public: + // Ints are stored in native format in one JavaCallArgument slot at *to. + static inline void put_int(jint from, intptr_t *to) { *(jint *)(to + 0 ) = from; } + static inline void put_int(jint from, intptr_t *to, int& pos) { *(jint *)(to + pos++) = from; } + static inline void put_int(jint *from, intptr_t *to, int& pos) { *(jint *)(to + pos++) = *from; } + + // Longs are stored in native format in one JavaCallArgument slot at + // *(to+1). + static inline void put_long(jlong from, intptr_t *to) { + *(jlong*) (to + 1) = from; + } + + static inline void put_long(jlong from, intptr_t *to, int& pos) { + *(jlong*) (to + 1 + pos) = from; + pos += 2; + } + + static inline void put_long(jlong *from, intptr_t *to, int& pos) { + *(jlong*) (to + 1 + pos) = *from; + pos += 2; + } + + // Oops are stored in native format in one JavaCallArgument slot at *to. + static inline void put_obj(const Handle& from_handle, intptr_t *to, int& pos) { *(to + pos++) = (intptr_t)from_handle.raw_value(); } + static inline void put_obj(jobject from_handle, intptr_t *to, int& pos) { *(to + pos++) = (intptr_t)from_handle; } + + // Floats are stored in native format in one JavaCallArgument slot at *to. + static inline void put_float(jfloat from, intptr_t *to) { *(jfloat *)(to + 0 ) = from; } + static inline void put_float(jfloat from, intptr_t *to, int& pos) { *(jfloat *)(to + pos++) = from; } + static inline void put_float(jfloat *from, intptr_t *to, int& pos) { *(jfloat *)(to + pos++) = *from; } + +#undef _JNI_SLOT_OFFSET +#define _JNI_SLOT_OFFSET 1 + // Doubles are stored in native word format in one JavaCallArgument + // slot at *(to+1). + static inline void put_double(jdouble from, intptr_t *to) { + *(jdouble*) (to + 1) = from; + } + + static inline void put_double(jdouble from, intptr_t *to, int& pos) { + *(jdouble*) (to + 1 + pos) = from; + pos += 2; + } + + static inline void put_double(jdouble *from, intptr_t *to, int& pos) { + *(jdouble*) (to + 1 + pos) = *from; + pos += 2; + } + + // The get_xxx routines, on the other hand, actually _do_ fetch + // java primitive types from the interpreter stack. + // No need to worry about alignment on Intel. + static inline jint get_int (intptr_t *from) { return *(jint *) from; } + static inline jlong get_long (intptr_t *from) { return *(jlong *) (from + _JNI_SLOT_OFFSET); } + static inline oop get_obj (intptr_t *from) { return *(oop *) from; } + static inline jfloat get_float (intptr_t *from) { return *(jfloat *) from; } + static inline jdouble get_double(intptr_t *from) { return *(jdouble *)(from + _JNI_SLOT_OFFSET); } +#undef _JNI_SLOT_OFFSET +}; + +#endif // CPU_RISCV_JNITYPES_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp new file mode 100644 index 0000000000000..0e7f3ea69cd3f --- /dev/null +++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp @@ -0,0 +1,4392 @@ +/* + * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/assembler.hpp" +#include "asm/assembler.inline.hpp" +#include "compiler/disassembler.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/barrierSetAssembler.hpp" +#include "gc/shared/cardTable.hpp" +#include "gc/shared/cardTableBarrierSet.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "interpreter/bytecodeHistogram.hpp" +#include "interpreter/interpreter.hpp" +#include "memory/resourceArea.hpp" +#include "memory/universe.hpp" +#include "nativeInst_riscv.hpp" +#include "oops/accessDecorators.hpp" +#include "oops/compressedOops.inline.hpp" +#include "oops/klass.inline.hpp" +#include "oops/oop.hpp" +#include "runtime/biasedLocking.hpp" +#include "runtime/interfaceSupport.inline.hpp" +#include "runtime/jniHandles.inline.hpp" +#include "runtime/sharedRuntime.hpp" +#include "runtime/stubRoutines.hpp" +#include "runtime/thread.hpp" +#include "utilities/powerOfTwo.hpp" +#ifdef COMPILER2 +#include "opto/compile.hpp" +#include "opto/node.hpp" +#include "opto/output.hpp" +#endif + +#ifdef PRODUCT +#define BLOCK_COMMENT(str) /* nothing */ +#else +#define BLOCK_COMMENT(str) block_comment(str) +#endif +#define BIND(label) bind(label); __ BLOCK_COMMENT(#label ":") + +static void pass_arg0(MacroAssembler* masm, Register arg) { + if (c_rarg0 != arg) { + masm->mv(c_rarg0, arg); + } +} + +static void pass_arg1(MacroAssembler* masm, Register arg) { + if (c_rarg1 != arg) { + masm->mv(c_rarg1, arg); + } +} + +static void pass_arg2(MacroAssembler* masm, Register arg) { + if (c_rarg2 != arg) { + masm->mv(c_rarg2, arg); + } +} + +static void pass_arg3(MacroAssembler* masm, Register arg) { + if (c_rarg3 != arg) { + masm->mv(c_rarg3, arg); + } +} + +void MacroAssembler::align(int modulus, int extra_offset) { + CompressibleRegion cr(this); + while ((offset() + extra_offset) % modulus != 0) { nop(); } +} + +void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { + call_VM_base(oop_result, noreg, noreg, entry_point, number_of_arguments, check_exceptions); +} + +// Implementation of call_VM versions + +void MacroAssembler::call_VM(Register oop_result, + address entry_point, + bool check_exceptions) { + call_VM_helper(oop_result, entry_point, 0, check_exceptions); +} + +void MacroAssembler::call_VM(Register oop_result, + address entry_point, + Register arg_1, + bool check_exceptions) { + pass_arg1(this, arg_1); + call_VM_helper(oop_result, entry_point, 1, check_exceptions); +} + +void MacroAssembler::call_VM(Register oop_result, + address entry_point, + Register arg_1, + Register arg_2, + bool check_exceptions) { + assert(arg_1 != c_rarg2, "smashed arg"); + pass_arg2(this, arg_2); + pass_arg1(this, arg_1); + call_VM_helper(oop_result, entry_point, 2, check_exceptions); +} + +void MacroAssembler::call_VM(Register oop_result, + address entry_point, + Register arg_1, + Register arg_2, + Register arg_3, + bool check_exceptions) { + assert(arg_1 != c_rarg3, "smashed arg"); + assert(arg_2 != c_rarg3, "smashed arg"); + pass_arg3(this, arg_3); + + assert(arg_1 != c_rarg2, "smashed arg"); + pass_arg2(this, arg_2); + + pass_arg1(this, arg_1); + call_VM_helper(oop_result, entry_point, 3, check_exceptions); +} + +void MacroAssembler::call_VM(Register oop_result, + Register last_java_sp, + address entry_point, + int number_of_arguments, + bool check_exceptions) { + call_VM_base(oop_result, xthread, last_java_sp, entry_point, number_of_arguments, check_exceptions); +} + +void MacroAssembler::call_VM(Register oop_result, + Register last_java_sp, + address entry_point, + Register arg_1, + bool check_exceptions) { + pass_arg1(this, arg_1); + call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); +} + +void MacroAssembler::call_VM(Register oop_result, + Register last_java_sp, + address entry_point, + Register arg_1, + Register arg_2, + bool check_exceptions) { + + assert(arg_1 != c_rarg2, "smashed arg"); + pass_arg2(this, arg_2); + pass_arg1(this, arg_1); + call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); +} + +void MacroAssembler::call_VM(Register oop_result, + Register last_java_sp, + address entry_point, + Register arg_1, + Register arg_2, + Register arg_3, + bool check_exceptions) { + assert(arg_1 != c_rarg3, "smashed arg"); + assert(arg_2 != c_rarg3, "smashed arg"); + pass_arg3(this, arg_3); + assert(arg_1 != c_rarg2, "smashed arg"); + pass_arg2(this, arg_2); + pass_arg1(this, arg_1); + call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); +} + +// these are no-ops overridden by InterpreterMacroAssembler +void MacroAssembler::check_and_handle_earlyret(Register java_thread) {} +void MacroAssembler::check_and_handle_popframe(Register java_thread) {} + +// Calls to C land +// +// When entering C land, the fp, & esp of the last Java frame have to be recorded +// in the (thread-local) JavaThread object. When leaving C land, the last Java fp +// has to be reset to 0. This is required to allow proper stack traversal. +void MacroAssembler::set_last_Java_frame(Register last_java_sp, + Register last_java_fp, + Register last_java_pc, + Register tmp) { + + if (last_java_pc->is_valid()) { + sd(last_java_pc, Address(xthread, + JavaThread::frame_anchor_offset() + + JavaFrameAnchor::last_Java_pc_offset())); + } + + // determine last_java_sp register + if (last_java_sp == sp) { + mv(tmp, sp); + last_java_sp = tmp; + } else if (!last_java_sp->is_valid()) { + last_java_sp = esp; + } + + sd(last_java_sp, Address(xthread, JavaThread::last_Java_sp_offset())); + + // last_java_fp is optional + if (last_java_fp->is_valid()) { + sd(last_java_fp, Address(xthread, JavaThread::last_Java_fp_offset())); + } +} + +void MacroAssembler::set_last_Java_frame(Register last_java_sp, + Register last_java_fp, + address last_java_pc, + Register tmp) { + assert(last_java_pc != NULL, "must provide a valid PC"); + + la(tmp, last_java_pc); + sd(tmp, Address(xthread, JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset())); + + set_last_Java_frame(last_java_sp, last_java_fp, noreg, tmp); +} + +void MacroAssembler::set_last_Java_frame(Register last_java_sp, + Register last_java_fp, + Label &L, + Register tmp) { + if (L.is_bound()) { + set_last_Java_frame(last_java_sp, last_java_fp, target(L), tmp); + } else { + L.add_patch_at(code(), locator()); + IncompressibleRegion ir(this); // the label address will be patched back. + set_last_Java_frame(last_java_sp, last_java_fp, pc() /* Patched later */, tmp); + } +} + +void MacroAssembler::reset_last_Java_frame(bool clear_fp) { + // we must set sp to zero to clear frame + sd(zr, Address(xthread, JavaThread::last_Java_sp_offset())); + + // must clear fp, so that compiled frames are not confused; it is + // possible that we need it only for debugging + if (clear_fp) { + sd(zr, Address(xthread, JavaThread::last_Java_fp_offset())); + } + + // Always clear the pc because it could have been set by make_walkable() + sd(zr, Address(xthread, JavaThread::last_Java_pc_offset())); +} + +void MacroAssembler::call_VM_base(Register oop_result, + Register java_thread, + Register last_java_sp, + address entry_point, + int number_of_arguments, + bool check_exceptions) { + // determine java_thread register + if (!java_thread->is_valid()) { + java_thread = xthread; + } + // determine last_java_sp register + if (!last_java_sp->is_valid()) { + last_java_sp = esp; + } + + // debugging support + assert(number_of_arguments >= 0 , "cannot have negative number of arguments"); + assert(java_thread == xthread, "unexpected register"); + + assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result"); + assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp"); + + // push java thread (becomes first argument of C function) + mv(c_rarg0, java_thread); + + // set last Java frame before call + assert(last_java_sp != fp, "can't use fp"); + + Label l; + set_last_Java_frame(last_java_sp, fp, l, t0); + + // do the call, remove parameters + MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments, &l); + + // reset last Java frame + // Only interpreter should have to clear fp + reset_last_Java_frame(true); + + // C++ interp handles this in the interpreter + check_and_handle_popframe(java_thread); + check_and_handle_earlyret(java_thread); + + if (check_exceptions) { + // check for pending exceptions (java_thread is set upon return) + ld(t0, Address(java_thread, in_bytes(Thread::pending_exception_offset()))); + Label ok; + beqz(t0, ok); + RuntimeAddress target(StubRoutines::forward_exception_entry()); + relocate(target.rspec(), [&] { + int32_t offset; + la_patchable(t0, target, offset); + jalr(x0, t0, offset); + }); + bind(ok); + } + + // get oop result if there is one and reset the value in the thread + if (oop_result->is_valid()) { + get_vm_result(oop_result, java_thread); + } +} + +void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) { + ld(oop_result, Address(java_thread, JavaThread::vm_result_offset())); + sd(zr, Address(java_thread, JavaThread::vm_result_offset())); + verify_oop(oop_result, "broken oop in call_VM_base"); +} + +void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) { + ld(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset())); + sd(zr, Address(java_thread, JavaThread::vm_result_2_offset())); +} + +void MacroAssembler::clinit_barrier(Register klass, Register tmp, Label* L_fast_path, Label* L_slow_path) { + assert(L_fast_path != NULL || L_slow_path != NULL, "at least one is required"); + assert_different_registers(klass, xthread, tmp); + + Label L_fallthrough, L_tmp; + if (L_fast_path == NULL) { + L_fast_path = &L_fallthrough; + } else if (L_slow_path == NULL) { + L_slow_path = &L_fallthrough; + } + + // Fast path check: class is fully initialized + lbu(tmp, Address(klass, InstanceKlass::init_state_offset())); + sub(tmp, tmp, InstanceKlass::fully_initialized); + beqz(tmp, *L_fast_path); + + // Fast path check: current thread is initializer thread + ld(tmp, Address(klass, InstanceKlass::init_thread_offset())); + + if (L_slow_path == &L_fallthrough) { + beq(xthread, tmp, *L_fast_path); + bind(*L_slow_path); + } else if (L_fast_path == &L_fallthrough) { + bne(xthread, tmp, *L_slow_path); + bind(*L_fast_path); + } else { + Unimplemented(); + } +} + +void MacroAssembler::verify_oop(Register reg, const char* s) { + if (!VerifyOops) { return; } + + // Pass register number to verify_oop_subroutine + const char* b = NULL; + { + ResourceMark rm; + stringStream ss; + ss.print("verify_oop: %s: %s", reg->name(), s); + b = code_string(ss.as_string()); + } + BLOCK_COMMENT("verify_oop {"); + + push_reg(RegSet::of(ra, t0, t1, c_rarg0), sp); + + mv(c_rarg0, reg); // c_rarg0 : x10 + { + // The length of the instruction sequence emitted should not depend + // on the address of the char buffer so that the size of mach nodes for + // scratch emit and normal emit matches. + IncompressibleRegion ir(this); // Fixed length + movptr(t0, (address) b); + } + + // call indirectly to solve generation ordering problem + ExternalAddress target(StubRoutines::verify_oop_subroutine_entry_address()); + relocate(target.rspec(), [&] { + int32_t offset; + la_patchable(t1, target, offset); + ld(t1, Address(t1, offset)); + }); + jalr(t1); + + pop_reg(RegSet::of(ra, t0, t1, c_rarg0), sp); + + BLOCK_COMMENT("} verify_oop"); +} + +void MacroAssembler::verify_oop_addr(Address addr, const char* s) { + if (!VerifyOops) { + return; + } + + const char* b = NULL; + { + ResourceMark rm; + stringStream ss; + ss.print("verify_oop_addr: %s", s); + b = code_string(ss.as_string()); + } + BLOCK_COMMENT("verify_oop_addr {"); + + push_reg(RegSet::of(ra, t0, t1, c_rarg0), sp); + + if (addr.uses(sp)) { + la(x10, addr); + ld(x10, Address(x10, 4 * wordSize)); + } else { + ld(x10, addr); + } + + { + // The length of the instruction sequence emitted should not depend + // on the address of the char buffer so that the size of mach nodes for + // scratch emit and normal emit matches. + IncompressibleRegion ir(this); // Fixed length + movptr(t0, (address) b); + } + + // call indirectly to solve generation ordering problem + ExternalAddress target(StubRoutines::verify_oop_subroutine_entry_address()); + relocate(target.rspec(), [&] { + int32_t offset; + la_patchable(t1, target, offset); + ld(t1, Address(t1, offset)); + }); + jalr(t1); + + pop_reg(RegSet::of(ra, t0, t1, c_rarg0), sp); + + BLOCK_COMMENT("} verify_oop_addr"); +} + +Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, + int extra_slot_offset) { + // cf. TemplateTable::prepare_invoke(), if (load_receiver). + int stackElementSize = Interpreter::stackElementSize; + int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0); +#ifdef ASSERT + int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1); + assert(offset1 - offset == stackElementSize, "correct arithmetic"); +#endif + if (arg_slot.is_constant()) { + return Address(esp, arg_slot.as_constant() * stackElementSize + offset); + } else { + assert_different_registers(t0, arg_slot.as_register()); + shadd(t0, arg_slot.as_register(), esp, t0, exact_log2(stackElementSize)); + return Address(t0, offset); + } +} + +#ifndef PRODUCT +extern "C" void findpc(intptr_t x); +#endif + +void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) +{ + // In order to get locks to work, we need to fake a in_VM state + if (ShowMessageBoxOnError) { + JavaThread* thread = JavaThread::current(); + JavaThreadState saved_state = thread->thread_state(); + thread->set_thread_state(_thread_in_vm); +#ifndef PRODUCT + if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { + ttyLocker ttyl; + BytecodeCounter::print(); + } +#endif + if (os::message_box(msg, "Execution stopped, print registers?")) { + ttyLocker ttyl; + tty->print_cr(" pc = 0x%016lx", pc); +#ifndef PRODUCT + tty->cr(); + findpc(pc); + tty->cr(); +#endif + tty->print_cr(" x0 = 0x%016lx", regs[0]); + tty->print_cr(" x1 = 0x%016lx", regs[1]); + tty->print_cr(" x2 = 0x%016lx", regs[2]); + tty->print_cr(" x3 = 0x%016lx", regs[3]); + tty->print_cr(" x4 = 0x%016lx", regs[4]); + tty->print_cr(" x5 = 0x%016lx", regs[5]); + tty->print_cr(" x6 = 0x%016lx", regs[6]); + tty->print_cr(" x7 = 0x%016lx", regs[7]); + tty->print_cr(" x8 = 0x%016lx", regs[8]); + tty->print_cr(" x9 = 0x%016lx", regs[9]); + tty->print_cr("x10 = 0x%016lx", regs[10]); + tty->print_cr("x11 = 0x%016lx", regs[11]); + tty->print_cr("x12 = 0x%016lx", regs[12]); + tty->print_cr("x13 = 0x%016lx", regs[13]); + tty->print_cr("x14 = 0x%016lx", regs[14]); + tty->print_cr("x15 = 0x%016lx", regs[15]); + tty->print_cr("x16 = 0x%016lx", regs[16]); + tty->print_cr("x17 = 0x%016lx", regs[17]); + tty->print_cr("x18 = 0x%016lx", regs[18]); + tty->print_cr("x19 = 0x%016lx", regs[19]); + tty->print_cr("x20 = 0x%016lx", regs[20]); + tty->print_cr("x21 = 0x%016lx", regs[21]); + tty->print_cr("x22 = 0x%016lx", regs[22]); + tty->print_cr("x23 = 0x%016lx", regs[23]); + tty->print_cr("x24 = 0x%016lx", regs[24]); + tty->print_cr("x25 = 0x%016lx", regs[25]); + tty->print_cr("x26 = 0x%016lx", regs[26]); + tty->print_cr("x27 = 0x%016lx", regs[27]); + tty->print_cr("x28 = 0x%016lx", regs[28]); + tty->print_cr("x30 = 0x%016lx", regs[30]); + tty->print_cr("x31 = 0x%016lx", regs[31]); + BREAKPOINT; + } + } + fatal("DEBUG MESSAGE: %s", msg); +} + +void MacroAssembler::resolve_jobject(Register value, Register thread, Register tmp) { + Label done, not_weak; + beqz(value, done); // Use NULL as-is. + + // Test for jweak tag. + test_bit(t0, value, exact_log2(JNIHandles::weak_tag_mask)); + beqz(t0, not_weak); + + // Resolve jweak. + access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, value, + Address(value, -JNIHandles::weak_tag_value), tmp, thread); + verify_oop(value); + j(done); + + bind(not_weak); + // Resolve (untagged) jobject. + access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, 0), tmp, thread); + verify_oop(value); + bind(done); +} + +void MacroAssembler::stop(const char* msg) { + BLOCK_COMMENT(msg); + illegal_instruction(Assembler::csr::time); + emit_int64((uintptr_t)msg); +} + +void MacroAssembler::unimplemented(const char* what) { + const char* buf = NULL; + { + ResourceMark rm; + stringStream ss; + ss.print("unimplemented: %s", what); + buf = code_string(ss.as_string()); + } + stop(buf); +} + +void MacroAssembler::emit_static_call_stub() { + IncompressibleRegion ir(this); // Fixed length: see CompiledStaticCall::to_interp_stub_size(). + // CompiledDirectStaticCall::set_to_interpreted knows the + // exact layout of this stub. + + mov_metadata(xmethod, (Metadata*)NULL); + + // Jump to the entry point of the i2c stub. + int32_t offset = 0; + movptr(t0, 0, offset); + jalr(x0, t0, offset); +} + +void MacroAssembler::call_VM_leaf_base(address entry_point, + int number_of_arguments, + Label *retaddr) { + push_reg(RegSet::of(t0, xmethod), sp); // push << t0 & xmethod >> to sp + call(entry_point); + if (retaddr != NULL) { + bind(*retaddr); + } + pop_reg(RegSet::of(t0, xmethod), sp); // pop << t0 & xmethod >> from sp +} + +void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) { + call_VM_leaf_base(entry_point, number_of_arguments); +} + +void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) { + pass_arg0(this, arg_0); + call_VM_leaf_base(entry_point, 1); +} + +void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { + pass_arg0(this, arg_0); + pass_arg1(this, arg_1); + call_VM_leaf_base(entry_point, 2); +} + +void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, + Register arg_1, Register arg_2) { + pass_arg0(this, arg_0); + pass_arg1(this, arg_1); + pass_arg2(this, arg_2); + call_VM_leaf_base(entry_point, 3); +} + +void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) { + pass_arg0(this, arg_0); + MacroAssembler::call_VM_leaf_base(entry_point, 1); +} + +void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { + + assert(arg_0 != c_rarg1, "smashed arg"); + pass_arg1(this, arg_1); + pass_arg0(this, arg_0); + MacroAssembler::call_VM_leaf_base(entry_point, 2); +} + +void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { + assert(arg_0 != c_rarg2, "smashed arg"); + assert(arg_1 != c_rarg2, "smashed arg"); + pass_arg2(this, arg_2); + assert(arg_0 != c_rarg1, "smashed arg"); + pass_arg1(this, arg_1); + pass_arg0(this, arg_0); + MacroAssembler::call_VM_leaf_base(entry_point, 3); +} + +void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { + assert(arg_0 != c_rarg3, "smashed arg"); + assert(arg_1 != c_rarg3, "smashed arg"); + assert(arg_2 != c_rarg3, "smashed arg"); + pass_arg3(this, arg_3); + assert(arg_0 != c_rarg2, "smashed arg"); + assert(arg_1 != c_rarg2, "smashed arg"); + pass_arg2(this, arg_2); + assert(arg_0 != c_rarg1, "smashed arg"); + pass_arg1(this, arg_1); + pass_arg0(this, arg_0); + MacroAssembler::call_VM_leaf_base(entry_point, 4); +} + +void MacroAssembler::la(Register Rd, const address dest) { + int64_t offset = dest - pc(); + if (is_simm32(offset)) { + auipc(Rd, (int32_t)offset + 0x800); //0x800, Note:the 11th sign bit + addi(Rd, Rd, ((int64_t)offset << 52) >> 52); + } else { + movptr(Rd, dest); + } +} + +void MacroAssembler::la(Register Rd, const Address &adr) { + switch (adr.getMode()) { + case Address::literal: { + relocInfo::relocType rtype = adr.rspec().reloc()->type(); + if (rtype == relocInfo::none) { + mv(Rd, (intptr_t)(adr.target())); + } else { + relocate(adr.rspec(), [&] { + movptr(Rd, adr.target()); + }); + } + break; + } + case Address::base_plus_offset: { + Address new_adr = legitimize_address(Rd, adr); + if (!(new_adr.base() == Rd && new_adr.offset() == 0)) { + addi(Rd, new_adr.base(), new_adr.offset()); + } + break; + } + default: + ShouldNotReachHere(); + } +} + +void MacroAssembler::la(Register Rd, Label &label) { + IncompressibleRegion ir(this); // the label address may be patched back. + la(Rd, target(label)); +} + +void MacroAssembler::li32(Register Rd, int32_t imm) { + // int32_t is in range 0x8000 0000 ~ 0x7fff ffff, and imm[31] is the sign bit + int64_t upper = imm, lower = imm; + lower = (imm << 20) >> 20; + upper -= lower; + upper = (int32_t)upper; + // lui Rd, imm[31:12] + imm[11] + lui(Rd, upper); + // use addiw to distinguish li32 to li64 + addiw(Rd, Rd, lower); +} + +void MacroAssembler::li64(Register Rd, int64_t imm) { + // Load upper 32 bits. upper = imm[63:32], but if imm[31] == 1 or + // (imm[31:20] == 0x7ff && imm[19] == 1), upper = imm[63:32] + 1. + int64_t lower = imm & 0xffffffff; + lower -= ((lower << 44) >> 44); + int64_t tmp_imm = ((uint64_t)(imm & 0xffffffff00000000)) + (uint64_t)lower; + int32_t upper = (tmp_imm - (int32_t)lower) >> 32; + + // Load upper 32 bits + int64_t up = upper, lo = upper; + lo = (lo << 52) >> 52; + up -= lo; + up = (int32_t)up; + lui(Rd, up); + addi(Rd, Rd, lo); + + // Load the rest 32 bits. + slli(Rd, Rd, 12); + addi(Rd, Rd, (int32_t)lower >> 20); + slli(Rd, Rd, 12); + lower = ((int32_t)imm << 12) >> 20; + addi(Rd, Rd, lower); + slli(Rd, Rd, 8); + lower = imm & 0xff; + addi(Rd, Rd, lower); +} + +void MacroAssembler::li(Register Rd, int64_t imm) { + // int64_t is in range 0x8000 0000 0000 0000 ~ 0x7fff ffff ffff ffff + // li -> c.li + if (do_compress() && (is_simm6(imm) && Rd != x0)) { + c_li(Rd, imm); + return; + } + + int shift = 12; + int64_t upper = imm, lower = imm; + // Split imm to a lower 12-bit sign-extended part and the remainder, + // because addi will sign-extend the lower imm. + lower = ((int32_t)imm << 20) >> 20; + upper -= lower; + + // Test whether imm is a 32-bit integer. + if (!(((imm) & ~(int64_t)0x7fffffff) == 0 || + (((imm) & ~(int64_t)0x7fffffff) == ~(int64_t)0x7fffffff))) { + while (((upper >> shift) & 1) == 0) { shift++; } + upper >>= shift; + li(Rd, upper); + slli(Rd, Rd, shift); + if (lower != 0) { + addi(Rd, Rd, lower); + } + } else { + // 32-bit integer + Register hi_Rd = zr; + if (upper != 0) { + lui(Rd, (int32_t)upper); + hi_Rd = Rd; + } + if (lower != 0 || hi_Rd == zr) { + addiw(Rd, hi_Rd, lower); + } + } +} + +#define INSN(NAME, REGISTER) \ + void MacroAssembler::NAME(const address dest, Register temp) { \ + assert_cond(dest != NULL); \ + int64_t distance = dest - pc(); \ + if (is_simm21(distance) && ((distance % 2) == 0)) { \ + Assembler::jal(REGISTER, distance); \ + } else { \ + assert(temp != noreg, "expecting a register"); \ + int32_t offset = 0; \ + movptr(temp, dest, offset); \ + Assembler::jalr(REGISTER, temp, offset); \ + } \ + } \ + + INSN(j, x0); + INSN(jal, x1); + +#undef INSN + +#define INSN(NAME, REGISTER) \ + void MacroAssembler::NAME(const Address &adr, Register temp) { \ + switch (adr.getMode()) { \ + case Address::literal: { \ + relocate(adr.rspec()); \ + NAME(adr.target(), temp); \ + break; \ + } \ + case Address::base_plus_offset: { \ + int32_t offset = ((int32_t)adr.offset() << 20) >> 20; \ + la(temp, Address(adr.base(), adr.offset() - offset)); \ + Assembler::jalr(REGISTER, temp, offset); \ + break; \ + } \ + default: \ + ShouldNotReachHere(); \ + } \ + } + + INSN(j, x0); + INSN(jal, x1); + +#undef INSN + +#define INSN(NAME) \ + void MacroAssembler::NAME(Register Rd, const address dest, Register temp) { \ + assert_cond(dest != NULL); \ + int64_t distance = dest - pc(); \ + if (is_simm21(distance) && ((distance % 2) == 0)) { \ + Assembler::NAME(Rd, distance); \ + } else { \ + assert_different_registers(Rd, temp); \ + int32_t offset = 0; \ + movptr(temp, dest, offset); \ + jalr(Rd, temp, offset); \ + } \ + } \ + void MacroAssembler::NAME(Register Rd, Label &L, Register temp) { \ + assert_different_registers(Rd, temp); \ + wrap_label(Rd, L, temp, &MacroAssembler::NAME); \ + } + + INSN(jal); + +#undef INSN + +#define INSN(NAME, REGISTER) \ + void MacroAssembler::NAME(Label &l, Register temp) { \ + jal(REGISTER, l, temp); \ + } \ + + INSN(j, x0); + INSN(jal, x1); + +#undef INSN + +void MacroAssembler::wrap_label(Register Rt, Label &L, Register tmp, load_insn_by_temp insn) { + if (L.is_bound()) { + (this->*insn)(Rt, target(L), tmp); + } else { + L.add_patch_at(code(), locator()); + (this->*insn)(Rt, pc(), tmp); + } +} + +void MacroAssembler::wrap_label(Register Rt, Label &L, jal_jalr_insn insn) { + if (L.is_bound()) { + (this->*insn)(Rt, target(L)); + } else { + L.add_patch_at(code(), locator()); + (this->*insn)(Rt, pc()); + } +} + +void MacroAssembler::wrap_label(Register r1, Register r2, Label &L, + compare_and_branch_insn insn, + compare_and_branch_label_insn neg_insn, bool is_far) { + if (is_far) { + Label done; + (this->*neg_insn)(r1, r2, done, /* is_far */ false); + j(L); + bind(done); + } else { + if (L.is_bound()) { + (this->*insn)(r1, r2, target(L)); + } else { + L.add_patch_at(code(), locator()); + (this->*insn)(r1, r2, pc()); + } + } +} + +#define INSN(NAME, NEG_INSN) \ + void MacroAssembler::NAME(Register Rs1, Register Rs2, Label &L, bool is_far) { \ + wrap_label(Rs1, Rs2, L, &MacroAssembler::NAME, &MacroAssembler::NEG_INSN, is_far); \ + } + + INSN(beq, bne); + INSN(bne, beq); + INSN(blt, bge); + INSN(bge, blt); + INSN(bltu, bgeu); + INSN(bgeu, bltu); + +#undef INSN + +#define INSN(NAME) \ + void MacroAssembler::NAME##z(Register Rs, const address dest) { \ + NAME(Rs, zr, dest); \ + } \ + void MacroAssembler::NAME##z(Register Rs, Label &l, bool is_far) { \ + NAME(Rs, zr, l, is_far); \ + } \ + + INSN(beq); + INSN(bne); + INSN(blt); + INSN(ble); + INSN(bge); + INSN(bgt); + +#undef INSN + +#define INSN(NAME, NEG_INSN) \ + void MacroAssembler::NAME(Register Rs, Register Rt, const address dest) { \ + NEG_INSN(Rt, Rs, dest); \ + } \ + void MacroAssembler::NAME(Register Rs, Register Rt, Label &l, bool is_far) { \ + NEG_INSN(Rt, Rs, l, is_far); \ + } + + INSN(bgt, blt); + INSN(ble, bge); + INSN(bgtu, bltu); + INSN(bleu, bgeu); + +#undef INSN + +// Float compare branch instructions + +#define INSN(NAME, FLOATCMP, BRANCH) \ + void MacroAssembler::float_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far, bool is_unordered) { \ + FLOATCMP##_s(t0, Rs1, Rs2); \ + BRANCH(t0, l, is_far); \ + } \ + void MacroAssembler::double_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far, bool is_unordered) { \ + FLOATCMP##_d(t0, Rs1, Rs2); \ + BRANCH(t0, l, is_far); \ + } + + INSN(beq, feq, bnez); + INSN(bne, feq, beqz); + +#undef INSN + + +#define INSN(NAME, FLOATCMP1, FLOATCMP2) \ + void MacroAssembler::float_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, \ + bool is_far, bool is_unordered) { \ + if (is_unordered) { \ + /* jump if either source is NaN or condition is expected */ \ + FLOATCMP2##_s(t0, Rs2, Rs1); \ + beqz(t0, l, is_far); \ + } else { \ + /* jump if no NaN in source and condition is expected */ \ + FLOATCMP1##_s(t0, Rs1, Rs2); \ + bnez(t0, l, is_far); \ + } \ + } \ + void MacroAssembler::double_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, \ + bool is_far, bool is_unordered) { \ + if (is_unordered) { \ + /* jump if either source is NaN or condition is expected */ \ + FLOATCMP2##_d(t0, Rs2, Rs1); \ + beqz(t0, l, is_far); \ + } else { \ + /* jump if no NaN in source and condition is expected */ \ + FLOATCMP1##_d(t0, Rs1, Rs2); \ + bnez(t0, l, is_far); \ + } \ + } + + INSN(ble, fle, flt); + INSN(blt, flt, fle); + +#undef INSN + +#define INSN(NAME, CMP) \ + void MacroAssembler::float_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, \ + bool is_far, bool is_unordered) { \ + float_##CMP(Rs2, Rs1, l, is_far, is_unordered); \ + } \ + void MacroAssembler::double_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, \ + bool is_far, bool is_unordered) { \ + double_##CMP(Rs2, Rs1, l, is_far, is_unordered); \ + } + + INSN(bgt, blt); + INSN(bge, ble); + +#undef INSN + + +#define INSN(NAME, CSR) \ + void MacroAssembler::NAME(Register Rd) { \ + csrr(Rd, CSR); \ + } + + INSN(rdinstret, CSR_INSTRET); + INSN(rdcycle, CSR_CYCLE); + INSN(rdtime, CSR_TIME); + INSN(frcsr, CSR_FCSR); + INSN(frrm, CSR_FRM); + INSN(frflags, CSR_FFLAGS); + +#undef INSN + +void MacroAssembler::csrr(Register Rd, unsigned csr) { + csrrs(Rd, csr, x0); +} + +#define INSN(NAME, OPFUN) \ + void MacroAssembler::NAME(unsigned csr, Register Rs) { \ + OPFUN(x0, csr, Rs); \ + } + + INSN(csrw, csrrw); + INSN(csrs, csrrs); + INSN(csrc, csrrc); + +#undef INSN + +#define INSN(NAME, OPFUN) \ + void MacroAssembler::NAME(unsigned csr, unsigned imm) { \ + OPFUN(x0, csr, imm); \ + } + + INSN(csrwi, csrrwi); + INSN(csrsi, csrrsi); + INSN(csrci, csrrci); + +#undef INSN + +#define INSN(NAME, CSR) \ + void MacroAssembler::NAME(Register Rd, Register Rs) { \ + csrrw(Rd, CSR, Rs); \ + } + + INSN(fscsr, CSR_FCSR); + INSN(fsrm, CSR_FRM); + INSN(fsflags, CSR_FFLAGS); + +#undef INSN + +#define INSN(NAME) \ + void MacroAssembler::NAME(Register Rs) { \ + NAME(x0, Rs); \ + } + + INSN(fscsr); + INSN(fsrm); + INSN(fsflags); + +#undef INSN + +void MacroAssembler::fsrmi(Register Rd, unsigned imm) { + guarantee(imm < 5, "Rounding Mode is invalid in Rounding Mode register"); + csrrwi(Rd, CSR_FRM, imm); +} + +void MacroAssembler::fsflagsi(Register Rd, unsigned imm) { + csrrwi(Rd, CSR_FFLAGS, imm); +} + +#define INSN(NAME) \ + void MacroAssembler::NAME(unsigned imm) { \ + NAME(x0, imm); \ + } + + INSN(fsrmi); + INSN(fsflagsi); + +#undef INSN + +void MacroAssembler::push_reg(Register Rs) +{ + addi(esp, esp, 0 - wordSize); + sd(Rs, Address(esp, 0)); +} + +void MacroAssembler::pop_reg(Register Rd) +{ + ld(Rd, Address(esp, 0)); + addi(esp, esp, wordSize); +} + +int MacroAssembler::bitset_to_regs(unsigned int bitset, unsigned char* regs) { + int count = 0; + // Scan bitset to accumulate register pairs + for (int reg = 31; reg >= 0; reg--) { + if ((1U << 31) & bitset) { + regs[count++] = reg; + } + bitset <<= 1; + } + return count; +} + +// Push integer registers in the bitset supplied. Don't push sp. +// Return the number of words pushed +int MacroAssembler::push_reg(unsigned int bitset, Register stack) { + DEBUG_ONLY(int words_pushed = 0;) + unsigned char regs[32]; + int count = bitset_to_regs(bitset, regs); + // reserve one slot to align for odd count + int offset = is_even(count) ? 0 : wordSize; + + if (count) { + addi(stack, stack, -count * wordSize - offset); + } + for (int i = count - 1; i >= 0; i--) { + sd(as_Register(regs[i]), Address(stack, (count - 1 - i) * wordSize + offset)); + DEBUG_ONLY(words_pushed++;) + } + + assert(words_pushed == count, "oops, pushed != count"); + + return count; +} + +int MacroAssembler::pop_reg(unsigned int bitset, Register stack) { + DEBUG_ONLY(int words_popped = 0;) + unsigned char regs[32]; + int count = bitset_to_regs(bitset, regs); + // reserve one slot to align for odd count + int offset = is_even(count) ? 0 : wordSize; + + for (int i = count - 1; i >= 0; i--) { + ld(as_Register(regs[i]), Address(stack, (count - 1 - i) * wordSize + offset)); + DEBUG_ONLY(words_popped++;) + } + + if (count) { + addi(stack, stack, count * wordSize + offset); + } + assert(words_popped == count, "oops, popped != count"); + + return count; +} + +// Push floating-point registers in the bitset supplied. +// Return the number of words pushed +int MacroAssembler::push_fp(unsigned int bitset, Register stack) { + DEBUG_ONLY(int words_pushed = 0;) + unsigned char regs[32]; + int count = bitset_to_regs(bitset, regs); + int push_slots = count + (count & 1); + + if (count) { + addi(stack, stack, -push_slots * wordSize); + } + + for (int i = count - 1; i >= 0; i--) { + fsd(as_FloatRegister(regs[i]), Address(stack, (push_slots - 1 - i) * wordSize)); + DEBUG_ONLY(words_pushed++;) + } + + assert(words_pushed == count, "oops, pushed(%d) != count(%d)", words_pushed, count); + + return count; +} + +int MacroAssembler::pop_fp(unsigned int bitset, Register stack) { + DEBUG_ONLY(int words_popped = 0;) + unsigned char regs[32]; + int count = bitset_to_regs(bitset, regs); + int pop_slots = count + (count & 1); + + for (int i = count - 1; i >= 0; i--) { + fld(as_FloatRegister(regs[i]), Address(stack, (pop_slots - 1 - i) * wordSize)); + DEBUG_ONLY(words_popped++;) + } + + if (count) { + addi(stack, stack, pop_slots * wordSize); + } + + assert(words_popped == count, "oops, popped(%d) != count(%d)", words_popped, count); + + return count; +} + +#ifdef COMPILER2 +// Push vector registers in the bitset supplied. +// Return the number of words pushed +int MacroAssembler::push_v(unsigned int bitset, Register stack) { + int vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE); + + // Scan bitset to accumulate register pairs + unsigned char regs[32]; + int count = bitset_to_regs(bitset, regs); + + for (int i = 0; i < count; i++) { + sub(stack, stack, vector_size_in_bytes); + vs1r_v(as_VectorRegister(regs[i]), stack); + } + + return count * vector_size_in_bytes / wordSize; +} + +int MacroAssembler::pop_v(unsigned int bitset, Register stack) { + int vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE); + + // Scan bitset to accumulate register pairs + unsigned char regs[32]; + int count = bitset_to_regs(bitset, regs); + + for (int i = count - 1; i >= 0; i--) { + vl1re8_v(as_VectorRegister(regs[i]), stack); + add(stack, stack, vector_size_in_bytes); + } + + return count * vector_size_in_bytes / wordSize; +} +#endif // COMPILER2 + +void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude) { + // Push integer registers x7, x10-x17, x28-x31. + push_reg(RegSet::of(x7) + RegSet::range(x10, x17) + RegSet::range(x28, x31) - exclude, sp); + + // Push float registers f0-f7, f10-f17, f28-f31. + addi(sp, sp, - wordSize * 20); + int offset = 0; + for (int i = 0; i < 32; i++) { + if (i <= f7->encoding() || i >= f28->encoding() || (i >= f10->encoding() && i <= f17->encoding())) { + fsd(as_FloatRegister(i), Address(sp, wordSize * (offset++))); + } + } +} + +void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude) { + int offset = 0; + for (int i = 0; i < 32; i++) { + if (i <= f7->encoding() || i >= f28->encoding() || (i >= f10->encoding() && i <= f17->encoding())) { + fld(as_FloatRegister(i), Address(sp, wordSize * (offset++))); + } + } + addi(sp, sp, wordSize * 20); + + pop_reg(RegSet::of(x7) + RegSet::range(x10, x17) + RegSet::range(x28, x31) - exclude, sp); +} + +void MacroAssembler::push_CPU_state(bool save_vectors, int vector_size_in_bytes) { + // integer registers, except zr(x0) & ra(x1) & sp(x2) & gp(x3) & tp(x4) + push_reg(RegSet::range(x5, x31), sp); + + // float registers + addi(sp, sp, - 32 * wordSize); + for (int i = 0; i < 32; i++) { + fsd(as_FloatRegister(i), Address(sp, i * wordSize)); + } + + // vector registers + if (save_vectors) { + sub(sp, sp, vector_size_in_bytes * VectorRegisterImpl::number_of_registers); + vsetvli(t0, x0, Assembler::e64, Assembler::m8); + for (int i = 0; i < VectorRegisterImpl::number_of_registers; i += 8) { + add(t0, sp, vector_size_in_bytes * i); + vse64_v(as_VectorRegister(i), t0); + } + } +} + +void MacroAssembler::pop_CPU_state(bool restore_vectors, int vector_size_in_bytes) { + // vector registers + if (restore_vectors) { + vsetvli(t0, x0, Assembler::e64, Assembler::m8); + for (int i = 0; i < VectorRegisterImpl::number_of_registers; i += 8) { + vle64_v(as_VectorRegister(i), sp); + add(sp, sp, vector_size_in_bytes * 8); + } + } + + // float registers + for (int i = 0; i < 32; i++) { + fld(as_FloatRegister(i), Address(sp, i * wordSize)); + } + addi(sp, sp, 32 * wordSize); + + // integer registers, except zr(x0) & ra(x1) & sp(x2) & gp(x3) & tp(x4) + pop_reg(RegSet::range(x5, x31), sp); +} + +static int patch_offset_in_jal(address branch, int64_t offset) { + assert(Assembler::is_simm21(offset) && ((offset % 2) == 0), + "offset is too large to be patched in one jal instruction!\n"); + Assembler::patch(branch, 31, 31, (offset >> 20) & 0x1); // offset[20] ==> branch[31] + Assembler::patch(branch, 30, 21, (offset >> 1) & 0x3ff); // offset[10:1] ==> branch[30:21] + Assembler::patch(branch, 20, 20, (offset >> 11) & 0x1); // offset[11] ==> branch[20] + Assembler::patch(branch, 19, 12, (offset >> 12) & 0xff); // offset[19:12] ==> branch[19:12] + return NativeInstruction::instruction_size; // only one instruction +} + +static int patch_offset_in_conditional_branch(address branch, int64_t offset) { + assert(Assembler::is_simm13(offset) && ((offset % 2) == 0), + "offset is too large to be patched in one beq/bge/bgeu/blt/bltu/bne instruction!\n"); + Assembler::patch(branch, 31, 31, (offset >> 12) & 0x1); // offset[12] ==> branch[31] + Assembler::patch(branch, 30, 25, (offset >> 5) & 0x3f); // offset[10:5] ==> branch[30:25] + Assembler::patch(branch, 7, 7, (offset >> 11) & 0x1); // offset[11] ==> branch[7] + Assembler::patch(branch, 11, 8, (offset >> 1) & 0xf); // offset[4:1] ==> branch[11:8] + return NativeInstruction::instruction_size; // only one instruction +} + +static int patch_offset_in_pc_relative(address branch, int64_t offset) { + const int PC_RELATIVE_INSTRUCTION_NUM = 2; // auipc, addi/jalr/load + Assembler::patch(branch, 31, 12, ((offset + 0x800) >> 12) & 0xfffff); // Auipc. offset[31:12] ==> branch[31:12] + Assembler::patch(branch + 4, 31, 20, offset & 0xfff); // Addi/Jalr/Load. offset[11:0] ==> branch[31:20] + return PC_RELATIVE_INSTRUCTION_NUM * NativeInstruction::instruction_size; +} + +static int patch_addr_in_movptr(address branch, address target) { + const int MOVPTR_INSTRUCTIONS_NUM = 6; // lui + addi + slli + addi + slli + addi/jalr/load + int32_t lower = ((intptr_t)target << 35) >> 35; + int64_t upper = ((intptr_t)target - lower) >> 29; + Assembler::patch(branch + 0, 31, 12, upper & 0xfffff); // Lui. target[48:29] + target[28] ==> branch[31:12] + Assembler::patch(branch + 4, 31, 20, (lower >> 17) & 0xfff); // Addi. target[28:17] ==> branch[31:20] + Assembler::patch(branch + 12, 31, 20, (lower >> 6) & 0x7ff); // Addi. target[16: 6] ==> branch[31:20] + Assembler::patch(branch + 20, 31, 20, lower & 0x3f); // Addi/Jalr/Load. target[ 5: 0] ==> branch[31:20] + return MOVPTR_INSTRUCTIONS_NUM * NativeInstruction::instruction_size; +} + +static int patch_imm_in_li64(address branch, address target) { + const int LI64_INSTRUCTIONS_NUM = 8; // lui + addi + slli + addi + slli + addi + slli + addi + int64_t lower = (intptr_t)target & 0xffffffff; + lower = lower - ((lower << 44) >> 44); + int64_t tmp_imm = ((uint64_t)((intptr_t)target & 0xffffffff00000000)) + (uint64_t)lower; + int32_t upper = (tmp_imm - (int32_t)lower) >> 32; + int64_t tmp_upper = upper, tmp_lower = upper; + tmp_lower = (tmp_lower << 52) >> 52; + tmp_upper -= tmp_lower; + tmp_upper >>= 12; + // Load upper 32 bits. Upper = target[63:32], but if target[31] = 1 or (target[31:20] == 0x7ff && target[19] == 1), + // upper = target[63:32] + 1. + Assembler::patch(branch + 0, 31, 12, tmp_upper & 0xfffff); // Lui. + Assembler::patch(branch + 4, 31, 20, tmp_lower & 0xfff); // Addi. + // Load the rest 32 bits. + Assembler::patch(branch + 12, 31, 20, ((int32_t)lower >> 20) & 0xfff); // Addi. + Assembler::patch(branch + 20, 31, 20, (((intptr_t)target << 44) >> 52) & 0xfff); // Addi. + Assembler::patch(branch + 28, 31, 20, (intptr_t)target & 0xff); // Addi. + return LI64_INSTRUCTIONS_NUM * NativeInstruction::instruction_size; +} + +static int patch_imm_in_li32(address branch, int32_t target) { + const int LI32_INSTRUCTIONS_NUM = 2; // lui + addiw + int64_t upper = (intptr_t)target; + int32_t lower = (((int32_t)target) << 20) >> 20; + upper -= lower; + upper = (int32_t)upper; + Assembler::patch(branch + 0, 31, 12, (upper >> 12) & 0xfffff); // Lui. + Assembler::patch(branch + 4, 31, 20, lower & 0xfff); // Addiw. + return LI32_INSTRUCTIONS_NUM * NativeInstruction::instruction_size; +} + +static long get_offset_of_jal(address insn_addr) { + assert_cond(insn_addr != NULL); + long offset = 0; + unsigned insn = *(unsigned*)insn_addr; + long val = (long)Assembler::sextract(insn, 31, 12); + offset |= ((val >> 19) & 0x1) << 20; + offset |= (val & 0xff) << 12; + offset |= ((val >> 8) & 0x1) << 11; + offset |= ((val >> 9) & 0x3ff) << 1; + offset = (offset << 43) >> 43; + return offset; +} + +static long get_offset_of_conditional_branch(address insn_addr) { + long offset = 0; + assert_cond(insn_addr != NULL); + unsigned insn = *(unsigned*)insn_addr; + offset = (long)Assembler::sextract(insn, 31, 31); + offset = (offset << 12) | (((long)(Assembler::sextract(insn, 7, 7) & 0x1)) << 11); + offset = offset | (((long)(Assembler::sextract(insn, 30, 25) & 0x3f)) << 5); + offset = offset | (((long)(Assembler::sextract(insn, 11, 8) & 0xf)) << 1); + offset = (offset << 41) >> 41; + return offset; +} + +static long get_offset_of_pc_relative(address insn_addr) { + long offset = 0; + assert_cond(insn_addr != NULL); + offset = ((long)(Assembler::sextract(((unsigned*)insn_addr)[0], 31, 12))) << 12; // Auipc. + offset += ((long)Assembler::sextract(((unsigned*)insn_addr)[1], 31, 20)); // Addi/Jalr/Load. + offset = (offset << 32) >> 32; + return offset; +} + +static address get_target_of_movptr(address insn_addr) { + assert_cond(insn_addr != NULL); + intptr_t target_address = (((int64_t)Assembler::sextract(((unsigned*)insn_addr)[0], 31, 12)) & 0xfffff) << 29; // Lui. + target_address += ((int64_t)Assembler::sextract(((unsigned*)insn_addr)[1], 31, 20)) << 17; // Addi. + target_address += ((int64_t)Assembler::sextract(((unsigned*)insn_addr)[3], 31, 20)) << 6; // Addi. + target_address += ((int64_t)Assembler::sextract(((unsigned*)insn_addr)[5], 31, 20)); // Addi/Jalr/Load. + return (address) target_address; +} + +static address get_target_of_li64(address insn_addr) { + assert_cond(insn_addr != NULL); + intptr_t target_address = (((int64_t)Assembler::sextract(((unsigned*)insn_addr)[0], 31, 12)) & 0xfffff) << 44; // Lui. + target_address += ((int64_t)Assembler::sextract(((unsigned*)insn_addr)[1], 31, 20)) << 32; // Addi. + target_address += ((int64_t)Assembler::sextract(((unsigned*)insn_addr)[3], 31, 20)) << 20; // Addi. + target_address += ((int64_t)Assembler::sextract(((unsigned*)insn_addr)[5], 31, 20)) << 8; // Addi. + target_address += ((int64_t)Assembler::sextract(((unsigned*)insn_addr)[7], 31, 20)); // Addi. + return (address)target_address; +} + +static address get_target_of_li32(address insn_addr) { + assert_cond(insn_addr != NULL); + intptr_t target_address = (((int64_t)Assembler::sextract(((unsigned*)insn_addr)[0], 31, 12)) & 0xfffff) << 12; // Lui. + target_address += ((int64_t)Assembler::sextract(((unsigned*)insn_addr)[1], 31, 20)); // Addiw. + return (address)target_address; +} + +// Patch any kind of instruction; there may be several instructions. +// Return the total length (in bytes) of the instructions. +int MacroAssembler::pd_patch_instruction_size(address branch, address target) { + assert_cond(branch != NULL); + int64_t offset = target - branch; + if (NativeInstruction::is_jal_at(branch)) { // jal + return patch_offset_in_jal(branch, offset); + } else if (NativeInstruction::is_branch_at(branch)) { // beq/bge/bgeu/blt/bltu/bne + return patch_offset_in_conditional_branch(branch, offset); + } else if (NativeInstruction::is_pc_relative_at(branch)) { // auipc, addi/jalr/load + return patch_offset_in_pc_relative(branch, offset); + } else if (NativeInstruction::is_movptr_at(branch)) { // movptr + return patch_addr_in_movptr(branch, target); + } else if (NativeInstruction::is_li64_at(branch)) { // li64 + return patch_imm_in_li64(branch, target); + } else if (NativeInstruction::is_li32_at(branch)) { // li32 + int64_t imm = (intptr_t)target; + return patch_imm_in_li32(branch, (int32_t)imm); + } else { +#ifdef ASSERT + tty->print_cr("pd_patch_instruction_size: instruction 0x%x at " INTPTR_FORMAT " could not be patched!\n", + *(unsigned*)branch, p2i(branch)); + Disassembler::decode(branch - 16, branch + 16); +#endif + ShouldNotReachHere(); + return -1; + } +} + +address MacroAssembler::target_addr_for_insn(address insn_addr) { + long offset = 0; + assert_cond(insn_addr != NULL); + if (NativeInstruction::is_jal_at(insn_addr)) { // jal + offset = get_offset_of_jal(insn_addr); + } else if (NativeInstruction::is_branch_at(insn_addr)) { // beq/bge/bgeu/blt/bltu/bne + offset = get_offset_of_conditional_branch(insn_addr); + } else if (NativeInstruction::is_pc_relative_at(insn_addr)) { // auipc, addi/jalr/load + offset = get_offset_of_pc_relative(insn_addr); + } else if (NativeInstruction::is_movptr_at(insn_addr)) { // movptr + return get_target_of_movptr(insn_addr); + } else if (NativeInstruction::is_li64_at(insn_addr)) { // li64 + return get_target_of_li64(insn_addr); + } else if (NativeInstruction::is_li32_at(insn_addr)) { // li32 + return get_target_of_li32(insn_addr); + } else { + ShouldNotReachHere(); + } + return address(((uintptr_t)insn_addr + offset)); +} + +int MacroAssembler::patch_oop(address insn_addr, address o) { + // OOPs are either narrow (32 bits) or wide (48 bits). We encode + // narrow OOPs by setting the upper 16 bits in the first + // instruction. + if (NativeInstruction::is_li32_at(insn_addr)) { + // Move narrow OOP + uint32_t n = CompressedOops::narrow_oop_value(cast_to_oop(o)); + return patch_imm_in_li32(insn_addr, (int32_t)n); + } else if (NativeInstruction::is_movptr_at(insn_addr)) { + // Move wide OOP + return patch_addr_in_movptr(insn_addr, o); + } + ShouldNotReachHere(); + return -1; +} + +void MacroAssembler::reinit_heapbase() { + if (UseCompressedOops) { + if (Universe::is_fully_initialized()) { + mv(xheapbase, CompressedOops::ptrs_base()); + } else { + ExternalAddress target((address)CompressedOops::ptrs_base_addr()); + relocate(target.rspec(), [&] { + int32_t offset; + la_patchable(xheapbase, target, offset); + ld(xheapbase, Address(xheapbase, offset)); + }); + } + } +} + +void MacroAssembler::movptr(Register Rd, address addr, int32_t &offset) { + int64_t imm64 = (int64_t)addr; +#ifndef PRODUCT + { + char buffer[64]; + snprintf(buffer, sizeof(buffer), "0x%" PRIx64, imm64); + block_comment(buffer); + } +#endif + assert((uintptr_t)imm64 < (1ull << 48), "48-bit overflow in address constant"); + // Load upper 31 bits + int64_t imm = imm64 >> 17; + int64_t upper = imm, lower = imm; + lower = (lower << 52) >> 52; + upper -= lower; + upper = (int32_t)upper; + lui(Rd, upper); + addi(Rd, Rd, lower); + + // Load the rest 17 bits. + slli(Rd, Rd, 11); + addi(Rd, Rd, (imm64 >> 6) & 0x7ff); + slli(Rd, Rd, 6); + + // This offset will be used by following jalr/ld. + offset = imm64 & 0x3f; +} + +void MacroAssembler::add(Register Rd, Register Rn, int64_t increment, Register temp) { + if (is_simm12(increment)) { + addi(Rd, Rn, increment); + } else { + assert_different_registers(Rn, temp); + li(temp, increment); + add(Rd, Rn, temp); + } +} + +void MacroAssembler::addw(Register Rd, Register Rn, int32_t increment, Register temp) { + if (is_simm12(increment)) { + addiw(Rd, Rn, increment); + } else { + assert_different_registers(Rn, temp); + li(temp, increment); + addw(Rd, Rn, temp); + } +} + +void MacroAssembler::sub(Register Rd, Register Rn, int64_t decrement, Register temp) { + if (is_simm12(-decrement)) { + addi(Rd, Rn, -decrement); + } else { + assert_different_registers(Rn, temp); + li(temp, decrement); + sub(Rd, Rn, temp); + } +} + +void MacroAssembler::subw(Register Rd, Register Rn, int32_t decrement, Register temp) { + if (is_simm12(-decrement)) { + addiw(Rd, Rn, -decrement); + } else { + assert_different_registers(Rn, temp); + li(temp, decrement); + subw(Rd, Rn, temp); + } +} + +void MacroAssembler::andrw(Register Rd, Register Rs1, Register Rs2) { + andr(Rd, Rs1, Rs2); + sign_extend(Rd, Rd, 32); +} + +void MacroAssembler::orrw(Register Rd, Register Rs1, Register Rs2) { + orr(Rd, Rs1, Rs2); + sign_extend(Rd, Rd, 32); +} + +void MacroAssembler::xorrw(Register Rd, Register Rs1, Register Rs2) { + xorr(Rd, Rs1, Rs2); + sign_extend(Rd, Rd, 32); +} + +// Note: load_unsigned_short used to be called load_unsigned_word. +int MacroAssembler::load_unsigned_short(Register dst, Address src) { + int off = offset(); + lhu(dst, src); + return off; +} + +int MacroAssembler::load_unsigned_byte(Register dst, Address src) { + int off = offset(); + lbu(dst, src); + return off; +} + +int MacroAssembler::load_signed_short(Register dst, Address src) { + int off = offset(); + lh(dst, src); + return off; +} + +int MacroAssembler::load_signed_byte(Register dst, Address src) { + int off = offset(); + lb(dst, src); + return off; +} + +void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) { + switch (size_in_bytes) { + case 8: ld(dst, src); break; + case 4: is_signed ? lw(dst, src) : lwu(dst, src); break; + case 2: is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break; + case 1: is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break; + default: ShouldNotReachHere(); + } +} + +void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) { + switch (size_in_bytes) { + case 8: sd(src, dst); break; + case 4: sw(src, dst); break; + case 2: sh(src, dst); break; + case 1: sb(src, dst); break; + default: ShouldNotReachHere(); + } +} + +// reverse bytes in halfword in lower 16 bits and sign-extend +// Rd[15:0] = Rs[7:0] Rs[15:8] (sign-extend to 64 bits) +void MacroAssembler::revb_h_h(Register Rd, Register Rs, Register tmp) { + if (UseZbb) { + rev8(Rd, Rs); + srai(Rd, Rd, 48); + return; + } + assert_different_registers(Rs, tmp); + assert_different_registers(Rd, tmp); + srli(tmp, Rs, 8); + andi(tmp, tmp, 0xFF); + slli(Rd, Rs, 56); + srai(Rd, Rd, 48); // sign-extend + orr(Rd, Rd, tmp); +} + +// reverse bytes in lower word and sign-extend +// Rd[31:0] = Rs[7:0] Rs[15:8] Rs[23:16] Rs[31:24] (sign-extend to 64 bits) +void MacroAssembler::revb_w_w(Register Rd, Register Rs, Register tmp1, Register tmp2) { + if (UseZbb) { + rev8(Rd, Rs); + srai(Rd, Rd, 32); + return; + } + assert_different_registers(Rs, tmp1, tmp2); + assert_different_registers(Rd, tmp1, tmp2); + revb_h_w_u(Rd, Rs, tmp1, tmp2); + slli(tmp2, Rd, 48); + srai(tmp2, tmp2, 32); // sign-extend + srli(Rd, Rd, 16); + orr(Rd, Rd, tmp2); +} + +// reverse bytes in halfword in lower 16 bits and zero-extend +// Rd[15:0] = Rs[7:0] Rs[15:8] (zero-extend to 64 bits) +void MacroAssembler::revb_h_h_u(Register Rd, Register Rs, Register tmp) { + if (UseZbb) { + rev8(Rd, Rs); + srli(Rd, Rd, 48); + return; + } + assert_different_registers(Rs, tmp); + assert_different_registers(Rd, tmp); + srli(tmp, Rs, 8); + andi(tmp, tmp, 0xFF); + andi(Rd, Rs, 0xFF); + slli(Rd, Rd, 8); + orr(Rd, Rd, tmp); +} + +// reverse bytes in halfwords in lower 32 bits and zero-extend +// Rd[31:0] = Rs[23:16] Rs[31:24] Rs[7:0] Rs[15:8] (zero-extend to 64 bits) +void MacroAssembler::revb_h_w_u(Register Rd, Register Rs, Register tmp1, Register tmp2) { + if (UseZbb) { + rev8(Rd, Rs); + rori(Rd, Rd, 32); + roriw(Rd, Rd, 16); + zero_extend(Rd, Rd, 32); + return; + } + assert_different_registers(Rs, tmp1, tmp2); + assert_different_registers(Rd, tmp1, tmp2); + srli(tmp2, Rs, 16); + revb_h_h_u(tmp2, tmp2, tmp1); + revb_h_h_u(Rd, Rs, tmp1); + slli(tmp2, tmp2, 16); + orr(Rd, Rd, tmp2); +} + +// This method is only used for revb_h +// Rd = Rs[47:0] Rs[55:48] Rs[63:56] +void MacroAssembler::revb_h_helper(Register Rd, Register Rs, Register tmp1, Register tmp2) { + assert_different_registers(Rs, tmp1, tmp2); + assert_different_registers(Rd, tmp1); + srli(tmp1, Rs, 48); + andi(tmp2, tmp1, 0xFF); + slli(tmp2, tmp2, 8); + srli(tmp1, tmp1, 8); + orr(tmp1, tmp1, tmp2); + slli(Rd, Rs, 16); + orr(Rd, Rd, tmp1); +} + +// reverse bytes in each halfword +// Rd[63:0] = Rs[55:48] Rs[63:56] Rs[39:32] Rs[47:40] Rs[23:16] Rs[31:24] Rs[7:0] Rs[15:8] +void MacroAssembler::revb_h(Register Rd, Register Rs, Register tmp1, Register tmp2) { + if (UseZbb) { + assert_different_registers(Rs, tmp1); + assert_different_registers(Rd, tmp1); + rev8(Rd, Rs); + zero_extend(tmp1, Rd, 32); + roriw(tmp1, tmp1, 16); + slli(tmp1, tmp1, 32); + srli(Rd, Rd, 32); + roriw(Rd, Rd, 16); + zero_extend(Rd, Rd, 32); + orr(Rd, Rd, tmp1); + return; + } + assert_different_registers(Rs, tmp1, tmp2); + assert_different_registers(Rd, tmp1, tmp2); + revb_h_helper(Rd, Rs, tmp1, tmp2); + for (int i = 0; i < 3; ++i) { + revb_h_helper(Rd, Rd, tmp1, tmp2); + } +} + +// reverse bytes in each word +// Rd[63:0] = Rs[39:32] Rs[47:40] Rs[55:48] Rs[63:56] Rs[7:0] Rs[15:8] Rs[23:16] Rs[31:24] +void MacroAssembler::revb_w(Register Rd, Register Rs, Register tmp1, Register tmp2) { + if (UseZbb) { + rev8(Rd, Rs); + rori(Rd, Rd, 32); + return; + } + assert_different_registers(Rs, tmp1, tmp2); + assert_different_registers(Rd, tmp1, tmp2); + revb(Rd, Rs, tmp1, tmp2); + ror_imm(Rd, Rd, 32); +} + +// reverse bytes in doubleword +// Rd[63:0] = Rs[7:0] Rs[15:8] Rs[23:16] Rs[31:24] Rs[39:32] Rs[47,40] Rs[55,48] Rs[63:56] +void MacroAssembler::revb(Register Rd, Register Rs, Register tmp1, Register tmp2) { + if (UseZbb) { + rev8(Rd, Rs); + return; + } + assert_different_registers(Rs, tmp1, tmp2); + assert_different_registers(Rd, tmp1, tmp2); + andi(tmp1, Rs, 0xFF); + slli(tmp1, tmp1, 8); + for (int step = 8; step < 56; step += 8) { + srli(tmp2, Rs, step); + andi(tmp2, tmp2, 0xFF); + orr(tmp1, tmp1, tmp2); + slli(tmp1, tmp1, 8); + } + srli(Rd, Rs, 56); + andi(Rd, Rd, 0xFF); + orr(Rd, tmp1, Rd); +} + +// rotate right with shift bits +void MacroAssembler::ror_imm(Register dst, Register src, uint32_t shift, Register tmp) +{ + if (UseZbb) { + rori(dst, src, shift); + return; + } + + assert_different_registers(dst, tmp); + assert_different_registers(src, tmp); + assert(shift < 64, "shift amount must be < 64"); + slli(tmp, src, 64 - shift); + srli(dst, src, shift); + orr(dst, dst, tmp); +} + +void MacroAssembler::andi(Register Rd, Register Rn, int64_t imm, Register tmp) { + if (is_simm12(imm)) { + and_imm12(Rd, Rn, imm); + } else { + assert_different_registers(Rn, tmp); + mv(tmp, imm); + andr(Rd, Rn, tmp); + } +} + +void MacroAssembler::orptr(Address adr, RegisterOrConstant src, Register tmp1, Register tmp2) { + ld(tmp1, adr); + if (src.is_register()) { + orr(tmp1, tmp1, src.as_register()); + } else { + if (is_simm12(src.as_constant())) { + ori(tmp1, tmp1, src.as_constant()); + } else { + assert_different_registers(tmp1, tmp2); + mv(tmp2, src.as_constant()); + orr(tmp1, tmp1, tmp2); + } + } + sd(tmp1, adr); +} + +void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp1, Register tmp2, Label &L) { + assert_different_registers(oop, trial_klass, tmp1, tmp2); + if (UseCompressedClassPointers) { + lwu(tmp1, Address(oop, oopDesc::klass_offset_in_bytes())); + if (CompressedKlassPointers::base() == NULL) { + slli(tmp1, tmp1, CompressedKlassPointers::shift()); + beq(trial_klass, tmp1, L); + return; + } + decode_klass_not_null(tmp1, tmp2); + } else { + ld(tmp1, Address(oop, oopDesc::klass_offset_in_bytes())); + } + beq(trial_klass, tmp1, L); +} + +// Move an oop into a register. immediate is true if we want +// immediate instructions and nmethod entry barriers are not enabled. +// i.e. we are not going to patch this instruction while the code is being +// executed by another thread. +void MacroAssembler::movoop(Register dst, jobject obj, bool immediate) { + int oop_index; + if (obj == NULL) { + oop_index = oop_recorder()->allocate_oop_index(obj); + } else { +#ifdef ASSERT + { + ThreadInVMfromUnknown tiv; + assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "should be real oop"); + } +#endif + oop_index = oop_recorder()->find_index(obj); + } + RelocationHolder rspec = oop_Relocation::spec(oop_index); + + // nmethod entry barrier necessitate using the constant pool. They have to be + // ordered with respected to oop access. + if (BarrierSet::barrier_set()->barrier_set_nmethod() != NULL || !immediate) { + address dummy = address(uintptr_t(pc()) & -wordSize); // A nearby aligned address + ld_constant(dst, Address(dummy, rspec)); + } else + mv(dst, Address((address)obj, rspec)); +} + +// Move a metadata address into a register. +void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { + int oop_index; + if (obj == NULL) { + oop_index = oop_recorder()->allocate_metadata_index(obj); + } else { + oop_index = oop_recorder()->find_index(obj); + } + RelocationHolder rspec = metadata_Relocation::spec(oop_index); + mv(dst, Address((address)obj, rspec)); +} + +// Writes to stack successive pages until offset reached to check for +// stack overflow + shadow pages. This clobbers tmp. +void MacroAssembler::bang_stack_size(Register size, Register tmp) { + assert_different_registers(tmp, size, t0); + // Bang stack for total size given plus shadow page size. + // Bang one page at a time because large size can bang beyond yellow and + // red zones. + mv(t0, os::vm_page_size()); + Label loop; + bind(loop); + sub(tmp, sp, t0); + subw(size, size, t0); + sd(size, Address(tmp)); + bgtz(size, loop); + + // Bang down shadow pages too. + // At this point, (tmp-0) is the last address touched, so don't + // touch it again. (It was touched as (tmp-pagesize) but then tmp + // was post-decremented.) Skip this address by starting at i=1, and + // touch a few more pages below. N.B. It is important to touch all + // the way down to and including i=StackShadowPages. + for (int i = 0; i < (int)(StackOverflow::stack_shadow_zone_size() / os::vm_page_size()) - 1; i++) { + // this could be any sized move but this is can be a debugging crumb + // so the bigger the better. + sub(tmp, tmp, os::vm_page_size()); + sd(size, Address(tmp, 0)); + } +} + +SkipIfEqual::SkipIfEqual(MacroAssembler* masm, const bool* flag_addr, bool value) { + int32_t offset = 0; + _masm = masm; + ExternalAddress target((address)flag_addr); + _masm->relocate(target.rspec(), [&] { + int32_t offset; + _masm->la_patchable(t0, target, offset); + _masm->lbu(t0, Address(t0, offset)); + }); + _masm->beqz(t0, _label); +} + +SkipIfEqual::~SkipIfEqual() { + _masm->bind(_label); + _masm = NULL; +} + +void MacroAssembler::load_mirror(Register dst, Register method, Register tmp) { + const int mirror_offset = in_bytes(Klass::java_mirror_offset()); + ld(dst, Address(xmethod, Method::const_offset())); + ld(dst, Address(dst, ConstMethod::constants_offset())); + ld(dst, Address(dst, ConstantPool::pool_holder_offset_in_bytes())); + ld(dst, Address(dst, mirror_offset)); + resolve_oop_handle(dst, tmp); +} + +void MacroAssembler::resolve_oop_handle(Register result, Register tmp) { + // OopHandle::resolve is an indirection. + assert_different_registers(result, tmp); + access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg); +} + +// ((WeakHandle)result).resolve() +void MacroAssembler::resolve_weak_handle(Register result, Register tmp) { + assert_different_registers(result, tmp); + Label resolved; + + // A null weak handle resolves to null. + beqz(result, resolved); + + // Only 64 bit platforms support GCs that require a tmp register + // Only IN_HEAP loads require a thread_tmp register + // WeakHandle::resolve is an indirection like jweak. + access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, + result, Address(result), tmp, noreg /* tmp_thread */); + bind(resolved); +} + +void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, + Register dst, Address src, + Register tmp1, Register thread_tmp) { + BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); + decorators = AccessInternal::decorator_fixup(decorators); + bool as_raw = (decorators & AS_RAW) != 0; + if (as_raw) { + bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp); + } else { + bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp); + } +} + +void MacroAssembler::null_check(Register reg, int offset) { + if (needs_explicit_null_check(offset)) { + // provoke OS NULL exception if reg = NULL by + // accessing M[reg] w/o changing any registers + // NOTE: this is plenty to provoke a segv + ld(zr, Address(reg, 0)); + } else { + // nothing to do, (later) access of M[reg + offset] + // will provoke OS NULL exception if reg = NULL + } +} + +void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, + Address dst, Register src, + Register tmp1, Register thread_tmp) { + BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); + decorators = AccessInternal::decorator_fixup(decorators); + bool as_raw = (decorators & AS_RAW) != 0; + if (as_raw) { + bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, thread_tmp); + } else { + bs->store_at(this, decorators, type, dst, src, tmp1, thread_tmp); + } +} + +// Algorithm must match CompressedOops::encode. +void MacroAssembler::encode_heap_oop(Register d, Register s) { + verify_oop(s, "broken oop in encode_heap_oop"); + if (CompressedOops::base() == NULL) { + if (CompressedOops::shift() != 0) { + assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); + srli(d, s, LogMinObjAlignmentInBytes); + } else { + mv(d, s); + } + } else { + Label notNull; + sub(d, s, xheapbase); + bgez(d, notNull); + mv(d, zr); + bind(notNull); + if (CompressedOops::shift() != 0) { + assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); + srli(d, d, CompressedOops::shift()); + } + } +} + +void MacroAssembler::load_klass(Register dst, Register src, Register tmp) { + assert_different_registers(dst, tmp); + assert_different_registers(src, tmp); + if (UseCompressedClassPointers) { + lwu(dst, Address(src, oopDesc::klass_offset_in_bytes())); + decode_klass_not_null(dst, tmp); + } else { + ld(dst, Address(src, oopDesc::klass_offset_in_bytes())); + } +} + +void MacroAssembler::store_klass(Register dst, Register src, Register tmp) { + // FIXME: Should this be a store release? concurrent gcs assumes + // klass length is valid if klass field is not null. + if (UseCompressedClassPointers) { + encode_klass_not_null(src, tmp); + sw(src, Address(dst, oopDesc::klass_offset_in_bytes())); + } else { + sd(src, Address(dst, oopDesc::klass_offset_in_bytes())); + } +} + +void MacroAssembler::store_klass_gap(Register dst, Register src) { + if (UseCompressedClassPointers) { + // Store to klass gap in destination + sw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes())); + } +} + +void MacroAssembler::decode_klass_not_null(Register r, Register tmp) { + assert_different_registers(r, tmp); + decode_klass_not_null(r, r, tmp); +} + +void MacroAssembler::decode_klass_not_null(Register dst, Register src, Register tmp) { + assert(UseCompressedClassPointers, "should only be used for compressed headers"); + + if (CompressedKlassPointers::base() == NULL) { + if (CompressedKlassPointers::shift() != 0) { + assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); + slli(dst, src, LogKlassAlignmentInBytes); + } else { + mv(dst, src); + } + return; + } + + Register xbase = dst; + if (dst == src) { + xbase = tmp; + } + + assert_different_registers(src, xbase); + mv(xbase, (uintptr_t)CompressedKlassPointers::base()); + + if (CompressedKlassPointers::shift() != 0) { + assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); + assert_different_registers(t0, xbase); + shadd(dst, src, xbase, t0, LogKlassAlignmentInBytes); + } else { + add(dst, xbase, src); + } +} + +void MacroAssembler::encode_klass_not_null(Register r, Register tmp) { + assert_different_registers(r, tmp); + encode_klass_not_null(r, r, tmp); +} + +void MacroAssembler::encode_klass_not_null(Register dst, Register src, Register tmp) { + assert(UseCompressedClassPointers, "should only be used for compressed headers"); + + if (CompressedKlassPointers::base() == NULL) { + if (CompressedKlassPointers::shift() != 0) { + assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); + srli(dst, src, LogKlassAlignmentInBytes); + } else { + mv(dst, src); + } + return; + } + + if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0 && + CompressedKlassPointers::shift() == 0) { + zero_extend(dst, src, 32); + return; + } + + Register xbase = dst; + if (dst == src) { + xbase = tmp; + } + + assert_different_registers(src, xbase); + mv(xbase, (uintptr_t)CompressedKlassPointers::base()); + sub(dst, src, xbase); + if (CompressedKlassPointers::shift() != 0) { + assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); + srli(dst, dst, LogKlassAlignmentInBytes); + } +} + +void MacroAssembler::decode_heap_oop_not_null(Register r) { + decode_heap_oop_not_null(r, r); +} + +void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) { + assert(UseCompressedOops, "should only be used for compressed headers"); + assert(Universe::heap() != NULL, "java heap should be initialized"); + // Cannot assert, unverified entry point counts instructions (see .ad file) + // vtableStubs also counts instructions in pd_code_size_limit. + // Also do not verify_oop as this is called by verify_oop. + if (CompressedOops::shift() != 0) { + assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); + slli(dst, src, LogMinObjAlignmentInBytes); + if (CompressedOops::base() != NULL) { + add(dst, xheapbase, dst); + } + } else { + assert(CompressedOops::base() == NULL, "sanity"); + mv(dst, src); + } +} + +void MacroAssembler::decode_heap_oop(Register d, Register s) { + if (CompressedOops::base() == NULL) { + if (CompressedOops::shift() != 0 || d != s) { + slli(d, s, CompressedOops::shift()); + } + } else { + Label done; + mv(d, s); + beqz(s, done); + shadd(d, s, xheapbase, d, LogMinObjAlignmentInBytes); + bind(done); + } + verify_oop(d, "broken oop in decode_heap_oop"); +} + +void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1, + Register thread_tmp, DecoratorSet decorators) { + access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp); +} + +void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1, + Register thread_tmp, DecoratorSet decorators) { + access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp); +} + +void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1, + Register thread_tmp, DecoratorSet decorators) { + access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL, dst, src, tmp1, thread_tmp); +} + +// Used for storing NULLs. +void MacroAssembler::store_heap_oop_null(Address dst) { + access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg); +} + +int MacroAssembler::corrected_idivl(Register result, Register rs1, Register rs2, + bool want_remainder) +{ + // Full implementation of Java idiv and irem. The function + // returns the (pc) offset of the div instruction - may be needed + // for implicit exceptions. + // + // input : rs1: dividend + // rs2: divisor + // + // result: either + // quotient (= rs1 idiv rs2) + // remainder (= rs1 irem rs2) + + + int idivl_offset = offset(); + if (!want_remainder) { + divw(result, rs1, rs2); + } else { + remw(result, rs1, rs2); // result = rs1 % rs2; + } + return idivl_offset; +} + +int MacroAssembler::corrected_idivq(Register result, Register rs1, Register rs2, + bool want_remainder) +{ + // Full implementation of Java ldiv and lrem. The function + // returns the (pc) offset of the div instruction - may be needed + // for implicit exceptions. + // + // input : rs1: dividend + // rs2: divisor + // + // result: either + // quotient (= rs1 idiv rs2) + // remainder (= rs1 irem rs2) + + int idivq_offset = offset(); + if (!want_remainder) { + div(result, rs1, rs2); + } else { + rem(result, rs1, rs2); // result = rs1 % rs2; + } + return idivq_offset; +} + +// Look up the method for a megamorpic invkkeinterface call. +// The target method is determined by . +// The receiver klass is in recv_klass. +// On success, the result will be in method_result, and execution falls through. +// On failure, execution transfers to the given label. +void MacroAssembler::lookup_interface_method(Register recv_klass, + Register intf_klass, + RegisterOrConstant itable_index, + Register method_result, + Register scan_tmp, + Label& L_no_such_interface, + bool return_method) { + assert_different_registers(recv_klass, intf_klass, scan_tmp); + assert_different_registers(method_result, intf_klass, scan_tmp); + assert(recv_klass != method_result || !return_method, + "recv_klass can be destroyed when mehtid isn't needed"); + assert(itable_index.is_constant() || itable_index.as_register() == method_result, + "caller must be same register for non-constant itable index as for method"); + + // Compute start of first itableOffsetEntry (which is at the end of the vtable). + int vtable_base = in_bytes(Klass::vtable_start_offset()); + int itentry_off = itableMethodEntry::method_offset_in_bytes(); + int scan_step = itableOffsetEntry::size() * wordSize; + int vte_size = vtableEntry::size_in_bytes(); + assert(vte_size == wordSize, "else adjust times_vte_scale"); + + lwu(scan_tmp, Address(recv_klass, Klass::vtable_length_offset())); + + // %%% Could store the aligned, prescaled offset in the klassoop. + shadd(scan_tmp, scan_tmp, recv_klass, scan_tmp, 3); + add(scan_tmp, scan_tmp, vtable_base); + + if (return_method) { + // Adjust recv_klass by scaled itable_index, so we can free itable_index. + assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); + if (itable_index.is_register()) { + slli(t0, itable_index.as_register(), 3); + } else { + mv(t0, itable_index.as_constant() << 3); + } + add(recv_klass, recv_klass, t0); + if (itentry_off) { + add(recv_klass, recv_klass, itentry_off); + } + } + + Label search, found_method; + + ld(method_result, Address(scan_tmp, itableOffsetEntry::interface_offset_in_bytes())); + beq(intf_klass, method_result, found_method); + bind(search); + // Check that the previous entry is non-null. A null entry means that + // the receiver class doens't implement the interface, and wasn't the + // same as when the caller was compiled. + beqz(method_result, L_no_such_interface, /* is_far */ true); + addi(scan_tmp, scan_tmp, scan_step); + ld(method_result, Address(scan_tmp, itableOffsetEntry::interface_offset_in_bytes())); + bne(intf_klass, method_result, search); + + bind(found_method); + + // Got a hit. + if (return_method) { + lwu(scan_tmp, Address(scan_tmp, itableOffsetEntry::offset_offset_in_bytes())); + add(method_result, recv_klass, scan_tmp); + ld(method_result, Address(method_result)); + } +} + +// virtual method calling +void MacroAssembler::lookup_virtual_method(Register recv_klass, + RegisterOrConstant vtable_index, + Register method_result) { + const int base = in_bytes(Klass::vtable_start_offset()); + assert(vtableEntry::size() * wordSize == 8, + "adjust the scaling in the code below"); + int vtable_offset_in_bytes = base + vtableEntry::method_offset_in_bytes(); + + if (vtable_index.is_register()) { + shadd(method_result, vtable_index.as_register(), recv_klass, method_result, LogBytesPerWord); + ld(method_result, Address(method_result, vtable_offset_in_bytes)); + } else { + vtable_offset_in_bytes += vtable_index.as_constant() * wordSize; + ld(method_result, form_address(method_result, recv_klass, vtable_offset_in_bytes)); + } +} + +void MacroAssembler::membar(uint32_t order_constraint) { + address prev = pc() - NativeMembar::instruction_size; + address last = code()->last_insn(); + + if (last != NULL && nativeInstruction_at(last)->is_membar() && prev == last) { + NativeMembar *bar = NativeMembar_at(prev); + // We are merging two memory barrier instructions. On RISCV we + // can do this simply by ORing them together. + bar->set_kind(bar->get_kind() | order_constraint); + BLOCK_COMMENT("merged membar"); + } else { + code()->set_last_insn(pc()); + + uint32_t predecessor = 0; + uint32_t successor = 0; + + membar_mask_to_pred_succ(order_constraint, predecessor, successor); + fence(predecessor, successor); + } +} + +// Form an addres from base + offset in Rd. Rd my or may not +// actually be used: you must use the Address that is returned. It +// is up to you to ensure that the shift provided mathces the size +// of your data. +Address MacroAssembler::form_address(Register Rd, Register base, int64_t byte_offset) { + if (is_simm12(byte_offset)) { // 12: imm in range 2^12 + return Address(base, byte_offset); + } + + assert_different_registers(Rd, base, noreg); + + // Do it the hard way + mv(Rd, byte_offset); + add(Rd, base, Rd); + return Address(Rd); +} + +void MacroAssembler::check_klass_subtype(Register sub_klass, + Register super_klass, + Register tmp_reg, + Label& L_success) { + Label L_failure; + check_klass_subtype_fast_path(sub_klass, super_klass, tmp_reg, &L_success, &L_failure, NULL); + check_klass_subtype_slow_path(sub_klass, super_klass, tmp_reg, noreg, &L_success, NULL); + bind(L_failure); +} + +void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool acquire, bool in_nmethod) { + ld(t0, Address(xthread, JavaThread::polling_word_offset())); + if (acquire) { + membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); + } + if (at_return) { + bgtu(in_nmethod ? sp : fp, t0, slow_path, true /* is_far */); + } else { + test_bit(t0, t0, exact_log2(SafepointMechanism::poll_bit())); + bnez(t0, slow_path, true /* is_far */); + } +} + +void MacroAssembler::cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp, + Label &succeed, Label *fail) { + // oldv holds comparison value + // newv holds value to write in exchange + // addr identifies memory word to compare against/update + Label retry_load, nope; + bind(retry_load); + // Load reserved from the memory location + lr_d(tmp, addr, Assembler::aqrl); + // Fail and exit if it is not what we expect + bne(tmp, oldv, nope); + // If the store conditional succeeds, tmp will be zero + sc_d(tmp, newv, addr, Assembler::rl); + beqz(tmp, succeed); + // Retry only when the store conditional failed + j(retry_load); + + bind(nope); + membar(AnyAny); + mv(oldv, tmp); + if (fail != NULL) { + j(*fail); + } +} + +void MacroAssembler::cmpxchg_obj_header(Register oldv, Register newv, Register obj, Register tmp, + Label &succeed, Label *fail) { + assert(oopDesc::mark_offset_in_bytes() == 0, "assumption"); + cmpxchgptr(oldv, newv, obj, tmp, succeed, fail); +} + +void MacroAssembler::load_reserved(Register addr, + enum operand_size size, + Assembler::Aqrl acquire) { + switch (size) { + case int64: + lr_d(t0, addr, acquire); + break; + case int32: + lr_w(t0, addr, acquire); + break; + case uint32: + lr_w(t0, addr, acquire); + zero_extend(t0, t0, 32); + break; + default: + ShouldNotReachHere(); + } +} + +void MacroAssembler::store_conditional(Register addr, + Register new_val, + enum operand_size size, + Assembler::Aqrl release) { + switch (size) { + case int64: + sc_d(t0, new_val, addr, release); + break; + case int32: + case uint32: + sc_w(t0, new_val, addr, release); + break; + default: + ShouldNotReachHere(); + } +} + + +void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expected, + Register new_val, + enum operand_size size, + Register tmp1, Register tmp2, Register tmp3) { + assert(size == int8 || size == int16, "unsupported operand size"); + + Register aligned_addr = t1, shift = tmp1, mask = tmp2, not_mask = tmp3; + + andi(shift, addr, 3); + slli(shift, shift, 3); + + andi(aligned_addr, addr, ~3); + + if (size == int8) { + mv(mask, 0xff); + } else { + // size == int16 case + mv(mask, -1); + zero_extend(mask, mask, 16); + } + sll(mask, mask, shift); + + xori(not_mask, mask, -1); + + sll(expected, expected, shift); + andr(expected, expected, mask); + + sll(new_val, new_val, shift); + andr(new_val, new_val, mask); +} + +// cmpxchg_narrow_value will kill t0, t1, expected, new_val and tmps. +// It's designed to implement compare and swap byte/boolean/char/short by lr.w/sc.w, +// which are forced to work with 4-byte aligned address. +void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected, + Register new_val, + enum operand_size size, + Assembler::Aqrl acquire, Assembler::Aqrl release, + Register result, bool result_as_bool, + Register tmp1, Register tmp2, Register tmp3) { + Register aligned_addr = t1, shift = tmp1, mask = tmp2, not_mask = tmp3, old = result, tmp = t0; + assert_different_registers(addr, old, mask, not_mask, new_val, expected, shift, tmp); + cmpxchg_narrow_value_helper(addr, expected, new_val, size, tmp1, tmp2, tmp3); + + Label retry, fail, done; + + bind(retry); + lr_w(old, aligned_addr, acquire); + andr(tmp, old, mask); + bne(tmp, expected, fail); + + andr(tmp, old, not_mask); + orr(tmp, tmp, new_val); + sc_w(tmp, tmp, aligned_addr, release); + bnez(tmp, retry); + + if (result_as_bool) { + mv(result, 1); + j(done); + + bind(fail); + mv(result, zr); + + bind(done); + } else { + andr(tmp, old, mask); + + bind(fail); + srl(result, tmp, shift); + + if (size == int8) { + sign_extend(result, result, 8); + } else { + // size == int16 case + sign_extend(result, result, 16); + } + } +} + +// weak_cmpxchg_narrow_value is a weak version of cmpxchg_narrow_value, to implement +// the weak CAS stuff. The major difference is that it just failed when store conditional +// failed. +void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected, + Register new_val, + enum operand_size size, + Assembler::Aqrl acquire, Assembler::Aqrl release, + Register result, + Register tmp1, Register tmp2, Register tmp3) { + Register aligned_addr = t1, shift = tmp1, mask = tmp2, not_mask = tmp3, old = result, tmp = t0; + assert_different_registers(addr, old, mask, not_mask, new_val, expected, shift, tmp); + cmpxchg_narrow_value_helper(addr, expected, new_val, size, tmp1, tmp2, tmp3); + + Label fail, done; + + lr_w(old, aligned_addr, acquire); + andr(tmp, old, mask); + bne(tmp, expected, fail); + + andr(tmp, old, not_mask); + orr(tmp, tmp, new_val); + sc_w(tmp, tmp, aligned_addr, release); + bnez(tmp, fail); + + // Success + mv(result, 1); + j(done); + + // Fail + bind(fail); + mv(result, zr); + + bind(done); +} + +void MacroAssembler::cmpxchg(Register addr, Register expected, + Register new_val, + enum operand_size size, + Assembler::Aqrl acquire, Assembler::Aqrl release, + Register result, bool result_as_bool) { + assert(size != int8 && size != int16, "unsupported operand size"); + + Label retry_load, done, ne_done; + bind(retry_load); + load_reserved(addr, size, acquire); + bne(t0, expected, ne_done); + store_conditional(addr, new_val, size, release); + bnez(t0, retry_load); + + // equal, succeed + if (result_as_bool) { + mv(result, 1); + } else { + mv(result, expected); + } + j(done); + + // not equal, failed + bind(ne_done); + if (result_as_bool) { + mv(result, zr); + } else { + mv(result, t0); + } + + bind(done); +} + +void MacroAssembler::cmpxchg_weak(Register addr, Register expected, + Register new_val, + enum operand_size size, + Assembler::Aqrl acquire, Assembler::Aqrl release, + Register result) { + Label fail, done; + load_reserved(addr, size, acquire); + bne(t0, expected, fail); + store_conditional(addr, new_val, size, release); + bnez(t0, fail); + + // Success + mv(result, 1); + j(done); + + // Fail + bind(fail); + mv(result, zr); + + bind(done); +} + +#define ATOMIC_OP(NAME, AOP, ACQUIRE, RELEASE) \ +void MacroAssembler::atomic_##NAME(Register prev, RegisterOrConstant incr, Register addr) { \ + prev = prev->is_valid() ? prev : zr; \ + if (incr.is_register()) { \ + AOP(prev, addr, incr.as_register(), (Assembler::Aqrl)(ACQUIRE | RELEASE)); \ + } else { \ + mv(t0, incr.as_constant()); \ + AOP(prev, addr, t0, (Assembler::Aqrl)(ACQUIRE | RELEASE)); \ + } \ + return; \ +} + +ATOMIC_OP(add, amoadd_d, Assembler::relaxed, Assembler::relaxed) +ATOMIC_OP(addw, amoadd_w, Assembler::relaxed, Assembler::relaxed) +ATOMIC_OP(addal, amoadd_d, Assembler::aq, Assembler::rl) +ATOMIC_OP(addalw, amoadd_w, Assembler::aq, Assembler::rl) + +#undef ATOMIC_OP + +#define ATOMIC_XCHG(OP, AOP, ACQUIRE, RELEASE) \ +void MacroAssembler::atomic_##OP(Register prev, Register newv, Register addr) { \ + prev = prev->is_valid() ? prev : zr; \ + AOP(prev, addr, newv, (Assembler::Aqrl)(ACQUIRE | RELEASE)); \ + return; \ +} + +ATOMIC_XCHG(xchg, amoswap_d, Assembler::relaxed, Assembler::relaxed) +ATOMIC_XCHG(xchgw, amoswap_w, Assembler::relaxed, Assembler::relaxed) +ATOMIC_XCHG(xchgal, amoswap_d, Assembler::aq, Assembler::rl) +ATOMIC_XCHG(xchgalw, amoswap_w, Assembler::aq, Assembler::rl) + +#undef ATOMIC_XCHG + +#define ATOMIC_XCHGU(OP1, OP2) \ +void MacroAssembler::atomic_##OP1(Register prev, Register newv, Register addr) { \ + atomic_##OP2(prev, newv, addr); \ + zero_extend(prev, prev, 32); \ + return; \ +} + +ATOMIC_XCHGU(xchgwu, xchgw) +ATOMIC_XCHGU(xchgalwu, xchgalw) + +#undef ATOMIC_XCHGU + +void MacroAssembler::far_jump(Address entry, CodeBuffer *cbuf, Register tmp) { + assert(ReservedCodeCacheSize < 4*G, "branch out of range"); + assert(CodeCache::find_blob(entry.target()) != NULL, + "destination of far call not found in code cache"); + IncompressibleRegion ir(this); // Fixed length: see MacroAssembler::far_branch_size() + if (far_branches()) { + // We can use auipc + jalr here because we know that the total size of + // the code cache cannot exceed 2Gb. + relocate(entry.rspec(), [&] { + int32_t offset; + la_patchable(tmp, entry, offset); + if (cbuf != NULL) { cbuf->set_insts_mark(); } + jalr(x0, tmp, offset); + }); + } else { + if (cbuf != NULL) { cbuf->set_insts_mark(); } + j(entry); + } +} + +void MacroAssembler::far_call(Address entry, CodeBuffer *cbuf, Register tmp) { + assert(ReservedCodeCacheSize < 4*G, "branch out of range"); + assert(CodeCache::find_blob(entry.target()) != NULL, + "destination of far call not found in code cache"); + IncompressibleRegion ir(this); // Fixed length: see MacroAssembler::far_branch_size() + if (far_branches()) { + // We can use auipc + jalr here because we know that the total size of + // the code cache cannot exceed 2Gb. + relocate(entry.rspec(), [&] { + int32_t offset; + la_patchable(tmp, entry, offset); + if (cbuf != NULL) { cbuf->set_insts_mark(); } + jalr(x1, tmp, offset); // link + }); + } else { + if (cbuf != NULL) { cbuf->set_insts_mark(); } + jal(entry); // link + } +} + +void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass, + Register super_klass, + Register tmp_reg, + Label* L_success, + Label* L_failure, + Label* L_slow_path, + Register super_check_offset) { + assert_different_registers(sub_klass, super_klass, tmp_reg); + bool must_load_sco = (super_check_offset == noreg); + if (must_load_sco) { + assert(tmp_reg != noreg, "supply either a temp or a register offset"); + } else { + assert_different_registers(sub_klass, super_klass, super_check_offset); + } + + Label L_fallthrough; + int label_nulls = 0; + if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; } + if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; } + if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; } + assert(label_nulls <= 1, "at most one NULL in batch"); + + int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); + int sco_offset = in_bytes(Klass::super_check_offset_offset()); + Address super_check_offset_addr(super_klass, sco_offset); + + // Hacked jmp, which may only be used just before L_fallthrough. +#define final_jmp(label) \ + if (&(label) == &L_fallthrough) { /*do nothing*/ } \ + else j(label) /*omit semi*/ + + // If the pointers are equal, we are done (e.g., String[] elements). + // This self-check enables sharing of secondary supertype arrays among + // non-primary types such as array-of-interface. Otherwise, each such + // type would need its own customized SSA. + // We move this check to the front fo the fast path because many + // type checks are in fact trivially successful in this manner, + // so we get a nicely predicted branch right at the start of the check. + beq(sub_klass, super_klass, *L_success); + + // Check the supertype display: + if (must_load_sco) { + lwu(tmp_reg, super_check_offset_addr); + super_check_offset = tmp_reg; + } + add(t0, sub_klass, super_check_offset); + Address super_check_addr(t0); + ld(t0, super_check_addr); // load displayed supertype + + // Ths check has worked decisively for primary supers. + // Secondary supers are sought in the super_cache ('super_cache_addr'). + // (Secondary supers are interfaces and very deeply nested subtypes.) + // This works in the same check above because of a tricky aliasing + // between the super_Cache and the primary super dispaly elements. + // (The 'super_check_addr' can address either, as the case requires.) + // Note that the cache is updated below if it does not help us find + // what we need immediately. + // So if it was a primary super, we can just fail immediately. + // Otherwise, it's the slow path for us (no success at this point). + + beq(super_klass, t0, *L_success); + mv(t1, sc_offset); + if (L_failure == &L_fallthrough) { + beq(super_check_offset, t1, *L_slow_path); + } else { + bne(super_check_offset, t1, *L_failure, /* is_far */ true); + final_jmp(*L_slow_path); + } + + bind(L_fallthrough); + +#undef final_jmp +} + +// Scans count pointer sized words at [addr] for occurence of value, +// generic +void MacroAssembler::repne_scan(Register addr, Register value, Register count, + Register tmp) { + Label Lloop, Lexit; + beqz(count, Lexit); + bind(Lloop); + ld(tmp, addr); + beq(value, tmp, Lexit); + add(addr, addr, wordSize); + sub(count, count, 1); + bnez(count, Lloop); + bind(Lexit); +} + +void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass, + Register super_klass, + Register tmp1_reg, + Register tmp2_reg, + Label* L_success, + Label* L_failure) { + assert_different_registers(sub_klass, super_klass, tmp1_reg); + if (tmp2_reg != noreg) { + assert_different_registers(sub_klass, super_klass, tmp1_reg, tmp2_reg, t0); + } +#define IS_A_TEMP(reg) ((reg) == tmp1_reg || (reg) == tmp2_reg) + + Label L_fallthrough; + int label_nulls = 0; + if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; } + if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; } + + assert(label_nulls <= 1, "at most one NULL in the batch"); + + // A couple of usefule fields in sub_klass: + int ss_offset = in_bytes(Klass::secondary_supers_offset()); + int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); + Address secondary_supers_addr(sub_klass, ss_offset); + Address super_cache_addr( sub_klass, sc_offset); + + BLOCK_COMMENT("check_klass_subtype_slow_path"); + + // Do a linear scan of the secondary super-klass chain. + // This code is rarely used, so simplicity is a virtue here. + // The repne_scan instruction uses fixed registers, which we must spill. + // Don't worry too much about pre-existing connecitons with the input regs. + + assert(sub_klass != x10, "killed reg"); // killed by mv(x10, super) + assert(sub_klass != x12, "killed reg"); // killed by la(x12, &pst_counter) + + RegSet pushed_registers; + if (!IS_A_TEMP(x12)) { + pushed_registers += x12; + } + if (!IS_A_TEMP(x15)) { + pushed_registers += x15; + } + + if (super_klass != x10) { + if (!IS_A_TEMP(x10)) { + pushed_registers += x10; + } + } + + push_reg(pushed_registers, sp); + + // Get super_klass value into x10 (even if it was in x15 or x12) + mv(x10, super_klass); + +#ifndef PRODUCT + mv(t1, (address)&SharedRuntime::_partial_subtype_ctr); + Address pst_counter_addr(t1); + ld(t0, pst_counter_addr); + add(t0, t0, 1); + sd(t0, pst_counter_addr); +#endif // PRODUCT + + // We will consult the secondary-super array. + ld(x15, secondary_supers_addr); + // Load the array length. + lwu(x12, Address(x15, Array::length_offset_in_bytes())); + // Skip to start of data. + add(x15, x15, Array::base_offset_in_bytes()); + + // Set t0 to an obvious invalid value, falling through by default + mv(t0, -1); + // Scan X12 words at [X15] for an occurrence of X10. + repne_scan(x15, x10, x12, t0); + + // pop will restore x10, so we should use a temp register to keep its value + mv(t1, x10); + + // Unspill the temp registers: + pop_reg(pushed_registers, sp); + + bne(t1, t0, *L_failure); + + // Success. Cache the super we found an proceed in triumph. + sd(super_klass, super_cache_addr); + + if (L_success != &L_fallthrough) { + j(*L_success); + } + +#undef IS_A_TEMP + + bind(L_fallthrough); +} + +// Defines obj, preserves var_size_in_bytes, okay for tmp2 == var_size_in_bytes. +void MacroAssembler::tlab_allocate(Register obj, + Register var_size_in_bytes, + int con_size_in_bytes, + Register tmp1, + Register tmp2, + Label& slow_case, + bool is_far) { + BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, tmp1, tmp2, slow_case, is_far); +} + +// Defines obj, preserves var_size_in_bytes +void MacroAssembler::eden_allocate(Register obj, + Register var_size_in_bytes, + int con_size_in_bytes, + Register tmp, + Label& slow_case, + bool is_far) { + BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->eden_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, tmp, slow_case, is_far); +} + + +// get_thread() can be called anywhere inside generated code so we +// need to save whatever non-callee save context might get clobbered +// by the call to Thread::current() or, indeed, the call setup code. +void MacroAssembler::get_thread(Register thread) { + // save all call-clobbered regs except thread + RegSet saved_regs = RegSet::range(x5, x7) + RegSet::range(x10, x17) + + RegSet::range(x28, x31) + ra - thread; + push_reg(saved_regs, sp); + + mv(ra, CAST_FROM_FN_PTR(address, Thread::current)); + jalr(ra); + if (thread != c_rarg0) { + mv(thread, c_rarg0); + } + + // restore pushed registers + pop_reg(saved_regs, sp); +} + +void MacroAssembler::load_byte_map_base(Register reg) { + CardTable::CardValue* byte_map_base = + ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base(); + mv(reg, (uint64_t)byte_map_base); +} + +void MacroAssembler::la_patchable(Register reg1, const Address &dest, int32_t &offset) { + unsigned long low_address = (uintptr_t)CodeCache::low_bound(); + unsigned long high_address = (uintptr_t)CodeCache::high_bound(); + unsigned long dest_address = (uintptr_t)dest.target(); + long offset_low = dest_address - low_address; + long offset_high = dest_address - high_address; + + assert(dest.getMode() == Address::literal, "la_patchable must be applied to a literal address"); + assert((uintptr_t)dest.target() < (1ull << 48), "bad address"); + + // RISC-V doesn't compute a page-aligned address, in order to partially + // compensate for the use of *signed* offsets in its base+disp12 + // addressing mode (RISC-V's PC-relative reach remains asymmetric + // [-(2G + 2K), 2G - 2k). + if (offset_high >= -((1L << 31) + (1L << 11)) && offset_low < (1L << 31) - (1L << 11)) { + int64_t distance = dest.target() - pc(); + auipc(reg1, (int32_t)distance + 0x800); + offset = ((int32_t)distance << 20) >> 20; + } else { + movptr(reg1, dest.target(), offset); + } +} + +void MacroAssembler::build_frame(int framesize) { + assert(framesize >= 2, "framesize must include space for FP/RA"); + assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment"); + sub(sp, sp, framesize); + sd(fp, Address(sp, framesize - 2 * wordSize)); + sd(ra, Address(sp, framesize - wordSize)); + if (PreserveFramePointer) { add(fp, sp, framesize); } +} + +void MacroAssembler::remove_frame(int framesize) { + assert(framesize >= 2, "framesize must include space for FP/RA"); + assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment"); + ld(fp, Address(sp, framesize - 2 * wordSize)); + ld(ra, Address(sp, framesize - wordSize)); + add(sp, sp, framesize); +} + +void MacroAssembler::reserved_stack_check() { + // testing if reserved zone needs to be enabled + Label no_reserved_zone_enabling; + + ld(t0, Address(xthread, JavaThread::reserved_stack_activation_offset())); + bltu(sp, t0, no_reserved_zone_enabling); + + enter(); // RA and FP are live. + mv(c_rarg0, xthread); + RuntimeAddress target(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone)); + relocate(target.rspec(), [&] { + int32_t offset; + la_patchable(t0, target, offset); + jalr(x1, t0, offset); + }); + leave(); + + // We have already removed our own frame. + // throw_delayed_StackOverflowError will think that it's been + // called by our caller. + target = RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry()); + relocate(target.rspec(), [&] { + int32_t offset; + la_patchable(t0, target, offset); + jalr(x0, t0, offset); + }); + should_not_reach_here(); + + bind(no_reserved_zone_enabling); +} + +void MacroAssembler::biased_locking_enter(Register lock_reg, + Register obj_reg, + Register swap_reg, + Register tmp_reg, + bool swap_reg_contains_mark, + Label& done, + Label* slow_case, + BiasedLockingCounters* counters, + Register flag) { + assert(UseBiasedLocking, "why call this otherwise?"); + assert_different_registers(lock_reg, obj_reg, swap_reg); + + if (PrintBiasedLockingStatistics && counters == NULL) { + counters = BiasedLocking::counters(); + } + + assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg, t0, flag); + assert(markWord::age_shift == markWord::lock_bits + markWord::biased_lock_bits, "biased locking makes assumptions about bit layout"); + Address mark_addr (obj_reg, oopDesc::mark_offset_in_bytes()); + + // Biased locking + // See whether the lock is currently biased toward our thread and + // whether the epoch is still valid + // Note that the runtime guarantees sufficient alignment of JavaThread + // pointers to allow age to be placed into low bits + // First check to see whether biasing is even enabled for this object + Label cas_label; + if (!swap_reg_contains_mark) { + ld(swap_reg, mark_addr); + } + andi(tmp_reg, swap_reg, markWord::biased_lock_mask_in_place); + xori(t0, tmp_reg, (u1)markWord::biased_lock_pattern); + bnez(t0, cas_label); // don't care flag unless jumping to done + // The bias pattern is present in the object's header. Need to check + // whether the bias owner and the epoch are both still current. + load_prototype_header(tmp_reg, obj_reg); + orr(tmp_reg, tmp_reg, xthread); + xorr(tmp_reg, tmp_reg, swap_reg); + andi(tmp_reg, tmp_reg, ~((int) markWord::age_mask_in_place)); + if (flag->is_valid()) { + mv(flag, tmp_reg); + } + + if (counters != NULL) { + Label around; + bnez(tmp_reg, around); + atomic_incw(Address((address)counters->biased_lock_entry_count_addr()), tmp_reg, t0); + j(done); + bind(around); + } else { + beqz(tmp_reg, done); + } + + Label try_revoke_bias; + Label try_rebias; + + // At this point we know the header has the bias pattern and + // that we are not the bias owner in the current epoch. We need to + // figure out more details about the state of the header in order to + // know what operations can be legally performed on the object's + // header. + + // If the low three bits in the xor result aren't clear, that means + // the prototype header is no longer biased and we have to revoke + // the bias on this object. + andi(t0, tmp_reg, markWord::biased_lock_mask_in_place); + bnez(t0, try_revoke_bias); + + // Biasing is still enabled for this data type. See whether the + // epoch of the current bias is still valid, meaning that the epoch + // bits of the mark word are equal to the epoch bits of the + // prototype header. (Note that the prototype header's epoch bits + // only change at a safepoint.) If not, attempt to rebias the object + // toward the current thread. Note that we must be absolutely sure + // that the current epoch is invalid in order to do this because + // otherwise the manipulations it performs on the mark word are + // illegal. + andi(t0, tmp_reg, markWord::epoch_mask_in_place); + bnez(t0, try_rebias); + + // The epoch of the current bias is still valid but we know nothing + // about the owner; it might be set or it might be clear. Try to + // acquire the bias of the object using an atomic operation. If this + // fails we will go in to the runtime to revoke the object's bias. + // Note that we first construct the presumed unbiased header so we + // don't accidentally blow away another thread's valid bias. + { + Label cas_success; + Label counter; + li(t0, (int64_t)(markWord::biased_lock_mask_in_place | markWord::age_mask_in_place | markWord::epoch_mask_in_place)); + andr(swap_reg, swap_reg, t0); + orr(tmp_reg, swap_reg, xthread); + cmpxchg_obj_header(swap_reg, tmp_reg, obj_reg, t0, cas_success, slow_case); + // cas failed here if slow_cas == NULL + if (flag->is_valid()) { + li(flag, 1); + j(counter); + } + + // If the biasing toward our thread failed, this means that + // another thread succeeded in biasing it toward itself and we + // need to revoke that bias. The revocation will occur in the + // interpreter runtime in the slow case. + bind(cas_success); + if (flag->is_valid()) { + li(flag, 0); + bind(counter); + } + + if (counters != NULL) { + atomic_incw(Address((address)counters->anonymously_biased_lock_entry_count_addr()), + tmp_reg, t0); + } + } + j(done); + + bind(try_rebias); + // At this point we know the epoch has expired, meaning that the + // current "bias owner", if any, is actually invalid. Under these + // circumstantces _only_, we are allowed to use the current header's + // value as the comparison value when doing the cas to acquire the + // bias in the current epoch. In other words, we allow transfer of + // the bias from one thread to another directly in this situation. + // + // FIXME: due to a lack of registers we currently blow away the age + // bias in this situation. Should attempt to preserve them. + { + Label cas_success; + Label counter; + load_prototype_header(tmp_reg, obj_reg); + orr(tmp_reg, tmp_reg, xthread); + cmpxchg_obj_header(swap_reg, tmp_reg, obj_reg, t0, cas_success, slow_case); + // cas failed here if slow_case == NULL + if (flag->is_valid()) { + li(flag, 1); + j(counter); + } + + // If the biasing toward our thread failed, then another thread + // succeeded in biasing it toward itself and we need to revoke that + // bias. The revocation will occur in the runtime in the slow case. + bind(cas_success); + if (flag->is_valid()) { + li(flag, 0); + bind(counter); + } + + if (counters != NULL) { + atomic_incw(Address((address)counters->rebiased_lock_entry_count_addr()), + tmp_reg, t0); + } + } + j(done); + + // don't care flag unless jumping to done + bind(try_revoke_bias); + // The prototype mark in the klass doesn't have the bias bit set any + // more, indicating that objects of this data type are not supposed + // to be biased any more. We are going to try to reset the mark of + // this object to the prototype value and fail through to the + // CAS-based locking scheme. Note that if our CAS fails, it means + // that another thread raced us for the privilege of revoking the + // bias of this particular object, so it's okay to continue in the + // normal locking code. + // + // FIXME: due to a lack of registers we currently blow away the age + // bits in this situation. Should attempt to preserve them. + { + Label cas_success, nope; + load_prototype_header(tmp_reg, obj_reg); + cmpxchg_obj_header(swap_reg, tmp_reg, obj_reg, t0, cas_success, &nope); + bind(cas_success); + + // Fall through to the normal CAS-based lock, because no matter what + // the result of the above CAS, some thread must have succeeded in + // removing the bias bit from the object's header. + if (counters != NULL) { + atomic_incw(Address((address)counters->revoked_lock_entry_count_addr()), + tmp_reg, t0); + } + bind(nope); + } + + bind(cas_label); +} + +void MacroAssembler::biased_locking_exit(Register obj_reg, Register tmp_reg, Label& done, Register flag) { + assert(UseBiasedLocking, "why call this otherwise"); + + // Check for biased locking unlock case, which is a no-op + // Note: we do not have to check the thread ID for two reasons. + // First, the interpreter checks for IllegalMonitorStateException at + // a higher level. Second, if the bias was revoke while we held the + // lock, the object could not be rebiased toward another, so + // the bias bit would be clear. + ld(tmp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes())); + andi(tmp_reg, tmp_reg, markWord::biased_lock_mask_in_place); + sub(tmp_reg, tmp_reg, markWord::biased_lock_pattern); + if (flag->is_valid()) { + mv(flag, tmp_reg); + } + beqz(tmp_reg, done); +} + +void MacroAssembler::load_prototype_header(Register dst, Register src) { + load_klass(dst, src); + ld(dst, Address(dst, Klass::prototype_header_offset())); +} + +void MacroAssembler::atomic_incw(Register counter_addr, Register tmp) { + Label retry_load; + bind(retry_load); + lr_w(tmp, counter_addr); + addw(tmp, tmp, 1); + sc_w(tmp, tmp, counter_addr); + bnez(tmp, retry_load); +} + +// Move the address of the polling page into dest. +void MacroAssembler::get_polling_page(Register dest, relocInfo::relocType rtype) { + ld(dest, Address(xthread, JavaThread::polling_page_offset())); +} + +// Read the polling page. The address of the polling page must +// already be in r. +void MacroAssembler::read_polling_page(Register r, int32_t offset, relocInfo::relocType rtype) { + relocate(rtype, [&] { + lwu(zr, Address(r, offset)); + }); +} + +void MacroAssembler::set_narrow_oop(Register dst, jobject obj) { +#ifdef ASSERT + { + ThreadInVMfromUnknown tiv; + assert (UseCompressedOops, "should only be used for compressed oops"); + assert (Universe::heap() != NULL, "java heap should be initialized"); + assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); + assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "should be real oop"); + } +#endif + int oop_index = oop_recorder()->find_index(obj); + relocate(oop_Relocation::spec(oop_index), [&] { + li32(dst, 0xDEADBEEF); + }); + zero_extend(dst, dst, 32); +} + +void MacroAssembler::set_narrow_klass(Register dst, Klass* k) { + assert (UseCompressedClassPointers, "should only be used for compressed headers"); + assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); + int index = oop_recorder()->find_index(k); + assert(!Universe::heap()->is_in(k), "should not be an oop"); + + narrowKlass nk = CompressedKlassPointers::encode(k); + relocate(metadata_Relocation::spec(index), [&] { + li32(dst, nk); + }); + zero_extend(dst, dst, 32); +} + +// Maybe emit a call via a trampoline. If the code cache is small +// trampolines won't be emitted. +address MacroAssembler::trampoline_call(Address entry, CodeBuffer* cbuf) { + assert(JavaThread::current()->is_Compiler_thread(), "just checking"); + assert(entry.rspec().type() == relocInfo::runtime_call_type || + entry.rspec().type() == relocInfo::opt_virtual_call_type || + entry.rspec().type() == relocInfo::static_call_type || + entry.rspec().type() == relocInfo::virtual_call_type, "wrong reloc type"); + + // We need a trampoline if branches are far. + if (far_branches()) { + bool in_scratch_emit_size = false; +#ifdef COMPILER2 + // We don't want to emit a trampoline if C2 is generating dummy + // code during its branch shortening phase. + CompileTask* task = ciEnv::current()->task(); + in_scratch_emit_size = + (task != NULL && is_c2_compile(task->comp_level()) && + Compile::current()->output()->in_scratch_emit_size()); +#endif + if (!in_scratch_emit_size) { + address stub = emit_trampoline_stub(offset(), entry.target()); + if (stub == NULL) { + postcond(pc() == badAddress); + return NULL; // CodeCache is full + } + } + } + + if (cbuf != NULL) { cbuf->set_insts_mark(); } +#ifdef ASSERT + if (entry.rspec().type() != relocInfo::runtime_call_type) { + assert_alignment(pc()); + } +#endif + relocate(entry.rspec(), [&] { + if (!far_branches()) { + jal(entry.target()); + } else { + jal(pc()); + } + }); + // just need to return a non-null address + postcond(pc() != badAddress); + return pc(); +} + +address MacroAssembler::ic_call(address entry, jint method_index) { + RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index); + IncompressibleRegion ir(this); // relocations + movptr(t1, (address)Universe::non_oop_word()); + assert_cond(entry != NULL); + return trampoline_call(Address(entry, rh)); +} + +// Emit a trampoline stub for a call to a target which is too far away. +// +// code sequences: +// +// call-site: +// branch-and-link to or +// +// Related trampoline stub for this call site in the stub section: +// load the call target from the constant pool +// branch (RA still points to the call site above) + +address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset, + address dest) { + address stub = start_a_stub(NativeInstruction::instruction_size + + NativeCallTrampolineStub::instruction_size); + if (stub == NULL) { + return NULL; // CodeBuffer::expand failed + } + + // Create a trampoline stub relocation which relates this trampoline stub + // with the call instruction at insts_call_instruction_offset in the + // instructions code-section. + + // Make sure the address of destination 8-byte aligned after 3 instructions. + align(wordSize, NativeCallTrampolineStub::data_offset); + + RelocationHolder rh = trampoline_stub_Relocation::spec(code()->insts()->start() + + insts_call_instruction_offset); + const int stub_start_offset = offset(); + relocate(rh, [&] { + // Now, create the trampoline stub's code: + // - load the call + // - call + Label target; + ld(t0, target); // auipc + ld + jr(t0); // jalr + bind(target); + assert(offset() - stub_start_offset == NativeCallTrampolineStub::data_offset, + "should be"); + assert(offset() % wordSize == 0, "bad alignment"); + emit_int64((int64_t)dest); + }); + + const address stub_start_addr = addr_at(stub_start_offset); + + assert(is_NativeCallTrampolineStub_at(stub_start_addr), "doesn't look like a trampoline"); + + end_a_stub(); + return stub_start_addr; +} + +Address MacroAssembler::add_memory_helper(const Address dst, Register tmp) { + switch (dst.getMode()) { + case Address::base_plus_offset: + // This is the expected mode, although we allow all the other + // forms below. + return form_address(tmp, dst.base(), dst.offset()); + default: + la(tmp, dst); + return Address(tmp); + } +} + +void MacroAssembler::increment(const Address dst, int64_t value, Register tmp1, Register tmp2) { + assert(((dst.getMode() == Address::base_plus_offset && + is_simm12(dst.offset())) || is_simm12(value)), + "invalid value and address mode combination"); + Address adr = add_memory_helper(dst, tmp2); + assert(!adr.uses(tmp1), "invalid dst for address increment"); + ld(tmp1, adr); + add(tmp1, tmp1, value, tmp2); + sd(tmp1, adr); +} + +void MacroAssembler::incrementw(const Address dst, int32_t value, Register tmp1, Register tmp2) { + assert(((dst.getMode() == Address::base_plus_offset && + is_simm12(dst.offset())) || is_simm12(value)), + "invalid value and address mode combination"); + Address adr = add_memory_helper(dst, tmp2); + assert(!adr.uses(tmp1), "invalid dst for address increment"); + lwu(tmp1, adr); + addw(tmp1, tmp1, value, tmp2); + sw(tmp1, adr); +} + +void MacroAssembler::decrement(const Address dst, int64_t value, Register tmp1, Register tmp2) { + assert(((dst.getMode() == Address::base_plus_offset && + is_simm12(dst.offset())) || is_simm12(value)), + "invalid value and address mode combination"); + Address adr = add_memory_helper(dst, tmp2); + assert(!adr.uses(tmp1), "invalid dst for address decrement"); + ld(tmp1, adr); + sub(tmp1, tmp1, value, tmp2); + sd(tmp1, adr); +} + +void MacroAssembler::decrementw(const Address dst, int32_t value, Register tmp1, Register tmp2) { + assert(((dst.getMode() == Address::base_plus_offset && + is_simm12(dst.offset())) || is_simm12(value)), + "invalid value and address mode combination"); + Address adr = add_memory_helper(dst, tmp2); + assert(!adr.uses(tmp1), "invalid dst for address decrement"); + lwu(tmp1, adr); + subw(tmp1, tmp1, value, tmp2); + sw(tmp1, adr); +} + +void MacroAssembler::cmpptr(Register src1, Address src2, Label& equal) { + assert_different_registers(src1, t0); + relocate(src2.rspec(), [&] { + int32_t offset; + la_patchable(t0, src2, offset); + ld(t0, Address(t0, offset)); + }); + beq(src1, t0, equal); +} + +void MacroAssembler::load_method_holder_cld(Register result, Register method) { + load_method_holder(result, method); + ld(result, Address(result, InstanceKlass::class_loader_data_offset())); +} + +void MacroAssembler::load_method_holder(Register holder, Register method) { + ld(holder, Address(method, Method::const_offset())); // ConstMethod* + ld(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool* + ld(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass* +} + +// string indexof +// compute index by trailing zeros +void MacroAssembler::compute_index(Register haystack, Register trailing_zeros, + Register match_mask, Register result, + Register ch2, Register tmp, + bool haystack_isL) { + int haystack_chr_shift = haystack_isL ? 0 : 1; + srl(match_mask, match_mask, trailing_zeros); + srli(match_mask, match_mask, 1); + srli(tmp, trailing_zeros, LogBitsPerByte); + if (!haystack_isL) andi(tmp, tmp, 0xE); + add(haystack, haystack, tmp); + ld(ch2, Address(haystack)); + if (!haystack_isL) srli(tmp, tmp, haystack_chr_shift); + add(result, result, tmp); +} + +// string indexof +// Find pattern element in src, compute match mask, +// only the first occurrence of 0x80/0x8000 at low bits is the valid match index +// match mask patterns and corresponding indices would be like: +// - 0x8080808080808080 (Latin1) +// - 7 6 5 4 3 2 1 0 (match index) +// - 0x8000800080008000 (UTF16) +// - 3 2 1 0 (match index) +void MacroAssembler::compute_match_mask(Register src, Register pattern, Register match_mask, + Register mask1, Register mask2) { + xorr(src, pattern, src); + sub(match_mask, src, mask1); + orr(src, src, mask2); + notr(src, src); + andr(match_mask, match_mask, src); +} + +#ifdef COMPILER2 +// Code for BigInteger::mulAdd instrinsic +// out = x10 +// in = x11 +// offset = x12 (already out.length-offset) +// len = x13 +// k = x14 +// tmp = x28 +// +// pseudo code from java implementation: +// long kLong = k & LONG_MASK; +// carry = 0; +// offset = out.length-offset - 1; +// for (int j = len - 1; j >= 0; j--) { +// product = (in[j] & LONG_MASK) * kLong + (out[offset] & LONG_MASK) + carry; +// out[offset--] = (int)product; +// carry = product >>> 32; +// } +// return (int)carry; +void MacroAssembler::mul_add(Register out, Register in, Register offset, + Register len, Register k, Register tmp) { + Label L_tail_loop, L_unroll, L_end; + mv(tmp, out); + mv(out, zr); + blez(len, L_end); + zero_extend(k, k, 32); + slliw(t0, offset, LogBytesPerInt); + add(offset, tmp, t0); + slliw(t0, len, LogBytesPerInt); + add(in, in, t0); + + const int unroll = 8; + mv(tmp, unroll); + blt(len, tmp, L_tail_loop); + bind(L_unroll); + for (int i = 0; i < unroll; i++) { + sub(in, in, BytesPerInt); + lwu(t0, Address(in, 0)); + mul(t1, t0, k); + add(t0, t1, out); + sub(offset, offset, BytesPerInt); + lwu(t1, Address(offset, 0)); + add(t0, t0, t1); + sw(t0, Address(offset, 0)); + srli(out, t0, 32); + } + subw(len, len, tmp); + bge(len, tmp, L_unroll); + + bind(L_tail_loop); + blez(len, L_end); + sub(in, in, BytesPerInt); + lwu(t0, Address(in, 0)); + mul(t1, t0, k); + add(t0, t1, out); + sub(offset, offset, BytesPerInt); + lwu(t1, Address(offset, 0)); + add(t0, t0, t1); + sw(t0, Address(offset, 0)); + srli(out, t0, 32); + subw(len, len, 1); + j(L_tail_loop); + + bind(L_end); +} + +// add two unsigned input and output carry +void MacroAssembler::cad(Register dst, Register src1, Register src2, Register carry) +{ + assert_different_registers(dst, carry); + assert_different_registers(dst, src2); + add(dst, src1, src2); + sltu(carry, dst, src2); +} + +// add two input with carry +void MacroAssembler::adc(Register dst, Register src1, Register src2, Register carry) { + assert_different_registers(dst, carry); + add(dst, src1, src2); + add(dst, dst, carry); +} + +// add two unsigned input with carry and output carry +void MacroAssembler::cadc(Register dst, Register src1, Register src2, Register carry) { + assert_different_registers(dst, src2); + adc(dst, src1, src2, carry); + sltu(carry, dst, src2); +} + +void MacroAssembler::add2_with_carry(Register final_dest_hi, Register dest_hi, Register dest_lo, + Register src1, Register src2, Register carry) { + cad(dest_lo, dest_lo, src1, carry); + add(dest_hi, dest_hi, carry); + cad(dest_lo, dest_lo, src2, carry); + add(final_dest_hi, dest_hi, carry); +} + +/** + * Multiply 32 bit by 32 bit first loop. + */ +void MacroAssembler::multiply_32_x_32_loop(Register x, Register xstart, Register x_xstart, + Register y, Register y_idx, Register z, + Register carry, Register product, + Register idx, Register kdx) { + // jlong carry, x[], y[], z[]; + // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx--, kdx--) { + // long product = y[idx] * x[xstart] + carry; + // z[kdx] = (int)product; + // carry = product >>> 32; + // } + // z[xstart] = (int)carry; + + Label L_first_loop, L_first_loop_exit; + blez(idx, L_first_loop_exit); + + shadd(t0, xstart, x, t0, LogBytesPerInt); + lwu(x_xstart, Address(t0, 0)); + + bind(L_first_loop); + subw(idx, idx, 1); + shadd(t0, idx, y, t0, LogBytesPerInt); + lwu(y_idx, Address(t0, 0)); + mul(product, x_xstart, y_idx); + add(product, product, carry); + srli(carry, product, 32); + subw(kdx, kdx, 1); + shadd(t0, kdx, z, t0, LogBytesPerInt); + sw(product, Address(t0, 0)); + bgtz(idx, L_first_loop); + + bind(L_first_loop_exit); +} + +/** + * Multiply 64 bit by 64 bit first loop. + */ +void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart, + Register y, Register y_idx, Register z, + Register carry, Register product, + Register idx, Register kdx) { + // + // jlong carry, x[], y[], z[]; + // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx--, kdx--) { + // huge_128 product = y[idx] * x[xstart] + carry; + // z[kdx] = (jlong)product; + // carry = (jlong)(product >>> 64); + // } + // z[xstart] = carry; + // + + Label L_first_loop, L_first_loop_exit; + Label L_one_x, L_one_y, L_multiply; + + subw(xstart, xstart, 1); + bltz(xstart, L_one_x); + + shadd(t0, xstart, x, t0, LogBytesPerInt); + ld(x_xstart, Address(t0, 0)); + ror_imm(x_xstart, x_xstart, 32); // convert big-endian to little-endian + + bind(L_first_loop); + subw(idx, idx, 1); + bltz(idx, L_first_loop_exit); + subw(idx, idx, 1); + bltz(idx, L_one_y); + + shadd(t0, idx, y, t0, LogBytesPerInt); + ld(y_idx, Address(t0, 0)); + ror_imm(y_idx, y_idx, 32); // convert big-endian to little-endian + bind(L_multiply); + + mulhu(t0, x_xstart, y_idx); + mul(product, x_xstart, y_idx); + cad(product, product, carry, t1); + adc(carry, t0, zr, t1); + + subw(kdx, kdx, 2); + ror_imm(product, product, 32); // back to big-endian + shadd(t0, kdx, z, t0, LogBytesPerInt); + sd(product, Address(t0, 0)); + + j(L_first_loop); + + bind(L_one_y); + lwu(y_idx, Address(y, 0)); + j(L_multiply); + + bind(L_one_x); + lwu(x_xstart, Address(x, 0)); + j(L_first_loop); + + bind(L_first_loop_exit); +} + +/** + * Multiply 128 bit by 128 bit. Unrolled inner loop. + * + */ +void MacroAssembler::multiply_128_x_128_loop(Register y, Register z, + Register carry, Register carry2, + Register idx, Register jdx, + Register yz_idx1, Register yz_idx2, + Register tmp, Register tmp3, Register tmp4, + Register tmp6, Register product_hi) { + // jlong carry, x[], y[], z[]; + // int kdx = xstart+1; + // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop + // huge_128 tmp3 = (y[idx+1] * product_hi) + z[kdx+idx+1] + carry; + // jlong carry2 = (jlong)(tmp3 >>> 64); + // huge_128 tmp4 = (y[idx] * product_hi) + z[kdx+idx] + carry2; + // carry = (jlong)(tmp4 >>> 64); + // z[kdx+idx+1] = (jlong)tmp3; + // z[kdx+idx] = (jlong)tmp4; + // } + // idx += 2; + // if (idx > 0) { + // yz_idx1 = (y[idx] * product_hi) + z[kdx+idx] + carry; + // z[kdx+idx] = (jlong)yz_idx1; + // carry = (jlong)(yz_idx1 >>> 64); + // } + // + + Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; + + srliw(jdx, idx, 2); + + bind(L_third_loop); + + subw(jdx, jdx, 1); + bltz(jdx, L_third_loop_exit); + subw(idx, idx, 4); + + shadd(t0, idx, y, t0, LogBytesPerInt); + ld(yz_idx2, Address(t0, 0)); + ld(yz_idx1, Address(t0, wordSize)); + + shadd(tmp6, idx, z, t0, LogBytesPerInt); + + ror_imm(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian + ror_imm(yz_idx2, yz_idx2, 32); + + ld(t1, Address(tmp6, 0)); + ld(t0, Address(tmp6, wordSize)); + + mul(tmp3, product_hi, yz_idx1); // yz_idx1 * product_hi -> tmp4:tmp3 + mulhu(tmp4, product_hi, yz_idx1); + + ror_imm(t0, t0, 32, tmp); // convert big-endian to little-endian + ror_imm(t1, t1, 32, tmp); + + mul(tmp, product_hi, yz_idx2); // yz_idx2 * product_hi -> carry2:tmp + mulhu(carry2, product_hi, yz_idx2); + + cad(tmp3, tmp3, carry, carry); + adc(tmp4, tmp4, zr, carry); + cad(tmp3, tmp3, t0, t0); + cadc(tmp4, tmp4, tmp, t0); + adc(carry, carry2, zr, t0); + cad(tmp4, tmp4, t1, carry2); + adc(carry, carry, zr, carry2); + + ror_imm(tmp3, tmp3, 32); // convert little-endian to big-endian + ror_imm(tmp4, tmp4, 32); + sd(tmp4, Address(tmp6, 0)); + sd(tmp3, Address(tmp6, wordSize)); + + j(L_third_loop); + + bind(L_third_loop_exit); + + andi(idx, idx, 0x3); + beqz(idx, L_post_third_loop_done); + + Label L_check_1; + subw(idx, idx, 2); + bltz(idx, L_check_1); + + shadd(t0, idx, y, t0, LogBytesPerInt); + ld(yz_idx1, Address(t0, 0)); + ror_imm(yz_idx1, yz_idx1, 32); + + mul(tmp3, product_hi, yz_idx1); // yz_idx1 * product_hi -> tmp4:tmp3 + mulhu(tmp4, product_hi, yz_idx1); + + shadd(t0, idx, z, t0, LogBytesPerInt); + ld(yz_idx2, Address(t0, 0)); + ror_imm(yz_idx2, yz_idx2, 32, tmp); + + add2_with_carry(carry, tmp4, tmp3, carry, yz_idx2, tmp); + + ror_imm(tmp3, tmp3, 32, tmp); + sd(tmp3, Address(t0, 0)); + + bind(L_check_1); + + andi(idx, idx, 0x1); + subw(idx, idx, 1); + bltz(idx, L_post_third_loop_done); + shadd(t0, idx, y, t0, LogBytesPerInt); + lwu(tmp4, Address(t0, 0)); + mul(tmp3, tmp4, product_hi); // tmp4 * product_hi -> carry2:tmp3 + mulhu(carry2, tmp4, product_hi); + + shadd(t0, idx, z, t0, LogBytesPerInt); + lwu(tmp4, Address(t0, 0)); + + add2_with_carry(carry2, carry2, tmp3, tmp4, carry, t0); + + shadd(t0, idx, z, t0, LogBytesPerInt); + sw(tmp3, Address(t0, 0)); + + slli(t0, carry2, 32); + srli(carry, tmp3, 32); + orr(carry, carry, t0); + + bind(L_post_third_loop_done); +} + +/** + * Code for BigInteger::multiplyToLen() intrinsic. + * + * x10: x + * x11: xlen + * x12: y + * x13: ylen + * x14: z + * x15: zlen + * x16: tmp1 + * x17: tmp2 + * x7: tmp3 + * x28: tmp4 + * x29: tmp5 + * x30: tmp6 + * x31: tmp7 + */ +void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, + Register z, Register zlen, + Register tmp1, Register tmp2, Register tmp3, Register tmp4, + Register tmp5, Register tmp6, Register product_hi) { + assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6); + + const Register idx = tmp1; + const Register kdx = tmp2; + const Register xstart = tmp3; + + const Register y_idx = tmp4; + const Register carry = tmp5; + const Register product = xlen; + const Register x_xstart = zlen; // reuse register + + mv(idx, ylen); // idx = ylen; + mv(kdx, zlen); // kdx = xlen+ylen; + mv(carry, zr); // carry = 0; + + Label L_multiply_64_x_64_loop, L_done; + + subw(xstart, xlen, 1); + bltz(xstart, L_done); + + const Register jdx = tmp1; + + if (AvoidUnalignedAccesses) { + // Check if x and y are both 8-byte aligned. + orr(t0, xlen, ylen); + test_bit(t0, t0, 0); + beqz(t0, L_multiply_64_x_64_loop); + + multiply_32_x_32_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx); + shadd(t0, xstart, z, t0, LogBytesPerInt); + sw(carry, Address(t0, 0)); + + Label L_second_loop_unaligned; + bind(L_second_loop_unaligned); + mv(carry, zr); + mv(jdx, ylen); + subw(xstart, xstart, 1); + bltz(xstart, L_done); + sub(sp, sp, 2 * wordSize); + sd(z, Address(sp, 0)); + sd(zr, Address(sp, wordSize)); + shadd(t0, xstart, z, t0, LogBytesPerInt); + addi(z, t0, 4); + shadd(t0, xstart, x, t0, LogBytesPerInt); + lwu(product, Address(t0, 0)); + Label L_third_loop, L_third_loop_exit; + + blez(jdx, L_third_loop_exit); + + bind(L_third_loop); + subw(jdx, jdx, 1); + shadd(t0, jdx, y, t0, LogBytesPerInt); + lwu(t0, Address(t0, 0)); + mul(t1, t0, product); + add(t0, t1, carry); + shadd(tmp6, jdx, z, t1, LogBytesPerInt); + lwu(t1, Address(tmp6, 0)); + add(t0, t0, t1); + sw(t0, Address(tmp6, 0)); + srli(carry, t0, 32); + bgtz(jdx, L_third_loop); + + bind(L_third_loop_exit); + ld(z, Address(sp, 0)); + addi(sp, sp, 2 * wordSize); + shadd(t0, xstart, z, t0, LogBytesPerInt); + sw(carry, Address(t0, 0)); + + j(L_second_loop_unaligned); + } + + bind(L_multiply_64_x_64_loop); + multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx); + + Label L_second_loop_aligned; + beqz(kdx, L_second_loop_aligned); + + Label L_carry; + subw(kdx, kdx, 1); + beqz(kdx, L_carry); + + shadd(t0, kdx, z, t0, LogBytesPerInt); + sw(carry, Address(t0, 0)); + srli(carry, carry, 32); + subw(kdx, kdx, 1); + + bind(L_carry); + shadd(t0, kdx, z, t0, LogBytesPerInt); + sw(carry, Address(t0, 0)); + + // Second and third (nested) loops. + // + // for (int i = xstart-1; i >= 0; i--) { // Second loop + // carry = 0; + // for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop + // long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) + + // (z[k] & LONG_MASK) + carry; + // z[k] = (int)product; + // carry = product >>> 32; + // } + // z[i] = (int)carry; + // } + // + // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = product_hi + + bind(L_second_loop_aligned); + mv(carry, zr); // carry = 0; + mv(jdx, ylen); // j = ystart+1 + + subw(xstart, xstart, 1); // i = xstart-1; + bltz(xstart, L_done); + + sub(sp, sp, 4 * wordSize); + sd(z, Address(sp, 0)); + + Label L_last_x; + shadd(t0, xstart, z, t0, LogBytesPerInt); + addi(z, t0, 4); + subw(xstart, xstart, 1); // i = xstart-1; + bltz(xstart, L_last_x); + + shadd(t0, xstart, x, t0, LogBytesPerInt); + ld(product_hi, Address(t0, 0)); + ror_imm(product_hi, product_hi, 32); // convert big-endian to little-endian + + Label L_third_loop_prologue; + bind(L_third_loop_prologue); + + sd(ylen, Address(sp, wordSize)); + sd(x, Address(sp, 2 * wordSize)); + sd(xstart, Address(sp, 3 * wordSize)); + multiply_128_x_128_loop(y, z, carry, x, jdx, ylen, product, + tmp2, x_xstart, tmp3, tmp4, tmp6, product_hi); + ld(z, Address(sp, 0)); + ld(ylen, Address(sp, wordSize)); + ld(x, Address(sp, 2 * wordSize)); + ld(xlen, Address(sp, 3 * wordSize)); // copy old xstart -> xlen + addi(sp, sp, 4 * wordSize); + + addiw(tmp3, xlen, 1); + shadd(t0, tmp3, z, t0, LogBytesPerInt); + sw(carry, Address(t0, 0)); + + subw(tmp3, tmp3, 1); + bltz(tmp3, L_done); + + srli(carry, carry, 32); + shadd(t0, tmp3, z, t0, LogBytesPerInt); + sw(carry, Address(t0, 0)); + j(L_second_loop_aligned); + + // Next infrequent code is moved outside loops. + bind(L_last_x); + lwu(product_hi, Address(x, 0)); + j(L_third_loop_prologue); + + bind(L_done); +} +#endif + +// Count bits of trailing zero chars from lsb to msb until first non-zero element. +// For LL case, one byte for one element, so shift 8 bits once, and for other case, +// shift 16 bits once. +void MacroAssembler::ctzc_bit(Register Rd, Register Rs, bool isLL, Register tmp1, Register tmp2) { + if (UseZbb) { + assert_different_registers(Rd, Rs, tmp1); + int step = isLL ? 8 : 16; + ctz(Rd, Rs); + andi(tmp1, Rd, step - 1); + sub(Rd, Rd, tmp1); + return; + } + + assert_different_registers(Rd, Rs, tmp1, tmp2); + Label Loop; + int step = isLL ? 8 : 16; + mv(Rd, -step); + mv(tmp2, Rs); + + bind(Loop); + addi(Rd, Rd, step); + andi(tmp1, tmp2, ((1 << step) - 1)); + srli(tmp2, tmp2, step); + beqz(tmp1, Loop); +} + +// This instruction reads adjacent 4 bytes from the lower half of source register, +// inflate into a register, for example: +// Rs: A7A6A5A4A3A2A1A0 +// Rd: 00A300A200A100A0 +void MacroAssembler::inflate_lo32(Register Rd, Register Rs, Register tmp1, Register tmp2) { + assert_different_registers(Rd, Rs, tmp1, tmp2); + + mv(tmp1, 0xFF); + mv(Rd, zr); + for (int i = 0; i <= 3; i++) { + andr(tmp2, Rs, tmp1); + if (i) { + slli(tmp2, tmp2, i * 8); + } + orr(Rd, Rd, tmp2); + if (i != 3) { + slli(tmp1, tmp1, 8); + } + } +} + +// This instruction reads adjacent 4 bytes from the upper half of source register, +// inflate into a register, for example: +// Rs: A7A6A5A4A3A2A1A0 +// Rd: 00A700A600A500A4 +void MacroAssembler::inflate_hi32(Register Rd, Register Rs, Register tmp1, Register tmp2) { + assert_different_registers(Rd, Rs, tmp1, tmp2); + + mv(tmp1, 0xFF00000000); + mv(Rd, zr); + for (int i = 0; i <= 3; i++) { + andr(tmp2, Rs, tmp1); + orr(Rd, Rd, tmp2); + srli(Rd, Rd, 8); + if (i != 3) { + slli(tmp1, tmp1, 8); + } + } +} + +// The size of the blocks erased by the zero_blocks stub. We must +// handle anything smaller than this ourselves in zero_words(). +const int MacroAssembler::zero_words_block_size = 8; + +// zero_words() is used by C2 ClearArray patterns. It is as small as +// possible, handling small word counts locally and delegating +// anything larger to the zero_blocks stub. It is expanded many times +// in compiled code, so it is important to keep it short. + +// ptr: Address of a buffer to be zeroed. +// cnt: Count in HeapWords. +// +// ptr, cnt, and t0 are clobbered. +address MacroAssembler::zero_words(Register ptr, Register cnt) { + assert(is_power_of_2(zero_words_block_size), "adjust this"); + assert(ptr == x28 && cnt == x29, "mismatch in register usage"); + assert_different_registers(cnt, t0); + + BLOCK_COMMENT("zero_words {"); + + mv(t0, zero_words_block_size); + Label around, done, done16; + bltu(cnt, t0, around); + { + RuntimeAddress zero_blocks = RuntimeAddress(StubRoutines::riscv::zero_blocks()); + assert(zero_blocks.target() != NULL, "zero_blocks stub has not been generated"); + if (StubRoutines::riscv::complete()) { + address tpc = trampoline_call(zero_blocks); + if (tpc == NULL) { + DEBUG_ONLY(reset_labels(around)); + postcond(pc() == badAddress); + return NULL; + } + } else { + jal(zero_blocks); + } + } + bind(around); + for (int i = zero_words_block_size >> 1; i > 1; i >>= 1) { + Label l; + test_bit(t0, cnt, exact_log2(i)); + beqz(t0, l); + for (int j = 0; j < i; j++) { + sd(zr, Address(ptr, 0)); + addi(ptr, ptr, 8); + } + bind(l); + } + { + Label l; + test_bit(t0, cnt, 0); + beqz(t0, l); + sd(zr, Address(ptr, 0)); + bind(l); + } + + BLOCK_COMMENT("} zero_words"); + postcond(pc() != badAddress); + return pc(); +} + +#define SmallArraySize (18 * BytesPerLong) + +// base: Address of a buffer to be zeroed, 8 bytes aligned. +// cnt: Immediate count in HeapWords. +void MacroAssembler::zero_words(Register base, u_int64_t cnt) { + assert_different_registers(base, t0, t1); + + BLOCK_COMMENT("zero_words {"); + + if (cnt <= SmallArraySize / BytesPerLong) { + for (int i = 0; i < (int)cnt; i++) { + sd(zr, Address(base, i * wordSize)); + } + } else { + const int unroll = 8; // Number of sd(zr, adr), instructions we'll unroll + int remainder = cnt % unroll; + for (int i = 0; i < remainder; i++) { + sd(zr, Address(base, i * wordSize)); + } + + Label loop; + Register cnt_reg = t0; + Register loop_base = t1; + cnt = cnt - remainder; + mv(cnt_reg, cnt); + add(loop_base, base, remainder * wordSize); + bind(loop); + sub(cnt_reg, cnt_reg, unroll); + for (int i = 0; i < unroll; i++) { + sd(zr, Address(loop_base, i * wordSize)); + } + add(loop_base, loop_base, unroll * wordSize); + bnez(cnt_reg, loop); + } + + BLOCK_COMMENT("} zero_words"); +} + +// base: Address of a buffer to be filled, 8 bytes aligned. +// cnt: Count in 8-byte unit. +// value: Value to be filled with. +// base will point to the end of the buffer after filling. +void MacroAssembler::fill_words(Register base, Register cnt, Register value) { +// Algorithm: +// +// t0 = cnt & 7 +// cnt -= t0 +// p += t0 +// switch (t0): +// switch start: +// do while cnt +// cnt -= 8 +// p[-8] = value +// case 7: +// p[-7] = value +// case 6: +// p[-6] = value +// // ... +// case 1: +// p[-1] = value +// case 0: +// p += 8 +// do-while end +// switch end + + assert_different_registers(base, cnt, value, t0, t1); + + Label fini, skip, entry, loop; + const int unroll = 8; // Number of sd instructions we'll unroll + + beqz(cnt, fini); + + andi(t0, cnt, unroll - 1); + sub(cnt, cnt, t0); + // align 8, so first sd n % 8 = mod, next loop sd 8 * n. + shadd(base, t0, base, t1, 3); + la(t1, entry); + slli(t0, t0, 2); // sd_inst_nums * 4; t0 is cnt % 8, so t1 = t1 - sd_inst_nums * 4, 4 is sizeof(inst) + sub(t1, t1, t0); + jr(t1); + + bind(loop); + add(base, base, unroll * 8); + for (int i = -unroll; i < 0; i++) { + sd(value, Address(base, i * 8)); + } + bind(entry); + sub(cnt, cnt, unroll); + bgez(cnt, loop); + + bind(fini); +} + +#define FCVT_SAFE(FLOATCVT, FLOATSIG) \ +void MacroAssembler::FLOATCVT##_safe(Register dst, FloatRegister src, Register tmp) { \ + Label done; \ + assert_different_registers(dst, tmp); \ + fclass_##FLOATSIG(tmp, src); \ + mv(dst, zr); \ + /* check if src is NaN */ \ + andi(tmp, tmp, 0b1100000000); \ + bnez(tmp, done); \ + FLOATCVT(dst, src); \ + bind(done); \ +} + +FCVT_SAFE(fcvt_w_s, s); +FCVT_SAFE(fcvt_l_s, s); +FCVT_SAFE(fcvt_w_d, d); +FCVT_SAFE(fcvt_l_d, d); + +#undef FCVT_SAFE + +#define FCMP(FLOATTYPE, FLOATSIG) \ +void MacroAssembler::FLOATTYPE##_compare(Register result, FloatRegister Rs1, \ + FloatRegister Rs2, int unordered_result) { \ + Label Ldone; \ + if (unordered_result < 0) { \ + /* we want -1 for unordered or less than, 0 for equal and 1 for greater than. */ \ + /* installs 1 if gt else 0 */ \ + flt_##FLOATSIG(result, Rs2, Rs1); \ + /* Rs1 > Rs2, install 1 */ \ + bgtz(result, Ldone); \ + feq_##FLOATSIG(result, Rs1, Rs2); \ + addi(result, result, -1); \ + /* Rs1 = Rs2, install 0 */ \ + /* NaN or Rs1 < Rs2, install -1 */ \ + bind(Ldone); \ + } else { \ + /* we want -1 for less than, 0 for equal and 1 for unordered or greater than. */ \ + /* installs 1 if gt or unordered else 0 */ \ + flt_##FLOATSIG(result, Rs1, Rs2); \ + /* Rs1 < Rs2, install -1 */ \ + bgtz(result, Ldone); \ + feq_##FLOATSIG(result, Rs1, Rs2); \ + addi(result, result, -1); \ + /* Rs1 = Rs2, install 0 */ \ + /* NaN or Rs1 > Rs2, install 1 */ \ + bind(Ldone); \ + neg(result, result); \ + } \ +} + +FCMP(float, s); +FCMP(double, d); + +#undef FCMP + +// Zero words; len is in bytes +// Destroys all registers except addr +// len must be a nonzero multiple of wordSize +void MacroAssembler::zero_memory(Register addr, Register len, Register tmp) { + assert_different_registers(addr, len, tmp, t0, t1); + +#ifdef ASSERT + { + Label L; + andi(t0, len, BytesPerWord - 1); + beqz(t0, L); + stop("len is not a multiple of BytesPerWord"); + bind(L); + } +#endif // ASSERT + +#ifndef PRODUCT + block_comment("zero memory"); +#endif // PRODUCT + + Label loop; + Label entry; + + // Algorithm: + // + // t0 = cnt & 7 + // cnt -= t0 + // p += t0 + // switch (t0) { + // do { + // cnt -= 8 + // p[-8] = 0 + // case 7: + // p[-7] = 0 + // case 6: + // p[-6] = 0 + // ... + // case 1: + // p[-1] = 0 + // case 0: + // p += 8 + // } while (cnt) + // } + + const int unroll = 8; // Number of sd(zr) instructions we'll unroll + + srli(len, len, LogBytesPerWord); + andi(t0, len, unroll - 1); // t0 = cnt % unroll + sub(len, len, t0); // cnt -= unroll + // tmp always points to the end of the region we're about to zero + shadd(tmp, t0, addr, t1, LogBytesPerWord); + la(t1, entry); + slli(t0, t0, 2); + sub(t1, t1, t0); + jr(t1); + bind(loop); + sub(len, len, unroll); + for (int i = -unroll; i < 0; i++) { + sd(zr, Address(tmp, i * wordSize)); + } + bind(entry); + add(tmp, tmp, unroll * wordSize); + bnez(len, loop); +} + +// shift left by shamt and add +// Rd = (Rs1 << shamt) + Rs2 +void MacroAssembler::shadd(Register Rd, Register Rs1, Register Rs2, Register tmp, int shamt) { + if (UseZba) { + if (shamt == 1) { + sh1add(Rd, Rs1, Rs2); + return; + } else if (shamt == 2) { + sh2add(Rd, Rs1, Rs2); + return; + } else if (shamt == 3) { + sh3add(Rd, Rs1, Rs2); + return; + } + } + + if (shamt != 0) { + slli(tmp, Rs1, shamt); + add(Rd, Rs2, tmp); + } else { + add(Rd, Rs1, Rs2); + } +} + +void MacroAssembler::zero_extend(Register dst, Register src, int bits) { + if (UseZba && bits == 32) { + zext_w(dst, src); + return; + } + + if (UseZbb && bits == 16) { + zext_h(dst, src); + return; + } + + if (bits == 8) { + zext_b(dst, src); + } else { + slli(dst, src, XLEN - bits); + srli(dst, dst, XLEN - bits); + } +} + +void MacroAssembler::sign_extend(Register dst, Register src, int bits) { + if (UseZbb) { + if (bits == 8) { + sext_b(dst, src); + return; + } else if (bits == 16) { + sext_h(dst, src); + return; + } + } + + if (bits == 32) { + sext_w(dst, src); + } else { + slli(dst, src, XLEN - bits); + srai(dst, dst, XLEN - bits); + } +} + +void MacroAssembler::cmp_l2i(Register dst, Register src1, Register src2, Register tmp) +{ + if (src1 == src2) { + mv(dst, zr); + return; + } + Label done; + Register left = src1; + Register right = src2; + if (dst == src1) { + assert_different_registers(dst, src2, tmp); + mv(tmp, src1); + left = tmp; + } else if (dst == src2) { + assert_different_registers(dst, src1, tmp); + mv(tmp, src2); + right = tmp; + } + + // installs 1 if gt else 0 + slt(dst, right, left); + bnez(dst, done); + slt(dst, left, right); + // dst = -1 if lt; else if eq , dst = 0 + neg(dst, dst); + bind(done); +} + +void MacroAssembler::test_bit(Register Rd, Register Rs, uint32_t bit_pos, Register tmp) { + assert(bit_pos < 64, "invalid bit range"); + if (UseZbs) { + bexti(Rd, Rs, bit_pos); + return; + } + andi(Rd, Rs, 1UL << bit_pos, tmp); +} diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp new file mode 100644 index 0000000000000..d92031192ab0a --- /dev/null +++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp @@ -0,0 +1,1310 @@ +/* + * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_MACROASSEMBLER_RISCV_HPP +#define CPU_RISCV_MACROASSEMBLER_RISCV_HPP + +#include "asm/assembler.inline.hpp" +#include "metaprogramming/enableIf.hpp" +#include "nativeInst_riscv.hpp" +#include "oops/compressedOops.hpp" +#include "utilities/powerOfTwo.hpp" + +// MacroAssembler extends Assembler by frequently used macros. +// +// Instructions for which a 'better' code sequence exists depending +// on arguments should also go in here. + +class MacroAssembler: public Assembler { + + public: + MacroAssembler(CodeBuffer* code) : Assembler(code) {} + + virtual ~MacroAssembler() {} + + void safepoint_poll(Label& slow_path, bool at_return, bool acquire, bool in_nmethod); + + // Alignment + void align(int modulus, int extra_offset = 0); + + static inline void assert_alignment(address pc, int alignment = NativeInstruction::instruction_size) { + assert(is_aligned(pc, alignment), "bad alignment"); + } + + // Stack frame creation/removal + // Note that SP must be updated to the right place before saving/restoring RA and FP + // because signal based thread suspend/resume could happen asynchronously. + void enter() { + addi(sp, sp, - 2 * wordSize); + sd(ra, Address(sp, wordSize)); + sd(fp, Address(sp)); + addi(fp, sp, 2 * wordSize); + } + + void leave() { + addi(sp, fp, - 2 * wordSize); + ld(fp, Address(sp)); + ld(ra, Address(sp, wordSize)); + addi(sp, sp, 2 * wordSize); + } + + + // Support for getting the JavaThread pointer (i.e.; a reference to thread-local information) + // The pointer will be loaded into the thread register. + void get_thread(Register thread); + + // Support for VM calls + // + // It is imperative that all calls into the VM are handled via the call_VM macros. + // They make sure that the stack linkage is setup correctly. call_VM's correspond + // to ENTRY/ENTRY_X entry points while call_VM_leaf's correspond to LEAF entry points. + + void call_VM(Register oop_result, + address entry_point, + bool check_exceptions = true); + void call_VM(Register oop_result, + address entry_point, + Register arg_1, + bool check_exceptions = true); + void call_VM(Register oop_result, + address entry_point, + Register arg_1, Register arg_2, + bool check_exceptions = true); + void call_VM(Register oop_result, + address entry_point, + Register arg_1, Register arg_2, Register arg_3, + bool check_exceptions = true); + + // Overloadings with last_Java_sp + void call_VM(Register oop_result, + Register last_java_sp, + address entry_point, + int number_of_arguments = 0, + bool check_exceptions = true); + void call_VM(Register oop_result, + Register last_java_sp, + address entry_point, + Register arg_1, + bool check_exceptions = true); + void call_VM(Register oop_result, + Register last_java_sp, + address entry_point, + Register arg_1, Register arg_2, + bool check_exceptions = true); + void call_VM(Register oop_result, + Register last_java_sp, + address entry_point, + Register arg_1, Register arg_2, Register arg_3, + bool check_exceptions = true); + + void get_vm_result(Register oop_result, Register java_thread); + void get_vm_result_2(Register metadata_result, Register java_thread); + + // These always tightly bind to MacroAssembler::call_VM_leaf_base + // bypassing the virtual implementation + void call_VM_leaf(address entry_point, + int number_of_arguments = 0); + void call_VM_leaf(address entry_point, + Register arg_0); + void call_VM_leaf(address entry_point, + Register arg_0, Register arg_1); + void call_VM_leaf(address entry_point, + Register arg_0, Register arg_1, Register arg_2); + + // These always tightly bind to MacroAssembler::call_VM_base + // bypassing the virtual implementation + void super_call_VM_leaf(address entry_point, Register arg_0); + void super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1); + void super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2); + void super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3); + + // last Java Frame (fills frame anchor) + void set_last_Java_frame(Register last_java_sp, Register last_java_fp, address last_java_pc, Register tmp); + void set_last_Java_frame(Register last_java_sp, Register last_java_fp, Label &last_java_pc, Register tmp); + void set_last_Java_frame(Register last_java_sp, Register last_java_fp, Register last_java_pc, Register tmp); + + // thread in the default location (xthread) + void reset_last_Java_frame(bool clear_fp); + + virtual void call_VM_leaf_base( + address entry_point, // the entry point + int number_of_arguments, // the number of arguments to pop after the call + Label* retaddr = NULL + ); + + virtual void call_VM_leaf_base( + address entry_point, // the entry point + int number_of_arguments, // the number of arguments to pop after the call + Label& retaddr) { + call_VM_leaf_base(entry_point, number_of_arguments, &retaddr); + } + + virtual void call_VM_base( // returns the register containing the thread upon return + Register oop_result, // where an oop-result ends up if any; use noreg otherwise + Register java_thread, // the thread if computed before ; use noreg otherwise + Register last_java_sp, // to set up last_Java_frame in stubs; use noreg otherwise + address entry_point, // the entry point + int number_of_arguments, // the number of arguments (w/o thread) to pop after the call + bool check_exceptions // whether to check for pending exceptions after return + ); + + void call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions); + + virtual void check_and_handle_earlyret(Register java_thread); + virtual void check_and_handle_popframe(Register java_thread); + + void resolve_weak_handle(Register result, Register tmp); + void resolve_oop_handle(Register result, Register tmp = x15); + void resolve_jobject(Register value, Register thread, Register tmp); + + void movoop(Register dst, jobject obj, bool immediate = false); + void mov_metadata(Register dst, Metadata* obj); + void bang_stack_size(Register size, Register tmp); + void set_narrow_oop(Register dst, jobject obj); + void set_narrow_klass(Register dst, Klass* k); + + void load_mirror(Register dst, Register method, Register tmp = x15); + void access_load_at(BasicType type, DecoratorSet decorators, Register dst, + Address src, Register tmp1, Register thread_tmp); + void access_store_at(BasicType type, DecoratorSet decorators, Address dst, + Register src, Register tmp1, Register thread_tmp); + void load_klass(Register dst, Register src, Register tmp = t0); + void store_klass(Register dst, Register src, Register tmp = t0); + void cmp_klass(Register oop, Register trial_klass, Register tmp1, Register tmp2, Label &L); + + void encode_klass_not_null(Register r, Register tmp = t0); + void decode_klass_not_null(Register r, Register tmp = t0); + void encode_klass_not_null(Register dst, Register src, Register tmp); + void decode_klass_not_null(Register dst, Register src, Register tmp); + void decode_heap_oop_not_null(Register r); + void decode_heap_oop_not_null(Register dst, Register src); + void decode_heap_oop(Register d, Register s); + void decode_heap_oop(Register r) { decode_heap_oop(r, r); } + void encode_heap_oop(Register d, Register s); + void encode_heap_oop(Register r) { encode_heap_oop(r, r); }; + void load_heap_oop(Register dst, Address src, Register tmp1 = noreg, + Register thread_tmp = noreg, DecoratorSet decorators = 0); + void load_heap_oop_not_null(Register dst, Address src, Register tmp1 = noreg, + Register thread_tmp = noreg, DecoratorSet decorators = 0); + void store_heap_oop(Address dst, Register src, Register tmp1 = noreg, + Register thread_tmp = noreg, DecoratorSet decorators = 0); + + void store_klass_gap(Register dst, Register src); + + // currently unimplemented + // Used for storing NULL. All other oop constants should be + // stored using routines that take a jobject. + void store_heap_oop_null(Address dst); + + // This dummy is to prevent a call to store_heap_oop from + // converting a zero (linke NULL) into a Register by giving + // the compiler two choices it can't resolve + + void store_heap_oop(Address dst, void* dummy); + + // Support for NULL-checks + // + // Generates code that causes a NULL OS exception if the content of reg is NULL. + // If the accessed location is M[reg + offset] and the offset is known, provide the + // offset. No explicit code generateion is needed if the offset is within a certain + // range (0 <= offset <= page_size). + + virtual void null_check(Register reg, int offset = -1); + static bool needs_explicit_null_check(intptr_t offset); + static bool uses_implicit_null_check(void* address); + + // idiv variant which deals with MINLONG as dividend and -1 as divisor + int corrected_idivl(Register result, Register rs1, Register rs2, + bool want_remainder); + int corrected_idivq(Register result, Register rs1, Register rs2, + bool want_remainder); + + // interface method calling + void lookup_interface_method(Register recv_klass, + Register intf_klass, + RegisterOrConstant itable_index, + Register method_result, + Register scan_tmp, + Label& no_such_interface, + bool return_method = true); + + // virtual method calling + // n.n. x86 allows RegisterOrConstant for vtable_index + void lookup_virtual_method(Register recv_klass, + RegisterOrConstant vtable_index, + Register method_result); + + // Form an addres from base + offset in Rd. Rd my or may not + // actually be used: you must use the Address that is returned. It + // is up to you to ensure that the shift provided mathces the size + // of your data. + Address form_address(Register Rd, Register base, int64_t byte_offset); + + // Sometimes we get misaligned loads and stores, usually from Unsafe + // accesses, and these can exceed the offset range. + Address legitimize_address(Register Rd, const Address &adr) { + if (adr.getMode() == Address::base_plus_offset) { + if (!is_simm12(adr.offset())) { + return form_address(Rd, adr.base(), adr.offset()); + } + } + return adr; + } + + // allocation + void tlab_allocate( + Register obj, // result: pointer to object after successful allocation + Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise + int con_size_in_bytes, // object size in bytes if known at compile time + Register tmp1, // temp register + Register tmp2, // temp register + Label& slow_case, // continuation point of fast allocation fails + bool is_far = false + ); + + void eden_allocate( + Register obj, // result: pointer to object after successful allocation + Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise + int con_size_in_bytes, // object size in bytes if known at compile time + Register tmp, // temp register + Label& slow_case, // continuation point if fast allocation fails + bool is_far = false + ); + + // Test sub_klass against super_klass, with fast and slow paths. + + // The fast path produces a tri-state answer: yes / no / maybe-slow. + // One of the three labels can be NULL, meaning take the fall-through. + // If super_check_offset is -1, the value is loaded up from super_klass. + // No registers are killed, except tmp_reg + void check_klass_subtype_fast_path(Register sub_klass, + Register super_klass, + Register tmp_reg, + Label* L_success, + Label* L_failure, + Label* L_slow_path, + Register super_check_offset = noreg); + + // The reset of the type cehck; must be wired to a corresponding fast path. + // It does not repeat the fast path logic, so don't use it standalone. + // The tmp1_reg and tmp2_reg can be noreg, if no temps are avaliable. + // Updates the sub's secondary super cache as necessary. + void check_klass_subtype_slow_path(Register sub_klass, + Register super_klass, + Register tmp1_reg, + Register tmp2_reg, + Label* L_success, + Label* L_failure); + + void check_klass_subtype(Register sub_klass, + Register super_klass, + Register tmp_reg, + Label& L_success); + + Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0); + + // only if +VerifyOops + void verify_oop(Register reg, const char* s = "broken oop"); + void verify_oop_addr(Address addr, const char* s = "broken oop addr"); + + void _verify_method_ptr(Register reg, const char* msg, const char* file, int line) {} + void _verify_klass_ptr(Register reg, const char* msg, const char* file, int line) {} + +#define verify_method_ptr(reg) _verify_method_ptr(reg, "broken method " #reg, __FILE__, __LINE__) +#define verify_klass_ptr(reg) _verify_method_ptr(reg, "broken klass " #reg, __FILE__, __LINE__) + + // A more convenient access to fence for our purposes + // We used four bit to indicate the read and write bits in the predecessors and successors, + // and extended i for r, o for w if UseConservativeFence enabled. + enum Membar_mask_bits { + StoreStore = 0b0101, // (pred = ow + succ = ow) + LoadStore = 0b1001, // (pred = ir + succ = ow) + StoreLoad = 0b0110, // (pred = ow + succ = ir) + LoadLoad = 0b1010, // (pred = ir + succ = ir) + AnyAny = LoadStore | StoreLoad // (pred = iorw + succ = iorw) + }; + + void membar(uint32_t order_constraint); + + static void membar_mask_to_pred_succ(uint32_t order_constraint, + uint32_t& predecessor, uint32_t& successor) { + predecessor = (order_constraint >> 2) & 0x3; + successor = order_constraint & 0x3; + + // extend rw -> iorw: + // 01(w) -> 0101(ow) + // 10(r) -> 1010(ir) + // 11(rw)-> 1111(iorw) + if (UseConservativeFence) { + predecessor |= predecessor << 2; + successor |= successor << 2; + } + } + + static int pred_succ_to_membar_mask(uint32_t predecessor, uint32_t successor) { + return ((predecessor & 0x3) << 2) | (successor & 0x3); + } + + // prints msg, dumps registers and stops execution + void stop(const char* msg); + + static void debug64(char* msg, int64_t pc, int64_t regs[]); + + void unimplemented(const char* what = ""); + + void should_not_reach_here() { stop("should not reach here"); } + + static address target_addr_for_insn(address insn_addr); + + // Required platform-specific helpers for Label::patch_instructions. + // They _shadow_ the declarations in AbstractAssembler, which are undefined. + static int pd_patch_instruction_size(address branch, address target); + static void pd_patch_instruction(address branch, address target, const char* file = NULL, int line = 0) { + pd_patch_instruction_size(branch, target); + } + static address pd_call_destination(address branch) { + return target_addr_for_insn(branch); + } + + static int patch_oop(address insn_addr, address o); + address emit_trampoline_stub(int insts_call_instruction_offset, address target); + void emit_static_call_stub(); + + // The following 4 methods return the offset of the appropriate move instruction + + // Support for fast byte/short loading with zero extension (depending on particular CPU) + int load_unsigned_byte(Register dst, Address src); + int load_unsigned_short(Register dst, Address src); + + // Support for fast byte/short loading with sign extension (depending on particular CPU) + int load_signed_byte(Register dst, Address src); + int load_signed_short(Register dst, Address src); + + // Load and store values by size and signed-ness + void load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2 = noreg); + void store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2 = noreg); + + public: + // Standard pseudo instructions + inline void nop() { + addi(x0, x0, 0); + } + + inline void mv(Register Rd, Register Rs) { + if (Rd != Rs) { + addi(Rd, Rs, 0); + } + } + + inline void notr(Register Rd, Register Rs) { + xori(Rd, Rs, -1); + } + + inline void neg(Register Rd, Register Rs) { + sub(Rd, x0, Rs); + } + + inline void negw(Register Rd, Register Rs) { + subw(Rd, x0, Rs); + } + + inline void sext_w(Register Rd, Register Rs) { + addiw(Rd, Rs, 0); + } + + inline void zext_b(Register Rd, Register Rs) { + andi(Rd, Rs, 0xFF); + } + + inline void seqz(Register Rd, Register Rs) { + sltiu(Rd, Rs, 1); + } + + inline void snez(Register Rd, Register Rs) { + sltu(Rd, x0, Rs); + } + + inline void sltz(Register Rd, Register Rs) { + slt(Rd, Rs, x0); + } + + inline void sgtz(Register Rd, Register Rs) { + slt(Rd, x0, Rs); + } + + // Bit-manipulation extension pseudo instructions + // zero extend word + inline void zext_w(Register Rd, Register Rs) { + add_uw(Rd, Rs, zr); + } + + // Floating-point data-processing pseudo instructions + inline void fmv_s(FloatRegister Rd, FloatRegister Rs) { + if (Rd != Rs) { + fsgnj_s(Rd, Rs, Rs); + } + } + + inline void fabs_s(FloatRegister Rd, FloatRegister Rs) { + fsgnjx_s(Rd, Rs, Rs); + } + + inline void fneg_s(FloatRegister Rd, FloatRegister Rs) { + fsgnjn_s(Rd, Rs, Rs); + } + + inline void fmv_d(FloatRegister Rd, FloatRegister Rs) { + if (Rd != Rs) { + fsgnj_d(Rd, Rs, Rs); + } + } + + inline void fabs_d(FloatRegister Rd, FloatRegister Rs) { + fsgnjx_d(Rd, Rs, Rs); + } + + inline void fneg_d(FloatRegister Rd, FloatRegister Rs) { + fsgnjn_d(Rd, Rs, Rs); + } + + // Control and status pseudo instructions + void rdinstret(Register Rd); // read instruction-retired counter + void rdcycle(Register Rd); // read cycle counter + void rdtime(Register Rd); // read time + void csrr(Register Rd, unsigned csr); // read csr + void csrw(unsigned csr, Register Rs); // write csr + void csrs(unsigned csr, Register Rs); // set bits in csr + void csrc(unsigned csr, Register Rs); // clear bits in csr + void csrwi(unsigned csr, unsigned imm); + void csrsi(unsigned csr, unsigned imm); + void csrci(unsigned csr, unsigned imm); + void frcsr(Register Rd); // read float-point csr + void fscsr(Register Rd, Register Rs); // swap float-point csr + void fscsr(Register Rs); // write float-point csr + void frrm(Register Rd); // read float-point rounding mode + void fsrm(Register Rd, Register Rs); // swap float-point rounding mode + void fsrm(Register Rs); // write float-point rounding mode + void fsrmi(Register Rd, unsigned imm); + void fsrmi(unsigned imm); + void frflags(Register Rd); // read float-point exception flags + void fsflags(Register Rd, Register Rs); // swap float-point exception flags + void fsflags(Register Rs); // write float-point exception flags + void fsflagsi(Register Rd, unsigned imm); + void fsflagsi(unsigned imm); + + // Control transfer pseudo instructions + void beqz(Register Rs, const address dest); + void bnez(Register Rs, const address dest); + void blez(Register Rs, const address dest); + void bgez(Register Rs, const address dest); + void bltz(Register Rs, const address dest); + void bgtz(Register Rs, const address dest); + + void j(Label &l, Register temp = t0); + void j(const address dest, Register temp = t0); + void j(const Address &adr, Register temp = t0); + void jal(Label &l, Register temp = t0); + void jal(const address dest, Register temp = t0); + void jal(const Address &adr, Register temp = t0); + void jal(Register Rd, Label &L, Register temp = t0); + void jal(Register Rd, const address dest, Register temp = t0); + + //label + void beqz(Register Rs, Label &l, bool is_far = false); + void bnez(Register Rs, Label &l, bool is_far = false); + void blez(Register Rs, Label &l, bool is_far = false); + void bgez(Register Rs, Label &l, bool is_far = false); + void bltz(Register Rs, Label &l, bool is_far = false); + void bgtz(Register Rs, Label &l, bool is_far = false); + + void beq (Register Rs1, Register Rs2, Label &L, bool is_far = false); + void bne (Register Rs1, Register Rs2, Label &L, bool is_far = false); + void blt (Register Rs1, Register Rs2, Label &L, bool is_far = false); + void bge (Register Rs1, Register Rs2, Label &L, bool is_far = false); + void bltu(Register Rs1, Register Rs2, Label &L, bool is_far = false); + void bgeu(Register Rs1, Register Rs2, Label &L, bool is_far = false); + + void bgt (Register Rs, Register Rt, const address dest); + void ble (Register Rs, Register Rt, const address dest); + void bgtu(Register Rs, Register Rt, const address dest); + void bleu(Register Rs, Register Rt, const address dest); + + void bgt (Register Rs, Register Rt, Label &l, bool is_far = false); + void ble (Register Rs, Register Rt, Label &l, bool is_far = false); + void bgtu(Register Rs, Register Rt, Label &l, bool is_far = false); + void bleu(Register Rs, Register Rt, Label &l, bool is_far = false); + +#define INSN_ENTRY_RELOC(result_type, header) \ + result_type header { \ + guarantee(rtype == relocInfo::internal_word_type, \ + "only internal_word_type relocs make sense here"); \ + relocate(InternalAddress(dest).rspec()); \ + IncompressibleRegion ir(this); /* relocations */ + +#define INSN(NAME) \ + void NAME(Register Rs1, Register Rs2, const address dest) { \ + assert_cond(dest != NULL); \ + int64_t offset = dest - pc(); \ + guarantee(is_simm13(offset) && ((offset % 2) == 0), "offset is invalid."); \ + Assembler::NAME(Rs1, Rs2, offset); \ + } \ + INSN_ENTRY_RELOC(void, NAME(Register Rs1, Register Rs2, address dest, relocInfo::relocType rtype)) \ + NAME(Rs1, Rs2, dest); \ + } + + INSN(beq); + INSN(bne); + INSN(bge); + INSN(bgeu); + INSN(blt); + INSN(bltu); + +#undef INSN + +#undef INSN_ENTRY_RELOC + + void float_beq(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false); + void float_bne(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false); + void float_ble(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false); + void float_bge(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false); + void float_blt(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false); + void float_bgt(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false); + + void double_beq(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false); + void double_bne(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false); + void double_ble(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false); + void double_bge(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false); + void double_blt(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false); + void double_bgt(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false); + +private: + int push_reg(unsigned int bitset, Register stack); + int pop_reg(unsigned int bitset, Register stack); + int push_fp(unsigned int bitset, Register stack); + int pop_fp(unsigned int bitset, Register stack); +#ifdef COMPILER2 + int push_v(unsigned int bitset, Register stack); + int pop_v(unsigned int bitset, Register stack); +#endif // COMPILER2 + +public: + void push_reg(Register Rs); + void pop_reg(Register Rd); + void push_reg(RegSet regs, Register stack) { if (regs.bits()) push_reg(regs.bits(), stack); } + void pop_reg(RegSet regs, Register stack) { if (regs.bits()) pop_reg(regs.bits(), stack); } + void push_fp(FloatRegSet regs, Register stack) { if (regs.bits()) push_fp(regs.bits(), stack); } + void pop_fp(FloatRegSet regs, Register stack) { if (regs.bits()) pop_fp(regs.bits(), stack); } +#ifdef COMPILER2 + void push_v(VectorRegSet regs, Register stack) { if (regs.bits()) push_v(regs.bits(), stack); } + void pop_v(VectorRegSet regs, Register stack) { if (regs.bits()) pop_v(regs.bits(), stack); } +#endif // COMPILER2 + + // Push and pop everything that might be clobbered by a native + // runtime call except t0 and t1. (They are always + // temporary registers, so we don't have to protect them.) + // Additional registers can be excluded in a passed RegSet. + void push_call_clobbered_registers_except(RegSet exclude); + void pop_call_clobbered_registers_except(RegSet exclude); + + void push_call_clobbered_registers() { + push_call_clobbered_registers_except(RegSet()); + } + void pop_call_clobbered_registers() { + pop_call_clobbered_registers_except(RegSet()); + } + + void push_CPU_state(bool save_vectors = false, int vector_size_in_bytes = 0); + void pop_CPU_state(bool restore_vectors = false, int vector_size_in_bytes = 0); + + // if heap base register is used - reinit it with the correct value + void reinit_heapbase(); + + void bind(Label& L) { + Assembler::bind(L); + // fences across basic blocks should not be merged + code()->clear_last_insn(); + } + + typedef void (MacroAssembler::* compare_and_branch_insn)(Register Rs1, Register Rs2, const address dest); + typedef void (MacroAssembler::* compare_and_branch_label_insn)(Register Rs1, Register Rs2, Label &L, bool is_far); + typedef void (MacroAssembler::* jal_jalr_insn)(Register Rt, address dest); + typedef void (MacroAssembler::* load_insn_by_temp)(Register Rt, address dest, Register temp); + + void wrap_label(Register r, Label &L, Register t, load_insn_by_temp insn); + void wrap_label(Register r, Label &L, jal_jalr_insn insn); + void wrap_label(Register r1, Register r2, Label &L, + compare_and_branch_insn insn, + compare_and_branch_label_insn neg_insn, bool is_far = false); + + void la(Register Rd, Label &label); + void la(Register Rd, const address dest); + void la(Register Rd, const Address &adr); + + void li32(Register Rd, int32_t imm); + void li64(Register Rd, int64_t imm); + void li (Register Rd, int64_t imm); // optimized load immediate + + // mv + void mv(Register Rd, address addr) { li(Rd, (int64_t)addr); } + void mv(Register Rd, address addr, int32_t &offset) { + // Split address into a lower 12-bit sign-extended offset and the remainder, + // so that the offset could be encoded in jalr or load/store instruction. + offset = ((int32_t)(int64_t)addr << 20) >> 20; + li(Rd, (int64_t)addr - offset); + } + + template::value)> + inline void mv(Register Rd, T o) { li(Rd, (int64_t)o); } + + void mv(Register Rd, Address dest) { + assert(dest.getMode() == Address::literal, "Address mode should be Address::literal"); + relocate(dest.rspec(), [&] { + movptr(Rd, dest.target()); + }); + } + + void mv(Register Rd, RegisterOrConstant src) { + if (src.is_register()) { + mv(Rd, src.as_register()); + } else { + mv(Rd, src.as_constant()); + } + } + + void movptr(Register Rd, address addr, int32_t &offset); + + void movptr(Register Rd, address addr) { + int offset = 0; + movptr(Rd, addr, offset); + addi(Rd, Rd, offset); + } + + inline void movptr(Register Rd, uintptr_t imm64) { + movptr(Rd, (address)imm64); + } + + // arith + void add (Register Rd, Register Rn, int64_t increment, Register temp = t0); + void addw(Register Rd, Register Rn, int32_t increment, Register temp = t0); + void sub (Register Rd, Register Rn, int64_t decrement, Register temp = t0); + void subw(Register Rd, Register Rn, int32_t decrement, Register temp = t0); + +#define INSN(NAME) \ + inline void NAME(Register Rd, Register Rs1, Register Rs2) { \ + Assembler::NAME(Rd, Rs1, Rs2); \ + } + + INSN(add); + INSN(addw); + INSN(sub); + INSN(subw); + +#undef INSN + + // logic + void andrw(Register Rd, Register Rs1, Register Rs2); + void orrw(Register Rd, Register Rs1, Register Rs2); + void xorrw(Register Rd, Register Rs1, Register Rs2); + + // revb + void revb_h_h(Register Rd, Register Rs, Register tmp = t0); // reverse bytes in halfword in lower 16 bits, sign-extend + void revb_w_w(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2 = t1); // reverse bytes in lower word, sign-extend + void revb_h_h_u(Register Rd, Register Rs, Register tmp = t0); // reverse bytes in halfword in lower 16 bits, zero-extend + void revb_h_w_u(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2 = t1); // reverse bytes in halfwords in lower 32 bits, zero-extend + void revb_h_helper(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2= t1); // reverse bytes in upper 16 bits (48:63) and move to lower + void revb_h(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2= t1); // reverse bytes in each halfword + void revb_w(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2= t1); // reverse bytes in each word + void revb(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2 = t1); // reverse bytes in doubleword + + void ror_imm(Register dst, Register src, uint32_t shift, Register tmp = t0); + void andi(Register Rd, Register Rn, int64_t imm, Register tmp = t0); + void orptr(Address adr, RegisterOrConstant src, Register tmp1 = t0, Register tmp2 = t1); + +// Load and Store Instructions +#define INSN_ENTRY_RELOC(result_type, header) \ + result_type header { \ + guarantee(rtype == relocInfo::internal_word_type, \ + "only internal_word_type relocs make sense here"); \ + relocate(InternalAddress(dest).rspec()); \ + IncompressibleRegion ir(this); /* relocations */ + +#define INSN(NAME) \ + void NAME(Register Rd, address dest) { \ + assert_cond(dest != NULL); \ + int64_t distance = dest - pc(); \ + if (is_simm32(distance)) { \ + auipc(Rd, (int32_t)distance + 0x800); \ + Assembler::NAME(Rd, Rd, ((int32_t)distance << 20) >> 20); \ + } else { \ + int32_t offset = 0; \ + movptr(Rd, dest, offset); \ + Assembler::NAME(Rd, Rd, offset); \ + } \ + } \ + INSN_ENTRY_RELOC(void, NAME(Register Rd, address dest, relocInfo::relocType rtype)) \ + NAME(Rd, dest); \ + } \ + void NAME(Register Rd, const Address &adr, Register temp = t0) { \ + switch (adr.getMode()) { \ + case Address::literal: { \ + relocate(adr.rspec()); \ + NAME(Rd, adr.target()); \ + break; \ + } \ + case Address::base_plus_offset: { \ + if (is_simm12(adr.offset())) { \ + Assembler::NAME(Rd, adr.base(), adr.offset()); \ + } else { \ + int32_t offset = ((int32_t)adr.offset() << 20) >> 20; \ + if (Rd == adr.base()) { \ + la(temp, Address(adr.base(), adr.offset() - offset)); \ + Assembler::NAME(Rd, temp, offset); \ + } else { \ + la(Rd, Address(adr.base(), adr.offset() - offset)); \ + Assembler::NAME(Rd, Rd, offset); \ + } \ + } \ + break; \ + } \ + default: \ + ShouldNotReachHere(); \ + } \ + } \ + void NAME(Register Rd, Label &L) { \ + wrap_label(Rd, L, &MacroAssembler::NAME); \ + } + + INSN(lb); + INSN(lbu); + INSN(lh); + INSN(lhu); + INSN(lw); + INSN(lwu); + INSN(ld); + +#undef INSN + +#define INSN(NAME) \ + void NAME(FloatRegister Rd, address dest, Register temp = t0) { \ + assert_cond(dest != NULL); \ + int64_t distance = dest - pc(); \ + if (is_simm32(distance)) { \ + auipc(temp, (int32_t)distance + 0x800); \ + Assembler::NAME(Rd, temp, ((int32_t)distance << 20) >> 20); \ + } else { \ + int32_t offset = 0; \ + movptr(temp, dest, offset); \ + Assembler::NAME(Rd, temp, offset); \ + } \ + } \ + INSN_ENTRY_RELOC(void, NAME(FloatRegister Rd, address dest, \ + relocInfo::relocType rtype, Register temp = t0)) \ + NAME(Rd, dest, temp); \ + } \ + void NAME(FloatRegister Rd, const Address &adr, Register temp = t0) { \ + switch (adr.getMode()) { \ + case Address::literal: { \ + relocate(adr.rspec()); \ + NAME(Rd, adr.target(), temp); \ + break; \ + } \ + case Address::base_plus_offset: { \ + if (is_simm12(adr.offset())) { \ + Assembler::NAME(Rd, adr.base(), adr.offset()); \ + } else { \ + int32_t offset = ((int32_t)adr.offset() << 20) >> 20; \ + la(temp, Address(adr.base(), adr.offset() - offset)); \ + Assembler::NAME(Rd, temp, offset); \ + } \ + break; \ + } \ + default: \ + ShouldNotReachHere(); \ + } \ + } + + INSN(flw); + INSN(fld); + +#undef INSN + +#define INSN(NAME, REGISTER) \ + INSN_ENTRY_RELOC(void, NAME(REGISTER Rs, address dest, \ + relocInfo::relocType rtype, Register temp = t0)) \ + NAME(Rs, dest, temp); \ + } + + INSN(sb, Register); + INSN(sh, Register); + INSN(sw, Register); + INSN(sd, Register); + INSN(fsw, FloatRegister); + INSN(fsd, FloatRegister); + +#undef INSN + +#define INSN(NAME) \ + void NAME(Register Rs, address dest, Register temp = t0) { \ + assert_cond(dest != NULL); \ + assert_different_registers(Rs, temp); \ + int64_t distance = dest - pc(); \ + if (is_simm32(distance)) { \ + auipc(temp, (int32_t)distance + 0x800); \ + Assembler::NAME(Rs, temp, ((int32_t)distance << 20) >> 20); \ + } else { \ + int32_t offset = 0; \ + movptr(temp, dest, offset); \ + Assembler::NAME(Rs, temp, offset); \ + } \ + } \ + void NAME(Register Rs, const Address &adr, Register temp = t0) { \ + switch (adr.getMode()) { \ + case Address::literal: { \ + assert_different_registers(Rs, temp); \ + relocate(adr.rspec()); \ + NAME(Rs, adr.target(), temp); \ + break; \ + } \ + case Address::base_plus_offset: { \ + if (is_simm12(adr.offset())) { \ + Assembler::NAME(Rs, adr.base(), adr.offset()); \ + } else { \ + assert_different_registers(Rs, temp); \ + int32_t offset = ((int32_t)adr.offset() << 20) >> 20; \ + la(temp, Address(adr.base(), adr.offset() - offset)); \ + Assembler::NAME(Rs, temp, offset); \ + } \ + break; \ + } \ + default: \ + ShouldNotReachHere(); \ + } \ + } + + INSN(sb); + INSN(sh); + INSN(sw); + INSN(sd); + +#undef INSN + +#define INSN(NAME) \ + void NAME(FloatRegister Rs, address dest, Register temp = t0) { \ + assert_cond(dest != NULL); \ + int64_t distance = dest - pc(); \ + if (is_simm32(distance)) { \ + auipc(temp, (int32_t)distance + 0x800); \ + Assembler::NAME(Rs, temp, ((int32_t)distance << 20) >> 20); \ + } else { \ + int32_t offset = 0; \ + movptr(temp, dest, offset); \ + Assembler::NAME(Rs, temp, offset); \ + } \ + } \ + void NAME(FloatRegister Rs, const Address &adr, Register temp = t0) { \ + switch (adr.getMode()) { \ + case Address::literal: { \ + relocate(adr.rspec()); \ + NAME(Rs, adr.target(), temp); \ + break; \ + } \ + case Address::base_plus_offset: { \ + if (is_simm12(adr.offset())) { \ + Assembler::NAME(Rs, adr.base(), adr.offset()); \ + } else { \ + int32_t offset = ((int32_t)adr.offset() << 20) >> 20; \ + la(temp, Address(adr.base(), adr.offset() - offset)); \ + Assembler::NAME(Rs, temp, offset); \ + } \ + break; \ + } \ + default: \ + ShouldNotReachHere(); \ + } \ + } + + INSN(fsw); + INSN(fsd); + +#undef INSN + +#undef INSN_ENTRY_RELOC + + void cmpxchg_obj_header(Register oldv, Register newv, Register obj, Register tmp, Label &succeed, Label *fail); + void cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp, Label &succeed, Label *fail); + void cmpxchg(Register addr, Register expected, + Register new_val, + enum operand_size size, + Assembler::Aqrl acquire, Assembler::Aqrl release, + Register result, bool result_as_bool = false); + void cmpxchg_weak(Register addr, Register expected, + Register new_val, + enum operand_size size, + Assembler::Aqrl acquire, Assembler::Aqrl release, + Register result); + void cmpxchg_narrow_value_helper(Register addr, Register expected, + Register new_val, + enum operand_size size, + Register tmp1, Register tmp2, Register tmp3); + void cmpxchg_narrow_value(Register addr, Register expected, + Register new_val, + enum operand_size size, + Assembler::Aqrl acquire, Assembler::Aqrl release, + Register result, bool result_as_bool, + Register tmp1, Register tmp2, Register tmp3); + void weak_cmpxchg_narrow_value(Register addr, Register expected, + Register new_val, + enum operand_size size, + Assembler::Aqrl acquire, Assembler::Aqrl release, + Register result, + Register tmp1, Register tmp2, Register tmp3); + + void atomic_add(Register prev, RegisterOrConstant incr, Register addr); + void atomic_addw(Register prev, RegisterOrConstant incr, Register addr); + void atomic_addal(Register prev, RegisterOrConstant incr, Register addr); + void atomic_addalw(Register prev, RegisterOrConstant incr, Register addr); + + void atomic_xchg(Register prev, Register newv, Register addr); + void atomic_xchgw(Register prev, Register newv, Register addr); + void atomic_xchgal(Register prev, Register newv, Register addr); + void atomic_xchgalw(Register prev, Register newv, Register addr); + void atomic_xchgwu(Register prev, Register newv, Register addr); + void atomic_xchgalwu(Register prev, Register newv, Register addr); + + void atomic_incw(Register counter_addr, Register tmp); + void atomic_incw(Address counter_addr, Register tmp1, Register tmp2) { + la(tmp1, counter_addr); + atomic_incw(tmp1, tmp2); + } + + // Biased locking support + // lock_reg and obj_reg must be loaded up with the appropriate values. + // swap_reg is killed. + // tmp_reg must be supplied and must not be t0 or t1 + // Optional slow case is for implementations (interpreter and C1) which branch to + // slow case directly. Leaves condition codes set for C2's Fast_Lock done. + // Returns offset of first potentially-faulting instruction for null + // check info (currently consumed only by C1). If + // swap_reg_contains_mark is true then returns -1 as it as assumed + // the calling code has already passed any potential faults. + void biased_locking_enter(Register lock_reg, Register obj_reg, + Register swap_reg, Register tmp_Reg, + bool swap_reg_contains_mark, + Label& done, Label* slow_case = NULL, + BiasedLockingCounters* counters = NULL, + Register flag = noreg); + void biased_locking_exit(Register obj_reg, Register tmp_reg, Label& done, Register flag = noreg); + + static bool far_branches() { + return ReservedCodeCacheSize > branch_range; + } + + // Jumps that can reach anywhere in the code cache. + // Trashes tmp. + void far_call(Address entry, CodeBuffer *cbuf = NULL, Register tmp = t0); + void far_jump(Address entry, CodeBuffer *cbuf = NULL, Register tmp = t0); + + static int far_branch_size() { + if (far_branches()) { + return 2 * 4; // auipc + jalr, see far_call() & far_jump() + } else { + return 4; + } + } + + void load_byte_map_base(Register reg); + + void bang_stack_with_offset(int offset) { + // stack grows down, caller passes positive offset + assert(offset > 0, "must bang with negative offset"); + sub(t0, sp, offset); + sd(zr, Address(t0)); + } + + void la_patchable(Register reg1, const Address &dest, int32_t &offset); + + virtual void _call_Unimplemented(address call_site) { + mv(t1, call_site); + } + + #define call_Unimplemented() _call_Unimplemented((address)__PRETTY_FUNCTION__) + + // Frame creation and destruction shared between JITs. + void build_frame(int framesize); + void remove_frame(int framesize); + + void reserved_stack_check(); + + void get_polling_page(Register dest, relocInfo::relocType rtype); + void read_polling_page(Register r, int32_t offset, relocInfo::relocType rtype); + + address trampoline_call(Address entry, CodeBuffer* cbuf = NULL); + address ic_call(address entry, jint method_index = 0); + + // Support for memory inc/dec + // n.b. increment/decrement calls with an Address destination will + // need to use a scratch register to load the value to be + // incremented. increment/decrement calls which add or subtract a + // constant value other than sign-extended 12-bit immediate will need + // to use a 2nd scratch register to hold the constant. so, an address + // increment/decrement may trash both t0 and t1. + + void increment(const Address dst, int64_t value = 1, Register tmp1 = t0, Register tmp2 = t1); + void incrementw(const Address dst, int32_t value = 1, Register tmp1 = t0, Register tmp2 = t1); + + void decrement(const Address dst, int64_t value = 1, Register tmp1 = t0, Register tmp2 = t1); + void decrementw(const Address dst, int32_t value = 1, Register tmp1 = t0, Register tmp2 = t1); + + void cmpptr(Register src1, Address src2, Label& equal); + + void clinit_barrier(Register klass, Register tmp, Label* L_fast_path = NULL, Label* L_slow_path = NULL); + void load_method_holder_cld(Register result, Register method); + void load_method_holder(Register holder, Register method); + + void compute_index(Register str1, Register trailing_zeros, Register match_mask, + Register result, Register char_tmp, Register tmp, + bool haystack_isL); + void compute_match_mask(Register src, Register pattern, Register match_mask, + Register mask1, Register mask2); + +#ifdef COMPILER2 + void mul_add(Register out, Register in, Register offset, + Register len, Register k, Register tmp); + void cad(Register dst, Register src1, Register src2, Register carry); + void cadc(Register dst, Register src1, Register src2, Register carry); + void adc(Register dst, Register src1, Register src2, Register carry); + void add2_with_carry(Register final_dest_hi, Register dest_hi, Register dest_lo, + Register src1, Register src2, Register carry); + void multiply_32_x_32_loop(Register x, Register xstart, Register x_xstart, + Register y, Register y_idx, Register z, + Register carry, Register product, + Register idx, Register kdx); + void multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart, + Register y, Register y_idx, Register z, + Register carry, Register product, + Register idx, Register kdx); + void multiply_128_x_128_loop(Register y, Register z, + Register carry, Register carry2, + Register idx, Register jdx, + Register yz_idx1, Register yz_idx2, + Register tmp, Register tmp3, Register tmp4, + Register tmp6, Register product_hi); + void multiply_to_len(Register x, Register xlen, Register y, Register ylen, + Register z, Register zlen, + Register tmp1, Register tmp2, Register tmp3, Register tmp4, + Register tmp5, Register tmp6, Register product_hi); +#endif + + void inflate_lo32(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2 = t1); + void inflate_hi32(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2 = t1); + + void ctzc_bit(Register Rd, Register Rs, bool isLL = false, Register tmp1 = t0, Register tmp2 = t1); + + void zero_words(Register base, u_int64_t cnt); + address zero_words(Register ptr, Register cnt); + void fill_words(Register base, Register cnt, Register value); + void zero_memory(Register addr, Register len, Register tmp); + + // shift left by shamt and add + void shadd(Register Rd, Register Rs1, Register Rs2, Register tmp, int shamt); + + // test single bit in Rs, result is set to Rd + void test_bit(Register Rd, Register Rs, uint32_t bit_pos, Register tmp = t0); + + // Here the float instructions with safe deal with some exceptions. + // e.g. convert from NaN, +Inf, -Inf to int, float, double + // will trigger exception, we need to deal with these situations + // to get correct results. + void fcvt_w_s_safe(Register dst, FloatRegister src, Register tmp = t0); + void fcvt_l_s_safe(Register dst, FloatRegister src, Register tmp = t0); + void fcvt_w_d_safe(Register dst, FloatRegister src, Register tmp = t0); + void fcvt_l_d_safe(Register dst, FloatRegister src, Register tmp = t0); + + // vector load/store unit-stride instructions + void vlex_v(VectorRegister vd, Register base, Assembler::SEW sew, VectorMask vm = unmasked) { + switch (sew) { + case Assembler::e64: + vle64_v(vd, base, vm); + break; + case Assembler::e32: + vle32_v(vd, base, vm); + break; + case Assembler::e16: + vle16_v(vd, base, vm); + break; + case Assembler::e8: // fall through + default: + vle8_v(vd, base, vm); + break; + } + } + + void vsex_v(VectorRegister store_data, Register base, Assembler::SEW sew, VectorMask vm = unmasked) { + switch (sew) { + case Assembler::e64: + vse64_v(store_data, base, vm); + break; + case Assembler::e32: + vse32_v(store_data, base, vm); + break; + case Assembler::e16: + vse16_v(store_data, base, vm); + break; + case Assembler::e8: // fall through + default: + vse8_v(store_data, base, vm); + break; + } + } + + // vector pseudo instructions + inline void vmnot_m(VectorRegister vd, VectorRegister vs) { + vmnand_mm(vd, vs, vs); + } + + inline void vncvt_x_x_w(VectorRegister vd, VectorRegister vs, VectorMask vm) { + vnsrl_wx(vd, vs, x0, vm); + } + + inline void vfneg_v(VectorRegister vd, VectorRegister vs) { + vfsgnjn_vv(vd, vs, vs); + } + + static const int zero_words_block_size; + + void cast_primitive_type(BasicType type, Register Rt) { + switch (type) { + case T_BOOLEAN: + sltu(Rt, zr, Rt); + break; + case T_CHAR : + zero_extend(Rt, Rt, 16); + break; + case T_BYTE : + sign_extend(Rt, Rt, 8); + break; + case T_SHORT : + sign_extend(Rt, Rt, 16); + break; + case T_INT : + sign_extend(Rt, Rt, 32); + break; + case T_LONG : /* nothing to do */ break; + case T_VOID : /* nothing to do */ break; + case T_FLOAT : /* nothing to do */ break; + case T_DOUBLE : /* nothing to do */ break; + default: ShouldNotReachHere(); + } + } + + // float cmp with unordered_result + void float_compare(Register result, FloatRegister Rs1, FloatRegister Rs2, int unordered_result); + void double_compare(Register result, FloatRegister Rs1, FloatRegister Rs2, int unordered_result); + + // Zero/Sign-extend + void zero_extend(Register dst, Register src, int bits); + void sign_extend(Register dst, Register src, int bits); + + // compare src1 and src2 and get -1/0/1 in dst. + // if [src1 > src2], dst = 1; + // if [src1 == src2], dst = 0; + // if [src1 < src2], dst = -1; + void cmp_l2i(Register dst, Register src1, Register src2, Register tmp = t0); + + void call(const address dest, Register temp = t0) { + assert_cond(dest != NULL); + assert(temp != noreg, "expecting a register"); + int32_t offset = 0; + mv(temp, dest, offset); + jalr(x1, temp, offset); + } + + inline void ret() { + jalr(x0, x1, 0); + } + +private: + +#ifdef ASSERT + // Template short-hand support to clean-up after a failed call to trampoline + // call generation (see trampoline_call() below), when a set of Labels must + // be reset (before returning). + template + void reset_labels(Label& lbl, More&... more) { + lbl.reset(); reset_labels(more...); + } + template + void reset_labels(Label& lbl) { + lbl.reset(); + } +#endif + void repne_scan(Register addr, Register value, Register count, Register tmp); + + void ld_constant(Register dest, const Address &const_addr) { + if (NearCpool) { + ld(dest, const_addr); + } else { + InternalAddress target(const_addr.target()); + relocate(target.rspec(), [&] { + int32_t offset; + la_patchable(dest, target, offset); + ld(dest, Address(dest, offset)); + }); + } + } + + int bitset_to_regs(unsigned int bitset, unsigned char* regs); + Address add_memory_helper(const Address dst, Register tmp); + + void load_reserved(Register addr, enum operand_size size, Assembler::Aqrl acquire); + void store_conditional(Register addr, Register new_val, enum operand_size size, Assembler::Aqrl release); + + void load_prototype_header(Register dst, Register src); +}; + +#ifdef ASSERT +inline bool AbstractAssembler::pd_check_instruction_mark() { return false; } +#endif + +/** + * class SkipIfEqual: + * + * Instantiating this class will result in assembly code being output that will + * jump around any code emitted between the creation of the instance and it's + * automatic destruction at the end of a scope block, depending on the value of + * the flag passed to the constructor, which will be checked at run-time. + */ +class SkipIfEqual { + private: + MacroAssembler* _masm; + Label _label; + + public: + SkipIfEqual(MacroAssembler*, const bool* flag_addr, bool value); + ~SkipIfEqual(); +}; + +#endif // CPU_RISCV_MACROASSEMBLER_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.inline.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.inline.hpp new file mode 100644 index 0000000000000..ef968ccd96d98 --- /dev/null +++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.inline.hpp @@ -0,0 +1,31 @@ +/* + * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_MACROASSEMBLER_RISCV_INLINE_HPP +#define CPU_RISCV_MACROASSEMBLER_RISCV_INLINE_HPP + +// Still empty. + +#endif // CPU_RISCV_MACROASSEMBLER_RISCV_INLINE_HPP diff --git a/src/hotspot/cpu/riscv/matcher_riscv.hpp b/src/hotspot/cpu/riscv/matcher_riscv.hpp new file mode 100644 index 0000000000000..23a75d2050260 --- /dev/null +++ b/src/hotspot/cpu/riscv/matcher_riscv.hpp @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_MATCHER_RISCV_HPP +#define CPU_RISCV_MATCHER_RISCV_HPP + + // Defined within class Matcher + + // false => size gets scaled to BytesPerLong, ok. + static const bool init_array_count_is_in_bytes = false; + + // Whether this platform implements the scalable vector feature + static const bool implements_scalable_vector = true; + + static const bool supports_scalable_vector() { + return UseRVV; + } + + // riscv supports misaligned vectors store/load. + static constexpr bool misaligned_vectors_ok() { + return true; + } + + // Whether code generation need accurate ConvI2L types. + static const bool convi2l_type_required = false; + + // Does the CPU require late expand (see block.cpp for description of late expand)? + static const bool require_postalloc_expand = false; + + // Do we need to mask the count passed to shift instructions or does + // the cpu only look at the lower 5/6 bits anyway? + static const bool need_masked_shift_count = false; + + // No support for generic vector operands. + static const bool supports_generic_vector_operands = false; + + static constexpr bool isSimpleConstant64(jlong value) { + // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?. + // Probably always true, even if a temp register is required. + return true; + } + + // Use conditional move (CMOVL) + static constexpr int long_cmove_cost() { + // long cmoves are no more expensive than int cmoves + return 0; + } + + static constexpr int float_cmove_cost() { + // float cmoves are no more expensive than int cmoves + return 0; + } + + // This affects two different things: + // - how Decode nodes are matched + // - how ImplicitNullCheck opportunities are recognized + // If true, the matcher will try to remove all Decodes and match them + // (as operands) into nodes. NullChecks are not prepared to deal with + // Decodes by final_graph_reshaping(). + // If false, final_graph_reshaping() forces the decode behind the Cmp + // for a NullCheck. The matcher matches the Decode node into a register. + // Implicit_null_check optimization moves the Decode along with the + // memory operation back up before the NullCheck. + static bool narrow_oop_use_complex_address() { + return CompressedOops::shift() == 0; + } + + static bool narrow_klass_use_complex_address() { + return false; + } + + static bool const_oop_prefer_decode() { + // Prefer ConN+DecodeN over ConP in simple compressed oops mode. + return CompressedOops::base() == NULL; + } + + static bool const_klass_prefer_decode() { + // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode. + return CompressedKlassPointers::base() == NULL; + } + + // Is it better to copy float constants, or load them directly from + // memory? Intel can load a float constant from a direct address, + // requiring no extra registers. Most RISCs will have to materialize + // an address into a register first, so they would do better to copy + // the constant from stack. + static const bool rematerialize_float_constants = false; + + // If CPU can load and store mis-aligned doubles directly then no + // fixup is needed. Else we split the double into 2 integer pieces + // and move it piece-by-piece. Only happens when passing doubles into + // C code as the Java calling convention forces doubles to be aligned. + static const bool misaligned_doubles_ok = true; + + // Advertise here if the CPU requires explicit rounding operations to implement strictfp mode. + static const bool strict_fp_requires_explicit_rounding = false; + + // Are floats converted to double when stored to stack during + // deoptimization? + static constexpr bool float_in_double() { return false; } + + // Do ints take an entire long register or just half? + // The relevant question is how the int is callee-saved: + // the whole long is written but de-opt'ing will have to extract + // the relevant 32 bits. + static const bool int_in_long = true; + + // Does the CPU supports vector variable shift instructions? + static constexpr bool supports_vector_variable_shifts(void) { + return false; + } + + // Does the CPU supports vector variable rotate instructions? + static constexpr bool supports_vector_variable_rotates(void) { + return false; + } + + // Does the CPU supports vector constant rotate instructions? + static constexpr bool supports_vector_constant_rotates(int shift) { + return false; + } + + // Does the CPU supports vector unsigned comparison instructions? + static const bool supports_vector_comparison_unsigned(int vlen, BasicType bt) { + return false; + } + + // Some microarchitectures have mask registers used on vectors + static const bool has_predicated_vectors(void) { + return false; + } + + // true means we have fast l2f convers + // false means that conversion is done by runtime call + static constexpr bool convL2FSupported(void) { + return true; + } + + // Implements a variant of EncodeISOArrayNode that encode ASCII only + static const bool supports_encode_ascii_array = false; + + // Returns pre-selection estimated size of a vector operation. + static int vector_op_pre_select_sz_estimate(int vopc, BasicType ety, int vlen) { + return 0; + } + +#endif // CPU_RISCV_MATCHER_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/methodHandles_riscv.cpp b/src/hotspot/cpu/riscv/methodHandles_riscv.cpp new file mode 100644 index 0000000000000..bd3530e2df482 --- /dev/null +++ b/src/hotspot/cpu/riscv/methodHandles_riscv.cpp @@ -0,0 +1,455 @@ +/* + * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.hpp" +#include "classfile/javaClasses.inline.hpp" +#include "classfile/vmClasses.hpp" +#include "interpreter/interpreter.hpp" +#include "interpreter/interpreterRuntime.hpp" +#include "memory/allocation.inline.hpp" +#include "prims/jvmtiExport.hpp" +#include "prims/methodHandles.hpp" +#include "runtime/flags/flagSetting.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/stubRoutines.hpp" + +#define __ _masm-> + +#ifdef PRODUCT +#define BLOCK_COMMENT(str) /* nothing */ +#else +#define BLOCK_COMMENT(str) __ block_comment(str) +#endif + +#define BIND(label) bind(label); BLOCK_COMMENT(#label ":") + +void MethodHandles::load_klass_from_Class(MacroAssembler* _masm, Register klass_reg) { + if (VerifyMethodHandles) { + verify_klass(_masm, klass_reg, VM_CLASS_ID(java_lang_Class), + "MH argument is a Class"); + } + __ ld(klass_reg, Address(klass_reg, java_lang_Class::klass_offset())); +} + +#ifdef ASSERT +static int check_nonzero(const char* xname, int x) { + assert(x != 0, "%s should be nonzero", xname); + return x; +} +#define NONZERO(x) check_nonzero(#x, x) +#else //ASSERT +#define NONZERO(x) (x) +#endif //PRODUCT + +#ifdef ASSERT +void MethodHandles::verify_klass(MacroAssembler* _masm, + Register obj, vmClassID klass_id, + const char* error_message) { + InstanceKlass** klass_addr = vmClasses::klass_addr_at(klass_id); + Klass* klass = vmClasses::klass_at(klass_id); + Register temp1 = t1; + Register temp2 = t0; // used by MacroAssembler::cmpptr + Label L_ok, L_bad; + BLOCK_COMMENT("verify_klass {"); + __ verify_oop(obj); + __ beqz(obj, L_bad); + __ push_reg(RegSet::of(temp1, temp2), sp); + __ load_klass(temp1, obj, temp2); + __ cmpptr(temp1, ExternalAddress((address) klass_addr), L_ok); + intptr_t super_check_offset = klass->super_check_offset(); + __ ld(temp1, Address(temp1, super_check_offset)); + __ cmpptr(temp1, ExternalAddress((address) klass_addr), L_ok); + __ pop_reg(RegSet::of(temp1, temp2), sp); + __ bind(L_bad); + __ stop(error_message); + __ BIND(L_ok); + __ pop_reg(RegSet::of(temp1, temp2), sp); + BLOCK_COMMENT("} verify_klass"); +} + +void MethodHandles::verify_ref_kind(MacroAssembler* _masm, int ref_kind, Register member_reg, Register temp) {} + +#endif //ASSERT + +void MethodHandles::jump_from_method_handle(MacroAssembler* _masm, Register method, Register temp, + bool for_compiler_entry) { + assert(method == xmethod, "interpreter calling convention"); + Label L_no_such_method; + __ beqz(xmethod, L_no_such_method); + __ verify_method_ptr(method); + + if (!for_compiler_entry && JvmtiExport::can_post_interpreter_events()) { + Label run_compiled_code; + // JVMTI events, such as single-stepping, are implemented partly by avoiding running + // compiled code in threads for which the event is enabled. Check here for + // interp_only_mode if these events CAN be enabled. + + __ lwu(t0, Address(xthread, JavaThread::interp_only_mode_offset())); + __ beqz(t0, run_compiled_code); + __ ld(t0, Address(method, Method::interpreter_entry_offset())); + __ jr(t0); + __ BIND(run_compiled_code); + } + + const ByteSize entry_offset = for_compiler_entry ? Method::from_compiled_offset() : + Method::from_interpreted_offset(); + __ ld(t0,Address(method, entry_offset)); + __ jr(t0); + __ bind(L_no_such_method); + __ far_jump(RuntimeAddress(StubRoutines::throw_AbstractMethodError_entry())); +} + +void MethodHandles::jump_to_lambda_form(MacroAssembler* _masm, + Register recv, Register method_temp, + Register temp2, + bool for_compiler_entry) { + BLOCK_COMMENT("jump_to_lambda_form {"); + // This is the initial entry point of a lazy method handle. + // After type checking, it picks up the invoker from the LambdaForm. + assert_different_registers(recv, method_temp, temp2); + assert(recv != noreg, "required register"); + assert(method_temp == xmethod, "required register for loading method"); + + // Load the invoker, as MH -> MH.form -> LF.vmentry + __ verify_oop(recv); + __ load_heap_oop(method_temp, Address(recv, NONZERO(java_lang_invoke_MethodHandle::form_offset())), temp2); + __ verify_oop(method_temp); + __ load_heap_oop(method_temp, Address(method_temp, NONZERO(java_lang_invoke_LambdaForm::vmentry_offset())), temp2); + __ verify_oop(method_temp); + __ load_heap_oop(method_temp, Address(method_temp, NONZERO(java_lang_invoke_MemberName::method_offset())), temp2); + __ verify_oop(method_temp); + __ access_load_at(T_ADDRESS, IN_HEAP, method_temp, Address(method_temp, NONZERO(java_lang_invoke_ResolvedMethodName::vmtarget_offset())), noreg, noreg); + + if (VerifyMethodHandles && !for_compiler_entry) { + // make sure recv is already on stack + __ ld(temp2, Address(method_temp, Method::const_offset())); + __ load_sized_value(temp2, + Address(temp2, ConstMethod::size_of_parameters_offset()), + sizeof(u2), /*is_signed*/ false); + Label L; + __ ld(t0, __ argument_address(temp2, -1)); + __ beq(recv, t0, L); + __ ld(x10, __ argument_address(temp2, -1)); + __ ebreak(); + __ BIND(L); + } + + jump_from_method_handle(_masm, method_temp, temp2, for_compiler_entry); + BLOCK_COMMENT("} jump_to_lambda_form"); +} + +// Code generation +address MethodHandles::generate_method_handle_interpreter_entry(MacroAssembler* _masm, + vmIntrinsics::ID iid) { + const bool not_for_compiler_entry = false; // this is the interpreter entry + assert(is_signature_polymorphic(iid), "expected invoke iid"); + if (iid == vmIntrinsics::_invokeGeneric || + iid == vmIntrinsics::_compiledLambdaForm) { + // Perhaps surprisingly, the symbolic references visible to Java are not directly used. + // They are linked to Java-generated adapters via MethodHandleNatives.linkMethod. + // They all allow an appendix argument. + __ ebreak(); // empty stubs make SG sick + return NULL; + } + + // No need in interpreter entry for linkToNative for now. + // Interpreter calls compiled entry through i2c. + if (iid == vmIntrinsics::_linkToNative) { + __ ebreak(); + return NULL; + } + + // x30: sender SP (must preserve; see prepare_to_jump_from_interpreted) + // xmethod: Method* + // x13: argument locator (parameter slot count, added to sp) + // x11: used as temp to hold mh or receiver + // x10, x29: garbage temps, blown away + Register argp = x13; // argument list ptr, live on error paths + Register mh = x11; // MH receiver; dies quickly and is recycled + + // here's where control starts out: + __ align(CodeEntryAlignment); + address entry_point = __ pc(); + + if (VerifyMethodHandles) { + assert(Method::intrinsic_id_size_in_bytes() == 2, "assuming Method::_intrinsic_id is u2"); + + Label L; + BLOCK_COMMENT("verify_intrinsic_id {"); + __ lhu(t0, Address(xmethod, Method::intrinsic_id_offset_in_bytes())); + __ mv(t1, (int) iid); + __ beq(t0, t1, L); + if (iid == vmIntrinsics::_linkToVirtual || + iid == vmIntrinsics::_linkToSpecial) { + // could do this for all kinds, but would explode assembly code size + trace_method_handle(_masm, "bad Method*::intrinsic_id"); + } + __ ebreak(); + __ bind(L); + BLOCK_COMMENT("} verify_intrinsic_id"); + } + + // First task: Find out how big the argument list is. + Address x13_first_arg_addr; + int ref_kind = signature_polymorphic_intrinsic_ref_kind(iid); + assert(ref_kind != 0 || iid == vmIntrinsics::_invokeBasic, "must be _invokeBasic or a linkTo intrinsic"); + if (ref_kind == 0 || MethodHandles::ref_kind_has_receiver(ref_kind)) { + __ ld(argp, Address(xmethod, Method::const_offset())); + __ load_sized_value(argp, + Address(argp, ConstMethod::size_of_parameters_offset()), + sizeof(u2), /*is_signed*/ false); + x13_first_arg_addr = __ argument_address(argp, -1); + } else { + DEBUG_ONLY(argp = noreg); + } + + if (!is_signature_polymorphic_static(iid)) { + __ ld(mh, x13_first_arg_addr); + DEBUG_ONLY(argp = noreg); + } + + // x13_first_arg_addr is live! + + trace_method_handle_interpreter_entry(_masm, iid); + if (iid == vmIntrinsics::_invokeBasic) { + generate_method_handle_dispatch(_masm, iid, mh, noreg, not_for_compiler_entry); + } else { + // Adjust argument list by popping the trailing MemberName argument. + Register recv = noreg; + if (MethodHandles::ref_kind_has_receiver(ref_kind)) { + // Load the receiver (not the MH; the actual MemberName's receiver) up from the interpreter stack. + __ ld(recv = x12, x13_first_arg_addr); + } + DEBUG_ONLY(argp = noreg); + Register xmember = xmethod; // MemberName ptr; incoming method ptr is dead now + __ pop_reg(xmember); // extract last argument + generate_method_handle_dispatch(_masm, iid, recv, xmember, not_for_compiler_entry); + } + + return entry_point; +} + + +void MethodHandles::generate_method_handle_dispatch(MacroAssembler* _masm, + vmIntrinsics::ID iid, + Register receiver_reg, + Register member_reg, + bool for_compiler_entry) { + assert(is_signature_polymorphic(iid), "expected invoke iid"); + // temps used in this code are not used in *either* compiled or interpreted calling sequences + Register temp1 = x7; + Register temp2 = x28; + Register temp3 = x29; // x30 is live by this point: it contains the sender SP + if (for_compiler_entry) { + assert(receiver_reg == (iid == vmIntrinsics::_linkToStatic ? noreg : j_rarg0), "only valid assignment"); + assert_different_registers(temp1, j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5, j_rarg6, j_rarg7); + assert_different_registers(temp2, j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5, j_rarg6, j_rarg7); + assert_different_registers(temp3, j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5, j_rarg6, j_rarg7); + } + + assert_different_registers(temp1, temp2, temp3, receiver_reg); + assert_different_registers(temp1, temp2, temp3, member_reg); + + if (iid == vmIntrinsics::_invokeBasic || iid == vmIntrinsics::_linkToNative) { + if (iid == vmIntrinsics::_linkToNative) { + assert(for_compiler_entry, "only compiler entry is supported"); + } + // indirect through MH.form.vmentry.vmtarget + jump_to_lambda_form(_masm, receiver_reg, xmethod, temp1, for_compiler_entry); + } else { + // The method is a member invoker used by direct method handles. + if (VerifyMethodHandles) { + // make sure the trailing argument really is a MemberName (caller responsibility) + verify_klass(_masm, member_reg, VM_CLASS_ID(java_lang_invoke_MemberName), + "MemberName required for invokeVirtual etc."); + } + + Address member_clazz( member_reg, NONZERO(java_lang_invoke_MemberName::clazz_offset())); + Address member_vmindex( member_reg, NONZERO(java_lang_invoke_MemberName::vmindex_offset())); + Address member_vmtarget( member_reg, NONZERO(java_lang_invoke_MemberName::method_offset())); + Address vmtarget_method( xmethod, NONZERO(java_lang_invoke_ResolvedMethodName::vmtarget_offset())); + + Register temp1_recv_klass = temp1; + if (iid != vmIntrinsics::_linkToStatic) { + __ verify_oop(receiver_reg); + if (iid == vmIntrinsics::_linkToSpecial) { + // Don't actually load the klass; just null-check the receiver. + __ null_check(receiver_reg); + } else { + // load receiver klass itself + __ null_check(receiver_reg, oopDesc::klass_offset_in_bytes()); + __ load_klass(temp1_recv_klass, receiver_reg); + __ verify_klass_ptr(temp1_recv_klass); + } + BLOCK_COMMENT("check_receiver {"); + // The receiver for the MemberName must be in receiver_reg. + // Check the receiver against the MemberName.clazz + if (VerifyMethodHandles && iid == vmIntrinsics::_linkToSpecial) { + // Did not load it above... + __ load_klass(temp1_recv_klass, receiver_reg); + __ verify_klass_ptr(temp1_recv_klass); + } + if (VerifyMethodHandles && iid != vmIntrinsics::_linkToInterface) { + Label L_ok; + Register temp2_defc = temp2; + __ load_heap_oop(temp2_defc, member_clazz, temp3); + load_klass_from_Class(_masm, temp2_defc); + __ verify_klass_ptr(temp2_defc); + __ check_klass_subtype(temp1_recv_klass, temp2_defc, temp3, L_ok); + // If we get here, the type check failed! + __ ebreak(); + __ bind(L_ok); + } + BLOCK_COMMENT("} check_receiver"); + } + if (iid == vmIntrinsics::_linkToSpecial || + iid == vmIntrinsics::_linkToStatic) { + DEBUG_ONLY(temp1_recv_klass = noreg); // these guys didn't load the recv_klass + } + + // Live registers at this point: + // member_reg - MemberName that was the trailing argument + // temp1_recv_klass - klass of stacked receiver, if needed + // x30 - interpreter linkage (if interpreted) + // x11 ... x10 - compiler arguments (if compiled) + + Label L_incompatible_class_change_error; + switch (iid) { + case vmIntrinsics::_linkToSpecial: + if (VerifyMethodHandles) { + verify_ref_kind(_masm, JVM_REF_invokeSpecial, member_reg, temp3); + } + __ load_heap_oop(xmethod, member_vmtarget); + __ access_load_at(T_ADDRESS, IN_HEAP, xmethod, vmtarget_method, noreg, noreg); + break; + + case vmIntrinsics::_linkToStatic: + if (VerifyMethodHandles) { + verify_ref_kind(_masm, JVM_REF_invokeStatic, member_reg, temp3); + } + __ load_heap_oop(xmethod, member_vmtarget); + __ access_load_at(T_ADDRESS, IN_HEAP, xmethod, vmtarget_method, noreg, noreg); + break; + + case vmIntrinsics::_linkToVirtual: + { + // same as TemplateTable::invokevirtual, + // minus the CP setup and profiling: + + if (VerifyMethodHandles) { + verify_ref_kind(_masm, JVM_REF_invokeVirtual, member_reg, temp3); + } + + // pick out the vtable index from the MemberName, and then we can discard it: + Register temp2_index = temp2; + __ access_load_at(T_ADDRESS, IN_HEAP, temp2_index, member_vmindex, noreg, noreg); + + if (VerifyMethodHandles) { + Label L_index_ok; + __ bgez(temp2_index, L_index_ok); + __ ebreak(); + __ BIND(L_index_ok); + } + + // Note: The verifier invariants allow us to ignore MemberName.clazz and vmtarget + // at this point. And VerifyMethodHandles has already checked clazz, if needed. + + // get target Method* & entry point + __ lookup_virtual_method(temp1_recv_klass, temp2_index, xmethod); + break; + } + + case vmIntrinsics::_linkToInterface: + { + // same as TemplateTable::invokeinterface + // (minus the CP setup and profiling, with different argument motion) + if (VerifyMethodHandles) { + verify_ref_kind(_masm, JVM_REF_invokeInterface, member_reg, temp3); + } + + Register temp3_intf = temp3; + __ load_heap_oop(temp3_intf, member_clazz); + load_klass_from_Class(_masm, temp3_intf); + __ verify_klass_ptr(temp3_intf); + + Register rindex = xmethod; + __ access_load_at(T_ADDRESS, IN_HEAP, rindex, member_vmindex, noreg, noreg); + if (VerifyMethodHandles) { + Label L; + __ bgez(rindex, L); + __ ebreak(); + __ bind(L); + } + + // given intf, index, and recv klass, dispatch to the implementation method + __ lookup_interface_method(temp1_recv_klass, temp3_intf, + // note: next two args must be the same: + rindex, xmethod, + temp2, + L_incompatible_class_change_error); + break; + } + + default: + fatal("unexpected intrinsic %d: %s", vmIntrinsics::as_int(iid), vmIntrinsics::name_at(iid)); + break; + } + + // live at this point: xmethod, x30 (if interpreted) + + // After figuring out which concrete method to call, jump into it. + // Note that this works in the interpreter with no data motion. + // But the compiled version will require that r2_recv be shifted out. + __ verify_method_ptr(xmethod); + jump_from_method_handle(_masm, xmethod, temp1, for_compiler_entry); + if (iid == vmIntrinsics::_linkToInterface) { + __ bind(L_incompatible_class_change_error); + __ far_jump(RuntimeAddress(StubRoutines::throw_IncompatibleClassChangeError_entry())); + } + } + +} + +#ifndef PRODUCT +void trace_method_handle_stub(const char* adaptername, + oopDesc* mh, + intptr_t* saved_regs, + intptr_t* entry_sp) { } + +// The stub wraps the arguments in a struct on the stack to avoid +// dealing with the different calling conventions for passing 6 +// arguments. +struct MethodHandleStubArguments { + const char* adaptername; + oopDesc* mh; + intptr_t* saved_regs; + intptr_t* entry_sp; +}; +void trace_method_handle_stub_wrapper(MethodHandleStubArguments* args) { } + +void MethodHandles::trace_method_handle(MacroAssembler* _masm, const char* adaptername) { } +#endif //PRODUCT diff --git a/src/hotspot/cpu/riscv/methodHandles_riscv.hpp b/src/hotspot/cpu/riscv/methodHandles_riscv.hpp new file mode 100644 index 0000000000000..f73aba29d6750 --- /dev/null +++ b/src/hotspot/cpu/riscv/methodHandles_riscv.hpp @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +// Platform-specific definitions for method handles. +// These definitions are inlined into class MethodHandles. + +// Adapters +enum /* platform_dependent_constants */ { + adapter_code_size = 32000 DEBUG_ONLY(+ 120000) +}; + +public: + + static void load_klass_from_Class(MacroAssembler* _masm, Register klass_reg); + + static void verify_klass(MacroAssembler* _masm, + Register obj, vmClassID klass_id, + const char* error_message = "wrong klass") NOT_DEBUG_RETURN; + + static void verify_method_handle(MacroAssembler* _masm, Register mh_reg) { + verify_klass(_masm, mh_reg, VM_CLASS_ID(java_lang_invoke_MethodHandle), + "reference is a MH"); + } + + static void verify_ref_kind(MacroAssembler* _masm, int ref_kind, Register member_reg, Register temp) NOT_DEBUG_RETURN; + + // Similar to InterpreterMacroAssembler::jump_from_interpreted. + // Takes care of special dispatch from single stepping too. + static void jump_from_method_handle(MacroAssembler* _masm, Register method, Register temp, + bool for_compiler_entry); + + static void jump_to_lambda_form(MacroAssembler* _masm, + Register recv, Register method_temp, + Register temp2, + bool for_compiler_entry); diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp new file mode 100644 index 0000000000000..675ed5be18992 --- /dev/null +++ b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp @@ -0,0 +1,441 @@ +/* + * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.hpp" +#include "code/compiledIC.hpp" +#include "memory/resourceArea.hpp" +#include "nativeInst_riscv.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/handles.hpp" +#include "runtime/orderAccess.hpp" +#include "runtime/safepoint.hpp" +#include "runtime/sharedRuntime.hpp" +#include "runtime/stubRoutines.hpp" +#include "utilities/ostream.hpp" +#ifdef COMPILER1 +#include "c1/c1_Runtime1.hpp" +#endif + +Register NativeInstruction::extract_rs1(address instr) { + assert_cond(instr != NULL); + return as_Register(Assembler::extract(((unsigned*)instr)[0], 19, 15)); +} + +Register NativeInstruction::extract_rs2(address instr) { + assert_cond(instr != NULL); + return as_Register(Assembler::extract(((unsigned*)instr)[0], 24, 20)); +} + +Register NativeInstruction::extract_rd(address instr) { + assert_cond(instr != NULL); + return as_Register(Assembler::extract(((unsigned*)instr)[0], 11, 7)); +} + +uint32_t NativeInstruction::extract_opcode(address instr) { + assert_cond(instr != NULL); + return Assembler::extract(((unsigned*)instr)[0], 6, 0); +} + +uint32_t NativeInstruction::extract_funct3(address instr) { + assert_cond(instr != NULL); + return Assembler::extract(((unsigned*)instr)[0], 14, 12); +} + +bool NativeInstruction::is_pc_relative_at(address instr) { + // auipc + jalr + // auipc + addi + // auipc + load + // auipc + fload_load + return (is_auipc_at(instr)) && + (is_addi_at(instr + instruction_size) || + is_jalr_at(instr + instruction_size) || + is_load_at(instr + instruction_size) || + is_float_load_at(instr + instruction_size)) && + check_pc_relative_data_dependency(instr); +} + +// ie:ld(Rd, Label) +bool NativeInstruction::is_load_pc_relative_at(address instr) { + return is_auipc_at(instr) && // auipc + is_ld_at(instr + instruction_size) && // ld + check_load_pc_relative_data_dependency(instr); +} + +bool NativeInstruction::is_movptr_at(address instr) { + return is_lui_at(instr) && // Lui + is_addi_at(instr + instruction_size) && // Addi + is_slli_shift_at(instr + instruction_size * 2, 11) && // Slli Rd, Rs, 11 + is_addi_at(instr + instruction_size * 3) && // Addi + is_slli_shift_at(instr + instruction_size * 4, 6) && // Slli Rd, Rs, 6 + (is_addi_at(instr + instruction_size * 5) || + is_jalr_at(instr + instruction_size * 5) || + is_load_at(instr + instruction_size * 5)) && // Addi/Jalr/Load + check_movptr_data_dependency(instr); +} + +bool NativeInstruction::is_li32_at(address instr) { + return is_lui_at(instr) && // lui + is_addiw_at(instr + instruction_size) && // addiw + check_li32_data_dependency(instr); +} + +bool NativeInstruction::is_li64_at(address instr) { + return is_lui_at(instr) && // lui + is_addi_at(instr + instruction_size) && // addi + is_slli_shift_at(instr + instruction_size * 2, 12) && // Slli Rd, Rs, 12 + is_addi_at(instr + instruction_size * 3) && // addi + is_slli_shift_at(instr + instruction_size * 4, 12) && // Slli Rd, Rs, 12 + is_addi_at(instr + instruction_size * 5) && // addi + is_slli_shift_at(instr + instruction_size * 6, 8) && // Slli Rd, Rs, 8 + is_addi_at(instr + instruction_size * 7) && // addi + check_li64_data_dependency(instr); +} + +void NativeCall::verify() { + assert(NativeCall::is_call_at((address)this), "unexpected code at call site"); +} + +address NativeCall::destination() const { + address addr = (address)this; + assert(NativeInstruction::is_jal_at(instruction_address()), "inst must be jal."); + address destination = MacroAssembler::target_addr_for_insn(instruction_address()); + + // Do we use a trampoline stub for this call? + CodeBlob* cb = CodeCache::find_blob_unsafe(addr); // Else we get assertion if nmethod is zombie. + assert(cb && cb->is_nmethod(), "sanity"); + nmethod *nm = (nmethod *)cb; + if (nm != NULL && nm->stub_contains(destination) && is_NativeCallTrampolineStub_at(destination)) { + // Yes we do, so get the destination from the trampoline stub. + const address trampoline_stub_addr = destination; + destination = nativeCallTrampolineStub_at(trampoline_stub_addr)->destination(); + } + + return destination; +} + +// Similar to replace_mt_safe, but just changes the destination. The +// important thing is that free-running threads are able to execute this +// call instruction at all times. +// +// Used in the runtime linkage of calls; see class CompiledIC. +// +// Add parameter assert_lock to switch off assertion +// during code generation, where no patching lock is needed. +void NativeCall::set_destination_mt_safe(address dest, bool assert_lock) { + assert(!assert_lock || + (Patching_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) || + CompiledICLocker::is_safe(addr_at(0)), + "concurrent code patching"); + + ResourceMark rm; + address addr_call = addr_at(0); + assert(NativeCall::is_call_at(addr_call), "unexpected code at call site"); + + // Patch the constant in the call's trampoline stub. + address trampoline_stub_addr = get_trampoline(); + if (trampoline_stub_addr != NULL) { + assert (!is_NativeCallTrampolineStub_at(dest), "chained trampolines"); + nativeCallTrampolineStub_at(trampoline_stub_addr)->set_destination(dest); + } + + // Patch the call. + if (Assembler::reachable_from_branch_at(addr_call, dest)) { + set_destination(dest); + } else { + assert (trampoline_stub_addr != NULL, "we need a trampoline"); + set_destination(trampoline_stub_addr); + } + + ICache::invalidate_range(addr_call, instruction_size); +} + +address NativeCall::get_trampoline() { + address call_addr = addr_at(0); + + CodeBlob *code = CodeCache::find_blob(call_addr); + assert(code != NULL, "Could not find the containing code blob"); + + address jal_destination = MacroAssembler::pd_call_destination(call_addr); + if (code != NULL && code->contains(jal_destination) && is_NativeCallTrampolineStub_at(jal_destination)) { + return jal_destination; + } + + if (code != NULL && code->is_nmethod()) { + return trampoline_stub_Relocation::get_trampoline_for(call_addr, (nmethod*)code); + } + + return NULL; +} + +// Inserts a native call instruction at a given pc +void NativeCall::insert(address code_pos, address entry) { Unimplemented(); } + +//------------------------------------------------------------------- + +void NativeMovConstReg::verify() { + if (!(nativeInstruction_at(instruction_address())->is_movptr() || + is_auipc_at(instruction_address()))) { + fatal("should be MOVPTR or AUIPC"); + } +} + +intptr_t NativeMovConstReg::data() const { + address addr = MacroAssembler::target_addr_for_insn(instruction_address()); + if (maybe_cpool_ref(instruction_address())) { + return *(intptr_t*)addr; + } else { + return (intptr_t)addr; + } +} + +void NativeMovConstReg::set_data(intptr_t x) { + if (maybe_cpool_ref(instruction_address())) { + address addr = MacroAssembler::target_addr_for_insn(instruction_address()); + *(intptr_t*)addr = x; + } else { + // Store x into the instruction stream. + MacroAssembler::pd_patch_instruction_size(instruction_address(), (address)x); + ICache::invalidate_range(instruction_address(), movptr_instruction_size); + } + + // Find and replace the oop/metadata corresponding to this + // instruction in oops section. + CodeBlob* cb = CodeCache::find_blob(instruction_address()); + nmethod* nm = cb->as_nmethod_or_null(); + if (nm != NULL) { + RelocIterator iter(nm, instruction_address(), next_instruction_address()); + while (iter.next()) { + if (iter.type() == relocInfo::oop_type) { + oop* oop_addr = iter.oop_reloc()->oop_addr(); + *oop_addr = cast_to_oop(x); + break; + } else if (iter.type() == relocInfo::metadata_type) { + Metadata** metadata_addr = iter.metadata_reloc()->metadata_addr(); + *metadata_addr = (Metadata*)x; + break; + } + } + } +} + +void NativeMovConstReg::print() { + tty->print_cr(PTR_FORMAT ": mov reg, " INTPTR_FORMAT, + p2i(instruction_address()), data()); +} + +//------------------------------------------------------------------- + +int NativeMovRegMem::offset() const { + Unimplemented(); + return 0; +} + +void NativeMovRegMem::set_offset(int x) { Unimplemented(); } + +void NativeMovRegMem::verify() { + Unimplemented(); +} + +//-------------------------------------------------------------------------------- + +void NativeJump::verify() { } + + +void NativeJump::check_verified_entry_alignment(address entry, address verified_entry) { + // Patching to not_entrant can happen while activations of the method are + // in use. The patching in that instance must happen only when certain + // alignment restrictions are true. These guarantees check those + // conditions. + + // Must be 4 bytes aligned + MacroAssembler::assert_alignment(verified_entry); +} + + +address NativeJump::jump_destination() const { + address dest = MacroAssembler::target_addr_for_insn(instruction_address()); + + // We use jump to self as the unresolved address which the inline + // cache code (and relocs) know about + // As a special case we also use sequence movptr(r,0), jalr(r,0) + // i.e. jump to 0 when we need leave space for a wide immediate + // load + + // return -1 if jump to self or to 0 + if ((dest == (address) this) || dest == 0) { + dest = (address) -1; + } + + return dest; +}; + +void NativeJump::set_jump_destination(address dest) { + // We use jump to self as the unresolved address which the inline + // cache code (and relocs) know about + if (dest == (address) -1) + dest = instruction_address(); + + MacroAssembler::pd_patch_instruction(instruction_address(), dest); + ICache::invalidate_range(instruction_address(), instruction_size); +} + +//------------------------------------------------------------------- + +address NativeGeneralJump::jump_destination() const { + NativeMovConstReg* move = nativeMovConstReg_at(instruction_address()); + address dest = (address) move->data(); + + // We use jump to self as the unresolved address which the inline + // cache code (and relocs) know about + // As a special case we also use jump to 0 when first generating + // a general jump + + // return -1 if jump to self or to 0 + if ((dest == (address) this) || dest == 0) { + dest = (address) -1; + } + + return dest; +} + +//------------------------------------------------------------------- + +bool NativeInstruction::is_safepoint_poll() { + return is_lwu_to_zr(address(this)); +} + +bool NativeInstruction::is_lwu_to_zr(address instr) { + assert_cond(instr != NULL); + return (extract_opcode(instr) == 0b0000011 && + extract_funct3(instr) == 0b110 && + extract_rd(instr) == zr); // zr +} + +// A 16-bit instruction with all bits ones is permanently reserved as an illegal instruction. +bool NativeInstruction::is_sigill_zombie_not_entrant() { + // jvmci + return uint_at(0) == 0xffffffff; +} + +void NativeIllegalInstruction::insert(address code_pos) { + assert_cond(code_pos != NULL); + *(juint*)code_pos = 0xffffffff; // all bits ones is permanently reserved as an illegal instruction +} + +bool NativeInstruction::is_stop() { + return uint_at(0) == 0xc0101073; // an illegal instruction, 'csrrw x0, time, x0' +} + +//------------------------------------------------------------------- + +// MT-safe inserting of a jump over a jump or a nop (used by +// nmethod::make_not_entrant_or_zombie) + +void NativeJump::patch_verified_entry(address entry, address verified_entry, address dest) { + + assert(dest == SharedRuntime::get_handle_wrong_method_stub(), "expected fixed destination of patch"); + + assert(nativeInstruction_at(verified_entry)->is_jump_or_nop() || + nativeInstruction_at(verified_entry)->is_sigill_zombie_not_entrant(), + "riscv cannot replace non-jump with jump"); + + check_verified_entry_alignment(entry, verified_entry); + + // Patch this nmethod atomically. + if (Assembler::reachable_from_branch_at(verified_entry, dest)) { + ptrdiff_t offset = dest - verified_entry; + guarantee(Assembler::is_simm21(offset) && ((offset % 2) == 0), + "offset is too large to be patched in one jal instruction."); // 1M + + uint32_t insn = 0; + address pInsn = (address)&insn; + Assembler::patch(pInsn, 31, 31, (offset >> 20) & 0x1); + Assembler::patch(pInsn, 30, 21, (offset >> 1) & 0x3ff); + Assembler::patch(pInsn, 20, 20, (offset >> 11) & 0x1); + Assembler::patch(pInsn, 19, 12, (offset >> 12) & 0xff); + Assembler::patch(pInsn, 11, 7, 0); // zero, no link jump + Assembler::patch(pInsn, 6, 0, 0b1101111); // j, (jal x0 offset) + *(unsigned int*)verified_entry = insn; + } else { + // We use an illegal instruction for marking a method as + // not_entrant or zombie. + NativeIllegalInstruction::insert(verified_entry); + } + + ICache::invalidate_range(verified_entry, instruction_size); +} + +void NativeGeneralJump::insert_unconditional(address code_pos, address entry) { + CodeBuffer cb(code_pos, instruction_size); + MacroAssembler a(&cb); + Assembler::IncompressibleRegion ir(&a); // Fixed length: see NativeGeneralJump::get_instruction_size() + + int32_t offset = 0; + a.movptr(t0, entry, offset); // lui, addi, slli, addi, slli + a.jalr(x0, t0, offset); // jalr + + ICache::invalidate_range(code_pos, instruction_size); +} + +// MT-safe patching of a long jump instruction. +void NativeGeneralJump::replace_mt_safe(address instr_addr, address code_buffer) { + ShouldNotCallThis(); +} + + +address NativeCallTrampolineStub::destination(nmethod *nm) const { + return ptr_at(data_offset); +} + +void NativeCallTrampolineStub::set_destination(address new_destination) { + set_ptr_at(data_offset, new_destination); + OrderAccess::release(); +} + +uint32_t NativeMembar::get_kind() { + uint32_t insn = uint_at(0); + + uint32_t predecessor = Assembler::extract(insn, 27, 24); + uint32_t successor = Assembler::extract(insn, 23, 20); + + return MacroAssembler::pred_succ_to_membar_mask(predecessor, successor); +} + +void NativeMembar::set_kind(uint32_t order_kind) { + uint32_t predecessor = 0; + uint32_t successor = 0; + + MacroAssembler::membar_mask_to_pred_succ(order_kind, predecessor, successor); + + uint32_t insn = uint_at(0); + address pInsn = (address) &insn; + Assembler::patch(pInsn, 27, 24, predecessor); + Assembler::patch(pInsn, 23, 20, successor); + + address membar = addr_at(0); + *(unsigned int*) membar = insn; +} diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp new file mode 100644 index 0000000000000..288ccc08d810c --- /dev/null +++ b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp @@ -0,0 +1,554 @@ +/* + * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2018, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_NATIVEINST_RISCV_HPP +#define CPU_RISCV_NATIVEINST_RISCV_HPP + +#include "asm/assembler.hpp" +#include "runtime/icache.hpp" +#include "runtime/os.hpp" + +// We have interfaces for the following instructions: +// - NativeInstruction +// - - NativeCall +// - - NativeMovConstReg +// - - NativeMovRegMem +// - - NativeJump +// - - NativeGeneralJump +// - - NativeIllegalInstruction +// - - NativeCallTrampolineStub +// - - NativeMembar + +// The base class for different kinds of native instruction abstractions. +// Provides the primitive operations to manipulate code relative to this. + +class NativeCall; + +class NativeInstruction { + friend class Relocation; + friend bool is_NativeCallTrampolineStub_at(address); + public: + enum { + instruction_size = 4, + compressed_instruction_size = 2, + }; + + juint encoding() const { + return uint_at(0); + } + + bool is_jal() const { return is_jal_at(addr_at(0)); } + bool is_movptr() const { return is_movptr_at(addr_at(0)); } + bool is_call() const { return is_call_at(addr_at(0)); } + bool is_jump() const { return is_jump_at(addr_at(0)); } + + static bool is_jal_at(address instr) { assert_cond(instr != NULL); return extract_opcode(instr) == 0b1101111; } + static bool is_jalr_at(address instr) { assert_cond(instr != NULL); return extract_opcode(instr) == 0b1100111 && extract_funct3(instr) == 0b000; } + static bool is_branch_at(address instr) { assert_cond(instr != NULL); return extract_opcode(instr) == 0b1100011; } + static bool is_ld_at(address instr) { assert_cond(instr != NULL); return is_load_at(instr) && extract_funct3(instr) == 0b011; } + static bool is_load_at(address instr) { assert_cond(instr != NULL); return extract_opcode(instr) == 0b0000011; } + static bool is_float_load_at(address instr) { assert_cond(instr != NULL); return extract_opcode(instr) == 0b0000111; } + static bool is_auipc_at(address instr) { assert_cond(instr != NULL); return extract_opcode(instr) == 0b0010111; } + static bool is_jump_at(address instr) { assert_cond(instr != NULL); return is_branch_at(instr) || is_jal_at(instr) || is_jalr_at(instr); } + static bool is_addi_at(address instr) { assert_cond(instr != NULL); return extract_opcode(instr) == 0b0010011 && extract_funct3(instr) == 0b000; } + static bool is_addiw_at(address instr) { assert_cond(instr != NULL); return extract_opcode(instr) == 0b0011011 && extract_funct3(instr) == 0b000; } + static bool is_lui_at(address instr) { assert_cond(instr != NULL); return extract_opcode(instr) == 0b0110111; } + static bool is_slli_shift_at(address instr, uint32_t shift) { + assert_cond(instr != NULL); + return (extract_opcode(instr) == 0b0010011 && // opcode field + extract_funct3(instr) == 0b001 && // funct3 field, select the type of operation + Assembler::extract(((unsigned*)instr)[0], 25, 20) == shift); // shamt field + } + + static Register extract_rs1(address instr); + static Register extract_rs2(address instr); + static Register extract_rd(address instr); + static uint32_t extract_opcode(address instr); + static uint32_t extract_funct3(address instr); + + // the instruction sequence of movptr is as below: + // lui + // addi + // slli + // addi + // slli + // addi/jalr/load + static bool check_movptr_data_dependency(address instr) { + address lui = instr; + address addi1 = lui + instruction_size; + address slli1 = addi1 + instruction_size; + address addi2 = slli1 + instruction_size; + address slli2 = addi2 + instruction_size; + address last_instr = slli2 + instruction_size; + return extract_rs1(addi1) == extract_rd(lui) && + extract_rs1(addi1) == extract_rd(addi1) && + extract_rs1(slli1) == extract_rd(addi1) && + extract_rs1(slli1) == extract_rd(slli1) && + extract_rs1(addi2) == extract_rd(slli1) && + extract_rs1(addi2) == extract_rd(addi2) && + extract_rs1(slli2) == extract_rd(addi2) && + extract_rs1(slli2) == extract_rd(slli2) && + extract_rs1(last_instr) == extract_rd(slli2); + } + + // the instruction sequence of li64 is as below: + // lui + // addi + // slli + // addi + // slli + // addi + // slli + // addi + static bool check_li64_data_dependency(address instr) { + address lui = instr; + address addi1 = lui + instruction_size; + address slli1 = addi1 + instruction_size; + address addi2 = slli1 + instruction_size; + address slli2 = addi2 + instruction_size; + address addi3 = slli2 + instruction_size; + address slli3 = addi3 + instruction_size; + address addi4 = slli3 + instruction_size; + return extract_rs1(addi1) == extract_rd(lui) && + extract_rs1(addi1) == extract_rd(addi1) && + extract_rs1(slli1) == extract_rd(addi1) && + extract_rs1(slli1) == extract_rd(slli1) && + extract_rs1(addi2) == extract_rd(slli1) && + extract_rs1(addi2) == extract_rd(addi2) && + extract_rs1(slli2) == extract_rd(addi2) && + extract_rs1(slli2) == extract_rd(slli2) && + extract_rs1(addi3) == extract_rd(slli2) && + extract_rs1(addi3) == extract_rd(addi3) && + extract_rs1(slli3) == extract_rd(addi3) && + extract_rs1(slli3) == extract_rd(slli3) && + extract_rs1(addi4) == extract_rd(slli3) && + extract_rs1(addi4) == extract_rd(addi4); + } + + // the instruction sequence of li32 is as below: + // lui + // addiw + static bool check_li32_data_dependency(address instr) { + address lui = instr; + address addiw = lui + instruction_size; + + return extract_rs1(addiw) == extract_rd(lui) && + extract_rs1(addiw) == extract_rd(addiw); + } + + // the instruction sequence of pc-relative is as below: + // auipc + // jalr/addi/load/float_load + static bool check_pc_relative_data_dependency(address instr) { + address auipc = instr; + address last_instr = auipc + instruction_size; + + return extract_rs1(last_instr) == extract_rd(auipc); + } + + // the instruction sequence of load_label is as below: + // auipc + // load + static bool check_load_pc_relative_data_dependency(address instr) { + address auipc = instr; + address load = auipc + instruction_size; + + return extract_rd(load) == extract_rd(auipc) && + extract_rs1(load) == extract_rd(load); + } + + static bool is_movptr_at(address instr); + static bool is_li32_at(address instr); + static bool is_li64_at(address instr); + static bool is_pc_relative_at(address branch); + static bool is_load_pc_relative_at(address branch); + + static bool is_call_at(address instr) { + if (is_jal_at(instr) || is_jalr_at(instr)) { + return true; + } + return false; + } + static bool is_lwu_to_zr(address instr); + + inline bool is_nop(); + inline bool is_jump_or_nop(); + bool is_safepoint_poll(); + bool is_sigill_zombie_not_entrant(); + bool is_stop(); + + protected: + address addr_at(int offset) const { return address(this) + offset; } + + jint int_at(int offset) const { return *(jint*) addr_at(offset); } + juint uint_at(int offset) const { return *(juint*) addr_at(offset); } + + address ptr_at(int offset) const { return *(address*) addr_at(offset); } + + oop oop_at (int offset) const { return *(oop*) addr_at(offset); } + + + void set_int_at(int offset, jint i) { *(jint*)addr_at(offset) = i; } + void set_uint_at(int offset, jint i) { *(juint*)addr_at(offset) = i; } + void set_ptr_at (int offset, address ptr) { *(address*) addr_at(offset) = ptr; } + void set_oop_at (int offset, oop o) { *(oop*) addr_at(offset) = o; } + + public: + + inline friend NativeInstruction* nativeInstruction_at(address addr); + + static bool maybe_cpool_ref(address instr) { + return is_auipc_at(instr); + } + + bool is_membar() { + return (uint_at(0) & 0x7f) == 0b1111 && extract_funct3(addr_at(0)) == 0; + } +}; + +inline NativeInstruction* nativeInstruction_at(address addr) { + return (NativeInstruction*)addr; +} + +// The natural type of an RISCV instruction is uint32_t +inline NativeInstruction* nativeInstruction_at(uint32_t *addr) { + return (NativeInstruction*)addr; +} + +inline NativeCall* nativeCall_at(address addr); +// The NativeCall is an abstraction for accessing/manipulating native +// call instructions (used to manipulate inline caches, primitive & +// DSO calls, etc.). + +class NativeCall: public NativeInstruction { + public: + enum RISCV_specific_constants { + instruction_size = 4, + instruction_offset = 0, + displacement_offset = 0, + return_address_offset = 4 + }; + + address instruction_address() const { return addr_at(instruction_offset); } + address next_instruction_address() const { return addr_at(return_address_offset); } + address return_address() const { return addr_at(return_address_offset); } + address destination() const; + + void set_destination(address dest) { + assert(is_jal(), "Should be jal instruction!"); + intptr_t offset = (intptr_t)(dest - instruction_address()); + assert((offset & 0x1) == 0, "bad alignment"); + assert(Assembler::is_simm21(offset), "encoding constraint"); + unsigned int insn = 0b1101111; // jal + address pInsn = (address)(&insn); + Assembler::patch(pInsn, 31, 31, (offset >> 20) & 0x1); + Assembler::patch(pInsn, 30, 21, (offset >> 1) & 0x3ff); + Assembler::patch(pInsn, 20, 20, (offset >> 11) & 0x1); + Assembler::patch(pInsn, 19, 12, (offset >> 12) & 0xff); + Assembler::patch(pInsn, 11, 7, ra->encoding()); // Rd must be x1, need ra + set_int_at(displacement_offset, insn); + } + + void verify_alignment() {} // do nothing on riscv + void verify(); + void print(); + + // Creation + inline friend NativeCall* nativeCall_at(address addr); + inline friend NativeCall* nativeCall_before(address return_address); + + static bool is_call_before(address return_address) { + return is_call_at(return_address - NativeCall::return_address_offset); + } + + // MT-safe patching of a call instruction. + static void insert(address code_pos, address entry); + + static void replace_mt_safe(address instr_addr, address code_buffer); + + // Similar to replace_mt_safe, but just changes the destination. The + // important thing is that free-running threads are able to execute + // this call instruction at all times. If the call is an immediate BL + // instruction we can simply rely on atomicity of 32-bit writes to + // make sure other threads will see no intermediate states. + + // We cannot rely on locks here, since the free-running threads must run at + // full speed. + // + // Used in the runtime linkage of calls; see class CompiledIC. + // (Cf. 4506997 and 4479829, where threads witnessed garbage displacements.) + + // The parameter assert_lock disables the assertion during code generation. + void set_destination_mt_safe(address dest, bool assert_lock = true); + + address get_trampoline(); +}; + +inline NativeCall* nativeCall_at(address addr) { + assert_cond(addr != NULL); + NativeCall* call = (NativeCall*)(addr - NativeCall::instruction_offset); + DEBUG_ONLY(call->verify()); + return call; +} + +inline NativeCall* nativeCall_before(address return_address) { + assert_cond(return_address != NULL); + NativeCall* call = (NativeCall*)(return_address - NativeCall::return_address_offset); + DEBUG_ONLY(call->verify()); + return call; +} + +// An interface for accessing/manipulating native mov reg, imm instructions. +// (used to manipulate inlined 64-bit data calls, etc.) +class NativeMovConstReg: public NativeInstruction { + public: + enum RISCV_specific_constants { + movptr_instruction_size = 6 * NativeInstruction::instruction_size, // lui, addi, slli, addi, slli, addi. See movptr(). + load_pc_relative_instruction_size = 2 * NativeInstruction::instruction_size, // auipc, ld + instruction_offset = 0, + displacement_offset = 0 + }; + + address instruction_address() const { return addr_at(instruction_offset); } + address next_instruction_address() const { + // if the instruction at 5 * instruction_size is addi, + // it means a lui + addi + slli + addi + slli + addi instruction sequence, + // and the next instruction address should be addr_at(6 * instruction_size). + // However, when the instruction at 5 * instruction_size isn't addi, + // the next instruction address should be addr_at(5 * instruction_size) + if (nativeInstruction_at(instruction_address())->is_movptr()) { + if (is_addi_at(addr_at(movptr_instruction_size - NativeInstruction::instruction_size))) { + // Assume: lui, addi, slli, addi, slli, addi + return addr_at(movptr_instruction_size); + } else { + // Assume: lui, addi, slli, addi, slli + return addr_at(movptr_instruction_size - NativeInstruction::instruction_size); + } + } else if (is_load_pc_relative_at(instruction_address())) { + // Assume: auipc, ld + return addr_at(load_pc_relative_instruction_size); + } + guarantee(false, "Unknown instruction in NativeMovConstReg"); + return NULL; + } + + intptr_t data() const; + void set_data(intptr_t x); + + void flush() { + if (!maybe_cpool_ref(instruction_address())) { + ICache::invalidate_range(instruction_address(), movptr_instruction_size); + } + } + + void verify(); + void print(); + + // Creation + inline friend NativeMovConstReg* nativeMovConstReg_at(address addr); + inline friend NativeMovConstReg* nativeMovConstReg_before(address addr); +}; + +inline NativeMovConstReg* nativeMovConstReg_at(address addr) { + assert_cond(addr != NULL); + NativeMovConstReg* test = (NativeMovConstReg*)(addr - NativeMovConstReg::instruction_offset); + DEBUG_ONLY(test->verify()); + return test; +} + +inline NativeMovConstReg* nativeMovConstReg_before(address addr) { + assert_cond(addr != NULL); + NativeMovConstReg* test = (NativeMovConstReg*)(addr - NativeMovConstReg::instruction_size - NativeMovConstReg::instruction_offset); + DEBUG_ONLY(test->verify()); + return test; +} + +// RISCV should not use C1 runtime patching, but still implement +// NativeMovRegMem to keep some compilers happy. +class NativeMovRegMem: public NativeInstruction { + public: + enum RISCV_specific_constants { + instruction_size = NativeInstruction::instruction_size, + instruction_offset = 0, + data_offset = 0, + next_instruction_offset = NativeInstruction::instruction_size + }; + + int instruction_start() const { return instruction_offset; } + + address instruction_address() const { return addr_at(instruction_offset); } + + int num_bytes_to_end_of_patch() const { return instruction_offset + instruction_size; } + + int offset() const; + + void set_offset(int x); + + void add_offset_in_bytes(int add_offset) { + set_offset(offset() + add_offset); + } + + void verify(); + void print(); + + private: + inline friend NativeMovRegMem* nativeMovRegMem_at(address addr); +}; + +inline NativeMovRegMem* nativeMovRegMem_at(address addr) { + NativeMovRegMem* test = (NativeMovRegMem*)(addr - NativeMovRegMem::instruction_offset); + DEBUG_ONLY(test->verify()); + return test; +} + +class NativeJump: public NativeInstruction { + public: + enum RISCV_specific_constants { + instruction_size = NativeInstruction::instruction_size, + instruction_offset = 0, + data_offset = 0, + next_instruction_offset = NativeInstruction::instruction_size + }; + + address instruction_address() const { return addr_at(instruction_offset); } + address next_instruction_address() const { return addr_at(instruction_size); } + address jump_destination() const; + void set_jump_destination(address dest); + + // Creation + inline friend NativeJump* nativeJump_at(address address); + + void verify(); + + // Insertion of native jump instruction + static void insert(address code_pos, address entry); + // MT-safe insertion of native jump at verified method entry + static void check_verified_entry_alignment(address entry, address verified_entry); + static void patch_verified_entry(address entry, address verified_entry, address dest); +}; + +inline NativeJump* nativeJump_at(address addr) { + NativeJump* jump = (NativeJump*)(addr - NativeJump::instruction_offset); + DEBUG_ONLY(jump->verify()); + return jump; +} + +class NativeGeneralJump: public NativeJump { +public: + enum RISCV_specific_constants { + instruction_size = 6 * NativeInstruction::instruction_size, // lui, addi, slli, addi, slli, jalr + instruction_offset = 0, + data_offset = 0, + next_instruction_offset = 6 * NativeInstruction::instruction_size // lui, addi, slli, addi, slli, jalr + }; + + address jump_destination() const; + + static void insert_unconditional(address code_pos, address entry); + static void replace_mt_safe(address instr_addr, address code_buffer); +}; + +inline NativeGeneralJump* nativeGeneralJump_at(address addr) { + assert_cond(addr != NULL); + NativeGeneralJump* jump = (NativeGeneralJump*)(addr); + debug_only(jump->verify();) + return jump; +} + +class NativeIllegalInstruction: public NativeInstruction { + public: + // Insert illegal opcode as specific address + static void insert(address code_pos); +}; + +inline bool NativeInstruction::is_nop() { + uint32_t insn = *(uint32_t*)addr_at(0); + return insn == 0x13; +} + +inline bool NativeInstruction::is_jump_or_nop() { + return is_nop() || is_jump(); +} + +// Call trampoline stubs. +class NativeCallTrampolineStub : public NativeInstruction { + public: + + enum RISCV_specific_constants { + // Refer to function emit_trampoline_stub. + instruction_size = 3 * NativeInstruction::instruction_size + wordSize, // auipc + ld + jr + target address + data_offset = 3 * NativeInstruction::instruction_size, // auipc + ld + jr + }; + + address destination(nmethod *nm = NULL) const; + void set_destination(address new_destination); + ptrdiff_t destination_offset() const; +}; + +inline bool is_NativeCallTrampolineStub_at(address addr) { + // Ensure that the stub is exactly + // ld t0, L--->auipc + ld + // jr t0 + // L: + + // judge inst + register + imm + // 1). check the instructions: auipc + ld + jalr + // 2). check if auipc[11:7] == t0 and ld[11:7] == t0 and ld[19:15] == t0 && jr[19:15] == t0 + // 3). check if the offset in ld[31:20] equals the data_offset + assert_cond(addr != NULL); + const int instr_size = NativeInstruction::instruction_size; + if (NativeInstruction::is_auipc_at(addr) && + NativeInstruction::is_ld_at(addr + instr_size) && + NativeInstruction::is_jalr_at(addr + 2 * instr_size) && + (NativeInstruction::extract_rd(addr) == x5) && + (NativeInstruction::extract_rd(addr + instr_size) == x5) && + (NativeInstruction::extract_rs1(addr + instr_size) == x5) && + (NativeInstruction::extract_rs1(addr + 2 * instr_size) == x5) && + (Assembler::extract(((unsigned*)addr)[1], 31, 20) == NativeCallTrampolineStub::data_offset)) { + return true; + } + return false; +} + +inline NativeCallTrampolineStub* nativeCallTrampolineStub_at(address addr) { + assert_cond(addr != NULL); + assert(is_NativeCallTrampolineStub_at(addr), "no call trampoline found"); + return (NativeCallTrampolineStub*)addr; +} + +class NativeMembar : public NativeInstruction { +public: + uint32_t get_kind(); + void set_kind(uint32_t order_kind); +}; + +inline NativeMembar *NativeMembar_at(address addr) { + assert_cond(addr != NULL); + assert(nativeInstruction_at(addr)->is_membar(), "no membar found"); + return (NativeMembar*)addr; +} + +#endif // CPU_RISCV_NATIVEINST_RISCV_HPP diff --git a/test/lib-test/jdk/test/whitebox/OldWhiteBox.java b/src/hotspot/cpu/riscv/registerMap_riscv.cpp similarity index 52% rename from test/lib-test/jdk/test/whitebox/OldWhiteBox.java rename to src/hotspot/cpu/riscv/registerMap_riscv.cpp index 677ed210f692b..26c1edc36ffc4 100644 --- a/test/lib-test/jdk/test/whitebox/OldWhiteBox.java +++ b/src/hotspot/cpu/riscv/registerMap_riscv.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -19,33 +20,26 @@ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. + * */ -/* - * @test id=without-inner-class - * @summary verify that sun.hotspot.WhiteBox class still can be used - * @library /test/lib - * @build sun.hotspot.WhiteBox - * @run driver jdk.test.lib.helpers.ClassFileInstaller sun.hotspot.WhiteBox - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI OldWhiteBox - */ - -/* - * @test id=with-inner-class - * @summary verify that sun.hotspot.WhiteBox class still can be used - * @library /test/lib - * @build sun.hotspot.WhiteBox - * @run driver jdk.test.lib.helpers.ClassFileInstaller sun.hotspot.WhiteBox sun.hotspot.WhiteBox$WhiteBoxPermission - * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI OldWhiteBox - */ - -import sun.hotspot.WhiteBox; +#include "precompiled.hpp" +#include "runtime/registerMap.hpp" +#include "vmreg_riscv.inline.hpp" -public class OldWhiteBox { - public static void main(String[] args) { - WhiteBox wb = WhiteBox.getWhiteBox(); - if (wb.getHeapOopSize() < 0) { - throw new Error("wb.getHeapOopSize() < 0"); - } +address RegisterMap::pd_location(VMReg base_reg, int slot_idx) const { + if (base_reg->is_VectorRegister()) { + assert(base_reg->is_concrete(), "must pass base reg"); + int base_reg_enc = (base_reg->value() - ConcreteRegisterImpl::max_fpr) / + VectorRegisterImpl::max_slots_per_register; + intptr_t offset_in_bytes = slot_idx * VMRegImpl::stack_slot_size; + address base_location = location(base_reg); + if (base_location != NULL) { + return base_location + offset_in_bytes; + } else { + return NULL; } + } else { + return location(base_reg->next(slot_idx)); + } } diff --git a/src/hotspot/cpu/riscv/registerMap_riscv.hpp b/src/hotspot/cpu/riscv/registerMap_riscv.hpp new file mode 100644 index 0000000000000..f34349811a9ca --- /dev/null +++ b/src/hotspot/cpu/riscv/registerMap_riscv.hpp @@ -0,0 +1,43 @@ +/* + * Copyright (c) 1998, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_REGISTERMAP_RISCV_HPP +#define CPU_RISCV_REGISTERMAP_RISCV_HPP + +// machine-dependent implemention for register maps + friend class frame; + + private: + // This is the hook for finding a register in an "well-known" location, + // such as a register block of a predetermined format. + address pd_location(VMReg reg) const { return NULL; } + address pd_location(VMReg base_reg, int slot_idx) const; + + // no PD state to clear or copy: + void pd_clear() {} + void pd_initialize() {} + void pd_initialize_from(const RegisterMap* map) {} + +#endif // CPU_RISCV_REGISTERMAP_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/register_definitions_riscv.cpp b/src/hotspot/cpu/riscv/register_definitions_riscv.cpp new file mode 100644 index 0000000000000..624d4b3e20193 --- /dev/null +++ b/src/hotspot/cpu/riscv/register_definitions_riscv.cpp @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2002, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/assembler.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "asm/register.hpp" +#include "interp_masm_riscv.hpp" +#include "register_riscv.hpp" + +REGISTER_DEFINITION(Register, noreg); + +REGISTER_DEFINITION(Register, x0); +REGISTER_DEFINITION(Register, x1); +REGISTER_DEFINITION(Register, x2); +REGISTER_DEFINITION(Register, x3); +REGISTER_DEFINITION(Register, x4); +REGISTER_DEFINITION(Register, x5); +REGISTER_DEFINITION(Register, x6); +REGISTER_DEFINITION(Register, x7); +REGISTER_DEFINITION(Register, x8); +REGISTER_DEFINITION(Register, x9); +REGISTER_DEFINITION(Register, x10); +REGISTER_DEFINITION(Register, x11); +REGISTER_DEFINITION(Register, x12); +REGISTER_DEFINITION(Register, x13); +REGISTER_DEFINITION(Register, x14); +REGISTER_DEFINITION(Register, x15); +REGISTER_DEFINITION(Register, x16); +REGISTER_DEFINITION(Register, x17); +REGISTER_DEFINITION(Register, x18); +REGISTER_DEFINITION(Register, x19); +REGISTER_DEFINITION(Register, x20); +REGISTER_DEFINITION(Register, x21); +REGISTER_DEFINITION(Register, x22); +REGISTER_DEFINITION(Register, x23); +REGISTER_DEFINITION(Register, x24); +REGISTER_DEFINITION(Register, x25); +REGISTER_DEFINITION(Register, x26); +REGISTER_DEFINITION(Register, x27); +REGISTER_DEFINITION(Register, x28); +REGISTER_DEFINITION(Register, x29); +REGISTER_DEFINITION(Register, x30); +REGISTER_DEFINITION(Register, x31); + +REGISTER_DEFINITION(FloatRegister, fnoreg); + +REGISTER_DEFINITION(FloatRegister, f0); +REGISTER_DEFINITION(FloatRegister, f1); +REGISTER_DEFINITION(FloatRegister, f2); +REGISTER_DEFINITION(FloatRegister, f3); +REGISTER_DEFINITION(FloatRegister, f4); +REGISTER_DEFINITION(FloatRegister, f5); +REGISTER_DEFINITION(FloatRegister, f6); +REGISTER_DEFINITION(FloatRegister, f7); +REGISTER_DEFINITION(FloatRegister, f8); +REGISTER_DEFINITION(FloatRegister, f9); +REGISTER_DEFINITION(FloatRegister, f10); +REGISTER_DEFINITION(FloatRegister, f11); +REGISTER_DEFINITION(FloatRegister, f12); +REGISTER_DEFINITION(FloatRegister, f13); +REGISTER_DEFINITION(FloatRegister, f14); +REGISTER_DEFINITION(FloatRegister, f15); +REGISTER_DEFINITION(FloatRegister, f16); +REGISTER_DEFINITION(FloatRegister, f17); +REGISTER_DEFINITION(FloatRegister, f18); +REGISTER_DEFINITION(FloatRegister, f19); +REGISTER_DEFINITION(FloatRegister, f20); +REGISTER_DEFINITION(FloatRegister, f21); +REGISTER_DEFINITION(FloatRegister, f22); +REGISTER_DEFINITION(FloatRegister, f23); +REGISTER_DEFINITION(FloatRegister, f24); +REGISTER_DEFINITION(FloatRegister, f25); +REGISTER_DEFINITION(FloatRegister, f26); +REGISTER_DEFINITION(FloatRegister, f27); +REGISTER_DEFINITION(FloatRegister, f28); +REGISTER_DEFINITION(FloatRegister, f29); +REGISTER_DEFINITION(FloatRegister, f30); +REGISTER_DEFINITION(FloatRegister, f31); + +REGISTER_DEFINITION(VectorRegister, vnoreg); + +REGISTER_DEFINITION(VectorRegister, v0); +REGISTER_DEFINITION(VectorRegister, v1); +REGISTER_DEFINITION(VectorRegister, v2); +REGISTER_DEFINITION(VectorRegister, v3); +REGISTER_DEFINITION(VectorRegister, v4); +REGISTER_DEFINITION(VectorRegister, v5); +REGISTER_DEFINITION(VectorRegister, v6); +REGISTER_DEFINITION(VectorRegister, v7); +REGISTER_DEFINITION(VectorRegister, v8); +REGISTER_DEFINITION(VectorRegister, v9); +REGISTER_DEFINITION(VectorRegister, v10); +REGISTER_DEFINITION(VectorRegister, v11); +REGISTER_DEFINITION(VectorRegister, v12); +REGISTER_DEFINITION(VectorRegister, v13); +REGISTER_DEFINITION(VectorRegister, v14); +REGISTER_DEFINITION(VectorRegister, v15); +REGISTER_DEFINITION(VectorRegister, v16); +REGISTER_DEFINITION(VectorRegister, v17); +REGISTER_DEFINITION(VectorRegister, v18); +REGISTER_DEFINITION(VectorRegister, v19); +REGISTER_DEFINITION(VectorRegister, v20); +REGISTER_DEFINITION(VectorRegister, v21); +REGISTER_DEFINITION(VectorRegister, v22); +REGISTER_DEFINITION(VectorRegister, v23); +REGISTER_DEFINITION(VectorRegister, v24); +REGISTER_DEFINITION(VectorRegister, v25); +REGISTER_DEFINITION(VectorRegister, v26); +REGISTER_DEFINITION(VectorRegister, v27); +REGISTER_DEFINITION(VectorRegister, v28); +REGISTER_DEFINITION(VectorRegister, v29); +REGISTER_DEFINITION(VectorRegister, v30); +REGISTER_DEFINITION(VectorRegister, v31); + +REGISTER_DEFINITION(Register, c_rarg0); +REGISTER_DEFINITION(Register, c_rarg1); +REGISTER_DEFINITION(Register, c_rarg2); +REGISTER_DEFINITION(Register, c_rarg3); +REGISTER_DEFINITION(Register, c_rarg4); +REGISTER_DEFINITION(Register, c_rarg5); +REGISTER_DEFINITION(Register, c_rarg6); +REGISTER_DEFINITION(Register, c_rarg7); + +REGISTER_DEFINITION(FloatRegister, c_farg0); +REGISTER_DEFINITION(FloatRegister, c_farg1); +REGISTER_DEFINITION(FloatRegister, c_farg2); +REGISTER_DEFINITION(FloatRegister, c_farg3); +REGISTER_DEFINITION(FloatRegister, c_farg4); +REGISTER_DEFINITION(FloatRegister, c_farg5); +REGISTER_DEFINITION(FloatRegister, c_farg6); +REGISTER_DEFINITION(FloatRegister, c_farg7); + +REGISTER_DEFINITION(Register, j_rarg0); +REGISTER_DEFINITION(Register, j_rarg1); +REGISTER_DEFINITION(Register, j_rarg2); +REGISTER_DEFINITION(Register, j_rarg3); +REGISTER_DEFINITION(Register, j_rarg4); +REGISTER_DEFINITION(Register, j_rarg5); +REGISTER_DEFINITION(Register, j_rarg6); +REGISTER_DEFINITION(Register, j_rarg7); + +REGISTER_DEFINITION(FloatRegister, j_farg0); +REGISTER_DEFINITION(FloatRegister, j_farg1); +REGISTER_DEFINITION(FloatRegister, j_farg2); +REGISTER_DEFINITION(FloatRegister, j_farg3); +REGISTER_DEFINITION(FloatRegister, j_farg4); +REGISTER_DEFINITION(FloatRegister, j_farg5); +REGISTER_DEFINITION(FloatRegister, j_farg6); +REGISTER_DEFINITION(FloatRegister, j_farg7); + +REGISTER_DEFINITION(Register, zr); +REGISTER_DEFINITION(Register, gp); +REGISTER_DEFINITION(Register, tp); +REGISTER_DEFINITION(Register, xmethod); +REGISTER_DEFINITION(Register, ra); +REGISTER_DEFINITION(Register, sp); +REGISTER_DEFINITION(Register, fp); +REGISTER_DEFINITION(Register, xheapbase); +REGISTER_DEFINITION(Register, xcpool); +REGISTER_DEFINITION(Register, xmonitors); +REGISTER_DEFINITION(Register, xlocals); +REGISTER_DEFINITION(Register, xthread); +REGISTER_DEFINITION(Register, xbcp); +REGISTER_DEFINITION(Register, xdispatch); +REGISTER_DEFINITION(Register, esp); + +REGISTER_DEFINITION(Register, t0); +REGISTER_DEFINITION(Register, t1); +REGISTER_DEFINITION(Register, t2); diff --git a/src/hotspot/cpu/riscv/register_riscv.cpp b/src/hotspot/cpu/riscv/register_riscv.cpp new file mode 100644 index 0000000000000..7e31ebe05dbeb --- /dev/null +++ b/src/hotspot/cpu/riscv/register_riscv.cpp @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "register_riscv.hpp" + +const int ConcreteRegisterImpl::max_gpr = RegisterImpl::number_of_registers * + RegisterImpl::max_slots_per_register; + +const int ConcreteRegisterImpl::max_fpr = + ConcreteRegisterImpl::max_gpr + + FloatRegisterImpl::number_of_registers * FloatRegisterImpl::max_slots_per_register; + +const int ConcreteRegisterImpl::max_vpr = + ConcreteRegisterImpl::max_fpr + + VectorRegisterImpl::number_of_registers * VectorRegisterImpl::max_slots_per_register; + + +const char* RegisterImpl::name() const { + static const char *const names[number_of_registers] = { + "zr", "ra", "sp", "gp", "tp", "t0", "t1", "t2", "fp", "x9", + "c_rarg0", "c_rarg1", "c_rarg2", "c_rarg3", "c_rarg4", "c_rarg5", "c_rarg6", "c_rarg7", + "x18", "x19", "esp", "xdispatch", "xbcp", "xthread", "xlocals", + "xmonitors", "xcpool", "xheapbase", "x28", "x29", "x30", "xmethod" + }; + return is_valid() ? names[encoding()] : "noreg"; +} + +const char* FloatRegisterImpl::name() const { + static const char *const names[number_of_registers] = { + "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", + "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", + "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", + "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31" + }; + return is_valid() ? names[encoding()] : "noreg"; +} + +const char* VectorRegisterImpl::name() const { + static const char *const names[number_of_registers] = { + "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", + "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31" + }; + return is_valid() ? names[encoding()] : "noreg"; +} diff --git a/src/hotspot/cpu/riscv/register_riscv.hpp b/src/hotspot/cpu/riscv/register_riscv.hpp new file mode 100644 index 0000000000000..8fa9e4237ffab --- /dev/null +++ b/src/hotspot/cpu/riscv/register_riscv.hpp @@ -0,0 +1,385 @@ +/* + * Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_REGISTER_RISCV_HPP +#define CPU_RISCV_REGISTER_RISCV_HPP + +#include "asm/register.hpp" + +#define CSR_FFLAGS 0x001 // Floating-Point Accrued Exceptions. +#define CSR_FRM 0x002 // Floating-Point Dynamic Rounding Mode. +#define CSR_FCSR 0x003 // Floating-Point Control and Status Register (frm + fflags). +#define CSR_VSTART 0x008 // Vector start position +#define CSR_VXSAT 0x009 // Fixed-Point Saturate Flag +#define CSR_VXRM 0x00A // Fixed-Point Rounding Mode +#define CSR_VCSR 0x00F // Vector control and status register +#define CSR_VL 0xC20 // Vector length +#define CSR_VTYPE 0xC21 // Vector data type register +#define CSR_VLENB 0xC22 // VLEN/8 (vector register length in bytes) +#define CSR_CYCLE 0xc00 // Cycle counter for RDCYCLE instruction. +#define CSR_TIME 0xc01 // Timer for RDTIME instruction. +#define CSR_INSTRET 0xc02 // Instructions-retired counter for RDINSTRET instruction. + +class VMRegImpl; +typedef VMRegImpl* VMReg; + +// Use Register as shortcut +class RegisterImpl; +typedef RegisterImpl* Register; + +inline Register as_Register(int encoding) { + return (Register)(intptr_t) encoding; +} + +class RegisterImpl: public AbstractRegisterImpl { + + public: + enum { + number_of_registers = 32, + max_slots_per_register = 2, + + // integer registers x8 - x15 and floating-point registers f8 - f15 are allocatable + // for compressed instructions. See Table 17.2 in spec. + compressed_register_base = 8, + compressed_register_top = 15, + }; + + // derived registers, offsets, and addresses + Register successor() const { return as_Register(encoding() + 1); } + + // construction + inline friend Register as_Register(int encoding); + + VMReg as_VMReg(); + + // accessors + int encoding() const { assert(is_valid(), "invalid register"); return encoding_nocheck(); } + bool is_valid() const { return 0 <= encoding_nocheck() && encoding_nocheck() < number_of_registers; } + const char* name() const; + int encoding_nocheck() const { return (intptr_t)this; } + + // Return the bit which represents this register. This is intended + // to be ORed into a bitmask: for usage see class AbstractRegSet below. + unsigned long bit(bool should_set = true) const { return should_set ? 1 << encoding() : 0; } + + // for rvc + int compressed_encoding() const { + assert(is_compressed_valid(), "invalid compressed register"); + return encoding() - compressed_register_base; + } + + int compressed_encoding_nocheck() const { + return encoding_nocheck() - compressed_register_base; + } + + bool is_compressed_valid() const { + return encoding_nocheck() >= compressed_register_base && + encoding_nocheck() <= compressed_register_top; + } +}; + +// The integer registers of the RISCV architecture + +CONSTANT_REGISTER_DECLARATION(Register, noreg, (-1)); + +CONSTANT_REGISTER_DECLARATION(Register, x0, (0)); +CONSTANT_REGISTER_DECLARATION(Register, x1, (1)); +CONSTANT_REGISTER_DECLARATION(Register, x2, (2)); +CONSTANT_REGISTER_DECLARATION(Register, x3, (3)); +CONSTANT_REGISTER_DECLARATION(Register, x4, (4)); +CONSTANT_REGISTER_DECLARATION(Register, x5, (5)); +CONSTANT_REGISTER_DECLARATION(Register, x6, (6)); +CONSTANT_REGISTER_DECLARATION(Register, x7, (7)); +CONSTANT_REGISTER_DECLARATION(Register, x8, (8)); +CONSTANT_REGISTER_DECLARATION(Register, x9, (9)); +CONSTANT_REGISTER_DECLARATION(Register, x10, (10)); +CONSTANT_REGISTER_DECLARATION(Register, x11, (11)); +CONSTANT_REGISTER_DECLARATION(Register, x12, (12)); +CONSTANT_REGISTER_DECLARATION(Register, x13, (13)); +CONSTANT_REGISTER_DECLARATION(Register, x14, (14)); +CONSTANT_REGISTER_DECLARATION(Register, x15, (15)); +CONSTANT_REGISTER_DECLARATION(Register, x16, (16)); +CONSTANT_REGISTER_DECLARATION(Register, x17, (17)); +CONSTANT_REGISTER_DECLARATION(Register, x18, (18)); +CONSTANT_REGISTER_DECLARATION(Register, x19, (19)); +CONSTANT_REGISTER_DECLARATION(Register, x20, (20)); +CONSTANT_REGISTER_DECLARATION(Register, x21, (21)); +CONSTANT_REGISTER_DECLARATION(Register, x22, (22)); +CONSTANT_REGISTER_DECLARATION(Register, x23, (23)); +CONSTANT_REGISTER_DECLARATION(Register, x24, (24)); +CONSTANT_REGISTER_DECLARATION(Register, x25, (25)); +CONSTANT_REGISTER_DECLARATION(Register, x26, (26)); +CONSTANT_REGISTER_DECLARATION(Register, x27, (27)); +CONSTANT_REGISTER_DECLARATION(Register, x28, (28)); +CONSTANT_REGISTER_DECLARATION(Register, x29, (29)); +CONSTANT_REGISTER_DECLARATION(Register, x30, (30)); +CONSTANT_REGISTER_DECLARATION(Register, x31, (31)); + +// Use FloatRegister as shortcut +class FloatRegisterImpl; +typedef FloatRegisterImpl* FloatRegister; + +inline FloatRegister as_FloatRegister(int encoding) { + return (FloatRegister)(intptr_t) encoding; +} + +// The implementation of floating point registers for the architecture +class FloatRegisterImpl: public AbstractRegisterImpl { + + public: + enum { + number_of_registers = 32, + max_slots_per_register = 2, + + // float registers in the range of [f8~f15] correspond to RVC. Please see Table 16.2 in spec. + compressed_register_base = 8, + compressed_register_top = 15, + }; + + // construction + inline friend FloatRegister as_FloatRegister(int encoding); + + VMReg as_VMReg(); + + // derived registers, offsets, and addresses + FloatRegister successor() const { + return as_FloatRegister((encoding() + 1)); + } + + // accessors + int encoding() const { assert(is_valid(), "invalid register"); return encoding_nocheck(); } + int encoding_nocheck() const { return (intptr_t)this; } + int is_valid() const { return 0 <= encoding_nocheck() && encoding_nocheck() < number_of_registers; } + const char* name() const; + + // for rvc + int compressed_encoding() const { + assert(is_compressed_valid(), "invalid compressed register"); + return encoding() - compressed_register_base; + } + + int compressed_encoding_nocheck() const { + return encoding_nocheck() - compressed_register_base; + } + + bool is_compressed_valid() const { + return encoding_nocheck() >= compressed_register_base && + encoding_nocheck() <= compressed_register_top; + } +}; + +// The float registers of the RISCV architecture + +CONSTANT_REGISTER_DECLARATION(FloatRegister, fnoreg , (-1)); + +CONSTANT_REGISTER_DECLARATION(FloatRegister, f0 , ( 0)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f1 , ( 1)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f2 , ( 2)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f3 , ( 3)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f4 , ( 4)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f5 , ( 5)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f6 , ( 6)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f7 , ( 7)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f8 , ( 8)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f9 , ( 9)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f10 , (10)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f11 , (11)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f12 , (12)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f13 , (13)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f14 , (14)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f15 , (15)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f16 , (16)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f17 , (17)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f18 , (18)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f19 , (19)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f20 , (20)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f21 , (21)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f22 , (22)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f23 , (23)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f24 , (24)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f25 , (25)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f26 , (26)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f27 , (27)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f28 , (28)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f29 , (29)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f30 , (30)); +CONSTANT_REGISTER_DECLARATION(FloatRegister, f31 , (31)); + +// Use VectorRegister as shortcut +class VectorRegisterImpl; +typedef VectorRegisterImpl* VectorRegister; + +inline VectorRegister as_VectorRegister(int encoding) { + return (VectorRegister)(intptr_t) encoding; +} + +// The implementation of vector registers for RVV +class VectorRegisterImpl: public AbstractRegisterImpl { + + public: + enum { + number_of_registers = 32, + max_slots_per_register = 4 + }; + + // construction + inline friend VectorRegister as_VectorRegister(int encoding); + + VMReg as_VMReg(); + + // derived registers, offsets, and addresses + VectorRegister successor() const { return as_VectorRegister(encoding() + 1); } + + // accessors + int encoding() const { assert(is_valid(), "invalid register"); return encoding_nocheck(); } + int encoding_nocheck() const { return (intptr_t)this; } + bool is_valid() const { return 0 <= encoding_nocheck() && encoding_nocheck() < number_of_registers; } + const char* name() const; + +}; + +// The vector registers of RVV +CONSTANT_REGISTER_DECLARATION(VectorRegister, vnoreg , (-1)); + +CONSTANT_REGISTER_DECLARATION(VectorRegister, v0 , ( 0)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v1 , ( 1)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v2 , ( 2)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v3 , ( 3)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v4 , ( 4)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v5 , ( 5)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v6 , ( 6)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v7 , ( 7)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v8 , ( 8)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v9 , ( 9)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v10 , (10)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v11 , (11)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v12 , (12)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v13 , (13)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v14 , (14)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v15 , (15)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v16 , (16)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v17 , (17)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v18 , (18)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v19 , (19)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v20 , (20)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v21 , (21)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v22 , (22)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v23 , (23)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v24 , (24)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v25 , (25)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v26 , (26)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v27 , (27)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v28 , (28)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v29 , (29)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v30 , (30)); +CONSTANT_REGISTER_DECLARATION(VectorRegister, v31 , (31)); + + +// Need to know the total number of registers of all sorts for SharedInfo. +// Define a class that exports it. +class ConcreteRegisterImpl : public AbstractRegisterImpl { + public: + enum { + // A big enough number for C2: all the registers plus flags + // This number must be large enough to cover REG_COUNT (defined by c2) registers. + // There is no requirement that any ordering here matches any ordering c2 gives + // it's optoregs. + + number_of_registers = (RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers + + FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers + + VectorRegisterImpl::max_slots_per_register * VectorRegisterImpl::number_of_registers) + }; + + // added to make it compile + static const int max_gpr; + static const int max_fpr; + static const int max_vpr; +}; + +// A set of registers +template +class AbstractRegSet { + uint32_t _bitset; + + AbstractRegSet(uint32_t bitset) : _bitset(bitset) { } + + public: + AbstractRegSet() : _bitset(0) { } + + AbstractRegSet(RegImpl r1) : _bitset(1 << r1->encoding()) { } + + AbstractRegSet operator+(const AbstractRegSet aSet) const { + AbstractRegSet result(_bitset | aSet._bitset); + return result; + } + + AbstractRegSet operator-(const AbstractRegSet aSet) const { + AbstractRegSet result(_bitset & ~aSet._bitset); + return result; + } + + AbstractRegSet &operator+=(const AbstractRegSet aSet) { + *this = *this + aSet; + return *this; + } + + AbstractRegSet &operator-=(const AbstractRegSet aSet) { + *this = *this - aSet; + return *this; + } + + static AbstractRegSet of(RegImpl r1) { + return AbstractRegSet(r1); + } + + static AbstractRegSet of(RegImpl r1, RegImpl r2) { + return of(r1) + r2; + } + + static AbstractRegSet of(RegImpl r1, RegImpl r2, RegImpl r3) { + return of(r1, r2) + r3; + } + + static AbstractRegSet of(RegImpl r1, RegImpl r2, RegImpl r3, RegImpl r4) { + return of(r1, r2, r3) + r4; + } + + static AbstractRegSet range(RegImpl start, RegImpl end) { + uint32_t bits = ~0; + bits <<= start->encoding(); + bits <<= (31 - end->encoding()); + bits >>= (31 - end->encoding()); + + return AbstractRegSet(bits); + } + + uint32_t bits() const { return _bitset; } +}; + +typedef AbstractRegSet RegSet; +typedef AbstractRegSet FloatRegSet; +typedef AbstractRegSet VectorRegSet; + +#endif // CPU_RISCV_REGISTER_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/relocInfo_riscv.cpp b/src/hotspot/cpu/riscv/relocInfo_riscv.cpp new file mode 100644 index 0000000000000..228a64eae2c64 --- /dev/null +++ b/src/hotspot/cpu/riscv/relocInfo_riscv.cpp @@ -0,0 +1,113 @@ +/* + * Copyright (c) 1998, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.hpp" +#include "code/relocInfo.hpp" +#include "nativeInst_riscv.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/safepoint.hpp" + +void Relocation::pd_set_data_value(address x, intptr_t o, bool verify_only) { + if (verify_only) { + return; + } + + int bytes; + + switch (type()) { + case relocInfo::oop_type: { + oop_Relocation *reloc = (oop_Relocation *)this; + // in movoop when BarrierSet::barrier_set()->barrier_set_nmethod() != NULL || !immediate + if (NativeInstruction::is_load_pc_relative_at(addr())) { + address constptr = (address)code()->oop_addr_at(reloc->oop_index()); + bytes = MacroAssembler::pd_patch_instruction_size(addr(), constptr); + assert(*(address*)constptr == x, "error in oop relocation"); + } else { + bytes = MacroAssembler::patch_oop(addr(), x); + } + break; + } + default: + bytes = MacroAssembler::pd_patch_instruction_size(addr(), x); + break; + } + ICache::invalidate_range(addr(), bytes); +} + +address Relocation::pd_call_destination(address orig_addr) { + assert(is_call(), "should be an address instruction here"); + if (NativeCall::is_call_at(addr())) { + address trampoline = nativeCall_at(addr())->get_trampoline(); + if (trampoline != NULL) { + return nativeCallTrampolineStub_at(trampoline)->destination(); + } + } + if (orig_addr != NULL) { + // the extracted address from the instructions in address orig_addr + address new_addr = MacroAssembler::pd_call_destination(orig_addr); + // If call is branch to self, don't try to relocate it, just leave it + // as branch to self. This happens during code generation if the code + // buffer expands. It will be relocated to the trampoline above once + // code generation is complete. + new_addr = (new_addr == orig_addr) ? addr() : new_addr; + return new_addr; + } + return MacroAssembler::pd_call_destination(addr()); +} + +void Relocation::pd_set_call_destination(address x) { + assert(is_call(), "should be an address instruction here"); + if (NativeCall::is_call_at(addr())) { + address trampoline = nativeCall_at(addr())->get_trampoline(); + if (trampoline != NULL) { + nativeCall_at(addr())->set_destination_mt_safe(x, /* assert_lock */false); + return; + } + } + MacroAssembler::pd_patch_instruction_size(addr(), x); + address pd_call = pd_call_destination(addr()); + assert(pd_call == x, "fail in reloc"); +} + +address* Relocation::pd_address_in_code() { + assert(NativeCall::is_load_pc_relative_at(addr()), "Not the expected instruction sequence!"); + return (address*)(MacroAssembler::target_addr_for_insn(addr())); +} + +address Relocation::pd_get_address_from_code() { + return MacroAssembler::pd_call_destination(addr()); +} + +void poll_Relocation::fix_relocation_after_move(const CodeBuffer* src, CodeBuffer* dest) { + if (NativeInstruction::maybe_cpool_ref(addr())) { + address old_addr = old_addr_for(addr(), src, dest); + MacroAssembler::pd_patch_instruction_size(addr(), MacroAssembler::target_addr_for_insn(old_addr)); + } +} + +void metadata_Relocation::pd_fix_value(address x) { +} diff --git a/src/hotspot/cpu/riscv/relocInfo_riscv.hpp b/src/hotspot/cpu/riscv/relocInfo_riscv.hpp new file mode 100644 index 0000000000000..840ed935d88b7 --- /dev/null +++ b/src/hotspot/cpu/riscv/relocInfo_riscv.hpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_RELOCINFO_RISCV_HPP +#define CPU_RISCV_RELOCINFO_RISCV_HPP + + // machine-dependent parts of class relocInfo + private: + enum { + // Relocations are byte-aligned. + offset_unit = 1, + // Must be at least 1 for RelocInfo::narrow_oop_in_const. + format_width = 1 + }; + + public: + + // This platform has no oops in the code that are not also + // listed in the oop section. + static bool mustIterateImmediateOopsInCode() { return false; } + +#endif // CPU_RISCV_RELOCINFO_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad new file mode 100644 index 0000000000000..3d59f61c83cba --- /dev/null +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -0,0 +1,10645 @@ +// +// Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. +// Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. +// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +// +// This code is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License version 2 only, as +// published by the Free Software Foundation. +// +// This code is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// version 2 for more details (a copy is included in the LICENSE file that +// accompanied this code). +// +// You should have received a copy of the GNU General Public License version +// 2 along with this work; if not, write to the Free Software Foundation, +// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +// +// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +// or visit www.oracle.com if you need additional information or have any +// questions. +// +// + +// RISCV Architecture Description File + +//----------REGISTER DEFINITION BLOCK------------------------------------------ +// This information is used by the matcher and the register allocator to +// describe individual registers and classes of registers within the target +// archtecture. + +register %{ +//----------Architecture Description Register Definitions---------------------- +// General Registers +// "reg_def" name ( register save type, C convention save type, +// ideal register type, encoding ); +// Register Save Types: +// +// NS = No-Save: The register allocator assumes that these registers +// can be used without saving upon entry to the method, & +// that they do not need to be saved at call sites. +// +// SOC = Save-On-Call: The register allocator assumes that these registers +// can be used without saving upon entry to the method, +// but that they must be saved at call sites. +// +// SOE = Save-On-Entry: The register allocator assumes that these registers +// must be saved before using them upon entry to the +// method, but they do not need to be saved at call +// sites. +// +// AS = Always-Save: The register allocator assumes that these registers +// must be saved before using them upon entry to the +// method, & that they must be saved at call sites. +// +// Ideal Register Type is used to determine how to save & restore a +// register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get +// spilled with LoadP/StoreP. If the register supports both, use Op_RegI. +// +// The encoding number is the actual bit-pattern placed into the opcodes. + +// We must define the 64 bit int registers in two 32 bit halves, the +// real lower register and a virtual upper half register. upper halves +// are used by the register allocator but are not actually supplied as +// operands to memory ops. +// +// follow the C1 compiler in making registers +// +// x7, x9-x17, x27-x31 volatile (caller save) +// x0-x4, x8, x23 system (no save, no allocate) +// x5-x6 non-allocatable (so we can use them as temporary regs) + +// +// as regards Java usage. we don't use any callee save registers +// because this makes it difficult to de-optimise a frame (see comment +// in x86 implementation of Deoptimization::unwind_callee_save_values) +// + +// General Registers + +reg_def R0 ( NS, NS, Op_RegI, 0, x0->as_VMReg() ); // zr +reg_def R0_H ( NS, NS, Op_RegI, 0, x0->as_VMReg()->next() ); +reg_def R1 ( NS, SOC, Op_RegI, 1, x1->as_VMReg() ); // ra +reg_def R1_H ( NS, SOC, Op_RegI, 1, x1->as_VMReg()->next() ); +reg_def R2 ( NS, SOE, Op_RegI, 2, x2->as_VMReg() ); // sp +reg_def R2_H ( NS, SOE, Op_RegI, 2, x2->as_VMReg()->next() ); +reg_def R3 ( NS, NS, Op_RegI, 3, x3->as_VMReg() ); // gp +reg_def R3_H ( NS, NS, Op_RegI, 3, x3->as_VMReg()->next() ); +reg_def R4 ( NS, NS, Op_RegI, 4, x4->as_VMReg() ); // tp +reg_def R4_H ( NS, NS, Op_RegI, 4, x4->as_VMReg()->next() ); +reg_def R7 ( SOC, SOC, Op_RegI, 7, x7->as_VMReg() ); +reg_def R7_H ( SOC, SOC, Op_RegI, 7, x7->as_VMReg()->next() ); +reg_def R8 ( NS, SOE, Op_RegI, 8, x8->as_VMReg() ); // fp +reg_def R8_H ( NS, SOE, Op_RegI, 8, x8->as_VMReg()->next() ); +reg_def R9 ( SOC, SOE, Op_RegI, 9, x9->as_VMReg() ); +reg_def R9_H ( SOC, SOE, Op_RegI, 9, x9->as_VMReg()->next() ); +reg_def R10 ( SOC, SOC, Op_RegI, 10, x10->as_VMReg() ); +reg_def R10_H ( SOC, SOC, Op_RegI, 10, x10->as_VMReg()->next()); +reg_def R11 ( SOC, SOC, Op_RegI, 11, x11->as_VMReg() ); +reg_def R11_H ( SOC, SOC, Op_RegI, 11, x11->as_VMReg()->next()); +reg_def R12 ( SOC, SOC, Op_RegI, 12, x12->as_VMReg() ); +reg_def R12_H ( SOC, SOC, Op_RegI, 12, x12->as_VMReg()->next()); +reg_def R13 ( SOC, SOC, Op_RegI, 13, x13->as_VMReg() ); +reg_def R13_H ( SOC, SOC, Op_RegI, 13, x13->as_VMReg()->next()); +reg_def R14 ( SOC, SOC, Op_RegI, 14, x14->as_VMReg() ); +reg_def R14_H ( SOC, SOC, Op_RegI, 14, x14->as_VMReg()->next()); +reg_def R15 ( SOC, SOC, Op_RegI, 15, x15->as_VMReg() ); +reg_def R15_H ( SOC, SOC, Op_RegI, 15, x15->as_VMReg()->next()); +reg_def R16 ( SOC, SOC, Op_RegI, 16, x16->as_VMReg() ); +reg_def R16_H ( SOC, SOC, Op_RegI, 16, x16->as_VMReg()->next()); +reg_def R17 ( SOC, SOC, Op_RegI, 17, x17->as_VMReg() ); +reg_def R17_H ( SOC, SOC, Op_RegI, 17, x17->as_VMReg()->next()); +reg_def R18 ( SOC, SOE, Op_RegI, 18, x18->as_VMReg() ); +reg_def R18_H ( SOC, SOE, Op_RegI, 18, x18->as_VMReg()->next()); +reg_def R19 ( SOC, SOE, Op_RegI, 19, x19->as_VMReg() ); +reg_def R19_H ( SOC, SOE, Op_RegI, 19, x19->as_VMReg()->next()); +reg_def R20 ( SOC, SOE, Op_RegI, 20, x20->as_VMReg() ); // caller esp +reg_def R20_H ( SOC, SOE, Op_RegI, 20, x20->as_VMReg()->next()); +reg_def R21 ( SOC, SOE, Op_RegI, 21, x21->as_VMReg() ); +reg_def R21_H ( SOC, SOE, Op_RegI, 21, x21->as_VMReg()->next()); +reg_def R22 ( SOC, SOE, Op_RegI, 22, x22->as_VMReg() ); +reg_def R22_H ( SOC, SOE, Op_RegI, 22, x22->as_VMReg()->next()); +reg_def R23 ( NS, SOE, Op_RegI, 23, x23->as_VMReg() ); // java thread +reg_def R23_H ( NS, SOE, Op_RegI, 23, x23->as_VMReg()->next()); +reg_def R24 ( SOC, SOE, Op_RegI, 24, x24->as_VMReg() ); +reg_def R24_H ( SOC, SOE, Op_RegI, 24, x24->as_VMReg()->next()); +reg_def R25 ( SOC, SOE, Op_RegI, 25, x25->as_VMReg() ); +reg_def R25_H ( SOC, SOE, Op_RegI, 25, x25->as_VMReg()->next()); +reg_def R26 ( SOC, SOE, Op_RegI, 26, x26->as_VMReg() ); +reg_def R26_H ( SOC, SOE, Op_RegI, 26, x26->as_VMReg()->next()); +reg_def R27 ( SOC, SOE, Op_RegI, 27, x27->as_VMReg() ); // heapbase +reg_def R27_H ( SOC, SOE, Op_RegI, 27, x27->as_VMReg()->next()); +reg_def R28 ( SOC, SOC, Op_RegI, 28, x28->as_VMReg() ); +reg_def R28_H ( SOC, SOC, Op_RegI, 28, x28->as_VMReg()->next()); +reg_def R29 ( SOC, SOC, Op_RegI, 29, x29->as_VMReg() ); +reg_def R29_H ( SOC, SOC, Op_RegI, 29, x29->as_VMReg()->next()); +reg_def R30 ( SOC, SOC, Op_RegI, 30, x30->as_VMReg() ); +reg_def R30_H ( SOC, SOC, Op_RegI, 30, x30->as_VMReg()->next()); +reg_def R31 ( SOC, SOC, Op_RegI, 31, x31->as_VMReg() ); +reg_def R31_H ( SOC, SOC, Op_RegI, 31, x31->as_VMReg()->next()); + +// ---------------------------- +// Float/Double Registers +// ---------------------------- + +// Double Registers + +// The rules of ADL require that double registers be defined in pairs. +// Each pair must be two 32-bit values, but not necessarily a pair of +// single float registers. In each pair, ADLC-assigned register numbers +// must be adjacent, with the lower number even. Finally, when the +// CPU stores such a register pair to memory, the word associated with +// the lower ADLC-assigned number must be stored to the lower address. + +// RISCV has 32 floating-point registers. Each can store a single +// or double precision floating-point value. + +// for Java use float registers f0-f31 are always save on call whereas +// the platform ABI treats f8-f9 and f18-f27 as callee save). Other +// float registers are SOC as per the platform spec + +reg_def F0 ( SOC, SOC, Op_RegF, 0, f0->as_VMReg() ); +reg_def F0_H ( SOC, SOC, Op_RegF, 0, f0->as_VMReg()->next() ); +reg_def F1 ( SOC, SOC, Op_RegF, 1, f1->as_VMReg() ); +reg_def F1_H ( SOC, SOC, Op_RegF, 1, f1->as_VMReg()->next() ); +reg_def F2 ( SOC, SOC, Op_RegF, 2, f2->as_VMReg() ); +reg_def F2_H ( SOC, SOC, Op_RegF, 2, f2->as_VMReg()->next() ); +reg_def F3 ( SOC, SOC, Op_RegF, 3, f3->as_VMReg() ); +reg_def F3_H ( SOC, SOC, Op_RegF, 3, f3->as_VMReg()->next() ); +reg_def F4 ( SOC, SOC, Op_RegF, 4, f4->as_VMReg() ); +reg_def F4_H ( SOC, SOC, Op_RegF, 4, f4->as_VMReg()->next() ); +reg_def F5 ( SOC, SOC, Op_RegF, 5, f5->as_VMReg() ); +reg_def F5_H ( SOC, SOC, Op_RegF, 5, f5->as_VMReg()->next() ); +reg_def F6 ( SOC, SOC, Op_RegF, 6, f6->as_VMReg() ); +reg_def F6_H ( SOC, SOC, Op_RegF, 6, f6->as_VMReg()->next() ); +reg_def F7 ( SOC, SOC, Op_RegF, 7, f7->as_VMReg() ); +reg_def F7_H ( SOC, SOC, Op_RegF, 7, f7->as_VMReg()->next() ); +reg_def F8 ( SOC, SOE, Op_RegF, 8, f8->as_VMReg() ); +reg_def F8_H ( SOC, SOE, Op_RegF, 8, f8->as_VMReg()->next() ); +reg_def F9 ( SOC, SOE, Op_RegF, 9, f9->as_VMReg() ); +reg_def F9_H ( SOC, SOE, Op_RegF, 9, f9->as_VMReg()->next() ); +reg_def F10 ( SOC, SOC, Op_RegF, 10, f10->as_VMReg() ); +reg_def F10_H ( SOC, SOC, Op_RegF, 10, f10->as_VMReg()->next() ); +reg_def F11 ( SOC, SOC, Op_RegF, 11, f11->as_VMReg() ); +reg_def F11_H ( SOC, SOC, Op_RegF, 11, f11->as_VMReg()->next() ); +reg_def F12 ( SOC, SOC, Op_RegF, 12, f12->as_VMReg() ); +reg_def F12_H ( SOC, SOC, Op_RegF, 12, f12->as_VMReg()->next() ); +reg_def F13 ( SOC, SOC, Op_RegF, 13, f13->as_VMReg() ); +reg_def F13_H ( SOC, SOC, Op_RegF, 13, f13->as_VMReg()->next() ); +reg_def F14 ( SOC, SOC, Op_RegF, 14, f14->as_VMReg() ); +reg_def F14_H ( SOC, SOC, Op_RegF, 14, f14->as_VMReg()->next() ); +reg_def F15 ( SOC, SOC, Op_RegF, 15, f15->as_VMReg() ); +reg_def F15_H ( SOC, SOC, Op_RegF, 15, f15->as_VMReg()->next() ); +reg_def F16 ( SOC, SOC, Op_RegF, 16, f16->as_VMReg() ); +reg_def F16_H ( SOC, SOC, Op_RegF, 16, f16->as_VMReg()->next() ); +reg_def F17 ( SOC, SOC, Op_RegF, 17, f17->as_VMReg() ); +reg_def F17_H ( SOC, SOC, Op_RegF, 17, f17->as_VMReg()->next() ); +reg_def F18 ( SOC, SOE, Op_RegF, 18, f18->as_VMReg() ); +reg_def F18_H ( SOC, SOE, Op_RegF, 18, f18->as_VMReg()->next() ); +reg_def F19 ( SOC, SOE, Op_RegF, 19, f19->as_VMReg() ); +reg_def F19_H ( SOC, SOE, Op_RegF, 19, f19->as_VMReg()->next() ); +reg_def F20 ( SOC, SOE, Op_RegF, 20, f20->as_VMReg() ); +reg_def F20_H ( SOC, SOE, Op_RegF, 20, f20->as_VMReg()->next() ); +reg_def F21 ( SOC, SOE, Op_RegF, 21, f21->as_VMReg() ); +reg_def F21_H ( SOC, SOE, Op_RegF, 21, f21->as_VMReg()->next() ); +reg_def F22 ( SOC, SOE, Op_RegF, 22, f22->as_VMReg() ); +reg_def F22_H ( SOC, SOE, Op_RegF, 22, f22->as_VMReg()->next() ); +reg_def F23 ( SOC, SOE, Op_RegF, 23, f23->as_VMReg() ); +reg_def F23_H ( SOC, SOE, Op_RegF, 23, f23->as_VMReg()->next() ); +reg_def F24 ( SOC, SOE, Op_RegF, 24, f24->as_VMReg() ); +reg_def F24_H ( SOC, SOE, Op_RegF, 24, f24->as_VMReg()->next() ); +reg_def F25 ( SOC, SOE, Op_RegF, 25, f25->as_VMReg() ); +reg_def F25_H ( SOC, SOE, Op_RegF, 25, f25->as_VMReg()->next() ); +reg_def F26 ( SOC, SOE, Op_RegF, 26, f26->as_VMReg() ); +reg_def F26_H ( SOC, SOE, Op_RegF, 26, f26->as_VMReg()->next() ); +reg_def F27 ( SOC, SOE, Op_RegF, 27, f27->as_VMReg() ); +reg_def F27_H ( SOC, SOE, Op_RegF, 27, f27->as_VMReg()->next() ); +reg_def F28 ( SOC, SOC, Op_RegF, 28, f28->as_VMReg() ); +reg_def F28_H ( SOC, SOC, Op_RegF, 28, f28->as_VMReg()->next() ); +reg_def F29 ( SOC, SOC, Op_RegF, 29, f29->as_VMReg() ); +reg_def F29_H ( SOC, SOC, Op_RegF, 29, f29->as_VMReg()->next() ); +reg_def F30 ( SOC, SOC, Op_RegF, 30, f30->as_VMReg() ); +reg_def F30_H ( SOC, SOC, Op_RegF, 30, f30->as_VMReg()->next() ); +reg_def F31 ( SOC, SOC, Op_RegF, 31, f31->as_VMReg() ); +reg_def F31_H ( SOC, SOC, Op_RegF, 31, f31->as_VMReg()->next() ); + +// ---------------------------- +// Vector Registers +// ---------------------------- + +// For RVV vector registers, we simply extend vector register size to 4 +// 'logical' slots. This is nominally 128 bits but it actually covers +// all possible 'physical' RVV vector register lengths from 128 ~ 1024 +// bits. The 'physical' RVV vector register length is detected during +// startup, so the register allocator is able to identify the correct +// number of bytes needed for an RVV spill/unspill. + +reg_def V0 ( SOC, SOC, Op_VecA, 0, v0->as_VMReg() ); +reg_def V0_H ( SOC, SOC, Op_VecA, 0, v0->as_VMReg()->next() ); +reg_def V0_J ( SOC, SOC, Op_VecA, 0, v0->as_VMReg()->next(2) ); +reg_def V0_K ( SOC, SOC, Op_VecA, 0, v0->as_VMReg()->next(3) ); + +reg_def V1 ( SOC, SOC, Op_VecA, 1, v1->as_VMReg() ); +reg_def V1_H ( SOC, SOC, Op_VecA, 1, v1->as_VMReg()->next() ); +reg_def V1_J ( SOC, SOC, Op_VecA, 1, v1->as_VMReg()->next(2) ); +reg_def V1_K ( SOC, SOC, Op_VecA, 1, v1->as_VMReg()->next(3) ); + +reg_def V2 ( SOC, SOC, Op_VecA, 2, v2->as_VMReg() ); +reg_def V2_H ( SOC, SOC, Op_VecA, 2, v2->as_VMReg()->next() ); +reg_def V2_J ( SOC, SOC, Op_VecA, 2, v2->as_VMReg()->next(2) ); +reg_def V2_K ( SOC, SOC, Op_VecA, 2, v2->as_VMReg()->next(3) ); + +reg_def V3 ( SOC, SOC, Op_VecA, 3, v3->as_VMReg() ); +reg_def V3_H ( SOC, SOC, Op_VecA, 3, v3->as_VMReg()->next() ); +reg_def V3_J ( SOC, SOC, Op_VecA, 3, v3->as_VMReg()->next(2) ); +reg_def V3_K ( SOC, SOC, Op_VecA, 3, v3->as_VMReg()->next(3) ); + +reg_def V4 ( SOC, SOC, Op_VecA, 4, v4->as_VMReg() ); +reg_def V4_H ( SOC, SOC, Op_VecA, 4, v4->as_VMReg()->next() ); +reg_def V4_J ( SOC, SOC, Op_VecA, 4, v4->as_VMReg()->next(2) ); +reg_def V4_K ( SOC, SOC, Op_VecA, 4, v4->as_VMReg()->next(3) ); + +reg_def V5 ( SOC, SOC, Op_VecA, 5, v5->as_VMReg() ); +reg_def V5_H ( SOC, SOC, Op_VecA, 5, v5->as_VMReg()->next() ); +reg_def V5_J ( SOC, SOC, Op_VecA, 5, v5->as_VMReg()->next(2) ); +reg_def V5_K ( SOC, SOC, Op_VecA, 5, v5->as_VMReg()->next(3) ); + +reg_def V6 ( SOC, SOC, Op_VecA, 6, v6->as_VMReg() ); +reg_def V6_H ( SOC, SOC, Op_VecA, 6, v6->as_VMReg()->next() ); +reg_def V6_J ( SOC, SOC, Op_VecA, 6, v6->as_VMReg()->next(2) ); +reg_def V6_K ( SOC, SOC, Op_VecA, 6, v6->as_VMReg()->next(3) ); + +reg_def V7 ( SOC, SOC, Op_VecA, 7, v7->as_VMReg() ); +reg_def V7_H ( SOC, SOC, Op_VecA, 7, v7->as_VMReg()->next() ); +reg_def V7_J ( SOC, SOC, Op_VecA, 7, v7->as_VMReg()->next(2) ); +reg_def V7_K ( SOC, SOC, Op_VecA, 7, v7->as_VMReg()->next(3) ); + +reg_def V8 ( SOC, SOC, Op_VecA, 8, v8->as_VMReg() ); +reg_def V8_H ( SOC, SOC, Op_VecA, 8, v8->as_VMReg()->next() ); +reg_def V8_J ( SOC, SOC, Op_VecA, 8, v8->as_VMReg()->next(2) ); +reg_def V8_K ( SOC, SOC, Op_VecA, 8, v8->as_VMReg()->next(3) ); + +reg_def V9 ( SOC, SOC, Op_VecA, 9, v9->as_VMReg() ); +reg_def V9_H ( SOC, SOC, Op_VecA, 9, v9->as_VMReg()->next() ); +reg_def V9_J ( SOC, SOC, Op_VecA, 9, v9->as_VMReg()->next(2) ); +reg_def V9_K ( SOC, SOC, Op_VecA, 9, v9->as_VMReg()->next(3) ); + +reg_def V10 ( SOC, SOC, Op_VecA, 10, v10->as_VMReg() ); +reg_def V10_H ( SOC, SOC, Op_VecA, 10, v10->as_VMReg()->next() ); +reg_def V10_J ( SOC, SOC, Op_VecA, 10, v10->as_VMReg()->next(2) ); +reg_def V10_K ( SOC, SOC, Op_VecA, 10, v10->as_VMReg()->next(3) ); + +reg_def V11 ( SOC, SOC, Op_VecA, 11, v11->as_VMReg() ); +reg_def V11_H ( SOC, SOC, Op_VecA, 11, v11->as_VMReg()->next() ); +reg_def V11_J ( SOC, SOC, Op_VecA, 11, v11->as_VMReg()->next(2) ); +reg_def V11_K ( SOC, SOC, Op_VecA, 11, v11->as_VMReg()->next(3) ); + +reg_def V12 ( SOC, SOC, Op_VecA, 12, v12->as_VMReg() ); +reg_def V12_H ( SOC, SOC, Op_VecA, 12, v12->as_VMReg()->next() ); +reg_def V12_J ( SOC, SOC, Op_VecA, 12, v12->as_VMReg()->next(2) ); +reg_def V12_K ( SOC, SOC, Op_VecA, 12, v12->as_VMReg()->next(3) ); + +reg_def V13 ( SOC, SOC, Op_VecA, 13, v13->as_VMReg() ); +reg_def V13_H ( SOC, SOC, Op_VecA, 13, v13->as_VMReg()->next() ); +reg_def V13_J ( SOC, SOC, Op_VecA, 13, v13->as_VMReg()->next(2) ); +reg_def V13_K ( SOC, SOC, Op_VecA, 13, v13->as_VMReg()->next(3) ); + +reg_def V14 ( SOC, SOC, Op_VecA, 14, v14->as_VMReg() ); +reg_def V14_H ( SOC, SOC, Op_VecA, 14, v14->as_VMReg()->next() ); +reg_def V14_J ( SOC, SOC, Op_VecA, 14, v14->as_VMReg()->next(2) ); +reg_def V14_K ( SOC, SOC, Op_VecA, 14, v14->as_VMReg()->next(3) ); + +reg_def V15 ( SOC, SOC, Op_VecA, 15, v15->as_VMReg() ); +reg_def V15_H ( SOC, SOC, Op_VecA, 15, v15->as_VMReg()->next() ); +reg_def V15_J ( SOC, SOC, Op_VecA, 15, v15->as_VMReg()->next(2) ); +reg_def V15_K ( SOC, SOC, Op_VecA, 15, v15->as_VMReg()->next(3) ); + +reg_def V16 ( SOC, SOC, Op_VecA, 16, v16->as_VMReg() ); +reg_def V16_H ( SOC, SOC, Op_VecA, 16, v16->as_VMReg()->next() ); +reg_def V16_J ( SOC, SOC, Op_VecA, 16, v16->as_VMReg()->next(2) ); +reg_def V16_K ( SOC, SOC, Op_VecA, 16, v16->as_VMReg()->next(3) ); + +reg_def V17 ( SOC, SOC, Op_VecA, 17, v17->as_VMReg() ); +reg_def V17_H ( SOC, SOC, Op_VecA, 17, v17->as_VMReg()->next() ); +reg_def V17_J ( SOC, SOC, Op_VecA, 17, v17->as_VMReg()->next(2) ); +reg_def V17_K ( SOC, SOC, Op_VecA, 17, v17->as_VMReg()->next(3) ); + +reg_def V18 ( SOC, SOC, Op_VecA, 18, v18->as_VMReg() ); +reg_def V18_H ( SOC, SOC, Op_VecA, 18, v18->as_VMReg()->next() ); +reg_def V18_J ( SOC, SOC, Op_VecA, 18, v18->as_VMReg()->next(2) ); +reg_def V18_K ( SOC, SOC, Op_VecA, 18, v18->as_VMReg()->next(3) ); + +reg_def V19 ( SOC, SOC, Op_VecA, 19, v19->as_VMReg() ); +reg_def V19_H ( SOC, SOC, Op_VecA, 19, v19->as_VMReg()->next() ); +reg_def V19_J ( SOC, SOC, Op_VecA, 19, v19->as_VMReg()->next(2) ); +reg_def V19_K ( SOC, SOC, Op_VecA, 19, v19->as_VMReg()->next(3) ); + +reg_def V20 ( SOC, SOC, Op_VecA, 20, v20->as_VMReg() ); +reg_def V20_H ( SOC, SOC, Op_VecA, 20, v20->as_VMReg()->next() ); +reg_def V20_J ( SOC, SOC, Op_VecA, 20, v20->as_VMReg()->next(2) ); +reg_def V20_K ( SOC, SOC, Op_VecA, 20, v20->as_VMReg()->next(3) ); + +reg_def V21 ( SOC, SOC, Op_VecA, 21, v21->as_VMReg() ); +reg_def V21_H ( SOC, SOC, Op_VecA, 21, v21->as_VMReg()->next() ); +reg_def V21_J ( SOC, SOC, Op_VecA, 21, v21->as_VMReg()->next(2) ); +reg_def V21_K ( SOC, SOC, Op_VecA, 21, v21->as_VMReg()->next(3) ); + +reg_def V22 ( SOC, SOC, Op_VecA, 22, v22->as_VMReg() ); +reg_def V22_H ( SOC, SOC, Op_VecA, 22, v22->as_VMReg()->next() ); +reg_def V22_J ( SOC, SOC, Op_VecA, 22, v22->as_VMReg()->next(2) ); +reg_def V22_K ( SOC, SOC, Op_VecA, 22, v22->as_VMReg()->next(3) ); + +reg_def V23 ( SOC, SOC, Op_VecA, 23, v23->as_VMReg() ); +reg_def V23_H ( SOC, SOC, Op_VecA, 23, v23->as_VMReg()->next() ); +reg_def V23_J ( SOC, SOC, Op_VecA, 23, v23->as_VMReg()->next(2) ); +reg_def V23_K ( SOC, SOC, Op_VecA, 23, v23->as_VMReg()->next(3) ); + +reg_def V24 ( SOC, SOC, Op_VecA, 24, v24->as_VMReg() ); +reg_def V24_H ( SOC, SOC, Op_VecA, 24, v24->as_VMReg()->next() ); +reg_def V24_J ( SOC, SOC, Op_VecA, 24, v24->as_VMReg()->next(2) ); +reg_def V24_K ( SOC, SOC, Op_VecA, 24, v24->as_VMReg()->next(3) ); + +reg_def V25 ( SOC, SOC, Op_VecA, 25, v25->as_VMReg() ); +reg_def V25_H ( SOC, SOC, Op_VecA, 25, v25->as_VMReg()->next() ); +reg_def V25_J ( SOC, SOC, Op_VecA, 25, v25->as_VMReg()->next(2) ); +reg_def V25_K ( SOC, SOC, Op_VecA, 25, v25->as_VMReg()->next(3) ); + +reg_def V26 ( SOC, SOC, Op_VecA, 26, v26->as_VMReg() ); +reg_def V26_H ( SOC, SOC, Op_VecA, 26, v26->as_VMReg()->next() ); +reg_def V26_J ( SOC, SOC, Op_VecA, 26, v26->as_VMReg()->next(2) ); +reg_def V26_K ( SOC, SOC, Op_VecA, 26, v26->as_VMReg()->next(3) ); + +reg_def V27 ( SOC, SOC, Op_VecA, 27, v27->as_VMReg() ); +reg_def V27_H ( SOC, SOC, Op_VecA, 27, v27->as_VMReg()->next() ); +reg_def V27_J ( SOC, SOC, Op_VecA, 27, v27->as_VMReg()->next(2) ); +reg_def V27_K ( SOC, SOC, Op_VecA, 27, v27->as_VMReg()->next(3) ); + +reg_def V28 ( SOC, SOC, Op_VecA, 28, v28->as_VMReg() ); +reg_def V28_H ( SOC, SOC, Op_VecA, 28, v28->as_VMReg()->next() ); +reg_def V28_J ( SOC, SOC, Op_VecA, 28, v28->as_VMReg()->next(2) ); +reg_def V28_K ( SOC, SOC, Op_VecA, 28, v28->as_VMReg()->next(3) ); + +reg_def V29 ( SOC, SOC, Op_VecA, 29, v29->as_VMReg() ); +reg_def V29_H ( SOC, SOC, Op_VecA, 29, v29->as_VMReg()->next() ); +reg_def V29_J ( SOC, SOC, Op_VecA, 29, v29->as_VMReg()->next(2) ); +reg_def V29_K ( SOC, SOC, Op_VecA, 29, v29->as_VMReg()->next(3) ); + +reg_def V30 ( SOC, SOC, Op_VecA, 30, v30->as_VMReg() ); +reg_def V30_H ( SOC, SOC, Op_VecA, 30, v30->as_VMReg()->next() ); +reg_def V30_J ( SOC, SOC, Op_VecA, 30, v30->as_VMReg()->next(2) ); +reg_def V30_K ( SOC, SOC, Op_VecA, 30, v30->as_VMReg()->next(3) ); + +reg_def V31 ( SOC, SOC, Op_VecA, 31, v31->as_VMReg() ); +reg_def V31_H ( SOC, SOC, Op_VecA, 31, v31->as_VMReg()->next() ); +reg_def V31_J ( SOC, SOC, Op_VecA, 31, v31->as_VMReg()->next(2) ); +reg_def V31_K ( SOC, SOC, Op_VecA, 31, v31->as_VMReg()->next(3) ); + +// ---------------------------- +// Special Registers +// ---------------------------- + +// On riscv, the physical flag register is missing, so we use t1 instead, +// to bridge the RegFlag semantics in share/opto + +reg_def RFLAGS (SOC, SOC, Op_RegFlags, 6, x6->as_VMReg() ); + +// Specify priority of register selection within phases of register +// allocation. Highest priority is first. A useful heuristic is to +// give registers a low priority when they are required by machine +// instructions, like EAX and EDX on I486, and choose no-save registers +// before save-on-call, & save-on-call before save-on-entry. Registers +// which participate in fixed calling sequences should come last. +// Registers which are used as pairs must fall on an even boundary. + +alloc_class chunk0( + // volatiles + R7, R7_H, + R28, R28_H, + R29, R29_H, + R30, R30_H, + R31, R31_H, + + // arg registers + R10, R10_H, + R11, R11_H, + R12, R12_H, + R13, R13_H, + R14, R14_H, + R15, R15_H, + R16, R16_H, + R17, R17_H, + + // non-volatiles + R9, R9_H, + R18, R18_H, + R19, R19_H, + R20, R20_H, + R21, R21_H, + R22, R22_H, + R24, R24_H, + R25, R25_H, + R26, R26_H, + + // non-allocatable registers + R23, R23_H, // java thread + R27, R27_H, // heapbase + R4, R4_H, // thread + R8, R8_H, // fp + R0, R0_H, // zero + R1, R1_H, // ra + R2, R2_H, // sp + R3, R3_H, // gp +); + +alloc_class chunk1( + + // no save + F0, F0_H, + F1, F1_H, + F2, F2_H, + F3, F3_H, + F4, F4_H, + F5, F5_H, + F6, F6_H, + F7, F7_H, + F28, F28_H, + F29, F29_H, + F30, F30_H, + F31, F31_H, + + // arg registers + F10, F10_H, + F11, F11_H, + F12, F12_H, + F13, F13_H, + F14, F14_H, + F15, F15_H, + F16, F16_H, + F17, F17_H, + + // non-volatiles + F8, F8_H, + F9, F9_H, + F18, F18_H, + F19, F19_H, + F20, F20_H, + F21, F21_H, + F22, F22_H, + F23, F23_H, + F24, F24_H, + F25, F25_H, + F26, F26_H, + F27, F27_H, +); + +alloc_class chunk2( + V0, V0_H, V0_J, V0_K, + V1, V1_H, V1_J, V1_K, + V2, V2_H, V2_J, V2_K, + V3, V3_H, V3_J, V3_K, + V4, V4_H, V4_J, V4_K, + V5, V5_H, V5_J, V5_K, + V6, V6_H, V6_J, V6_K, + V7, V7_H, V7_J, V7_K, + V8, V8_H, V8_J, V8_K, + V9, V9_H, V9_J, V9_K, + V10, V10_H, V10_J, V10_K, + V11, V11_H, V11_J, V11_K, + V12, V12_H, V12_J, V12_K, + V13, V13_H, V13_J, V13_K, + V14, V14_H, V14_J, V14_K, + V15, V15_H, V15_J, V15_K, + V16, V16_H, V16_J, V16_K, + V17, V17_H, V17_J, V17_K, + V18, V18_H, V18_J, V18_K, + V19, V19_H, V19_J, V19_K, + V20, V20_H, V20_J, V20_K, + V21, V21_H, V21_J, V21_K, + V22, V22_H, V22_J, V22_K, + V23, V23_H, V23_J, V23_K, + V24, V24_H, V24_J, V24_K, + V25, V25_H, V25_J, V25_K, + V26, V26_H, V26_J, V26_K, + V27, V27_H, V27_J, V27_K, + V28, V28_H, V28_J, V28_K, + V29, V29_H, V29_J, V29_K, + V30, V30_H, V30_J, V30_K, + V31, V31_H, V31_J, V31_K, +); + +alloc_class chunk3(RFLAGS); + +//----------Architecture Description Register Classes-------------------------- +// Several register classes are automatically defined based upon information in +// this architecture description. +// 1) reg_class inline_cache_reg ( /* as def'd in frame section */ ) +// 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ ) +// + +// Class for all 32 bit general purpose registers +reg_class all_reg32( + R0, + R1, + R2, + R3, + R4, + R7, + R8, + R9, + R10, + R11, + R12, + R13, + R14, + R15, + R16, + R17, + R18, + R19, + R20, + R21, + R22, + R23, + R24, + R25, + R26, + R27, + R28, + R29, + R30, + R31 +); + +// Class for any 32 bit integer registers (excluding zr) +reg_class any_reg32 %{ + return _ANY_REG32_mask; +%} + +// Singleton class for R10 int register +reg_class int_r10_reg(R10); + +// Singleton class for R12 int register +reg_class int_r12_reg(R12); + +// Singleton class for R13 int register +reg_class int_r13_reg(R13); + +// Singleton class for R14 int register +reg_class int_r14_reg(R14); + +// Class for all long integer registers +reg_class all_reg( + R0, R0_H, + R1, R1_H, + R2, R2_H, + R3, R3_H, + R4, R4_H, + R7, R7_H, + R8, R8_H, + R9, R9_H, + R10, R10_H, + R11, R11_H, + R12, R12_H, + R13, R13_H, + R14, R14_H, + R15, R15_H, + R16, R16_H, + R17, R17_H, + R18, R18_H, + R19, R19_H, + R20, R20_H, + R21, R21_H, + R22, R22_H, + R23, R23_H, + R24, R24_H, + R25, R25_H, + R26, R26_H, + R27, R27_H, + R28, R28_H, + R29, R29_H, + R30, R30_H, + R31, R31_H +); + +// Class for all long integer registers (excluding zr) +reg_class any_reg %{ + return _ANY_REG_mask; +%} + +// Class for non-allocatable 32 bit registers +reg_class non_allocatable_reg32( + R0, // zr + R1, // ra + R2, // sp + R3, // gp + R4, // tp + R23 // java thread +); + +// Class for non-allocatable 64 bit registers +reg_class non_allocatable_reg( + R0, R0_H, // zr + R1, R1_H, // ra + R2, R2_H, // sp + R3, R3_H, // gp + R4, R4_H, // tp + R23, R23_H // java thread +); + +reg_class no_special_reg32 %{ + return _NO_SPECIAL_REG32_mask; +%} + +reg_class no_special_reg %{ + return _NO_SPECIAL_REG_mask; +%} + +reg_class ptr_reg %{ + return _PTR_REG_mask; +%} + +reg_class no_special_ptr_reg %{ + return _NO_SPECIAL_PTR_REG_mask; +%} + +// Class for 64 bit register r10 +reg_class r10_reg( + R10, R10_H +); + +// Class for 64 bit register r11 +reg_class r11_reg( + R11, R11_H +); + +// Class for 64 bit register r12 +reg_class r12_reg( + R12, R12_H +); + +// Class for 64 bit register r13 +reg_class r13_reg( + R13, R13_H +); + +// Class for 64 bit register r14 +reg_class r14_reg( + R14, R14_H +); + +// Class for 64 bit register r15 +reg_class r15_reg( + R15, R15_H +); + +// Class for 64 bit register r16 +reg_class r16_reg( + R16, R16_H +); + +// Class for method register +reg_class method_reg( + R31, R31_H +); + +// Class for heapbase register +reg_class heapbase_reg( + R27, R27_H +); + +// Class for java thread register +reg_class java_thread_reg( + R23, R23_H +); + +reg_class r28_reg( + R28, R28_H +); + +reg_class r29_reg( + R29, R29_H +); + +reg_class r30_reg( + R30, R30_H +); + +// Class for zero registesr +reg_class zr_reg( + R0, R0_H +); + +// Class for thread register +reg_class thread_reg( + R4, R4_H +); + +// Class for frame pointer register +reg_class fp_reg( + R8, R8_H +); + +// Class for link register +reg_class ra_reg( + R1, R1_H +); + +// Class for long sp register +reg_class sp_reg( + R2, R2_H +); + +// Class for all float registers +reg_class float_reg( + F0, + F1, + F2, + F3, + F4, + F5, + F6, + F7, + F8, + F9, + F10, + F11, + F12, + F13, + F14, + F15, + F16, + F17, + F18, + F19, + F20, + F21, + F22, + F23, + F24, + F25, + F26, + F27, + F28, + F29, + F30, + F31 +); + +// Double precision float registers have virtual `high halves' that +// are needed by the allocator. +// Class for all double registers +reg_class double_reg( + F0, F0_H, + F1, F1_H, + F2, F2_H, + F3, F3_H, + F4, F4_H, + F5, F5_H, + F6, F6_H, + F7, F7_H, + F8, F8_H, + F9, F9_H, + F10, F10_H, + F11, F11_H, + F12, F12_H, + F13, F13_H, + F14, F14_H, + F15, F15_H, + F16, F16_H, + F17, F17_H, + F18, F18_H, + F19, F19_H, + F20, F20_H, + F21, F21_H, + F22, F22_H, + F23, F23_H, + F24, F24_H, + F25, F25_H, + F26, F26_H, + F27, F27_H, + F28, F28_H, + F29, F29_H, + F30, F30_H, + F31, F31_H +); + +// Class for all RVV vector registers +reg_class vectora_reg( + V1, V1_H, V1_J, V1_K, + V2, V2_H, V2_J, V2_K, + V3, V3_H, V3_J, V3_K, + V4, V4_H, V4_J, V4_K, + V5, V5_H, V5_J, V5_K, + V6, V6_H, V6_J, V6_K, + V7, V7_H, V7_J, V7_K, + V8, V8_H, V8_J, V8_K, + V9, V9_H, V9_J, V9_K, + V10, V10_H, V10_J, V10_K, + V11, V11_H, V11_J, V11_K, + V12, V12_H, V12_J, V12_K, + V13, V13_H, V13_J, V13_K, + V14, V14_H, V14_J, V14_K, + V15, V15_H, V15_J, V15_K, + V16, V16_H, V16_J, V16_K, + V17, V17_H, V17_J, V17_K, + V18, V18_H, V18_J, V18_K, + V19, V19_H, V19_J, V19_K, + V20, V20_H, V20_J, V20_K, + V21, V21_H, V21_J, V21_K, + V22, V22_H, V22_J, V22_K, + V23, V23_H, V23_J, V23_K, + V24, V24_H, V24_J, V24_K, + V25, V25_H, V25_J, V25_K, + V26, V26_H, V26_J, V26_K, + V27, V27_H, V27_J, V27_K, + V28, V28_H, V28_J, V28_K, + V29, V29_H, V29_J, V29_K, + V30, V30_H, V30_J, V30_K, + V31, V31_H, V31_J, V31_K +); + +// Class for 64 bit register f0 +reg_class f0_reg( + F0, F0_H +); + +// Class for 64 bit register f1 +reg_class f1_reg( + F1, F1_H +); + +// Class for 64 bit register f2 +reg_class f2_reg( + F2, F2_H +); + +// Class for 64 bit register f3 +reg_class f3_reg( + F3, F3_H +); + +// class for vector register v1 +reg_class v1_reg( + V1, V1_H, V1_J, V1_K +); + +// class for vector register v2 +reg_class v2_reg( + V2, V2_H, V2_J, V2_K +); + +// class for vector register v3 +reg_class v3_reg( + V3, V3_H, V3_J, V3_K +); + +// class for vector register v4 +reg_class v4_reg( + V4, V4_H, V4_J, V4_K +); + +// class for vector register v5 +reg_class v5_reg( + V5, V5_H, V5_J, V5_K +); + +// class for condition codes +reg_class reg_flags(RFLAGS); +%} + +//----------DEFINITION BLOCK--------------------------------------------------- +// Define name --> value mappings to inform the ADLC of an integer valued name +// Current support includes integer values in the range [0, 0x7FFFFFFF] +// Format: +// int_def ( , ); +// Generated Code in ad_.hpp +// #define () +// // value == +// Generated code in ad_.cpp adlc_verification() +// assert( == , "Expect () to equal "); +// + +// we follow the ppc-aix port in using a simple cost model which ranks +// register operations as cheap, memory ops as more expensive and +// branches as most expensive. the first two have a low as well as a +// normal cost. huge cost appears to be a way of saying don't do +// something + +definitions %{ + // The default cost (of a register move instruction). + int_def DEFAULT_COST ( 100, 100); + int_def ALU_COST ( 100, 1 * DEFAULT_COST); // unknown, const, arith, shift, slt, + // multi, auipc, nop, logical, move + int_def LOAD_COST ( 300, 3 * DEFAULT_COST); // load, fpload + int_def STORE_COST ( 100, 1 * DEFAULT_COST); // store, fpstore + int_def XFER_COST ( 300, 3 * DEFAULT_COST); // mfc, mtc, fcvt, fmove, fcmp + int_def BRANCH_COST ( 200, 2 * DEFAULT_COST); // branch, jmp, call + int_def IMUL_COST ( 1000, 10 * DEFAULT_COST); // imul + int_def IDIVSI_COST ( 3400, 34 * DEFAULT_COST); // idivdi + int_def IDIVDI_COST ( 6600, 66 * DEFAULT_COST); // idivsi + int_def FMUL_SINGLE_COST ( 500, 5 * DEFAULT_COST); // fmul, fmadd + int_def FMUL_DOUBLE_COST ( 700, 7 * DEFAULT_COST); // fmul, fmadd + int_def FDIV_COST ( 2000, 20 * DEFAULT_COST); // fdiv + int_def FSQRT_COST ( 2500, 25 * DEFAULT_COST); // fsqrt + int_def VOLATILE_REF_COST ( 1000, 10 * DEFAULT_COST); +%} + + + +//----------SOURCE BLOCK------------------------------------------------------- +// This is a block of C++ code which provides values, functions, and +// definitions necessary in the rest of the architecture description + +source_hpp %{ + +#include "asm/macroAssembler.hpp" +#include "gc/shared/barrierSetAssembler.hpp" +#include "gc/shared/cardTable.hpp" +#include "gc/shared/cardTableBarrierSet.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "opto/addnode.hpp" +#include "opto/convertnode.hpp" +#include "runtime/objectMonitor.hpp" + +extern RegMask _ANY_REG32_mask; +extern RegMask _ANY_REG_mask; +extern RegMask _PTR_REG_mask; +extern RegMask _NO_SPECIAL_REG32_mask; +extern RegMask _NO_SPECIAL_REG_mask; +extern RegMask _NO_SPECIAL_PTR_REG_mask; + +class CallStubImpl { + + //-------------------------------------------------------------- + //---< Used for optimization in Compile::shorten_branches >--- + //-------------------------------------------------------------- + + public: + // Size of call trampoline stub. + static uint size_call_trampoline() { + return 0; // no call trampolines on this platform + } + + // number of relocations needed by a call trampoline stub + static uint reloc_call_trampoline() { + return 0; // no call trampolines on this platform + } +}; + +class HandlerImpl { + + public: + + static int emit_exception_handler(CodeBuffer &cbuf); + static int emit_deopt_handler(CodeBuffer& cbuf); + + static uint size_exception_handler() { + return MacroAssembler::far_branch_size(); + } + + static uint size_deopt_handler() { + // count auipc + far branch + return NativeInstruction::instruction_size + MacroAssembler::far_branch_size(); + } +}; + +class Node::PD { +public: + enum NodeFlags { + _last_flag = Node::_last_flag + }; +}; + +bool is_CAS(int opcode, bool maybe_volatile); + +// predicate controlling translation of CompareAndSwapX +bool needs_acquiring_load_reserved(const Node *load); + +// predicate controlling addressing modes +bool size_fits_all_mem_uses(AddPNode* addp, int shift); +%} + +source %{ + +// Derived RegMask with conditionally allocatable registers + +RegMask _ANY_REG32_mask; +RegMask _ANY_REG_mask; +RegMask _PTR_REG_mask; +RegMask _NO_SPECIAL_REG32_mask; +RegMask _NO_SPECIAL_REG_mask; +RegMask _NO_SPECIAL_PTR_REG_mask; + +void reg_mask_init() { + + _ANY_REG32_mask = _ALL_REG32_mask; + _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(x0->as_VMReg())); + + _ANY_REG_mask = _ALL_REG_mask; + _ANY_REG_mask.SUBTRACT(_ZR_REG_mask); + + _PTR_REG_mask = _ALL_REG_mask; + _PTR_REG_mask.SUBTRACT(_ZR_REG_mask); + + _NO_SPECIAL_REG32_mask = _ALL_REG32_mask; + _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask); + + _NO_SPECIAL_REG_mask = _ALL_REG_mask; + _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask); + + _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask; + _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask); + + // x27 is not allocatable when compressed oops is on + if (UseCompressedOops) { + _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(x27->as_VMReg())); + _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask); + _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask); + } + + // x8 is not allocatable when PreserveFramePointer is on + if (PreserveFramePointer) { + _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(x8->as_VMReg())); + _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask); + _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask); + } +} + +void PhaseOutput::pd_perform_mach_node_analysis() { +} + +int MachNode::pd_alignment_required() const { + return 1; +} + +int MachNode::compute_padding(int current_offset) const { + return 0; +} + +// is_CAS(int opcode, bool maybe_volatile) +// +// return true if opcode is one of the possible CompareAndSwapX +// values otherwise false. +bool is_CAS(int opcode, bool maybe_volatile) +{ + switch (opcode) { + // We handle these + case Op_CompareAndSwapI: + case Op_CompareAndSwapL: + case Op_CompareAndSwapP: + case Op_CompareAndSwapN: + case Op_ShenandoahCompareAndSwapP: + case Op_ShenandoahCompareAndSwapN: + case Op_CompareAndSwapB: + case Op_CompareAndSwapS: + case Op_GetAndSetI: + case Op_GetAndSetL: + case Op_GetAndSetP: + case Op_GetAndSetN: + case Op_GetAndAddI: + case Op_GetAndAddL: + return true; + case Op_CompareAndExchangeI: + case Op_CompareAndExchangeN: + case Op_CompareAndExchangeB: + case Op_CompareAndExchangeS: + case Op_CompareAndExchangeL: + case Op_CompareAndExchangeP: + case Op_WeakCompareAndSwapB: + case Op_WeakCompareAndSwapS: + case Op_WeakCompareAndSwapI: + case Op_WeakCompareAndSwapL: + case Op_WeakCompareAndSwapP: + case Op_WeakCompareAndSwapN: + case Op_ShenandoahWeakCompareAndSwapP: + case Op_ShenandoahWeakCompareAndSwapN: + case Op_ShenandoahCompareAndExchangeP: + case Op_ShenandoahCompareAndExchangeN: + return maybe_volatile; + default: + return false; + } +} + +// predicate controlling translation of CAS +// +// returns true if CAS needs to use an acquiring load otherwise false +bool needs_acquiring_load_reserved(const Node *n) +{ + assert(n != NULL && is_CAS(n->Opcode(), true), "expecting a compare and swap"); + + LoadStoreNode* ldst = n->as_LoadStore(); + if (n != NULL && is_CAS(n->Opcode(), false)) { + assert(ldst != NULL && ldst->trailing_membar() != NULL, "expected trailing membar"); + } else { + return ldst != NULL && ldst->trailing_membar() != NULL; + } + // so we can just return true here + return true; +} +#define __ _masm. + +// advance declarations for helper functions to convert register +// indices to register objects + +// the ad file has to provide implementations of certain methods +// expected by the generic code +// +// REQUIRED FUNCTIONALITY + +//============================================================================= + +// !!!!! Special hack to get all types of calls to specify the byte offset +// from the start of the call to the point where the return address +// will point. + +int MachCallStaticJavaNode::ret_addr_offset() +{ + // jal + return 1 * NativeInstruction::instruction_size; +} + +int MachCallDynamicJavaNode::ret_addr_offset() +{ + return 7 * NativeInstruction::instruction_size; // movptr, jal +} + +int MachCallRuntimeNode::ret_addr_offset() { + // for generated stubs the call will be + // jal(addr) + // or with far branches + // jal(trampoline_stub) + // for real runtime callouts it will be 11 instructions + // see riscv_enc_java_to_runtime + // la(t1, retaddr) -> auipc + addi + // la(t0, RuntimeAddress(addr)) -> lui + addi + slli + addi + slli + addi + // addi(sp, sp, -2 * wordSize) -> addi + // sd(t1, Address(sp, wordSize)) -> sd + // jalr(t0) -> jalr + CodeBlob *cb = CodeCache::find_blob(_entry_point); + if (cb != NULL) { + return 1 * NativeInstruction::instruction_size; + } else { + return 11 * NativeInstruction::instruction_size; + } +} + +int MachCallNativeNode::ret_addr_offset() { + Unimplemented(); + return -1; +} + +// +// Compute padding required for nodes which need alignment +// + +// With RVC a call instruction may get 2-byte aligned. +// The address of the call instruction needs to be 4-byte aligned to +// ensure that it does not span a cache line so that it can be patched. +int CallStaticJavaDirectNode::compute_padding(int current_offset) const +{ + // to make sure the address of jal 4-byte aligned. + return align_up(current_offset, alignment_required()) - current_offset; +} + +// With RVC a call instruction may get 2-byte aligned. +// The address of the call instruction needs to be 4-byte aligned to +// ensure that it does not span a cache line so that it can be patched. +int CallDynamicJavaDirectNode::compute_padding(int current_offset) const +{ + // skip the movptr in MacroAssembler::ic_call(): + // lui + addi + slli + addi + slli + addi + // Though movptr() has already 4-byte aligned with or without RVC, + // We need to prevent from further changes by explicitly calculating the size. + const int movptr_size = 6 * NativeInstruction::instruction_size; + current_offset += movptr_size; + // to make sure the address of jal 4-byte aligned. + return align_up(current_offset, alignment_required()) - current_offset; +} + +//============================================================================= + +#ifndef PRODUCT +void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const { + assert_cond(st != NULL); + st->print("BREAKPOINT"); +} +#endif + +void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const { + C2_MacroAssembler _masm(&cbuf); + __ ebreak(); +} + +uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const { + return MachNode::size(ra_); +} + +//============================================================================= + +#ifndef PRODUCT + void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const { + st->print("nop \t# %d bytes pad for loops and calls", _count); + } +#endif + + void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const { + C2_MacroAssembler _masm(&cbuf); + Assembler::CompressibleRegion cr(&_masm); // nops shall be 2-byte under RVC for alignment purposes. + for (int i = 0; i < _count; i++) { + __ nop(); + } + } + + uint MachNopNode::size(PhaseRegAlloc*) const { + return _count * (UseRVC ? NativeInstruction::compressed_instruction_size : NativeInstruction::instruction_size); + } + +//============================================================================= +const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty; + +int ConstantTable::calculate_table_base_offset() const { + return 0; // absolute addressing, no offset +} + +bool MachConstantBaseNode::requires_postalloc_expand() const { return false; } +void MachConstantBaseNode::postalloc_expand(GrowableArray *nodes, PhaseRegAlloc *ra_) { + ShouldNotReachHere(); +} + +void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const { + // Empty encoding +} + +uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const { + return 0; +} + +#ifndef PRODUCT +void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const { + assert_cond(st != NULL); + st->print("-- \t// MachConstantBaseNode (empty encoding)"); +} +#endif + +#ifndef PRODUCT +void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const { + assert_cond(st != NULL && ra_ != NULL); + Compile* C = ra_->C; + + int framesize = C->output()->frame_slots() << LogBytesPerInt; + + if (C->output()->need_stack_bang(framesize)) { + st->print("# stack bang size=%d\n\t", framesize); + } + + st->print("sd fp, [sp, #%d]\n\t", - 2 * wordSize); + st->print("sd ra, [sp, #%d]\n\t", - wordSize); + if (PreserveFramePointer) { st->print("sub fp, sp, #%d\n\t", 2 * wordSize); } + st->print("sub sp, sp, #%d\n\t", framesize); + + if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) { + st->print("ld t0, [guard]\n\t"); + st->print("membar LoadLoad\n\t"); + st->print("ld t1, [xthread, #thread_disarmed_offset]\n\t"); + st->print("beq t0, t1, skip\n\t"); + st->print("jalr #nmethod_entry_barrier_stub\n\t"); + st->print("j skip\n\t"); + st->print("guard: int\n\t"); + st->print("skip:\n\t"); + } +} +#endif + +void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const { + assert_cond(ra_ != NULL); + Compile* C = ra_->C; + C2_MacroAssembler _masm(&cbuf); + + // n.b. frame size includes space for return pc and fp + const int framesize = C->output()->frame_size_in_bytes(); + + // insert a nop at the start of the prolog so we can patch in a + // branch if we need to invalidate the method later + { + Assembler::IncompressibleRegion ir(&_masm); // keep the nop as 4 bytes for patching. + MacroAssembler::assert_alignment(__ pc()); + __ nop(); // 4 bytes + } + + assert_cond(C != NULL); + + if (C->clinit_barrier_on_entry()) { + assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started"); + + Label L_skip_barrier; + + __ mov_metadata(t1, C->method()->holder()->constant_encoding()); + __ clinit_barrier(t1, t0, &L_skip_barrier); + __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); + __ bind(L_skip_barrier); + } + + int bangsize = C->output()->bang_size_in_bytes(); + if (C->output()->need_stack_bang(bangsize)) { + __ generate_stack_overflow_check(bangsize); + } + + __ build_frame(framesize); + + if (C->stub_function() == NULL) { + BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->nmethod_entry_barrier(&_masm); + } + + if (VerifyStackAtCalls) { + Unimplemented(); + } + + C->output()->set_frame_complete(cbuf.insts_size()); + + if (C->has_mach_constant_base_node()) { + // NOTE: We set the table base offset here because users might be + // emitted before MachConstantBaseNode. + ConstantTable& constant_table = C->output()->constant_table(); + constant_table.set_table_base_offset(constant_table.calculate_table_base_offset()); + } +} + +uint MachPrologNode::size(PhaseRegAlloc* ra_) const +{ + assert_cond(ra_ != NULL); + return MachNode::size(ra_); // too many variables; just compute it + // the hard way +} + +int MachPrologNode::reloc() const +{ + return 0; +} + +//============================================================================= + +#ifndef PRODUCT +void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const { + assert_cond(st != NULL && ra_ != NULL); + Compile* C = ra_->C; + assert_cond(C != NULL); + int framesize = C->output()->frame_size_in_bytes(); + + st->print("# pop frame %d\n\t", framesize); + + if (framesize == 0) { + st->print("ld ra, [sp,#%d]\n\t", (2 * wordSize)); + st->print("ld fp, [sp,#%d]\n\t", (3 * wordSize)); + st->print("add sp, sp, #%d\n\t", (2 * wordSize)); + } else { + st->print("add sp, sp, #%d\n\t", framesize); + st->print("ld ra, [sp,#%d]\n\t", - 2 * wordSize); + st->print("ld fp, [sp,#%d]\n\t", - wordSize); + } + + if (do_polling() && C->is_method_compilation()) { + st->print("# test polling word\n\t"); + st->print("ld t0, [xthread,#%d]\n\t", in_bytes(JavaThread::polling_word_offset())); + st->print("bgtu sp, t0, #slow_path"); + } +} +#endif + +void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const { + assert_cond(ra_ != NULL); + Compile* C = ra_->C; + C2_MacroAssembler _masm(&cbuf); + assert_cond(C != NULL); + int framesize = C->output()->frame_size_in_bytes(); + + __ remove_frame(framesize); + + if (StackReservedPages > 0 && C->has_reserved_stack_access()) { + __ reserved_stack_check(); + } + + if (do_polling() && C->is_method_compilation()) { + Label dummy_label; + Label* code_stub = &dummy_label; + if (!C->output()->in_scratch_emit_size()) { + code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset()); + } + __ relocate(relocInfo::poll_return_type); + __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */); + } +} + +uint MachEpilogNode::size(PhaseRegAlloc *ra_) const { + assert_cond(ra_ != NULL); + // Variable size. Determine dynamically. + return MachNode::size(ra_); +} + +int MachEpilogNode::reloc() const { + // Return number of relocatable values contained in this instruction. + return 1; // 1 for polling page. +} +const Pipeline * MachEpilogNode::pipeline() const { + return MachNode::pipeline_class(); +} + +//============================================================================= + +// Figure out which register class each belongs in: rc_int, rc_float or +// rc_stack. +enum RC { rc_bad, rc_int, rc_float, rc_vector, rc_stack }; + +static enum RC rc_class(OptoReg::Name reg) { + + if (reg == OptoReg::Bad) { + return rc_bad; + } + + // we have 30 int registers * 2 halves + // (t0 and t1 are omitted) + int slots_of_int_registers = RegisterImpl::max_slots_per_register * (RegisterImpl::number_of_registers - 2); + if (reg < slots_of_int_registers) { + return rc_int; + } + + // we have 32 float register * 2 halves + int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers; + if (reg < slots_of_int_registers + slots_of_float_registers) { + return rc_float; + } + + // we have 32 vector register * 4 halves + int slots_of_vector_registers = VectorRegisterImpl::max_slots_per_register * VectorRegisterImpl::number_of_registers; + if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_vector_registers) { + return rc_vector; + } + + // Between vector regs & stack is the flags regs. + assert(OptoReg::is_stack(reg), "blow up if spilling flags"); + + return rc_stack; +} + +uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const { + assert_cond(ra_ != NULL); + Compile* C = ra_->C; + + // Get registers to move. + OptoReg::Name src_hi = ra_->get_reg_second(in(1)); + OptoReg::Name src_lo = ra_->get_reg_first(in(1)); + OptoReg::Name dst_hi = ra_->get_reg_second(this); + OptoReg::Name dst_lo = ra_->get_reg_first(this); + + enum RC src_hi_rc = rc_class(src_hi); + enum RC src_lo_rc = rc_class(src_lo); + enum RC dst_hi_rc = rc_class(dst_hi); + enum RC dst_lo_rc = rc_class(dst_lo); + + assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register"); + + if (src_hi != OptoReg::Bad) { + assert((src_lo & 1) == 0 && src_lo + 1 == src_hi && + (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi, + "expected aligned-adjacent pairs"); + } + + if (src_lo == dst_lo && src_hi == dst_hi) { + return 0; // Self copy, no move. + } + + bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi && + (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi; + int src_offset = ra_->reg2offset(src_lo); + int dst_offset = ra_->reg2offset(dst_lo); + + if (bottom_type()->isa_vect() != NULL) { + uint ireg = ideal_reg(); + if (ireg == Op_VecA && cbuf) { + C2_MacroAssembler _masm(cbuf); + int vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE); + if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) { + // stack to stack + __ spill_copy_vector_stack_to_stack(src_offset, dst_offset, + vector_reg_size_in_bytes); + } else if (src_lo_rc == rc_vector && dst_lo_rc == rc_stack) { + // vpr to stack + __ spill(as_VectorRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo)); + } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_vector) { + // stack to vpr + __ unspill(as_VectorRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo)); + } else if (src_lo_rc == rc_vector && dst_lo_rc == rc_vector) { + // vpr to vpr + __ vmv1r_v(as_VectorRegister(Matcher::_regEncode[dst_lo]), as_VectorRegister(Matcher::_regEncode[src_lo])); + } else { + ShouldNotReachHere(); + } + } + } else if (cbuf != NULL) { + C2_MacroAssembler _masm(cbuf); + switch (src_lo_rc) { + case rc_int: + if (dst_lo_rc == rc_int) { // gpr --> gpr copy + if (!is64 && this->ideal_reg() != Op_RegI) { // zero extended for narrow oop or klass + __ zero_extend(as_Register(Matcher::_regEncode[dst_lo]), as_Register(Matcher::_regEncode[src_lo]), 32); + } else { + __ mv(as_Register(Matcher::_regEncode[dst_lo]), as_Register(Matcher::_regEncode[src_lo])); + } + } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy + if (is64) { + __ fmv_d_x(as_FloatRegister(Matcher::_regEncode[dst_lo]), + as_Register(Matcher::_regEncode[src_lo])); + } else { + __ fmv_w_x(as_FloatRegister(Matcher::_regEncode[dst_lo]), + as_Register(Matcher::_regEncode[src_lo])); + } + } else { // gpr --> stack spill + assert(dst_lo_rc == rc_stack, "spill to bad register class"); + __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset); + } + break; + case rc_float: + if (dst_lo_rc == rc_int) { // fpr --> gpr copy + if (is64) { + __ fmv_x_d(as_Register(Matcher::_regEncode[dst_lo]), + as_FloatRegister(Matcher::_regEncode[src_lo])); + } else { + __ fmv_x_w(as_Register(Matcher::_regEncode[dst_lo]), + as_FloatRegister(Matcher::_regEncode[src_lo])); + } + } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy + if (is64) { + __ fmv_d(as_FloatRegister(Matcher::_regEncode[dst_lo]), + as_FloatRegister(Matcher::_regEncode[src_lo])); + } else { + __ fmv_s(as_FloatRegister(Matcher::_regEncode[dst_lo]), + as_FloatRegister(Matcher::_regEncode[src_lo])); + } + } else { // fpr --> stack spill + assert(dst_lo_rc == rc_stack, "spill to bad register class"); + __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]), + is64, dst_offset); + } + break; + case rc_stack: + if (dst_lo_rc == rc_int) { // stack --> gpr load + if (this->ideal_reg() == Op_RegI) { + __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset); + } else { // // zero extended for narrow oop or klass + __ unspillu(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset); + } + } else if (dst_lo_rc == rc_float) { // stack --> fpr load + __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]), + is64, src_offset); + } else { // stack --> stack copy + assert(dst_lo_rc == rc_stack, "spill to bad register class"); + if (this->ideal_reg() == Op_RegI) { + __ unspill(t0, is64, src_offset); + } else { // zero extended for narrow oop or klass + __ unspillu(t0, is64, src_offset); + } + __ spill(t0, is64, dst_offset); + } + break; + default: + ShouldNotReachHere(); + } + } + + if (st != NULL) { + st->print("spill "); + if (src_lo_rc == rc_stack) { + st->print("[sp, #%d] -> ", src_offset); + } else { + st->print("%s -> ", Matcher::regName[src_lo]); + } + if (dst_lo_rc == rc_stack) { + st->print("[sp, #%d]", dst_offset); + } else { + st->print("%s", Matcher::regName[dst_lo]); + } + if (bottom_type()->isa_vect() != NULL) { + int vsize = 0; + if (ideal_reg() == Op_VecA) { + vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8; + } else { + ShouldNotReachHere(); + } + st->print("\t# vector spill size = %d", vsize); + } else { + st->print("\t# spill size = %d", is64 ? 64 : 32); + } + } + + return 0; +} + +#ifndef PRODUCT +void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const { + if (ra_ == NULL) { + st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx); + } else { + implementation(NULL, ra_, false, st); + } +} +#endif + +void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const { + implementation(&cbuf, ra_, false, NULL); +} + +uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const { + return MachNode::size(ra_); +} + +//============================================================================= + +#ifndef PRODUCT +void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const { + assert_cond(ra_ != NULL && st != NULL); + int offset = ra_->reg2offset(in_RegMask(0).find_first_elem()); + int reg = ra_->get_reg_first(this); + st->print("add %s, sp, #%d\t# box lock", + Matcher::regName[reg], offset); +} +#endif + +void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const { + C2_MacroAssembler _masm(&cbuf); + Assembler::IncompressibleRegion ir(&_masm); // Fixed length: see BoxLockNode::size() + + assert_cond(ra_ != NULL); + int offset = ra_->reg2offset(in_RegMask(0).find_first_elem()); + int reg = ra_->get_encode(this); + + if (Assembler::is_simm12(offset)) { + __ addi(as_Register(reg), sp, offset); + } else { + __ li32(t0, offset); + __ add(as_Register(reg), sp, t0); + } +} + +uint BoxLockNode::size(PhaseRegAlloc *ra_) const { + // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_). + int offset = ra_->reg2offset(in_RegMask(0).find_first_elem()); + + if (Assembler::is_simm12(offset)) { + return NativeInstruction::instruction_size; + } else { + return 3 * NativeInstruction::instruction_size; // lui + addiw + add; + } +} + +//============================================================================= + +#ifndef PRODUCT +void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const +{ + assert_cond(st != NULL); + st->print_cr("# MachUEPNode"); + if (UseCompressedClassPointers) { + st->print_cr("\tlwu t0, [j_rarg0, oopDesc::klass_offset_in_bytes()]\t# compressed klass"); + if (CompressedKlassPointers::shift() != 0) { + st->print_cr("\tdecode_klass_not_null t0, t0"); + } + } else { + st->print_cr("\tld t0, [j_rarg0, oopDesc::klass_offset_in_bytes()]\t# compressed klass"); + } + st->print_cr("\tbeq t0, t1, ic_hit"); + st->print_cr("\tj, SharedRuntime::_ic_miss_stub\t # Inline cache check"); + st->print_cr("\tic_hit:"); +} +#endif + +void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const +{ + // This is the unverified entry point. + C2_MacroAssembler _masm(&cbuf); + + Label skip; + __ cmp_klass(j_rarg0, t1, t0, t2 /* call-clobbered t2 as a tmp */, skip); + __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub())); + __ bind(skip); + + // These NOPs are critical so that verified entry point is properly + // 4 bytes aligned for patching by NativeJump::patch_verified_entry() + __ align(NativeInstruction::instruction_size); +} + +uint MachUEPNode::size(PhaseRegAlloc* ra_) const +{ + assert_cond(ra_ != NULL); + return MachNode::size(ra_); +} + +// REQUIRED EMIT CODE + +//============================================================================= + +// Emit exception handler code. +int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf) +{ + // la_patchable t0, #exception_blob_entry_point + // jr (offset)t0 + // or + // j #exception_blob_entry_point + // Note that the code buffer's insts_mark is always relative to insts. + // That's why we must use the macroassembler to generate a handler. + C2_MacroAssembler _masm(&cbuf); + address base = __ start_a_stub(size_exception_handler()); + if (base == NULL) { + ciEnv::current()->record_failure("CodeCache is full"); + return 0; // CodeBuffer::expand failed + } + int offset = __ offset(); + __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point())); + assert(__ offset() - offset <= (int) size_exception_handler(), "overflow"); + __ end_a_stub(); + return offset; +} + +// Emit deopt handler code. +int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) +{ + // Note that the code buffer's insts_mark is always relative to insts. + // That's why we must use the macroassembler to generate a handler. + C2_MacroAssembler _masm(&cbuf); + address base = __ start_a_stub(size_deopt_handler()); + if (base == NULL) { + ciEnv::current()->record_failure("CodeCache is full"); + return 0; // CodeBuffer::expand failed + } + int offset = __ offset(); + + __ auipc(ra, 0); + __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); + + assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow"); + __ end_a_stub(); + return offset; + +} +// REQUIRED MATCHER CODE + +//============================================================================= + +const bool Matcher::match_rule_supported(int opcode) { + if (!has_match_rule(opcode)) { + return false; + } + + switch (opcode) { + case Op_CacheWB: // fall through + case Op_CacheWBPreSync: // fall through + case Op_CacheWBPostSync: + if (!VM_Version::supports_data_cache_line_flush()) { + return false; + } + break; + + case Op_StrCompressedCopy: // fall through + case Op_StrInflatedCopy: // fall through + case Op_HasNegatives: + return UseRVV; + + case Op_EncodeISOArray: + return UseRVV && SpecialEncodeISOArray; + + case Op_PopCountI: + case Op_PopCountL: + return UsePopCountInstruction; + + case Op_RotateRight: + case Op_RotateLeft: + case Op_CountLeadingZerosI: + case Op_CountLeadingZerosL: + case Op_CountTrailingZerosI: + case Op_CountTrailingZerosL: + return UseZbb; + } + + return true; // Per default match rules are supported. +} + +// Identify extra cases that we might want to provide match rules for vector nodes and +// other intrinsics guarded with vector length (vlen) and element type (bt). +const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) { + if (!match_rule_supported(opcode) || !vector_size_supported(bt, vlen)) { + return false; + } + + return op_vec_supported(opcode); +} + +const RegMask* Matcher::predicate_reg_mask(void) { + return NULL; +} + +const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) { + return NULL; +} + +// Vector calling convention not yet implemented. +const bool Matcher::supports_vector_calling_convention(void) { + return false; +} + +OptoRegPair Matcher::vector_return_value(uint ideal_reg) { + Unimplemented(); + return OptoRegPair(0, 0); +} + +// Is this branch offset short enough that a short branch can be used? +// +// NOTE: If the platform does not provide any short branch variants, then +// this method should return false for offset 0. +// |---label(L1)-----| +// |-----------------| +// |-----------------|----------eq: float------------------- +// |-----------------| // far_cmpD_branch | cmpD_branch +// |------- ---------| feq; | feq; +// |-far_cmpD_branch-| beqz done; | bnez L; +// |-----------------| j L; | +// |-----------------| bind(done); | +// |-----------------|-------------------------------------- +// |-----------------| // so shortBrSize = br_size - 4; +// |-----------------| // so offs = offset - shortBrSize + 4; +// |---label(L2)-----| +bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) { + // The passed offset is relative to address of the branch. + int shortBrSize = br_size - 4; + int offs = offset - shortBrSize + 4; + return (-4096 <= offs && offs < 4096); +} + +// Vector width in bytes. +const int Matcher::vector_width_in_bytes(BasicType bt) { + if (UseRVV) { + // The MaxVectorSize should have been set by detecting RVV max vector register size when check UseRVV. + // MaxVectorSize == VM_Version::_initial_vector_length + return MaxVectorSize; + } + return 0; +} + +// Limits on vector size (number of elements) loaded into vector. +const int Matcher::max_vector_size(const BasicType bt) { + return vector_width_in_bytes(bt) / type2aelembytes(bt); +} +const int Matcher::min_vector_size(const BasicType bt) { + return max_vector_size(bt); +} + +// Vector ideal reg. +const uint Matcher::vector_ideal_reg(int len) { + assert(MaxVectorSize >= len, ""); + if (UseRVV) { + return Op_VecA; + } + + ShouldNotReachHere(); + return 0; +} + +const int Matcher::scalable_vector_reg_size(const BasicType bt) { + return Matcher::max_vector_size(bt); +} + +MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) { + ShouldNotReachHere(); // generic vector operands not supported + return NULL; +} + +bool Matcher::is_generic_reg2reg_move(MachNode* m) { + ShouldNotReachHere(); // generic vector operands not supported + return false; +} + +bool Matcher::is_generic_vector(MachOper* opnd) { + ShouldNotReachHere(); // generic vector operands not supported + return false; +} + +// Return whether or not this register is ever used as an argument. +// This function is used on startup to build the trampoline stubs in +// generateOptoStub. Registers not mentioned will be killed by the VM +// call in the trampoline, and arguments in those registers not be +// available to the callee. +bool Matcher::can_be_java_arg(int reg) +{ + return + reg == R10_num || reg == R10_H_num || + reg == R11_num || reg == R11_H_num || + reg == R12_num || reg == R12_H_num || + reg == R13_num || reg == R13_H_num || + reg == R14_num || reg == R14_H_num || + reg == R15_num || reg == R15_H_num || + reg == R16_num || reg == R16_H_num || + reg == R17_num || reg == R17_H_num || + reg == F10_num || reg == F10_H_num || + reg == F11_num || reg == F11_H_num || + reg == F12_num || reg == F12_H_num || + reg == F13_num || reg == F13_H_num || + reg == F14_num || reg == F14_H_num || + reg == F15_num || reg == F15_H_num || + reg == F16_num || reg == F16_H_num || + reg == F17_num || reg == F17_H_num; +} + +bool Matcher::is_spillable_arg(int reg) +{ + return can_be_java_arg(reg); +} + +const int Matcher::float_pressure(int default_pressure_threshold) { + return default_pressure_threshold; +} + +bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) { + return false; +} + +RegMask Matcher::divI_proj_mask() { + ShouldNotReachHere(); + return RegMask(); +} + +// Register for MODI projection of divmodI. +RegMask Matcher::modI_proj_mask() { + ShouldNotReachHere(); + return RegMask(); +} + +// Register for DIVL projection of divmodL. +RegMask Matcher::divL_proj_mask() { + ShouldNotReachHere(); + return RegMask(); +} + +// Register for MODL projection of divmodL. +RegMask Matcher::modL_proj_mask() { + ShouldNotReachHere(); + return RegMask(); +} + +const RegMask Matcher::method_handle_invoke_SP_save_mask() { + return FP_REG_mask(); +} + +bool size_fits_all_mem_uses(AddPNode* addp, int shift) { + assert_cond(addp != NULL); + for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) { + Node* u = addp->fast_out(i); + if (u != NULL && u->is_Mem()) { + int opsize = u->as_Mem()->memory_size(); + assert(opsize > 0, "unexpected memory operand size"); + if (u->as_Mem()->memory_size() != (1 << shift)) { + return false; + } + } + } + return true; +} + +// Should the Matcher clone input 'm' of node 'n'? +bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) { + assert_cond(m != NULL); + if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con) + mstack.push(m, Visit); // m = ShiftCntV + return true; + } + return false; +} + +// Should the Matcher clone shifts on addressing modes, expecting them +// to be subsumed into complex addressing expressions or compute them +// into registers? +bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) { + return clone_base_plus_offset_address(m, mstack, address_visited); +} + +%} + + + +//----------ENCODING BLOCK----------------------------------------------------- +// This block specifies the encoding classes used by the compiler to +// output byte streams. Encoding classes are parameterized macros +// used by Machine Instruction Nodes in order to generate the bit +// encoding of the instruction. Operands specify their base encoding +// interface with the interface keyword. There are currently +// supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, & +// COND_INTER. REG_INTER causes an operand to generate a function +// which returns its register number when queried. CONST_INTER causes +// an operand to generate a function which returns the value of the +// constant when queried. MEMORY_INTER causes an operand to generate +// four functions which return the Base Register, the Index Register, +// the Scale Value, and the Offset Value of the operand when queried. +// COND_INTER causes an operand to generate six functions which return +// the encoding code (ie - encoding bits for the instruction) +// associated with each basic boolean condition for a conditional +// instruction. +// +// Instructions specify two basic values for encoding. Again, a +// function is available to check if the constant displacement is an +// oop. They use the ins_encode keyword to specify their encoding +// classes (which must be a sequence of enc_class names, and their +// parameters, specified in the encoding block), and they use the +// opcode keyword to specify, in order, their primary, secondary, and +// tertiary opcode. Only the opcode sections which a particular +// instruction needs for encoding need to be specified. +encode %{ + // BEGIN Non-volatile memory access + + enc_class riscv_enc_li_imm(iRegIorL dst, immIorL src) %{ + C2_MacroAssembler _masm(&cbuf); + int64_t con = (int64_t)$src$$constant; + Register dst_reg = as_Register($dst$$reg); + __ mv(dst_reg, con); + %} + + enc_class riscv_enc_mov_p(iRegP dst, immP src) %{ + C2_MacroAssembler _masm(&cbuf); + Register dst_reg = as_Register($dst$$reg); + address con = (address)$src$$constant; + if (con == NULL || con == (address)1) { + ShouldNotReachHere(); + } else { + relocInfo::relocType rtype = $src->constant_reloc(); + if (rtype == relocInfo::oop_type) { + __ movoop(dst_reg, (jobject)con, /*immediate*/true); + } else if (rtype == relocInfo::metadata_type) { + __ mov_metadata(dst_reg, (Metadata*)con); + } else { + assert(rtype == relocInfo::none, "unexpected reloc type"); + __ mv(dst_reg, $src$$constant); + } + } + %} + + enc_class riscv_enc_mov_p1(iRegP dst) %{ + C2_MacroAssembler _masm(&cbuf); + Register dst_reg = as_Register($dst$$reg); + __ mv(dst_reg, 1); + %} + + enc_class riscv_enc_mov_byte_map_base(iRegP dst) %{ + C2_MacroAssembler _masm(&cbuf); + __ load_byte_map_base($dst$$Register); + %} + + enc_class riscv_enc_mov_n(iRegN dst, immN src) %{ + C2_MacroAssembler _masm(&cbuf); + Register dst_reg = as_Register($dst$$reg); + address con = (address)$src$$constant; + if (con == NULL) { + ShouldNotReachHere(); + } else { + relocInfo::relocType rtype = $src->constant_reloc(); + assert(rtype == relocInfo::oop_type, "unexpected reloc type"); + __ set_narrow_oop(dst_reg, (jobject)con); + } + %} + + enc_class riscv_enc_mov_zero(iRegNorP dst) %{ + C2_MacroAssembler _masm(&cbuf); + Register dst_reg = as_Register($dst$$reg); + __ mv(dst_reg, zr); + %} + + enc_class riscv_enc_mov_nk(iRegN dst, immNKlass src) %{ + C2_MacroAssembler _masm(&cbuf); + Register dst_reg = as_Register($dst$$reg); + address con = (address)$src$$constant; + if (con == NULL) { + ShouldNotReachHere(); + } else { + relocInfo::relocType rtype = $src->constant_reloc(); + assert(rtype == relocInfo::metadata_type, "unexpected reloc type"); + __ set_narrow_klass(dst_reg, (Klass *)con); + } + %} + + enc_class riscv_enc_cmpxchgw(iRegINoSp res, memory mem, iRegI oldval, iRegI newval) %{ + C2_MacroAssembler _masm(&cbuf); + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + %} + + enc_class riscv_enc_cmpxchgn(iRegINoSp res, memory mem, iRegI oldval, iRegI newval) %{ + C2_MacroAssembler _masm(&cbuf); + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + %} + + enc_class riscv_enc_cmpxchg(iRegINoSp res, memory mem, iRegL oldval, iRegL newval) %{ + C2_MacroAssembler _masm(&cbuf); + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + %} + + enc_class riscv_enc_cmpxchgw_acq(iRegINoSp res, memory mem, iRegI oldval, iRegI newval) %{ + C2_MacroAssembler _masm(&cbuf); + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + %} + + enc_class riscv_enc_cmpxchgn_acq(iRegINoSp res, memory mem, iRegI oldval, iRegI newval) %{ + C2_MacroAssembler _masm(&cbuf); + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + %} + + enc_class riscv_enc_cmpxchg_acq(iRegINoSp res, memory mem, iRegL oldval, iRegL newval) %{ + C2_MacroAssembler _masm(&cbuf); + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + %} + + // compare and branch instruction encodings + + enc_class riscv_enc_j(label lbl) %{ + C2_MacroAssembler _masm(&cbuf); + Label* L = $lbl$$label; + __ j(*L); + %} + + enc_class riscv_enc_far_cmpULtGe_imm0_branch(cmpOpULtGe cmp, iRegIorL op1, label lbl) %{ + C2_MacroAssembler _masm(&cbuf); + Label* L = $lbl$$label; + switch ($cmp$$cmpcode) { + case(BoolTest::ge): + __ j(*L); + break; + case(BoolTest::lt): + break; + default: + Unimplemented(); + } + %} + + // call instruction encodings + + enc_class riscv_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result) %{ + Register sub_reg = as_Register($sub$$reg); + Register super_reg = as_Register($super$$reg); + Register temp_reg = as_Register($temp$$reg); + Register result_reg = as_Register($result$$reg); + Register cr_reg = t1; + + Label miss; + Label done; + C2_MacroAssembler _masm(&cbuf); + __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg, + NULL, &miss); + if ($primary) { + __ mv(result_reg, zr); + } else { + __ mv(cr_reg, zr); + __ j(done); + } + + __ bind(miss); + if (!$primary) { + __ mv(cr_reg, 1); + } + + __ bind(done); + %} + + enc_class riscv_enc_java_static_call(method meth) %{ + C2_MacroAssembler _masm(&cbuf); + Assembler::IncompressibleRegion ir(&_masm); // Fixed length: see ret_addr_offset + + address addr = (address)$meth$$method; + address call = NULL; + assert_cond(addr != NULL); + if (!_method) { + // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap. + call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf); + if (call == NULL) { + ciEnv::current()->record_failure("CodeCache is full"); + return; + } + } else { + int method_index = resolved_method_index(cbuf); + RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index) + : static_call_Relocation::spec(method_index); + call = __ trampoline_call(Address(addr, rspec), &cbuf); + if (call == NULL) { + ciEnv::current()->record_failure("CodeCache is full"); + return; + } + + // Emit stub for static call + address stub = CompiledStaticCall::emit_to_interp_stub(cbuf); + if (stub == NULL) { + ciEnv::current()->record_failure("CodeCache is full"); + return; + } + } + %} + + enc_class riscv_enc_java_dynamic_call(method meth) %{ + C2_MacroAssembler _masm(&cbuf); + Assembler::IncompressibleRegion ir(&_masm); // Fixed length: see ret_addr_offset + int method_index = resolved_method_index(cbuf); + address call = __ ic_call((address)$meth$$method, method_index); + if (call == NULL) { + ciEnv::current()->record_failure("CodeCache is full"); + return; + } + %} + + enc_class riscv_enc_call_epilog() %{ + C2_MacroAssembler _masm(&cbuf); + if (VerifyStackAtCalls) { + // Check that stack depth is unchanged: find majik cookie on stack + __ call_Unimplemented(); + } + %} + + enc_class riscv_enc_java_to_runtime(method meth) %{ + C2_MacroAssembler _masm(&cbuf); + Assembler::IncompressibleRegion ir(&_masm); // Fixed length: see ret_addr_offset + + // some calls to generated routines (arraycopy code) are scheduled + // by C2 as runtime calls. if so we can call them using a jr (they + // will be in a reachable segment) otherwise we have to use a jalr + // which loads the absolute address into a register. + address entry = (address)$meth$$method; + CodeBlob *cb = CodeCache::find_blob(entry); + if (cb != NULL) { + address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type)); + if (call == NULL) { + ciEnv::current()->record_failure("CodeCache is full"); + return; + } + } else { + Label retaddr; + __ la(t1, retaddr); + __ la(t0, RuntimeAddress(entry)); + // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc() + __ addi(sp, sp, -2 * wordSize); + __ sd(t1, Address(sp, wordSize)); + __ jalr(t0); + __ bind(retaddr); + __ addi(sp, sp, 2 * wordSize); + } + %} + + // using the cr register as the bool result: 0 for success; others failed. + enc_class riscv_enc_fast_lock(iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2) %{ + C2_MacroAssembler _masm(&cbuf); + Register flag = t1; + Register oop = as_Register($object$$reg); + Register box = as_Register($box$$reg); + Register disp_hdr = as_Register($tmp1$$reg); + Register tmp = as_Register($tmp2$$reg); + Label cont; + Label object_has_monitor; + + assert_different_registers(oop, box, tmp, disp_hdr, t0); + + // Load markWord from object into displaced_header. + __ ld(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes())); + + if (DiagnoseSyncOnValueBasedClasses != 0) { + __ load_klass(flag, oop); + __ lwu(flag, Address(flag, Klass::access_flags_offset())); + __ test_bit(flag, flag, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS), tmp /* tmp */); + __ bnez(flag, cont, true /* is_far */); + } + + if (UseBiasedLocking && !UseOptoBiasInlining) { + __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont, NULL /* slow_case */, NULL, flag); + } + + // Check for existing monitor + __ test_bit(t0, disp_hdr, exact_log2(markWord::monitor_value)); + __ bnez(t0, object_has_monitor); + + // Set tmp to be (markWord of object | UNLOCK_VALUE). + __ ori(tmp, disp_hdr, markWord::unlocked_value); + + // Initialize the box. (Must happen before we update the object mark!) + __ sd(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes())); + + // Compare object markWord with an unlocked value (tmp) and if + // equal exchange the stack address of our box with object markWord. + // On failure disp_hdr contains the possibly locked markWord. + __ cmpxchg(/*memory address*/oop, /*expected value*/tmp, /*new value*/box, Assembler::int64, Assembler::aq, + Assembler::rl, /*result*/disp_hdr); + __ mv(flag, zr); + __ beq(disp_hdr, tmp, cont); // prepare zero flag and goto cont if we won the cas + + assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); + + // If the compare-and-exchange succeeded, then we found an unlocked + // object, will have now locked it will continue at label cont + // We did not see an unlocked object so try the fast recursive case. + + // Check if the owner is self by comparing the value in the + // markWord of object (disp_hdr) with the stack pointer. + __ sub(disp_hdr, disp_hdr, sp); + __ mv(tmp, (intptr_t) (~(os::vm_page_size()-1) | (uintptr_t)markWord::lock_mask_in_place)); + // If (mark & lock_mask) == 0 and mark - sp < page_size, we are stack-locking and goto cont, + // hence we can store 0 as the displaced header in the box, which indicates that it is a + // recursive lock. + __ andr(tmp/*==0?*/, disp_hdr, tmp); + __ sd(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes())); + __ mv(flag, tmp); // we can use the value of tmp as the result here + + __ j(cont); + + // Handle existing monitor. + __ bind(object_has_monitor); + // The object's monitor m is unlocked iff m->owner == NULL, + // otherwise m->owner may contain a thread or a stack address. + // + // Try to CAS m->owner from NULL to current thread. + __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes() - markWord::monitor_value)); + __ cmpxchg(/*memory address*/tmp, /*expected value*/zr, /*new value*/xthread, Assembler::int64, Assembler::aq, + Assembler::rl, /*result*/flag); // cas succeeds if flag == zr(expected) + + // Store a non-null value into the box to avoid looking like a re-entrant + // lock. The fast-path monitor unlock code checks for + // markWord::monitor_value so use markWord::unused_mark which has the + // relevant bit set, and also matches ObjectSynchronizer::slow_enter. + __ mv(tmp, (address)markWord::unused_mark().value()); + __ sd(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes())); + + __ beqz(flag, cont); // CAS success means locking succeeded + + __ bne(flag, xthread, cont); // Check for recursive locking + + // Recursive lock case + __ mv(flag, zr); + __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1, t0, tmp); + + __ bind(cont); + %} + + // using cr flag to indicate the fast_unlock result: 0 for success; others failed. + enc_class riscv_enc_fast_unlock(iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2) %{ + C2_MacroAssembler _masm(&cbuf); + Register flag = t1; + Register oop = as_Register($object$$reg); + Register box = as_Register($box$$reg); + Register disp_hdr = as_Register($tmp1$$reg); + Register tmp = as_Register($tmp2$$reg); + Label cont; + Label object_has_monitor; + + assert_different_registers(oop, box, tmp, disp_hdr, flag); + + if (UseBiasedLocking && !UseOptoBiasInlining) { + __ biased_locking_exit(oop, tmp, cont, flag); + } + + // Find the lock address and load the displaced header from the stack. + __ ld(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes())); + + // If the displaced header is 0, we have a recursive unlock. + __ mv(flag, disp_hdr); + __ beqz(disp_hdr, cont); + + // Handle existing monitor. + __ ld(tmp, Address(oop, oopDesc::mark_offset_in_bytes())); + __ test_bit(t0, tmp, exact_log2(markWord::monitor_value)); + __ bnez(t0, object_has_monitor); + + // Check if it is still a light weight lock, this is true if we + // see the stack address of the basicLock in the markWord of the + // object. + + __ cmpxchg(/*memory address*/oop, /*expected value*/box, /*new value*/disp_hdr, Assembler::int64, Assembler::relaxed, + Assembler::rl, /*result*/tmp); + __ xorr(flag, box, tmp); // box == tmp if cas succeeds + __ j(cont); + + assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); + + // Handle existing monitor. + __ bind(object_has_monitor); + STATIC_ASSERT(markWord::monitor_value <= INT_MAX); + __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor + __ ld(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes())); + + Label notRecursive; + __ beqz(disp_hdr, notRecursive); // Will be 0 if not recursive. + + // Recursive lock + __ addi(disp_hdr, disp_hdr, -1); + __ sd(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes())); + __ mv(flag, zr); + __ j(cont); + + __ bind(notRecursive); + __ ld(flag, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes())); + __ ld(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes())); + __ orr(flag, flag, disp_hdr); // Will be 0 if both are 0. + __ bnez(flag, cont); + // need a release store here + __ la(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes())); + __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore); + __ sd(zr, Address(tmp)); // set unowned + + __ bind(cont); + %} + + // arithmetic encodings + + enc_class riscv_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{ + C2_MacroAssembler _masm(&cbuf); + Register dst_reg = as_Register($dst$$reg); + Register src1_reg = as_Register($src1$$reg); + Register src2_reg = as_Register($src2$$reg); + __ corrected_idivl(dst_reg, src1_reg, src2_reg, false); + %} + + enc_class riscv_enc_div(iRegI dst, iRegI src1, iRegI src2) %{ + C2_MacroAssembler _masm(&cbuf); + Register dst_reg = as_Register($dst$$reg); + Register src1_reg = as_Register($src1$$reg); + Register src2_reg = as_Register($src2$$reg); + __ corrected_idivq(dst_reg, src1_reg, src2_reg, false); + %} + + enc_class riscv_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{ + C2_MacroAssembler _masm(&cbuf); + Register dst_reg = as_Register($dst$$reg); + Register src1_reg = as_Register($src1$$reg); + Register src2_reg = as_Register($src2$$reg); + __ corrected_idivl(dst_reg, src1_reg, src2_reg, true); + %} + + enc_class riscv_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{ + C2_MacroAssembler _masm(&cbuf); + Register dst_reg = as_Register($dst$$reg); + Register src1_reg = as_Register($src1$$reg); + Register src2_reg = as_Register($src2$$reg); + __ corrected_idivq(dst_reg, src1_reg, src2_reg, true); + %} + + enc_class riscv_enc_tail_call(iRegP jump_target) %{ + C2_MacroAssembler _masm(&cbuf); + Register target_reg = as_Register($jump_target$$reg); + __ jr(target_reg); + %} + + enc_class riscv_enc_tail_jmp(iRegP jump_target) %{ + C2_MacroAssembler _masm(&cbuf); + Register target_reg = as_Register($jump_target$$reg); + // exception oop should be in x10 + // ret addr has been popped into ra + // callee expects it in x13 + __ mv(x13, ra); + __ jr(target_reg); + %} + + enc_class riscv_enc_rethrow() %{ + C2_MacroAssembler _masm(&cbuf); + __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub())); + %} + + enc_class riscv_enc_ret() %{ + C2_MacroAssembler _masm(&cbuf); + __ ret(); + %} + +%} + +//----------FRAME-------------------------------------------------------------- +// Definition of frame structure and management information. +// +// S T A C K L A Y O U T Allocators stack-slot number +// | (to get allocators register number +// G Owned by | | v add OptoReg::stack0()) +// r CALLER | | +// o | +--------+ pad to even-align allocators stack-slot +// w V | pad0 | numbers; owned by CALLER +// t -----------+--------+----> Matcher::_in_arg_limit, unaligned +// h ^ | in | 5 +// | | args | 4 Holes in incoming args owned by SELF +// | | | | 3 +// | | +--------+ +// V | | old out| Empty on Intel, window on Sparc +// | old |preserve| Must be even aligned. +// | SP-+--------+----> Matcher::_old_SP, even aligned +// | | in | 3 area for Intel ret address +// Owned by |preserve| Empty on Sparc. +// SELF +--------+ +// | | pad2 | 2 pad to align old SP +// | +--------+ 1 +// | | locks | 0 +// | +--------+----> OptoReg::stack0(), even aligned +// | | pad1 | 11 pad to align new SP +// | +--------+ +// | | | 10 +// | | spills | 9 spills +// V | | 8 (pad0 slot for callee) +// -----------+--------+----> Matcher::_out_arg_limit, unaligned +// ^ | out | 7 +// | | args | 6 Holes in outgoing args owned by CALLEE +// Owned by +--------+ +// CALLEE | new out| 6 Empty on Intel, window on Sparc +// | new |preserve| Must be even-aligned. +// | SP-+--------+----> Matcher::_new_SP, even aligned +// | | | +// +// Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is +// known from SELF's arguments and the Java calling convention. +// Region 6-7 is determined per call site. +// Note 2: If the calling convention leaves holes in the incoming argument +// area, those holes are owned by SELF. Holes in the outgoing area +// are owned by the CALLEE. Holes should not be nessecary in the +// incoming area, as the Java calling convention is completely under +// the control of the AD file. Doubles can be sorted and packed to +// avoid holes. Holes in the outgoing arguments may be nessecary for +// varargs C calling conventions. +// Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is +// even aligned with pad0 as needed. +// Region 6 is even aligned. Region 6-7 is NOT even aligned; +// (the latter is true on Intel but is it false on RISCV?) +// region 6-11 is even aligned; it may be padded out more so that +// the region from SP to FP meets the minimum stack alignment. +// Note 4: For I2C adapters, the incoming FP may not meet the minimum stack +// alignment. Region 11, pad1, may be dynamically extended so that +// SP meets the minimum alignment. + +frame %{ + // These three registers define part of the calling convention + // between compiled code and the interpreter. + + // Inline Cache Register or methodOop for I2C. + inline_cache_reg(R31); + + // Optional: name the operand used by cisc-spilling to access [stack_pointer + offset] + cisc_spilling_operand_name(indOffset); + + // Number of stack slots consumed by locking an object + // generate Compile::sync_stack_slots + // VMRegImpl::slots_per_word = wordSize / stack_slot_size = 8 / 4 = 2 + sync_stack_slots(1 * VMRegImpl::slots_per_word); + + // Compiled code's Frame Pointer + frame_pointer(R2); + + // Interpreter stores its frame pointer in a register which is + // stored to the stack by I2CAdaptors. + // I2CAdaptors convert from interpreted java to compiled java. + interpreter_frame_pointer(R8); + + // Stack alignment requirement + stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes) + + // Number of outgoing stack slots killed above the out_preserve_stack_slots + // for calls to C. Supports the var-args backing area for register parms. + varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes / BytesPerInt); + + // The after-PROLOG location of the return address. Location of + // return address specifies a type (REG or STACK) and a number + // representing the register number (i.e. - use a register name) or + // stack slot. + // Ret Addr is on stack in slot 0 if no locks or verification or alignment. + // Otherwise, it is above the locks and verification slot and alignment word + // TODO this may well be correct but need to check why that - 2 is there + // ppc port uses 0 but we definitely need to allow for fixed_slots + // which folds in the space used for monitors + return_addr(STACK - 2 + + align_up((Compile::current()->in_preserve_stack_slots() + + Compile::current()->fixed_slots()), + stack_alignment_in_slots())); + + // Location of compiled Java return values. Same as C for now. + return_value + %{ + assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL, + "only return normal values"); + + static const int lo[Op_RegL + 1] = { // enum name + 0, // Op_Node + 0, // Op_Set + R10_num, // Op_RegN + R10_num, // Op_RegI + R10_num, // Op_RegP + F10_num, // Op_RegF + F10_num, // Op_RegD + R10_num // Op_RegL + }; + + static const int hi[Op_RegL + 1] = { // enum name + 0, // Op_Node + 0, // Op_Set + OptoReg::Bad, // Op_RegN + OptoReg::Bad, // Op_RegI + R10_H_num, // Op_RegP + OptoReg::Bad, // Op_RegF + F10_H_num, // Op_RegD + R10_H_num // Op_RegL + }; + + return OptoRegPair(hi[ideal_reg], lo[ideal_reg]); + %} +%} + +//----------ATTRIBUTES--------------------------------------------------------- +//----------Operand Attributes------------------------------------------------- +op_attrib op_cost(1); // Required cost attribute + +//----------Instruction Attributes--------------------------------------------- +ins_attrib ins_cost(DEFAULT_COST); // Required cost attribute +ins_attrib ins_size(32); // Required size attribute (in bits) +ins_attrib ins_short_branch(0); // Required flag: is this instruction + // a non-matching short branch variant + // of some long branch? +ins_attrib ins_alignment(4); // Required alignment attribute (must + // be a power of 2) specifies the + // alignment that some part of the + // instruction (not necessarily the + // start) requires. If > 1, a + // compute_padding() function must be + // provided for the instruction + +//----------OPERANDS----------------------------------------------------------- +// Operand definitions must precede instruction definitions for correct parsing +// in the ADLC because operands constitute user defined types which are used in +// instruction definitions. + +//----------Simple Operands---------------------------------------------------- + +// Integer operands 32 bit +// 32 bit immediate +operand immI() +%{ + match(ConI); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// 32 bit zero +operand immI0() +%{ + predicate(n->get_int() == 0); + match(ConI); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// 32 bit unit increment +operand immI_1() +%{ + predicate(n->get_int() == 1); + match(ConI); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// 32 bit unit decrement +operand immI_M1() +%{ + predicate(n->get_int() == -1); + match(ConI); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// Unsigned Integer Immediate: 6-bit int, greater than 32 +operand uimmI6_ge32() %{ + predicate(((unsigned int)(n->get_int()) < 64) && (n->get_int() >= 32)); + match(ConI); + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +operand immI_le_4() +%{ + predicate(n->get_int() <= 4); + match(ConI); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +operand immI_16() +%{ + predicate(n->get_int() == 16); + match(ConI); + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +operand immI_24() +%{ + predicate(n->get_int() == 24); + match(ConI); + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +operand immI_31() +%{ + predicate(n->get_int() == 31); + match(ConI); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +operand immI_63() +%{ + predicate(n->get_int() == 63); + match(ConI); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// 32 bit integer valid for add immediate +operand immIAdd() +%{ + predicate(Assembler::is_simm12((int64_t)n->get_int())); + match(ConI); + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// 32 bit integer valid for sub immediate +operand immISub() +%{ + predicate(Assembler::is_simm12(-(int64_t)n->get_int())); + match(ConI); + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// 5 bit signed value. +operand immI5() +%{ + predicate(n->get_int() <= 15 && n->get_int() >= -16); + match(ConI); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// 5 bit signed value (simm5) +operand immL5() +%{ + predicate(n->get_long() <= 15 && n->get_long() >= -16); + match(ConL); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// Integer operands 64 bit +// 64 bit immediate +operand immL() +%{ + match(ConL); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// 64 bit zero +operand immL0() +%{ + predicate(n->get_long() == 0); + match(ConL); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// Pointer operands +// Pointer Immediate +operand immP() +%{ + match(ConP); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// NULL Pointer Immediate +operand immP0() +%{ + predicate(n->get_ptr() == 0); + match(ConP); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// Pointer Immediate One +// this is used in object initialization (initial object header) +operand immP_1() +%{ + predicate(n->get_ptr() == 1); + match(ConP); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// Card Table Byte Map Base +operand immByteMapBase() +%{ + // Get base of card map + predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) && + (CardTable::CardValue*)n->get_ptr() == + ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base()); + match(ConP); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// Int Immediate: low 16-bit mask +operand immI_16bits() +%{ + predicate(n->get_int() == 0xFFFF); + match(ConI); + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +operand immIpowerOf2() %{ + predicate(is_power_of_2((juint)(n->get_int()))); + match(ConI); + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// Long Immediate: low 32-bit mask +operand immL_32bits() +%{ + predicate(n->get_long() == 0xFFFFFFFFL); + match(ConL); + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// 64 bit unit decrement +operand immL_M1() +%{ + predicate(n->get_long() == -1); + match(ConL); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + + +// 32 bit offset of pc in thread anchor + +operand immL_pc_off() +%{ + predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) + + in_bytes(JavaFrameAnchor::last_Java_pc_offset())); + match(ConL); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// 64 bit integer valid for add immediate +operand immLAdd() +%{ + predicate(Assembler::is_simm12(n->get_long())); + match(ConL); + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// 64 bit integer valid for sub immediate +operand immLSub() +%{ + predicate(Assembler::is_simm12(-(n->get_long()))); + match(ConL); + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// Narrow pointer operands +// Narrow Pointer Immediate +operand immN() +%{ + match(ConN); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// Narrow NULL Pointer Immediate +operand immN0() +%{ + predicate(n->get_narrowcon() == 0); + match(ConN); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +operand immNKlass() +%{ + match(ConNKlass); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// Float and Double operands +// Double Immediate +operand immD() +%{ + match(ConD); + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// Double Immediate: +0.0d +operand immD0() +%{ + predicate(jlong_cast(n->getd()) == 0); + match(ConD); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// Float Immediate +operand immF() +%{ + match(ConF); + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// Float Immediate: +0.0f. +operand immF0() +%{ + predicate(jint_cast(n->getf()) == 0); + match(ConF); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +operand immIOffset() +%{ + predicate(Assembler::is_simm12(n->get_int())); + match(ConI); + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +operand immLOffset() +%{ + predicate(Assembler::is_simm12(n->get_long())); + match(ConL); + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// Scale values +operand immIScale() +%{ + predicate(1 <= n->get_int() && (n->get_int() <= 3)); + match(ConI); + + op_cost(0); + format %{ %} + interface(CONST_INTER); +%} + +// Integer 32 bit Register Operands +operand iRegI() +%{ + constraint(ALLOC_IN_RC(any_reg32)); + match(RegI); + match(iRegINoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Integer 32 bit Register not Special +operand iRegINoSp() +%{ + constraint(ALLOC_IN_RC(no_special_reg32)); + match(RegI); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Register R10 only +operand iRegI_R10() +%{ + constraint(ALLOC_IN_RC(int_r10_reg)); + match(RegI); + match(iRegINoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Register R12 only +operand iRegI_R12() +%{ + constraint(ALLOC_IN_RC(int_r12_reg)); + match(RegI); + match(iRegINoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Register R13 only +operand iRegI_R13() +%{ + constraint(ALLOC_IN_RC(int_r13_reg)); + match(RegI); + match(iRegINoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Register R14 only +operand iRegI_R14() +%{ + constraint(ALLOC_IN_RC(int_r14_reg)); + match(RegI); + match(iRegINoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Integer 64 bit Register Operands +operand iRegL() +%{ + constraint(ALLOC_IN_RC(any_reg)); + match(RegL); + match(iRegLNoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Integer 64 bit Register not Special +operand iRegLNoSp() +%{ + constraint(ALLOC_IN_RC(no_special_reg)); + match(RegL); + match(iRegL_R10); + format %{ %} + interface(REG_INTER); +%} + +// Long 64 bit Register R28 only +operand iRegL_R28() +%{ + constraint(ALLOC_IN_RC(r28_reg)); + match(RegL); + match(iRegLNoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Long 64 bit Register R29 only +operand iRegL_R29() +%{ + constraint(ALLOC_IN_RC(r29_reg)); + match(RegL); + match(iRegLNoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Long 64 bit Register R30 only +operand iRegL_R30() +%{ + constraint(ALLOC_IN_RC(r30_reg)); + match(RegL); + match(iRegLNoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Pointer Register Operands +// Pointer Register +operand iRegP() +%{ + constraint(ALLOC_IN_RC(ptr_reg)); + match(RegP); + match(iRegPNoSp); + match(iRegP_R10); + match(iRegP_R15); + match(javaThread_RegP); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Pointer 64 bit Register not Special +operand iRegPNoSp() +%{ + constraint(ALLOC_IN_RC(no_special_ptr_reg)); + match(RegP); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +operand iRegP_R10() +%{ + constraint(ALLOC_IN_RC(r10_reg)); + match(RegP); + // match(iRegP); + match(iRegPNoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Pointer 64 bit Register R11 only +operand iRegP_R11() +%{ + constraint(ALLOC_IN_RC(r11_reg)); + match(RegP); + match(iRegPNoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +operand iRegP_R12() +%{ + constraint(ALLOC_IN_RC(r12_reg)); + match(RegP); + // match(iRegP); + match(iRegPNoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Pointer 64 bit Register R13 only +operand iRegP_R13() +%{ + constraint(ALLOC_IN_RC(r13_reg)); + match(RegP); + match(iRegPNoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +operand iRegP_R14() +%{ + constraint(ALLOC_IN_RC(r14_reg)); + match(RegP); + // match(iRegP); + match(iRegPNoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +operand iRegP_R15() +%{ + constraint(ALLOC_IN_RC(r15_reg)); + match(RegP); + // match(iRegP); + match(iRegPNoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +operand iRegP_R16() +%{ + constraint(ALLOC_IN_RC(r16_reg)); + match(RegP); + // match(iRegP); + match(iRegPNoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Pointer 64 bit Register R28 only +operand iRegP_R28() +%{ + constraint(ALLOC_IN_RC(r28_reg)); + match(RegP); + match(iRegPNoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Pointer Register Operands +// Narrow Pointer Register +operand iRegN() +%{ + constraint(ALLOC_IN_RC(any_reg32)); + match(RegN); + match(iRegNNoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Integer 64 bit Register not Special +operand iRegNNoSp() +%{ + constraint(ALLOC_IN_RC(no_special_reg32)); + match(RegN); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// heap base register -- used for encoding immN0 +operand iRegIHeapbase() +%{ + constraint(ALLOC_IN_RC(heapbase_reg)); + match(RegI); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Long 64 bit Register R10 only +operand iRegL_R10() +%{ + constraint(ALLOC_IN_RC(r10_reg)); + match(RegL); + match(iRegLNoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Float Register +// Float register operands +operand fRegF() +%{ + constraint(ALLOC_IN_RC(float_reg)); + match(RegF); + + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Double Register +// Double register operands +operand fRegD() +%{ + constraint(ALLOC_IN_RC(double_reg)); + match(RegD); + + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Generic vector class. This will be used for +// all vector operands. +operand vReg() +%{ + constraint(ALLOC_IN_RC(vectora_reg)); + match(VecA); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +operand vReg_V1() +%{ + constraint(ALLOC_IN_RC(v1_reg)); + match(VecA); + match(vReg); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +operand vReg_V2() +%{ + constraint(ALLOC_IN_RC(v2_reg)); + match(VecA); + match(vReg); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +operand vReg_V3() +%{ + constraint(ALLOC_IN_RC(v3_reg)); + match(VecA); + match(vReg); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +operand vReg_V4() +%{ + constraint(ALLOC_IN_RC(v4_reg)); + match(VecA); + match(vReg); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +operand vReg_V5() +%{ + constraint(ALLOC_IN_RC(v5_reg)); + match(VecA); + match(vReg); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +// Java Thread Register +operand javaThread_RegP(iRegP reg) +%{ + constraint(ALLOC_IN_RC(java_thread_reg)); // java_thread_reg + match(reg); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +//----------Memory Operands---------------------------------------------------- +// RISCV has only base_plus_offset and literal address mode, so no need to use +// index and scale. Here set index as 0xffffffff and scale as 0x0. +operand indirect(iRegP reg) +%{ + constraint(ALLOC_IN_RC(ptr_reg)); + match(reg); + op_cost(0); + format %{ "[$reg]" %} + interface(MEMORY_INTER) %{ + base($reg); + index(0xffffffff); + scale(0x0); + disp(0x0); + %} +%} + +operand indOffI(iRegP reg, immIOffset off) +%{ + constraint(ALLOC_IN_RC(ptr_reg)); + match(AddP reg off); + op_cost(0); + format %{ "[$reg, $off]" %} + interface(MEMORY_INTER) %{ + base($reg); + index(0xffffffff); + scale(0x0); + disp($off); + %} +%} + +operand indOffL(iRegP reg, immLOffset off) +%{ + constraint(ALLOC_IN_RC(ptr_reg)); + match(AddP reg off); + op_cost(0); + format %{ "[$reg, $off]" %} + interface(MEMORY_INTER) %{ + base($reg); + index(0xffffffff); + scale(0x0); + disp($off); + %} +%} + +operand indirectN(iRegN reg) +%{ + predicate(CompressedOops::shift() == 0); + constraint(ALLOC_IN_RC(ptr_reg)); + match(DecodeN reg); + op_cost(0); + format %{ "[$reg]\t# narrow" %} + interface(MEMORY_INTER) %{ + base($reg); + index(0xffffffff); + scale(0x0); + disp(0x0); + %} +%} + +operand indOffIN(iRegN reg, immIOffset off) +%{ + predicate(CompressedOops::shift() == 0); + constraint(ALLOC_IN_RC(ptr_reg)); + match(AddP (DecodeN reg) off); + op_cost(0); + format %{ "[$reg, $off]\t# narrow" %} + interface(MEMORY_INTER) %{ + base($reg); + index(0xffffffff); + scale(0x0); + disp($off); + %} +%} + +operand indOffLN(iRegN reg, immLOffset off) +%{ + predicate(CompressedOops::shift() == 0); + constraint(ALLOC_IN_RC(ptr_reg)); + match(AddP (DecodeN reg) off); + op_cost(0); + format %{ "[$reg, $off]\t# narrow" %} + interface(MEMORY_INTER) %{ + base($reg); + index(0xffffffff); + scale(0x0); + disp($off); + %} +%} + +// RISCV opto stubs need to write to the pc slot in the thread anchor +operand thread_anchor_pc(javaThread_RegP reg, immL_pc_off off) +%{ + constraint(ALLOC_IN_RC(ptr_reg)); + match(AddP reg off); + op_cost(0); + format %{ "[$reg, $off]" %} + interface(MEMORY_INTER) %{ + base($reg); + index(0xffffffff); + scale(0x0); + disp($off); + %} +%} + + +//----------Special Memory Operands-------------------------------------------- +// Stack Slot Operand - This operand is used for loading and storing temporary +// values on the stack where a match requires a value to +// flow through memory. +operand stackSlotI(sRegI reg) +%{ + constraint(ALLOC_IN_RC(stack_slots)); + // No match rule because this operand is only generated in matching + // match(RegI); + format %{ "[$reg]" %} + interface(MEMORY_INTER) %{ + base(0x02); // RSP + index(0xffffffff); // No Index + scale(0x0); // No Scale + disp($reg); // Stack Offset + %} +%} + +operand stackSlotF(sRegF reg) +%{ + constraint(ALLOC_IN_RC(stack_slots)); + // No match rule because this operand is only generated in matching + // match(RegF); + format %{ "[$reg]" %} + interface(MEMORY_INTER) %{ + base(0x02); // RSP + index(0xffffffff); // No Index + scale(0x0); // No Scale + disp($reg); // Stack Offset + %} +%} + +operand stackSlotD(sRegD reg) +%{ + constraint(ALLOC_IN_RC(stack_slots)); + // No match rule because this operand is only generated in matching + // match(RegD); + format %{ "[$reg]" %} + interface(MEMORY_INTER) %{ + base(0x02); // RSP + index(0xffffffff); // No Index + scale(0x0); // No Scale + disp($reg); // Stack Offset + %} +%} + +operand stackSlotL(sRegL reg) +%{ + constraint(ALLOC_IN_RC(stack_slots)); + // No match rule because this operand is only generated in matching + // match(RegL); + format %{ "[$reg]" %} + interface(MEMORY_INTER) %{ + base(0x02); // RSP + index(0xffffffff); // No Index + scale(0x0); // No Scale + disp($reg); // Stack Offset + %} +%} + +// Special operand allowing long args to int ops to be truncated for free + +operand iRegL2I(iRegL reg) %{ + + op_cost(0); + + match(ConvL2I reg); + + format %{ "l2i($reg)" %} + + interface(REG_INTER) +%} + + +// Comparison Operands +// NOTE: Label is a predefined operand which should not be redefined in +// the AD file. It is generically handled within the ADLC. + +//----------Conditional Branch Operands---------------------------------------- +// Comparison Op - This is the operation of the comparison, and is limited to +// the following set of codes: +// L (<), LE (<=), G (>), GE (>=), E (==), NE (!=) +// +// Other attributes of the comparison, such as unsignedness, are specified +// by the comparison instruction that sets a condition code flags register. +// That result is represented by a flags operand whose subtype is appropriate +// to the unsignedness (etc.) of the comparison. +// +// Later, the instruction which matches both the Comparison Op (a Bool) and +// the flags (produced by the Cmp) specifies the coding of the comparison op +// by matching a specific subtype of Bool operand below, such as cmpOpU. + + +// used for signed integral comparisons and fp comparisons +operand cmpOp() +%{ + match(Bool); + + format %{ "" %} + + // the values in interface derives from struct BoolTest::mask + interface(COND_INTER) %{ + equal(0x0, "eq"); + greater(0x1, "gt"); + overflow(0x2, "overflow"); + less(0x3, "lt"); + not_equal(0x4, "ne"); + less_equal(0x5, "le"); + no_overflow(0x6, "no_overflow"); + greater_equal(0x7, "ge"); + %} +%} + +// used for unsigned integral comparisons +operand cmpOpU() +%{ + match(Bool); + + format %{ "" %} + // the values in interface derives from struct BoolTest::mask + interface(COND_INTER) %{ + equal(0x0, "eq"); + greater(0x1, "gtu"); + overflow(0x2, "overflow"); + less(0x3, "ltu"); + not_equal(0x4, "ne"); + less_equal(0x5, "leu"); + no_overflow(0x6, "no_overflow"); + greater_equal(0x7, "geu"); + %} +%} + +// used for certain integral comparisons which can be +// converted to bxx instructions +operand cmpOpEqNe() +%{ + match(Bool); + op_cost(0); + predicate(n->as_Bool()->_test._test == BoolTest::ne || + n->as_Bool()->_test._test == BoolTest::eq); + + format %{ "" %} + interface(COND_INTER) %{ + equal(0x0, "eq"); + greater(0x1, "gt"); + overflow(0x2, "overflow"); + less(0x3, "lt"); + not_equal(0x4, "ne"); + less_equal(0x5, "le"); + no_overflow(0x6, "no_overflow"); + greater_equal(0x7, "ge"); + %} +%} + +operand cmpOpULtGe() +%{ + match(Bool); + op_cost(0); + predicate(n->as_Bool()->_test._test == BoolTest::lt || + n->as_Bool()->_test._test == BoolTest::ge); + + format %{ "" %} + interface(COND_INTER) %{ + equal(0x0, "eq"); + greater(0x1, "gtu"); + overflow(0x2, "overflow"); + less(0x3, "ltu"); + not_equal(0x4, "ne"); + less_equal(0x5, "leu"); + no_overflow(0x6, "no_overflow"); + greater_equal(0x7, "geu"); + %} +%} + +operand cmpOpUEqNeLeGt() +%{ + match(Bool); + op_cost(0); + predicate(n->as_Bool()->_test._test == BoolTest::ne || + n->as_Bool()->_test._test == BoolTest::eq || + n->as_Bool()->_test._test == BoolTest::le || + n->as_Bool()->_test._test == BoolTest::gt); + + format %{ "" %} + interface(COND_INTER) %{ + equal(0x0, "eq"); + greater(0x1, "gtu"); + overflow(0x2, "overflow"); + less(0x3, "ltu"); + not_equal(0x4, "ne"); + less_equal(0x5, "leu"); + no_overflow(0x6, "no_overflow"); + greater_equal(0x7, "geu"); + %} +%} + + +// Flags register, used as output of compare logic +operand rFlagsReg() +%{ + constraint(ALLOC_IN_RC(reg_flags)); + match(RegFlags); + + op_cost(0); + format %{ "RFLAGS" %} + interface(REG_INTER); +%} + +// Special Registers + +// Method Register +operand inline_cache_RegP(iRegP reg) +%{ + constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg + match(reg); + match(iRegPNoSp); + op_cost(0); + format %{ %} + interface(REG_INTER); +%} + +//----------OPERAND CLASSES---------------------------------------------------- +// Operand Classes are groups of operands that are used as to simplify +// instruction definitions by not requiring the AD writer to specify +// separate instructions for every form of operand when the +// instruction accepts multiple operand types with the same basic +// encoding and format. The classic case of this is memory operands. + +// memory is used to define read/write location for load/store +// instruction defs. we can turn a memory op into an Address + +opclass memory(indirect, indOffI, indOffL, indirectN, indOffIN, indOffLN); + +// iRegIorL2I is used for src inputs in rules for 32 bit int (I) +// operations. it allows the src to be either an iRegI or a (ConvL2I +// iRegL). in the latter case the l2i normally planted for a ConvL2I +// can be elided because the 32-bit instruction will just employ the +// lower 32 bits anyway. +// +// n.b. this does not elide all L2I conversions. if the truncated +// value is consumed by more than one operation then the ConvL2I +// cannot be bundled into the consuming nodes so an l2i gets planted +// (actually an addiw $dst, $src, 0) and the downstream instructions +// consume the result of the L2I as an iRegI input. That's a shame since +// the addiw is actually redundant but its not too costly. + +opclass iRegIorL2I(iRegI, iRegL2I); +opclass iRegIorL(iRegI, iRegL); +opclass iRegNorP(iRegN, iRegP); +opclass iRegILNP(iRegI, iRegL, iRegN, iRegP); +opclass iRegILNPNoSp(iRegINoSp, iRegLNoSp, iRegNNoSp, iRegPNoSp); +opclass immIorL(immI, immL); + +//----------PIPELINE----------------------------------------------------------- +// Rules which define the behavior of the target architectures pipeline. + +// For specific pipelines, e.g. generic RISC-V, define the stages of that pipeline +//pipe_desc(ID, EX, MEM, WR); +#define ID S0 +#define EX S1 +#define MEM S2 +#define WR S3 + +// Integer ALU reg operation +pipeline %{ + +attributes %{ + // RISC-V instructions are of fixed length + fixed_size_instructions; // Fixed size instructions TODO does + max_instructions_per_bundle = 2; // Generic RISC-V 1, Sifive Series 7 2 + // RISC-V instructions come in 32-bit word units + instruction_unit_size = 4; // An instruction is 4 bytes long + instruction_fetch_unit_size = 64; // The processor fetches one line + instruction_fetch_units = 1; // of 64 bytes + + // List of nop instructions + nops( MachNop ); +%} + +// We don't use an actual pipeline model so don't care about resources +// or description. we do use pipeline classes to introduce fixed +// latencies + +//----------RESOURCES---------------------------------------------------------- +// Resources are the functional units available to the machine + +// Generic RISC-V pipeline +// 1 decoder +// 1 instruction decoded per cycle +// 1 load/store ops per cycle, 1 branch, 1 FPU +// 1 mul, 1 div + +resources ( DECODE, + ALU, + MUL, + DIV, + BRANCH, + LDST, + FPU); + +//----------PIPELINE DESCRIPTION----------------------------------------------- +// Pipeline Description specifies the stages in the machine's pipeline + +// Define the pipeline as a generic 6 stage pipeline +pipe_desc(S0, S1, S2, S3, S4, S5); + +//----------PIPELINE CLASSES--------------------------------------------------- +// Pipeline Classes describe the stages in which input and output are +// referenced by the hardware pipeline. + +pipe_class fp_dop_reg_reg_s(fRegF dst, fRegF src1, fRegF src2) +%{ + single_instruction; + src1 : S1(read); + src2 : S2(read); + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_dop_reg_reg_d(fRegD dst, fRegD src1, fRegD src2) +%{ + src1 : S1(read); + src2 : S2(read); + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_uop_s(fRegF dst, fRegF src) +%{ + single_instruction; + src : S1(read); + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_uop_d(fRegD dst, fRegD src) +%{ + single_instruction; + src : S1(read); + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_d2f(fRegF dst, fRegD src) +%{ + single_instruction; + src : S1(read); + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_f2d(fRegD dst, fRegF src) +%{ + single_instruction; + src : S1(read); + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_f2i(iRegINoSp dst, fRegF src) +%{ + single_instruction; + src : S1(read); + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_f2l(iRegLNoSp dst, fRegF src) +%{ + single_instruction; + src : S1(read); + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_i2f(fRegF dst, iRegIorL2I src) +%{ + single_instruction; + src : S1(read); + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_l2f(fRegF dst, iRegL src) +%{ + single_instruction; + src : S1(read); + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_d2i(iRegINoSp dst, fRegD src) +%{ + single_instruction; + src : S1(read); + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_d2l(iRegLNoSp dst, fRegD src) +%{ + single_instruction; + src : S1(read); + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_i2d(fRegD dst, iRegIorL2I src) +%{ + single_instruction; + src : S1(read); + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_l2d(fRegD dst, iRegIorL2I src) +%{ + single_instruction; + src : S1(read); + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_div_s(fRegF dst, fRegF src1, fRegF src2) +%{ + single_instruction; + src1 : S1(read); + src2 : S2(read); + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_div_d(fRegD dst, fRegD src1, fRegD src2) +%{ + single_instruction; + src1 : S1(read); + src2 : S2(read); + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_sqrt_s(fRegF dst, fRegF src1, fRegF src2) +%{ + single_instruction; + src1 : S1(read); + src2 : S2(read); + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_sqrt_d(fRegD dst, fRegD src1, fRegD src2) +%{ + single_instruction; + src1 : S1(read); + src2 : S2(read); + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_load_constant_s(fRegF dst) +%{ + single_instruction; + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_load_constant_d(fRegD dst) +%{ + single_instruction; + dst : S5(write); + DECODE : ID; + FPU : S5; +%} + +pipe_class fp_load_mem_s(fRegF dst, memory mem) +%{ + single_instruction; + mem : S1(read); + dst : S5(write); + DECODE : ID; + LDST : MEM; +%} + +pipe_class fp_load_mem_d(fRegD dst, memory mem) +%{ + single_instruction; + mem : S1(read); + dst : S5(write); + DECODE : ID; + LDST : MEM; +%} + +pipe_class fp_store_reg_s(fRegF src, memory mem) +%{ + single_instruction; + src : S1(read); + mem : S5(write); + DECODE : ID; + LDST : MEM; +%} + +pipe_class fp_store_reg_d(fRegD src, memory mem) +%{ + single_instruction; + src : S1(read); + mem : S5(write); + DECODE : ID; + LDST : MEM; +%} + +//------- Integer ALU operations -------------------------- + +// Integer ALU reg-reg operation +// Operands needs in ID, result generated in EX +// E.g. ADD Rd, Rs1, Rs2 +pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2) +%{ + single_instruction; + dst : EX(write); + src1 : ID(read); + src2 : ID(read); + DECODE : ID; + ALU : EX; +%} + +// Integer ALU reg operation with constant shift +// E.g. SLLI Rd, Rs1, #shift +pipe_class ialu_reg_shift(iRegI dst, iRegI src1) +%{ + single_instruction; + dst : EX(write); + src1 : ID(read); + DECODE : ID; + ALU : EX; +%} + +// Integer ALU reg-reg operation with variable shift +// both operands must be available in ID +// E.g. SLL Rd, Rs1, Rs2 +pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2) +%{ + single_instruction; + dst : EX(write); + src1 : ID(read); + src2 : ID(read); + DECODE : ID; + ALU : EX; +%} + +// Integer ALU reg operation +// E.g. NEG Rd, Rs2 +pipe_class ialu_reg(iRegI dst, iRegI src) +%{ + single_instruction; + dst : EX(write); + src : ID(read); + DECODE : ID; + ALU : EX; +%} + +// Integer ALU reg immediate operation +// E.g. ADDI Rd, Rs1, #imm +pipe_class ialu_reg_imm(iRegI dst, iRegI src1) +%{ + single_instruction; + dst : EX(write); + src1 : ID(read); + DECODE : ID; + ALU : EX; +%} + +// Integer ALU immediate operation (no source operands) +// E.g. LI Rd, #imm +pipe_class ialu_imm(iRegI dst) +%{ + single_instruction; + dst : EX(write); + DECODE : ID; + ALU : EX; +%} + +//------- Multiply pipeline operations -------------------- + +// Multiply reg-reg +// E.g. MULW Rd, Rs1, Rs2 +pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2) +%{ + single_instruction; + dst : WR(write); + src1 : ID(read); + src2 : ID(read); + DECODE : ID; + MUL : WR; +%} + +// E.g. MUL RD, Rs1, Rs2 +pipe_class lmul_reg_reg(iRegL dst, iRegL src1, iRegL src2) +%{ + single_instruction; + fixed_latency(3); // Maximum latency for 64 bit mul + dst : WR(write); + src1 : ID(read); + src2 : ID(read); + DECODE : ID; + MUL : WR; +%} + +//------- Divide pipeline operations -------------------- + +// E.g. DIVW Rd, Rs1, Rs2 +pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2) +%{ + single_instruction; + fixed_latency(8); // Maximum latency for 32 bit divide + dst : WR(write); + src1 : ID(read); + src2 : ID(read); + DECODE : ID; + DIV : WR; +%} + +// E.g. DIV RD, Rs1, Rs2 +pipe_class ldiv_reg_reg(iRegL dst, iRegL src1, iRegL src2) +%{ + single_instruction; + fixed_latency(16); // Maximum latency for 64 bit divide + dst : WR(write); + src1 : ID(read); + src2 : ID(read); + DECODE : ID; + DIV : WR; +%} + +//------- Load pipeline operations ------------------------ + +// Load - reg, mem +// E.g. LA Rd, mem +pipe_class iload_reg_mem(iRegI dst, memory mem) +%{ + single_instruction; + dst : WR(write); + mem : ID(read); + DECODE : ID; + LDST : MEM; +%} + +// Load - reg, reg +// E.g. LD Rd, Rs +pipe_class iload_reg_reg(iRegI dst, iRegI src) +%{ + single_instruction; + dst : WR(write); + src : ID(read); + DECODE : ID; + LDST : MEM; +%} + +//------- Store pipeline operations ----------------------- + +// Store - zr, mem +// E.g. SD zr, mem +pipe_class istore_mem(memory mem) +%{ + single_instruction; + mem : ID(read); + DECODE : ID; + LDST : MEM; +%} + +// Store - reg, mem +// E.g. SD Rs, mem +pipe_class istore_reg_mem(iRegI src, memory mem) +%{ + single_instruction; + mem : ID(read); + src : EX(read); + DECODE : ID; + LDST : MEM; +%} + +// Store - reg, reg +// E.g. SD Rs2, Rs1 +pipe_class istore_reg_reg(iRegI dst, iRegI src) +%{ + single_instruction; + dst : ID(read); + src : EX(read); + DECODE : ID; + LDST : MEM; +%} + +//------- Control transfer pipeline operations ------------ + +// Branch +pipe_class pipe_branch() +%{ + single_instruction; + DECODE : ID; + BRANCH : EX; +%} + +// Branch +pipe_class pipe_branch_reg(iRegI src) +%{ + single_instruction; + src : ID(read); + DECODE : ID; + BRANCH : EX; +%} + +// Compare & Branch +// E.g. BEQ Rs1, Rs2, L +pipe_class pipe_cmp_branch(iRegI src1, iRegI src2) +%{ + single_instruction; + src1 : ID(read); + src2 : ID(read); + DECODE : ID; + BRANCH : EX; +%} + +// E.g. BEQZ Rs, L +pipe_class pipe_cmpz_branch(iRegI src) +%{ + single_instruction; + src : ID(read); + DECODE : ID; + BRANCH : EX; +%} + +//------- Synchronisation operations ---------------------- +// Any operation requiring serialization +// E.g. FENCE/Atomic Ops/Load Acquire/Store Release +pipe_class pipe_serial() +%{ + single_instruction; + force_serialization; + fixed_latency(16); + DECODE : ID; + LDST : MEM; +%} + +pipe_class pipe_slow() +%{ + instruction_count(10); + multiple_bundles; + force_serialization; + fixed_latency(16); + DECODE : ID; + LDST : MEM; +%} + +// Empty pipeline class +pipe_class pipe_class_empty() +%{ + single_instruction; + fixed_latency(0); +%} + +// Default pipeline class. +pipe_class pipe_class_default() +%{ + single_instruction; + fixed_latency(2); +%} + +// Pipeline class for compares. +pipe_class pipe_class_compare() +%{ + single_instruction; + fixed_latency(16); +%} + +// Pipeline class for memory operations. +pipe_class pipe_class_memory() +%{ + single_instruction; + fixed_latency(16); +%} + +// Pipeline class for call. +pipe_class pipe_class_call() +%{ + single_instruction; + fixed_latency(100); +%} + +// Define the class for the Nop node. +define %{ + MachNop = pipe_class_empty; +%} +%} +//----------INSTRUCTIONS------------------------------------------------------- +// +// match -- States which machine-independent subtree may be replaced +// by this instruction. +// ins_cost -- The estimated cost of this instruction is used by instruction +// selection to identify a minimum cost tree of machine +// instructions that matches a tree of machine-independent +// instructions. +// format -- A string providing the disassembly for this instruction. +// The value of an instruction's operand may be inserted +// by referring to it with a '$' prefix. +// opcode -- Three instruction opcodes may be provided. These are referred +// to within an encode class as $primary, $secondary, and $tertiary +// rrspectively. The primary opcode is commonly used to +// indicate the type of machine instruction, while secondary +// and tertiary are often used for prefix options or addressing +// modes. +// ins_encode -- A list of encode classes with parameters. The encode class +// name must have been defined in an 'enc_class' specification +// in the encode section of the architecture description. + +// ============================================================================ +// Memory (Load/Store) Instructions + +// Load Instructions + +// Load Byte (8 bit signed) +instruct loadB(iRegINoSp dst, memory mem) +%{ + match(Set dst (LoadB mem)); + + ins_cost(LOAD_COST); + format %{ "lb $dst, $mem\t# byte, #@loadB" %} + + ins_encode %{ + __ lb(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Byte (8 bit signed) into long +instruct loadB2L(iRegLNoSp dst, memory mem) +%{ + match(Set dst (ConvI2L (LoadB mem))); + + ins_cost(LOAD_COST); + format %{ "lb $dst, $mem\t# byte, #@loadB2L" %} + + ins_encode %{ + __ lb(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Byte (8 bit unsigned) +instruct loadUB(iRegINoSp dst, memory mem) +%{ + match(Set dst (LoadUB mem)); + + ins_cost(LOAD_COST); + format %{ "lbu $dst, $mem\t# byte, #@loadUB" %} + + ins_encode %{ + __ lbu(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Byte (8 bit unsigned) into long +instruct loadUB2L(iRegLNoSp dst, memory mem) +%{ + match(Set dst (ConvI2L (LoadUB mem))); + + ins_cost(LOAD_COST); + format %{ "lbu $dst, $mem\t# byte, #@loadUB2L" %} + + ins_encode %{ + __ lbu(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Short (16 bit signed) +instruct loadS(iRegINoSp dst, memory mem) +%{ + match(Set dst (LoadS mem)); + + ins_cost(LOAD_COST); + format %{ "lh $dst, $mem\t# short, #@loadS" %} + + ins_encode %{ + __ lh(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Short (16 bit signed) into long +instruct loadS2L(iRegLNoSp dst, memory mem) +%{ + match(Set dst (ConvI2L (LoadS mem))); + + ins_cost(LOAD_COST); + format %{ "lh $dst, $mem\t# short, #@loadS2L" %} + + ins_encode %{ + __ lh(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Char (16 bit unsigned) +instruct loadUS(iRegINoSp dst, memory mem) +%{ + match(Set dst (LoadUS mem)); + + ins_cost(LOAD_COST); + format %{ "lhu $dst, $mem\t# short, #@loadUS" %} + + ins_encode %{ + __ lhu(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Short/Char (16 bit unsigned) into long +instruct loadUS2L(iRegLNoSp dst, memory mem) +%{ + match(Set dst (ConvI2L (LoadUS mem))); + + ins_cost(LOAD_COST); + format %{ "lhu $dst, $mem\t# short, #@loadUS2L" %} + + ins_encode %{ + __ lhu(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Integer (32 bit signed) +instruct loadI(iRegINoSp dst, memory mem) +%{ + match(Set dst (LoadI mem)); + + ins_cost(LOAD_COST); + format %{ "lw $dst, $mem\t# int, #@loadI" %} + + ins_encode %{ + __ lw(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Integer (32 bit signed) into long +instruct loadI2L(iRegLNoSp dst, memory mem) +%{ + match(Set dst (ConvI2L (LoadI mem))); + + ins_cost(LOAD_COST); + format %{ "lw $dst, $mem\t# int, #@loadI2L" %} + + ins_encode %{ + __ lw(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Integer (32 bit unsigned) into long +instruct loadUI2L(iRegLNoSp dst, memory mem, immL_32bits mask) +%{ + match(Set dst (AndL (ConvI2L (LoadI mem)) mask)); + + ins_cost(LOAD_COST); + format %{ "lwu $dst, $mem\t# int, #@loadUI2L" %} + + ins_encode %{ + __ lwu(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Long (64 bit signed) +instruct loadL(iRegLNoSp dst, memory mem) +%{ + match(Set dst (LoadL mem)); + + ins_cost(LOAD_COST); + format %{ "ld $dst, $mem\t# int, #@loadL" %} + + ins_encode %{ + __ ld(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Range +instruct loadRange(iRegINoSp dst, memory mem) +%{ + match(Set dst (LoadRange mem)); + + ins_cost(LOAD_COST); + format %{ "lwu $dst, $mem\t# range, #@loadRange" %} + + ins_encode %{ + __ lwu(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Pointer +instruct loadP(iRegPNoSp dst, memory mem) +%{ + match(Set dst (LoadP mem)); + predicate(n->as_Load()->barrier_data() == 0); + + ins_cost(LOAD_COST); + format %{ "ld $dst, $mem\t# ptr, #@loadP" %} + + ins_encode %{ + __ ld(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Compressed Pointer +instruct loadN(iRegNNoSp dst, memory mem) +%{ + match(Set dst (LoadN mem)); + + ins_cost(LOAD_COST); + format %{ "lwu $dst, $mem\t# loadN, compressed ptr, #@loadN" %} + + ins_encode %{ + __ lwu(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Klass Pointer +instruct loadKlass(iRegPNoSp dst, memory mem) +%{ + match(Set dst (LoadKlass mem)); + + ins_cost(LOAD_COST); + format %{ "ld $dst, $mem\t# class, #@loadKlass" %} + + ins_encode %{ + __ ld(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Narrow Klass Pointer +instruct loadNKlass(iRegNNoSp dst, memory mem) +%{ + match(Set dst (LoadNKlass mem)); + + ins_cost(LOAD_COST); + format %{ "lwu $dst, $mem\t# loadNKlass, compressed class ptr, #@loadNKlass" %} + + ins_encode %{ + __ lwu(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(iload_reg_mem); +%} + +// Load Float +instruct loadF(fRegF dst, memory mem) +%{ + match(Set dst (LoadF mem)); + + ins_cost(LOAD_COST); + format %{ "flw $dst, $mem\t# float, #@loadF" %} + + ins_encode %{ + __ flw(as_FloatRegister($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(fp_load_mem_s); +%} + +// Load Double +instruct loadD(fRegD dst, memory mem) +%{ + match(Set dst (LoadD mem)); + + ins_cost(LOAD_COST); + format %{ "fld $dst, $mem\t# double, #@loadD" %} + + ins_encode %{ + __ fld(as_FloatRegister($dst$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(fp_load_mem_d); +%} + +// Load Int Constant +instruct loadConI(iRegINoSp dst, immI src) +%{ + match(Set dst src); + + ins_cost(ALU_COST); + format %{ "li $dst, $src\t# int, #@loadConI" %} + + ins_encode(riscv_enc_li_imm(dst, src)); + + ins_pipe(ialu_imm); +%} + +// Load Long Constant +instruct loadConL(iRegLNoSp dst, immL src) +%{ + match(Set dst src); + + ins_cost(ALU_COST); + format %{ "li $dst, $src\t# long, #@loadConL" %} + + ins_encode(riscv_enc_li_imm(dst, src)); + + ins_pipe(ialu_imm); +%} + +// Load Pointer Constant +instruct loadConP(iRegPNoSp dst, immP con) +%{ + match(Set dst con); + + ins_cost(ALU_COST); + format %{ "mv $dst, $con\t# ptr, #@loadConP" %} + + ins_encode(riscv_enc_mov_p(dst, con)); + + ins_pipe(ialu_imm); +%} + +// Load Null Pointer Constant +instruct loadConP0(iRegPNoSp dst, immP0 con) +%{ + match(Set dst con); + + ins_cost(ALU_COST); + format %{ "mv $dst, $con\t# NULL ptr, #@loadConP0" %} + + ins_encode(riscv_enc_mov_zero(dst)); + + ins_pipe(ialu_imm); +%} + +// Load Pointer Constant One +instruct loadConP1(iRegPNoSp dst, immP_1 con) +%{ + match(Set dst con); + + ins_cost(ALU_COST); + format %{ "mv $dst, $con\t# load ptr constant one, #@loadConP1" %} + + ins_encode(riscv_enc_mov_p1(dst)); + + ins_pipe(ialu_imm); +%} + +// Load Byte Map Base Constant +instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con) +%{ + match(Set dst con); + ins_cost(ALU_COST); + format %{ "mv $dst, $con\t# Byte Map Base, #@loadByteMapBase" %} + + ins_encode(riscv_enc_mov_byte_map_base(dst)); + + ins_pipe(ialu_imm); +%} + +// Load Narrow Pointer Constant +instruct loadConN(iRegNNoSp dst, immN con) +%{ + match(Set dst con); + + ins_cost(ALU_COST * 4); + format %{ "mv $dst, $con\t# compressed ptr, #@loadConN" %} + + ins_encode(riscv_enc_mov_n(dst, con)); + + ins_pipe(ialu_imm); +%} + +// Load Narrow Null Pointer Constant +instruct loadConN0(iRegNNoSp dst, immN0 con) +%{ + match(Set dst con); + + ins_cost(ALU_COST); + format %{ "mv $dst, $con\t# compressed NULL ptr, #@loadConN0" %} + + ins_encode(riscv_enc_mov_zero(dst)); + + ins_pipe(ialu_imm); +%} + +// Load Narrow Klass Constant +instruct loadConNKlass(iRegNNoSp dst, immNKlass con) +%{ + match(Set dst con); + + ins_cost(ALU_COST * 6); + format %{ "mv $dst, $con\t# compressed klass ptr, #@loadConNKlass" %} + + ins_encode(riscv_enc_mov_nk(dst, con)); + + ins_pipe(ialu_imm); +%} + +// Load Float Constant +instruct loadConF(fRegF dst, immF con) %{ + match(Set dst con); + + ins_cost(LOAD_COST); + format %{ + "flw $dst, [$constantaddress]\t# load from constant table: float=$con, #@loadConF" + %} + + ins_encode %{ + __ flw(as_FloatRegister($dst$$reg), $constantaddress($con)); + %} + + ins_pipe(fp_load_constant_s); +%} + +instruct loadConF0(fRegF dst, immF0 con) %{ + match(Set dst con); + + ins_cost(XFER_COST); + + format %{ "fmv.w.x $dst, zr\t# float, #@loadConF0" %} + + ins_encode %{ + __ fmv_w_x(as_FloatRegister($dst$$reg), zr); + %} + + ins_pipe(fp_load_constant_s); +%} + +// Load Double Constant +instruct loadConD(fRegD dst, immD con) %{ + match(Set dst con); + + ins_cost(LOAD_COST); + format %{ + "fld $dst, [$constantaddress]\t# load from constant table: double=$con, #@loadConD" + %} + + ins_encode %{ + __ fld(as_FloatRegister($dst$$reg), $constantaddress($con)); + %} + + ins_pipe(fp_load_constant_d); +%} + +instruct loadConD0(fRegD dst, immD0 con) %{ + match(Set dst con); + + ins_cost(XFER_COST); + + format %{ "fmv.d.x $dst, zr\t# double, #@loadConD0" %} + + ins_encode %{ + __ fmv_d_x(as_FloatRegister($dst$$reg), zr); + %} + + ins_pipe(fp_load_constant_d); +%} + +// Store Instructions +// Store CMS card-mark Immediate +instruct storeimmCM0(immI0 zero, memory mem) +%{ + match(Set mem (StoreCM mem zero)); + + ins_cost(STORE_COST); + format %{ "storestore (elided)\n\t" + "sb zr, $mem\t# byte, #@storeimmCM0" %} + + ins_encode %{ + __ sb(zr, Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(istore_mem); +%} + +// Store CMS card-mark Immediate with intervening StoreStore +// needed when using CMS with no conditional card marking +instruct storeimmCM0_ordered(immI0 zero, memory mem) +%{ + match(Set mem (StoreCM mem zero)); + + ins_cost(ALU_COST + STORE_COST); + format %{ "membar(StoreStore)\n\t" + "sb zr, $mem\t# byte, #@storeimmCM0_ordered" %} + + ins_encode %{ + __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore); + __ sb(zr, Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(istore_mem); +%} + +// Store Byte +instruct storeB(iRegIorL2I src, memory mem) +%{ + match(Set mem (StoreB mem src)); + + ins_cost(STORE_COST); + format %{ "sb $src, $mem\t# byte, #@storeB" %} + + ins_encode %{ + __ sb(as_Register($src$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(istore_reg_mem); +%} + +instruct storeimmB0(immI0 zero, memory mem) +%{ + match(Set mem (StoreB mem zero)); + + ins_cost(STORE_COST); + format %{ "sb zr, $mem\t# byte, #@storeimmB0" %} + + ins_encode %{ + __ sb(zr, Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(istore_mem); +%} + +// Store Char/Short +instruct storeC(iRegIorL2I src, memory mem) +%{ + match(Set mem (StoreC mem src)); + + ins_cost(STORE_COST); + format %{ "sh $src, $mem\t# short, #@storeC" %} + + ins_encode %{ + __ sh(as_Register($src$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(istore_reg_mem); +%} + +instruct storeimmC0(immI0 zero, memory mem) +%{ + match(Set mem (StoreC mem zero)); + + ins_cost(STORE_COST); + format %{ "sh zr, $mem\t# short, #@storeimmC0" %} + + ins_encode %{ + __ sh(zr, Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(istore_mem); +%} + +// Store Integer +instruct storeI(iRegIorL2I src, memory mem) +%{ + match(Set mem(StoreI mem src)); + + ins_cost(STORE_COST); + format %{ "sw $src, $mem\t# int, #@storeI" %} + + ins_encode %{ + __ sw(as_Register($src$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(istore_reg_mem); +%} + +instruct storeimmI0(immI0 zero, memory mem) +%{ + match(Set mem(StoreI mem zero)); + + ins_cost(STORE_COST); + format %{ "sw zr, $mem\t# int, #@storeimmI0" %} + + ins_encode %{ + __ sw(zr, Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(istore_mem); +%} + +// Store Long (64 bit signed) +instruct storeL(iRegL src, memory mem) +%{ + match(Set mem (StoreL mem src)); + + ins_cost(STORE_COST); + format %{ "sd $src, $mem\t# long, #@storeL" %} + + ins_encode %{ + __ sd(as_Register($src$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(istore_reg_mem); +%} + +// Store Long (64 bit signed) +instruct storeimmL0(immL0 zero, memory mem) +%{ + match(Set mem (StoreL mem zero)); + + ins_cost(STORE_COST); + format %{ "sd zr, $mem\t# long, #@storeimmL0" %} + + ins_encode %{ + __ sd(zr, Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(istore_mem); +%} + +// Store Pointer +instruct storeP(iRegP src, memory mem) +%{ + match(Set mem (StoreP mem src)); + + ins_cost(STORE_COST); + format %{ "sd $src, $mem\t# ptr, #@storeP" %} + + ins_encode %{ + __ sd(as_Register($src$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(istore_reg_mem); +%} + +// Store Pointer +instruct storeimmP0(immP0 zero, memory mem) +%{ + match(Set mem (StoreP mem zero)); + + ins_cost(STORE_COST); + format %{ "sd zr, $mem\t# ptr, #@storeimmP0" %} + + ins_encode %{ + __ sd(zr, Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(istore_mem); +%} + +// Store Compressed Pointer +instruct storeN(iRegN src, memory mem) +%{ + match(Set mem (StoreN mem src)); + + ins_cost(STORE_COST); + format %{ "sw $src, $mem\t# compressed ptr, #@storeN" %} + + ins_encode %{ + __ sw(as_Register($src$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(istore_reg_mem); +%} + +instruct storeImmN0(immN0 zero, memory mem) +%{ + match(Set mem (StoreN mem zero)); + + ins_cost(STORE_COST); + format %{ "sw zr, $mem\t# compressed ptr, #@storeImmN0" %} + + ins_encode %{ + __ sw(zr, Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(istore_reg_mem); +%} + +// Store Float +instruct storeF(fRegF src, memory mem) +%{ + match(Set mem (StoreF mem src)); + + ins_cost(STORE_COST); + format %{ "fsw $src, $mem\t# float, #@storeF" %} + + ins_encode %{ + __ fsw(as_FloatRegister($src$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(fp_store_reg_s); +%} + +// Store Double +instruct storeD(fRegD src, memory mem) +%{ + match(Set mem (StoreD mem src)); + + ins_cost(STORE_COST); + format %{ "fsd $src, $mem\t# double, #@storeD" %} + + ins_encode %{ + __ fsd(as_FloatRegister($src$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(fp_store_reg_d); +%} + +// Store Compressed Klass Pointer +instruct storeNKlass(iRegN src, memory mem) +%{ + match(Set mem (StoreNKlass mem src)); + + ins_cost(STORE_COST); + format %{ "sw $src, $mem\t# compressed klass ptr, #@storeNKlass" %} + + ins_encode %{ + __ sw(as_Register($src$$reg), Address(as_Register($mem$$base), $mem$$disp)); + %} + + ins_pipe(istore_reg_mem); +%} + +// ============================================================================ +// Atomic operation instructions +// +// Intel and SPARC both implement Ideal Node LoadPLocked and +// Store{PIL}Conditional instructions using a normal load for the +// LoadPLocked and a CAS for the Store{PIL}Conditional. +// +// The ideal code appears only to use LoadPLocked/storePConditional as a +// pair to lock object allocations from Eden space when not using +// TLABs. +// +// There does not appear to be a Load{IL}Locked Ideal Node and the +// Ideal code appears to use Store{IL}Conditional as an alias for CAS +// and to use StoreIConditional only for 32-bit and StoreLConditional +// only for 64-bit. +// +// We implement LoadPLocked and storePConditional instructions using, +// respectively the RISCV hw load-reserve and store-conditional +// instructions. Whereas we must implement each of +// Store{IL}Conditional using a CAS which employs a pair of +// instructions comprising a load-reserve followed by a +// store-conditional. + + +// Locked-load (load reserved) of the current heap-top +// used when updating the eden heap top +// implemented using lr_d on RISCV64 +instruct loadPLocked(iRegPNoSp dst, indirect mem) +%{ + match(Set dst (LoadPLocked mem)); + + ins_cost(ALU_COST * 2 + LOAD_COST); + + format %{ "lr.d $dst, $mem\t# ptr load reserved, #@loadPLocked" %} + + ins_encode %{ + __ la(t0, Address(as_Register($mem$$base), $mem$$disp)); + __ lr_d($dst$$Register, t0, Assembler::aq); + %} + + ins_pipe(pipe_serial); +%} + +// Conditional-store of the updated heap-top. +// Used during allocation of the shared heap. +// implemented using sc_d on RISCV64. +instruct storePConditional(memory heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr) +%{ + match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval))); + + ins_cost(ALU_COST * 2 + STORE_COST); + + format %{ + "sc_d t1, $newval $heap_top_ptr,\t# ptr store conditional, #@storePConditional" + %} + + ins_encode %{ + __ la(t0, Address(as_Register($heap_top_ptr$$base), $heap_top_ptr$$disp)); + __ sc_d($cr$$Register, $newval$$Register, t0, Assembler::rl); + %} + + ins_pipe(pipe_serial); +%} + +instruct storeLConditional(indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) +%{ + match(Set cr (StoreLConditional mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + 2 * BRANCH_COST); + + format %{ + "cmpxchg t1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval" + "xorr $cr, $cr, $oldval\t# $cr == 0 on successful write, #@storeLConditional" + %} + + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $cr$$Register); + __ xorr($cr$$Register,$cr$$Register, $oldval$$Register); + %} + + ins_pipe(pipe_slow); +%} + +// storeIConditional also has acquire semantics, for no better reason +// than matching storeLConditional. +instruct storeIConditional(indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) +%{ + match(Set cr (StoreIConditional mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2); + + format %{ + "cmpxchgw t1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval" + "xorr $cr, $cr, $oldval\t# $cr == 0 on successful write, #@storeIConditional" + %} + + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $cr$$Register); + __ xorr($cr$$Register,$cr$$Register, $oldval$$Register); + %} + + ins_pipe(pipe_slow); +%} + +// standard CompareAndSwapX when we are using barriers +// these have higher priority than the rules selected by a predicate +instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +%{ + match(Set res (CompareAndSwapB mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + ALU_COST * 10 + BRANCH_COST * 4); + + effect(TEMP_DEF res, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); + + format %{ + "cmpxchg $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval\n\t" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapB" + %} + + ins_encode %{ + __ cmpxchg_narrow_value(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8, + Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register, + true /* result as bool */, $tmp1$$Register, $tmp2$$Register, $tmp3$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +%{ + match(Set res (CompareAndSwapS mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + ALU_COST * 11 + BRANCH_COST * 4); + + effect(TEMP_DEF res, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); + + format %{ + "cmpxchg $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval\n\t" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapS" + %} + + ins_encode %{ + __ cmpxchg_narrow_value(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16, + Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register, + true /* result as bool */, $tmp1$$Register, $tmp2$$Register, $tmp3$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) +%{ + match(Set res (CompareAndSwapI mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4); + + format %{ + "cmpxchg $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval\n\t" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapI" + %} + + ins_encode(riscv_enc_cmpxchgw(res, mem, oldval, newval)); + + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval) +%{ + match(Set res (CompareAndSwapL mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4); + + format %{ + "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval\n\t" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapL" + %} + + ins_encode(riscv_enc_cmpxchg(res, mem, oldval, newval)); + + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval) +%{ + predicate(n->as_LoadStore()->barrier_data() == 0); + + match(Set res (CompareAndSwapP mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4); + + format %{ + "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval\n\t" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapP" + %} + + ins_encode(riscv_enc_cmpxchg(res, mem, oldval, newval)); + + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval) +%{ + match(Set res (CompareAndSwapN mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + ALU_COST * 8 + BRANCH_COST * 4); + + format %{ + "cmpxchg $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval\n\t" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapN" + %} + + ins_encode(riscv_enc_cmpxchgn(res, mem, oldval, newval)); + + ins_pipe(pipe_slow); +%} + +// alternative CompareAndSwapX when we are eliding barriers +instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set res (CompareAndSwapB mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + ALU_COST * 10 + BRANCH_COST * 4); + + effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); + + format %{ + "cmpxchg_acq $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval\n\t" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapBAcq" + %} + + ins_encode %{ + __ cmpxchg_narrow_value(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8, + Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register, + true /* result as bool */, $tmp1$$Register, $tmp2$$Register, $tmp3$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set res (CompareAndSwapS mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + ALU_COST * 11 + BRANCH_COST * 4); + + effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); + + format %{ + "cmpxchg_acq $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval\n\t" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapSAcq" + %} + + ins_encode %{ + __ cmpxchg_narrow_value(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16, + Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register, + true /* result as bool */, $tmp1$$Register, $tmp2$$Register, $tmp3$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set res (CompareAndSwapI mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4); + + format %{ + "cmpxchg_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval\n\t" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapIAcq" + %} + + ins_encode(riscv_enc_cmpxchgw_acq(res, mem, oldval, newval)); + + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set res (CompareAndSwapL mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4); + + format %{ + "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval\n\t" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapLAcq" + %} + + ins_encode(riscv_enc_cmpxchg_acq(res, mem, oldval, newval)); + + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval) +%{ + predicate(needs_acquiring_load_reserved(n) && (n->as_LoadStore()->barrier_data() == 0)); + + match(Set res (CompareAndSwapP mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4); + + format %{ + "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval\n\t" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapPAcq" + %} + + ins_encode(riscv_enc_cmpxchg_acq(res, mem, oldval, newval)); + + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set res (CompareAndSwapN mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + ALU_COST * 8 + BRANCH_COST * 4); + + format %{ + "cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval\n\t" + "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapNAcq" + %} + + ins_encode(riscv_enc_cmpxchgn_acq(res, mem, oldval, newval)); + + ins_pipe(pipe_slow); +%} + +// Sundry CAS operations. Note that release is always true, +// regardless of the memory ordering of the CAS. This is because we +// need the volatile case to be sequentially consistent but there is +// no trailing StoreLoad barrier emitted by C2. Unfortunately we +// can't check the type of memory ordering here, so we always emit a +// sc_d(w) with rl bit set. +instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +%{ + match(Set res (CompareAndExchangeB mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 5); + + effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); + + format %{ + "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeB" + %} + + ins_encode %{ + __ cmpxchg_narrow_value(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register, + /*result_as_bool*/ false, $tmp1$$Register, $tmp2$$Register, $tmp3$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +%{ + match(Set res (CompareAndExchangeS mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 6); + + effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); + + format %{ + "cmpxchg $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeS" + %} + + ins_encode %{ + __ cmpxchg_narrow_value(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register, + /*result_as_bool*/ false, $tmp1$$Register, $tmp2$$Register, $tmp3$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) +%{ + match(Set res (CompareAndExchangeI mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST); + + effect(TEMP_DEF res); + + format %{ + "cmpxchg $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeI" + %} + + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval) +%{ + match(Set res (CompareAndExchangeL mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST); + + effect(TEMP_DEF res); + + format %{ + "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeL" + %} + + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval) +%{ + match(Set res (CompareAndExchangeN mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 3); + + effect(TEMP_DEF res); + + format %{ + "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeN" + %} + + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval) +%{ + predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST); + + effect(TEMP_DEF res); + + format %{ + "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeP" + %} + + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set res (CompareAndExchangeB mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 5); + + effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); + + format %{ + "cmpxchg_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeBAcq" + %} + + ins_encode %{ + __ cmpxchg_narrow_value(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register, + /*result_as_bool*/ false, $tmp1$$Register, $tmp2$$Register, $tmp3$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set res (CompareAndExchangeS mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 6); + + effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); + + format %{ + "cmpxchg_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeSAcq" + %} + + ins_encode %{ + __ cmpxchg_narrow_value(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register, + /*result_as_bool*/ false, $tmp1$$Register, $tmp2$$Register, $tmp3$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set res (CompareAndExchangeI mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST); + + effect(TEMP_DEF res); + + format %{ + "cmpxchg_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeIAcq" + %} + + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set res (CompareAndExchangeL mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST); + + effect(TEMP_DEF res); + + format %{ + "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeLAcq" + %} + + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set res (CompareAndExchangeN mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST); + + effect(TEMP_DEF res); + + format %{ + "cmpxchg_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeNAcq" + %} + + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval) +%{ + predicate(needs_acquiring_load_reserved(n) && (n->as_LoadStore()->barrier_data() == 0)); + + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST); + + effect(TEMP_DEF res); + + format %{ + "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangePAcq" + %} + + ins_encode %{ + __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +%{ + match(Set res (WeakCompareAndSwapB mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 6); + + effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); + + format %{ + "cmpxchg_weak $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t" + "# $res == 1 when success, #@weakCompareAndSwapB" + %} + + ins_encode %{ + __ weak_cmpxchg_narrow_value(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register, + $tmp1$$Register, $tmp2$$Register, $tmp3$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +%{ + match(Set res (WeakCompareAndSwapS mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 7); + + effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); + + format %{ + "cmpxchg_weak $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t" + "# $res == 1 when success, #@weakCompareAndSwapS" + %} + + ins_encode %{ + __ weak_cmpxchg_narrow_value(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register, + $tmp1$$Register, $tmp2$$Register, $tmp3$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) +%{ + match(Set res (WeakCompareAndSwapI mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2); + + format %{ + "cmpxchg_weak $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval\n\t" + "# $res == 1 when success, #@weakCompareAndSwapI" + %} + + ins_encode %{ + __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval) +%{ + match(Set res (WeakCompareAndSwapL mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2); + + format %{ + "cmpxchg_weak $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval\n\t" + "# $res == 1 when success, #@weakCompareAndSwapL" + %} + + ins_encode %{ + __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval) +%{ + match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 4); + + format %{ + "cmpxchg_weak $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval\n\t" + "# $res == 1 when success, #@weakCompareAndSwapN" + %} + + ins_encode %{ + __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval) +%{ + predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2); + + format %{ + "cmpxchg_weak $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval\n\t" + "# $res == 1 when success, #@weakCompareAndSwapP" + %} + + ins_encode %{ + __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set res (WeakCompareAndSwapB mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 6); + + effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); + + format %{ + "cmpxchg_weak_acq $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t" + "# $res == 1 when success, #@weakCompareAndSwapBAcq" + %} + + ins_encode %{ + __ weak_cmpxchg_narrow_value(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register, + $tmp1$$Register, $tmp2$$Register, $tmp3$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval, + iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set res (WeakCompareAndSwapS mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 7); + + effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3); + + format %{ + "cmpxchg_weak_acq $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t" + "# $res == 1 when success, #@weakCompareAndSwapSAcq" + %} + + ins_encode %{ + __ weak_cmpxchg_narrow_value(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register, + $tmp1$$Register, $tmp2$$Register, $tmp3$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set res (WeakCompareAndSwapI mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2); + + format %{ + "cmpxchg_weak_acq $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval\n\t" + "# $res == 1 when success, #@weakCompareAndSwapIAcq" + %} + + ins_encode %{ + __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set res (WeakCompareAndSwapL mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2); + + format %{ + "cmpxchg_weak_acq $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval\n\t" + "# $res == 1 when success, #@weakCompareAndSwapLAcq" + %} + + ins_encode %{ + __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 4); + + format %{ + "cmpxchg_weak_acq $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval\n\t" + "# $res == 1 when success, #@weakCompareAndSwapNAcq" + %} + + ins_encode %{ + __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval) +%{ + predicate(needs_acquiring_load_reserved(n) && (n->as_LoadStore()->barrier_data() == 0)); + + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + + ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2); + + format %{ + "cmpxchg_weak_acq $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval\n\t" + "\t# $res == 1 when success, #@weakCompareAndSwapPAcq" + %} + + ins_encode %{ + __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) +%{ + match(Set prev (GetAndSetI mem newv)); + + ins_cost(ALU_COST); + + format %{ "atomic_xchgw $prev, $newv, [$mem]\t#@get_and_setI" %} + + ins_encode %{ + __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) +%{ + match(Set prev (GetAndSetL mem newv)); + + ins_cost(ALU_COST); + + format %{ "atomic_xchg $prev, $newv, [$mem]\t#@get_and_setL" %} + + ins_encode %{ + __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) +%{ + match(Set prev (GetAndSetN mem newv)); + + ins_cost(ALU_COST); + + format %{ "atomic_xchgwu $prev, $newv, [$mem]\t#@get_and_setN" %} + + ins_encode %{ + __ atomic_xchgwu($prev$$Register, $newv$$Register, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) +%{ + predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set prev (GetAndSetP mem newv)); + + ins_cost(ALU_COST); + + format %{ "atomic_xchg $prev, $newv, [$mem]\t#@get_and_setP" %} + + ins_encode %{ + __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set prev (GetAndSetI mem newv)); + + ins_cost(ALU_COST); + + format %{ "atomic_xchgw_acq $prev, $newv, [$mem]\t#@get_and_setIAcq" %} + + ins_encode %{ + __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set prev (GetAndSetL mem newv)); + + ins_cost(ALU_COST); + + format %{ "atomic_xchg_acq $prev, $newv, [$mem]\t#@get_and_setLAcq" %} + + ins_encode %{ + __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set prev (GetAndSetN mem newv)); + + ins_cost(ALU_COST); + + format %{ "atomic_xchgwu_acq $prev, $newv, [$mem]\t#@get_and_setNAcq" %} + + ins_encode %{ + __ atomic_xchgalwu($prev$$Register, $newv$$Register, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) +%{ + predicate(needs_acquiring_load_reserved(n) && (n->as_LoadStore()->barrier_data() == 0)); + + match(Set prev (GetAndSetP mem newv)); + + ins_cost(ALU_COST); + + format %{ "atomic_xchg_acq $prev, $newv, [$mem]\t#@get_and_setPAcq" %} + + ins_encode %{ + __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) +%{ + match(Set newval (GetAndAddL mem incr)); + + ins_cost(ALU_COST); + + format %{ "get_and_addL $newval, [$mem], $incr\t#@get_and_addL" %} + + ins_encode %{ + __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) +%{ + predicate(n->as_LoadStore()->result_not_used()); + + match(Set dummy (GetAndAddL mem incr)); + + ins_cost(ALU_COST); + + format %{ "get_and_addL [$mem], $incr\t#@get_and_addL_no_res" %} + + ins_encode %{ + __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAdd incr) +%{ + match(Set newval (GetAndAddL mem incr)); + + ins_cost(ALU_COST); + + format %{ "get_and_addL $newval, [$mem], $incr\t#@get_and_addLi" %} + + ins_encode %{ + __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAdd incr) +%{ + predicate(n->as_LoadStore()->result_not_used()); + + match(Set dummy (GetAndAddL mem incr)); + + ins_cost(ALU_COST); + + format %{ "get_and_addL [$mem], $incr\t#@get_and_addLi_no_res" %} + + ins_encode %{ + __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) +%{ + match(Set newval (GetAndAddI mem incr)); + + ins_cost(ALU_COST); + + format %{ "get_and_addI $newval, [$mem], $incr\t#@get_and_addI" %} + + ins_encode %{ + __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) +%{ + predicate(n->as_LoadStore()->result_not_used()); + + match(Set dummy (GetAndAddI mem incr)); + + ins_cost(ALU_COST); + + format %{ "get_and_addI [$mem], $incr\t#@get_and_addI_no_res" %} + + ins_encode %{ + __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAdd incr) +%{ + match(Set newval (GetAndAddI mem incr)); + + ins_cost(ALU_COST); + + format %{ "get_and_addI $newval, [$mem], $incr\t#@get_and_addIi" %} + + ins_encode %{ + __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAdd incr) +%{ + predicate(n->as_LoadStore()->result_not_used()); + + match(Set dummy (GetAndAddI mem incr)); + + ins_cost(ALU_COST); + + format %{ "get_and_addI [$mem], $incr\t#@get_and_addIi_no_res" %} + + ins_encode %{ + __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set newval (GetAndAddL mem incr)); + + ins_cost(ALU_COST); + + format %{ "get_and_addL_acq $newval, [$mem], $incr\t#@get_and_addLAcq" %} + + ins_encode %{ + __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{ + predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_reserved(n)); + + match(Set dummy (GetAndAddL mem incr)); + + ins_cost(ALU_COST); + + format %{ "get_and_addL_acq [$mem], $incr\t#@get_and_addL_no_resAcq" %} + + ins_encode %{ + __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAdd incr) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set newval (GetAndAddL mem incr)); + + ins_cost(ALU_COST); + + format %{ "get_and_addL_acq $newval, [$mem], $incr\t#@get_and_addLiAcq" %} + + ins_encode %{ + __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAdd incr) +%{ + predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_reserved(n)); + + match(Set dummy (GetAndAddL mem incr)); + + ins_cost(ALU_COST); + + format %{ "get_and_addL_acq [$mem], $incr\t#@get_and_addLi_no_resAcq" %} + + ins_encode %{ + __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set newval (GetAndAddI mem incr)); + + ins_cost(ALU_COST); + + format %{ "get_and_addI_acq $newval, [$mem], $incr\t#@get_and_addIAcq" %} + + ins_encode %{ + __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) +%{ + predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_reserved(n)); + + match(Set dummy (GetAndAddI mem incr)); + + ins_cost(ALU_COST); + + format %{ "get_and_addI_acq [$mem], $incr\t#@get_and_addI_no_resAcq" %} + + ins_encode %{ + __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAdd incr) +%{ + predicate(needs_acquiring_load_reserved(n)); + + match(Set newval (GetAndAddI mem incr)); + + ins_cost(ALU_COST); + + format %{ "get_and_addI_acq $newval, [$mem], $incr\t#@get_and_addIiAcq" %} + + ins_encode %{ + __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAdd incr) +%{ + predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_reserved(n)); + + match(Set dummy (GetAndAddI mem incr)); + + ins_cost(ALU_COST); + + format %{ "get_and_addI_acq [$mem], $incr\t#@get_and_addIi_no_resAcq" %} + + ins_encode %{ + __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base)); + %} + + ins_pipe(pipe_serial); +%} + +// ============================================================================ +// Arithmetic Instructions +// + +// Integer Addition + +// TODO +// these currently employ operations which do not set CR and hence are +// not flagged as killing CR but we would like to isolate the cases +// where we want to set flags from those where we don't. need to work +// out how to do that. +instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{ + match(Set dst (AddI src1 src2)); + + ins_cost(ALU_COST); + format %{ "addw $dst, $src1, $src2\t#@addI_reg_reg" %} + + ins_encode %{ + __ addw(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg); +%} + +instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAdd src2) %{ + match(Set dst (AddI src1 src2)); + + ins_cost(ALU_COST); + format %{ "addiw $dst, $src1, $src2\t#@addI_reg_imm" %} + + ins_encode %{ + int32_t con = (int32_t)$src2$$constant; + __ addiw(as_Register($dst$$reg), + as_Register($src1$$reg), + $src2$$constant); + %} + + ins_pipe(ialu_reg_imm); +%} + +instruct addI_reg_imm_l2i(iRegINoSp dst, iRegL src1, immIAdd src2) %{ + match(Set dst (AddI (ConvL2I src1) src2)); + + ins_cost(ALU_COST); + format %{ "addiw $dst, $src1, $src2\t#@addI_reg_imm_l2i" %} + + ins_encode %{ + __ addiw(as_Register($dst$$reg), + as_Register($src1$$reg), + $src2$$constant); + %} + + ins_pipe(ialu_reg_imm); +%} + +// Pointer Addition +instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{ + match(Set dst (AddP src1 src2)); + + ins_cost(ALU_COST); + format %{ "add $dst, $src1, $src2\t# ptr, #@addP_reg_reg" %} + + ins_encode %{ + __ add(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg); +%} + +// If we shift more than 32 bits, we need not convert I2L. +instruct lShiftL_regI_immGE32(iRegLNoSp dst, iRegI src, uimmI6_ge32 scale) %{ + match(Set dst (LShiftL (ConvI2L src) scale)); + ins_cost(ALU_COST); + format %{ "slli $dst, $src, $scale & 63\t#@lShiftL_regI_immGE32" %} + + ins_encode %{ + __ slli(as_Register($dst$$reg), as_Register($src$$reg), $scale$$constant & 63); + %} + + ins_pipe(ialu_reg_shift); +%} + +// Pointer Immediate Addition +// n.b. this needs to be more expensive than using an indirect memory +// operand +instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAdd src2) %{ + match(Set dst (AddP src1 src2)); + ins_cost(ALU_COST); + format %{ "addi $dst, $src1, $src2\t# ptr, #@addP_reg_imm" %} + + ins_encode %{ + // src2 is imm, so actually call the addi + __ add(as_Register($dst$$reg), + as_Register($src1$$reg), + $src2$$constant); + %} + + ins_pipe(ialu_reg_imm); +%} + +// Long Addition +instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{ + match(Set dst (AddL src1 src2)); + ins_cost(ALU_COST); + format %{ "add $dst, $src1, $src2\t#@addL_reg_reg" %} + + ins_encode %{ + __ add(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg); +%} + +// No constant pool entries requiredLong Immediate Addition. +instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAdd src2) %{ + match(Set dst (AddL src1 src2)); + ins_cost(ALU_COST); + format %{ "addi $dst, $src1, $src2\t#@addL_reg_imm" %} + + ins_encode %{ + // src2 is imm, so actually call the addi + __ add(as_Register($dst$$reg), + as_Register($src1$$reg), + $src2$$constant); + %} + + ins_pipe(ialu_reg_imm); +%} + +// Integer Subtraction +instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{ + match(Set dst (SubI src1 src2)); + + ins_cost(ALU_COST); + format %{ "subw $dst, $src1, $src2\t#@subI_reg_reg" %} + + ins_encode %{ + __ subw(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg); +%} + +// Immediate Subtraction +instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immISub src2) %{ + match(Set dst (SubI src1 src2)); + + ins_cost(ALU_COST); + format %{ "addiw $dst, $src1, -$src2\t#@subI_reg_imm" %} + + ins_encode %{ + // src2 is imm, so actually call the addiw + __ subw(as_Register($dst$$reg), + as_Register($src1$$reg), + $src2$$constant); + %} + + ins_pipe(ialu_reg_imm); +%} + +// Long Subtraction +instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{ + match(Set dst (SubL src1 src2)); + ins_cost(ALU_COST); + format %{ "sub $dst, $src1, $src2\t#@subL_reg_reg" %} + + ins_encode %{ + __ sub(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg); +%} + +// No constant pool entries requiredLong Immediate Subtraction. +instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLSub src2) %{ + match(Set dst (SubL src1 src2)); + ins_cost(ALU_COST); + format %{ "addi $dst, $src1, -$src2\t#@subL_reg_imm" %} + + ins_encode %{ + // src2 is imm, so actually call the addi + __ sub(as_Register($dst$$reg), + as_Register($src1$$reg), + $src2$$constant); + %} + + ins_pipe(ialu_reg_imm); +%} + +// Integer Negation (special case for sub) + +instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero) %{ + match(Set dst (SubI zero src)); + ins_cost(ALU_COST); + format %{ "subw $dst, x0, $src\t# int, #@negI_reg" %} + + ins_encode %{ + // actually call the subw + __ negw(as_Register($dst$$reg), + as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg); +%} + +// Long Negation + +instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero) %{ + match(Set dst (SubL zero src)); + ins_cost(ALU_COST); + format %{ "sub $dst, x0, $src\t# long, #@negL_reg" %} + + ins_encode %{ + // actually call the sub + __ neg(as_Register($dst$$reg), + as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg); +%} + +// Integer Multiply + +instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{ + match(Set dst (MulI src1 src2)); + ins_cost(IMUL_COST); + format %{ "mulw $dst, $src1, $src2\t#@mulI" %} + + //this means 2 word multi, and no sign extend to 64 bits + ins_encode %{ + // riscv64 mulw will sign-extension to high 32 bits in dst reg + __ mulw(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(imul_reg_reg); +%} + +// Long Multiply + +instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{ + match(Set dst (MulL src1 src2)); + ins_cost(IMUL_COST); + format %{ "mul $dst, $src1, $src2\t#@mulL" %} + + ins_encode %{ + __ mul(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(lmul_reg_reg); +%} + +instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2) +%{ + match(Set dst (MulHiL src1 src2)); + ins_cost(IMUL_COST); + format %{ "mulh $dst, $src1, $src2\t# mulhi, #@mulHiL_rReg" %} + + ins_encode %{ + __ mulh(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(lmul_reg_reg); +%} + +// Integer Divide + +instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{ + match(Set dst (DivI src1 src2)); + ins_cost(IDIVSI_COST); + format %{ "divw $dst, $src1, $src2\t#@divI"%} + + ins_encode(riscv_enc_divw(dst, src1, src2)); + ins_pipe(idiv_reg_reg); +%} + +instruct signExtract(iRegINoSp dst, iRegIorL2I src1, immI_31 div1, immI_31 div2) %{ + match(Set dst (URShiftI (RShiftI src1 div1) div2)); + ins_cost(ALU_COST); + format %{ "srliw $dst, $src1, $div1\t# int signExtract, #@signExtract" %} + + ins_encode %{ + __ srliw(as_Register($dst$$reg), as_Register($src1$$reg), 31); + %} + ins_pipe(ialu_reg_shift); +%} + +// Long Divide + +instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{ + match(Set dst (DivL src1 src2)); + ins_cost(IDIVDI_COST); + format %{ "div $dst, $src1, $src2\t#@divL" %} + + ins_encode(riscv_enc_div(dst, src1, src2)); + ins_pipe(ldiv_reg_reg); +%} + +instruct signExtractL(iRegLNoSp dst, iRegL src1, immI_63 div1, immI_63 div2) %{ + match(Set dst (URShiftL (RShiftL src1 div1) div2)); + ins_cost(ALU_COST); + format %{ "srli $dst, $src1, $div1\t# long signExtract, #@signExtractL" %} + + ins_encode %{ + __ srli(as_Register($dst$$reg), as_Register($src1$$reg), 63); + %} + ins_pipe(ialu_reg_shift); +%} + +// Integer Remainder + +instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{ + match(Set dst (ModI src1 src2)); + ins_cost(IDIVSI_COST); + format %{ "remw $dst, $src1, $src2\t#@modI" %} + + ins_encode(riscv_enc_modw(dst, src1, src2)); + ins_pipe(ialu_reg_reg); +%} + +// Long Remainder + +instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{ + match(Set dst (ModL src1 src2)); + ins_cost(IDIVDI_COST); + format %{ "rem $dst, $src1, $src2\t#@modL" %} + + ins_encode(riscv_enc_mod(dst, src1, src2)); + ins_pipe(ialu_reg_reg); +%} + +// Integer Shifts + +// Shift Left Register +// In RV64I, only the low 5 bits of src2 are considered for the shift amount +instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{ + match(Set dst (LShiftI src1 src2)); + ins_cost(ALU_COST); + format %{ "sllw $dst, $src1, $src2\t#@lShiftI_reg_reg" %} + + ins_encode %{ + __ sllw(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg_vshift); +%} + +// Shift Left Immediate +instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{ + match(Set dst (LShiftI src1 src2)); + ins_cost(ALU_COST); + format %{ "slliw $dst, $src1, ($src2 & 0x1f)\t#@lShiftI_reg_imm" %} + + ins_encode %{ + // the shift amount is encoded in the lower + // 5 bits of the I-immediate field for RV32I + __ slliw(as_Register($dst$$reg), + as_Register($src1$$reg), + (unsigned) $src2$$constant & 0x1f); + %} + + ins_pipe(ialu_reg_shift); +%} + +// Shift Right Logical Register +// In RV64I, only the low 5 bits of src2 are considered for the shift amount +instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{ + match(Set dst (URShiftI src1 src2)); + ins_cost(ALU_COST); + format %{ "srlw $dst, $src1, $src2\t#@urShiftI_reg_reg" %} + + ins_encode %{ + __ srlw(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg_vshift); +%} + +// Shift Right Logical Immediate +instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{ + match(Set dst (URShiftI src1 src2)); + ins_cost(ALU_COST); + format %{ "srliw $dst, $src1, ($src2 & 0x1f)\t#@urShiftI_reg_imm" %} + + ins_encode %{ + // the shift amount is encoded in the lower + // 6 bits of the I-immediate field for RV64I + __ srliw(as_Register($dst$$reg), + as_Register($src1$$reg), + (unsigned) $src2$$constant & 0x1f); + %} + + ins_pipe(ialu_reg_shift); +%} + +// Shift Right Arithmetic Register +// In RV64I, only the low 5 bits of src2 are considered for the shift amount +instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{ + match(Set dst (RShiftI src1 src2)); + ins_cost(ALU_COST); + format %{ "sraw $dst, $src1, $src2\t#@rShiftI_reg_reg" %} + + ins_encode %{ + // riscv will sign-ext dst high 32 bits + __ sraw(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg_vshift); +%} + +// Shift Right Arithmetic Immediate +instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{ + match(Set dst (RShiftI src1 src2)); + ins_cost(ALU_COST); + format %{ "sraiw $dst, $src1, ($src2 & 0x1f)\t#@rShiftI_reg_imm" %} + + ins_encode %{ + // riscv will sign-ext dst high 32 bits + __ sraiw(as_Register($dst$$reg), + as_Register($src1$$reg), + (unsigned) $src2$$constant & 0x1f); + %} + + ins_pipe(ialu_reg_shift); +%} + +// Long Shifts + +// Shift Left Register +// In RV64I, only the low 6 bits of src2 are considered for the shift amount +instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{ + match(Set dst (LShiftL src1 src2)); + + ins_cost(ALU_COST); + format %{ "sll $dst, $src1, $src2\t#@lShiftL_reg_reg" %} + + ins_encode %{ + __ sll(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg_vshift); +%} + +// Shift Left Immediate +instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{ + match(Set dst (LShiftL src1 src2)); + + ins_cost(ALU_COST); + format %{ "slli $dst, $src1, ($src2 & 0x3f)\t#@lShiftL_reg_imm" %} + + ins_encode %{ + // the shift amount is encoded in the lower + // 6 bits of the I-immediate field for RV64I + __ slli(as_Register($dst$$reg), + as_Register($src1$$reg), + (unsigned) $src2$$constant & 0x3f); + %} + + ins_pipe(ialu_reg_shift); +%} + +// Shift Right Logical Register +// In RV64I, only the low 6 bits of src2 are considered for the shift amount +instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{ + match(Set dst (URShiftL src1 src2)); + + ins_cost(ALU_COST); + format %{ "srl $dst, $src1, $src2\t#@urShiftL_reg_reg" %} + + ins_encode %{ + __ srl(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg_vshift); +%} + +// Shift Right Logical Immediate +instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{ + match(Set dst (URShiftL src1 src2)); + + ins_cost(ALU_COST); + format %{ "srli $dst, $src1, ($src2 & 0x3f)\t#@urShiftL_reg_imm" %} + + ins_encode %{ + // the shift amount is encoded in the lower + // 6 bits of the I-immediate field for RV64I + __ srli(as_Register($dst$$reg), + as_Register($src1$$reg), + (unsigned) $src2$$constant & 0x3f); + %} + + ins_pipe(ialu_reg_shift); +%} + +// A special-case pattern for card table stores. +instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{ + match(Set dst (URShiftL (CastP2X src1) src2)); + + ins_cost(ALU_COST); + format %{ "srli $dst, p2x($src1), ($src2 & 0x3f)\t#@urShiftP_reg_imm" %} + + ins_encode %{ + // the shift amount is encoded in the lower + // 6 bits of the I-immediate field for RV64I + __ srli(as_Register($dst$$reg), + as_Register($src1$$reg), + (unsigned) $src2$$constant & 0x3f); + %} + + ins_pipe(ialu_reg_shift); +%} + +// Shift Right Arithmetic Register +// In RV64I, only the low 6 bits of src2 are considered for the shift amount +instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{ + match(Set dst (RShiftL src1 src2)); + + ins_cost(ALU_COST); + format %{ "sra $dst, $src1, $src2\t#@rShiftL_reg_reg" %} + + ins_encode %{ + __ sra(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg_vshift); +%} + +// Shift Right Arithmetic Immediate +instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{ + match(Set dst (RShiftL src1 src2)); + + ins_cost(ALU_COST); + format %{ "srai $dst, $src1, ($src2 & 0x3f)\t#@rShiftL_reg_imm" %} + + ins_encode %{ + // the shift amount is encoded in the lower + // 6 bits of the I-immediate field for RV64I + __ srai(as_Register($dst$$reg), + as_Register($src1$$reg), + (unsigned) $src2$$constant & 0x3f); + %} + + ins_pipe(ialu_reg_shift); +%} + +instruct regI_not_reg(iRegINoSp dst, iRegI src1, immI_M1 m1) %{ + match(Set dst (XorI src1 m1)); + ins_cost(ALU_COST); + format %{ "xori $dst, $src1, -1\t#@regI_not_reg" %} + + ins_encode %{ + __ xori(as_Register($dst$$reg), as_Register($src1$$reg), -1); + %} + + ins_pipe(ialu_reg_imm); +%} + +instruct regL_not_reg(iRegLNoSp dst, iRegL src1, immL_M1 m1) %{ + match(Set dst (XorL src1 m1)); + ins_cost(ALU_COST); + format %{ "xori $dst, $src1, -1\t#@regL_not_reg" %} + + ins_encode %{ + __ xori(as_Register($dst$$reg), as_Register($src1$$reg), -1); + %} + + ins_pipe(ialu_reg_imm); +%} + + +// ============================================================================ +// Floating Point Arithmetic Instructions + +instruct addF_reg_reg(fRegF dst, fRegF src1, fRegF src2) %{ + match(Set dst (AddF src1 src2)); + + ins_cost(DEFAULT_COST * 5); + format %{ "fadd.s $dst, $src1, $src2\t#@addF_reg_reg" %} + + ins_encode %{ + __ fadd_s(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), + as_FloatRegister($src2$$reg)); + %} + + ins_pipe(fp_dop_reg_reg_s); +%} + +instruct addD_reg_reg(fRegD dst, fRegD src1, fRegD src2) %{ + match(Set dst (AddD src1 src2)); + + ins_cost(DEFAULT_COST * 5); + format %{ "fadd.d $dst, $src1, $src2\t#@addD_reg_reg" %} + + ins_encode %{ + __ fadd_d(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), + as_FloatRegister($src2$$reg)); + %} + + ins_pipe(fp_dop_reg_reg_d); +%} + +instruct subF_reg_reg(fRegF dst, fRegF src1, fRegF src2) %{ + match(Set dst (SubF src1 src2)); + + ins_cost(DEFAULT_COST * 5); + format %{ "fsub.s $dst, $src1, $src2\t#@subF_reg_reg" %} + + ins_encode %{ + __ fsub_s(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), + as_FloatRegister($src2$$reg)); + %} + + ins_pipe(fp_dop_reg_reg_s); +%} + +instruct subD_reg_reg(fRegD dst, fRegD src1, fRegD src2) %{ + match(Set dst (SubD src1 src2)); + + ins_cost(DEFAULT_COST * 5); + format %{ "fsub.d $dst, $src1, $src2\t#@subD_reg_reg" %} + + ins_encode %{ + __ fsub_d(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), + as_FloatRegister($src2$$reg)); + %} + + ins_pipe(fp_dop_reg_reg_d); +%} + +instruct mulF_reg_reg(fRegF dst, fRegF src1, fRegF src2) %{ + match(Set dst (MulF src1 src2)); + + ins_cost(FMUL_SINGLE_COST); + format %{ "fmul.s $dst, $src1, $src2\t#@mulF_reg_reg" %} + + ins_encode %{ + __ fmul_s(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), + as_FloatRegister($src2$$reg)); + %} + + ins_pipe(fp_dop_reg_reg_s); +%} + +instruct mulD_reg_reg(fRegD dst, fRegD src1, fRegD src2) %{ + match(Set dst (MulD src1 src2)); + + ins_cost(FMUL_DOUBLE_COST); + format %{ "fmul.d $dst, $src1, $src2\t#@mulD_reg_reg" %} + + ins_encode %{ + __ fmul_d(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), + as_FloatRegister($src2$$reg)); + %} + + ins_pipe(fp_dop_reg_reg_d); +%} + +// src1 * src2 + src3 +instruct maddF_reg_reg(fRegF dst, fRegF src1, fRegF src2, fRegF src3) %{ + predicate(UseFMA); + match(Set dst (FmaF src3 (Binary src1 src2))); + + ins_cost(FMUL_SINGLE_COST); + format %{ "fmadd.s $dst, $src1, $src2, $src3\t#@maddF_reg_reg" %} + + ins_encode %{ + __ fmadd_s(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), + as_FloatRegister($src2$$reg), + as_FloatRegister($src3$$reg)); + %} + + ins_pipe(pipe_class_default); +%} + +// src1 * src2 + src3 +instruct maddD_reg_reg(fRegD dst, fRegD src1, fRegD src2, fRegD src3) %{ + predicate(UseFMA); + match(Set dst (FmaD src3 (Binary src1 src2))); + + ins_cost(FMUL_DOUBLE_COST); + format %{ "fmadd.d $dst, $src1, $src2, $src3\t#@maddD_reg_reg" %} + + ins_encode %{ + __ fmadd_d(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), + as_FloatRegister($src2$$reg), + as_FloatRegister($src3$$reg)); + %} + + ins_pipe(pipe_class_default); +%} + +// src1 * src2 - src3 +instruct msubF_reg_reg(fRegF dst, fRegF src1, fRegF src2, fRegF src3) %{ + predicate(UseFMA); + match(Set dst (FmaF (NegF src3) (Binary src1 src2))); + + ins_cost(FMUL_SINGLE_COST); + format %{ "fmsub.s $dst, $src1, $src2, $src3\t#@msubF_reg_reg" %} + + ins_encode %{ + __ fmsub_s(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), + as_FloatRegister($src2$$reg), + as_FloatRegister($src3$$reg)); + %} + + ins_pipe(pipe_class_default); +%} + +// src1 * src2 - src3 +instruct msubD_reg_reg(fRegD dst, fRegD src1, fRegD src2, fRegD src3) %{ + predicate(UseFMA); + match(Set dst (FmaD (NegD src3) (Binary src1 src2))); + + ins_cost(FMUL_DOUBLE_COST); + format %{ "fmsub.d $dst, $src1, $src2, $src3\t#@msubD_reg_reg" %} + + ins_encode %{ + __ fmsub_d(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), + as_FloatRegister($src2$$reg), + as_FloatRegister($src3$$reg)); + %} + + ins_pipe(pipe_class_default); +%} + +// -src1 * src2 + src3 +instruct nmsubF_reg_reg(fRegF dst, fRegF src1, fRegF src2, fRegF src3) %{ + predicate(UseFMA); + match(Set dst (FmaF src3 (Binary (NegF src1) src2))); + match(Set dst (FmaF src3 (Binary src1 (NegF src2)))); + + ins_cost(FMUL_SINGLE_COST); + format %{ "fnmsub.s $dst, $src1, $src2, $src3\t#@nmsubF_reg_reg" %} + + ins_encode %{ + __ fnmsub_s(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), + as_FloatRegister($src2$$reg), + as_FloatRegister($src3$$reg)); + %} + + ins_pipe(pipe_class_default); +%} + +// -src1 * src2 + src3 +instruct nmsubD_reg_reg(fRegD dst, fRegD src1, fRegD src2, fRegD src3) %{ + predicate(UseFMA); + match(Set dst (FmaD src3 (Binary (NegD src1) src2))); + match(Set dst (FmaD src3 (Binary src1 (NegD src2)))); + + ins_cost(FMUL_DOUBLE_COST); + format %{ "fnmsub.d $dst, $src1, $src2, $src3\t#@nmsubD_reg_reg" %} + + ins_encode %{ + __ fnmsub_d(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), + as_FloatRegister($src2$$reg), + as_FloatRegister($src3$$reg)); + %} + + ins_pipe(pipe_class_default); +%} + +// -src1 * src2 - src3 +instruct nmaddF_reg_reg(fRegF dst, fRegF src1, fRegF src2, fRegF src3) %{ + predicate(UseFMA); + match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2))); + match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2)))); + + ins_cost(FMUL_SINGLE_COST); + format %{ "fnmadd.s $dst, $src1, $src2, $src3\t#@nmaddF_reg_reg" %} + + ins_encode %{ + __ fnmadd_s(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), + as_FloatRegister($src2$$reg), + as_FloatRegister($src3$$reg)); + %} + + ins_pipe(pipe_class_default); +%} + +// -src1 * src2 - src3 +instruct nmaddD_reg_reg(fRegD dst, fRegD src1, fRegD src2, fRegD src3) %{ + predicate(UseFMA); + match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2))); + match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2)))); + + ins_cost(FMUL_DOUBLE_COST); + format %{ "fnmadd.d $dst, $src1, $src2, $src3\t#@nmaddD_reg_reg" %} + + ins_encode %{ + __ fnmadd_d(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), + as_FloatRegister($src2$$reg), + as_FloatRegister($src3$$reg)); + %} + + ins_pipe(pipe_class_default); +%} + +// Math.max(FF)F +instruct maxF_reg_reg(fRegF dst, fRegF src1, fRegF src2, rFlagsReg cr) %{ + match(Set dst (MaxF src1 src2)); + effect(TEMP_DEF dst, KILL cr); + + format %{ "maxF $dst, $src1, $src2" %} + + ins_encode %{ + __ minmax_FD(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg), + false /* is_double */, false /* is_min */); + %} + + ins_pipe(pipe_class_default); +%} + +// Math.min(FF)F +instruct minF_reg_reg(fRegF dst, fRegF src1, fRegF src2, rFlagsReg cr) %{ + match(Set dst (MinF src1 src2)); + effect(TEMP_DEF dst, KILL cr); + + format %{ "minF $dst, $src1, $src2" %} + + ins_encode %{ + __ minmax_FD(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg), + false /* is_double */, true /* is_min */); + %} + + ins_pipe(pipe_class_default); +%} + +// Math.max(DD)D +instruct maxD_reg_reg(fRegD dst, fRegD src1, fRegD src2, rFlagsReg cr) %{ + match(Set dst (MaxD src1 src2)); + effect(TEMP_DEF dst, KILL cr); + + format %{ "maxD $dst, $src1, $src2" %} + + ins_encode %{ + __ minmax_FD(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg), + true /* is_double */, false /* is_min */); + %} + + ins_pipe(pipe_class_default); +%} + +// Math.min(DD)D +instruct minD_reg_reg(fRegD dst, fRegD src1, fRegD src2, rFlagsReg cr) %{ + match(Set dst (MinD src1 src2)); + effect(TEMP_DEF dst, KILL cr); + + format %{ "minD $dst, $src1, $src2" %} + + ins_encode %{ + __ minmax_FD(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg), + true /* is_double */, true /* is_min */); + %} + + ins_pipe(pipe_class_default); +%} + +instruct divF_reg_reg(fRegF dst, fRegF src1, fRegF src2) %{ + match(Set dst (DivF src1 src2)); + + ins_cost(FDIV_COST); + format %{ "fdiv.s $dst, $src1, $src2\t#@divF_reg_reg" %} + + ins_encode %{ + __ fdiv_s(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), + as_FloatRegister($src2$$reg)); + %} + + ins_pipe(fp_div_s); +%} + +instruct divD_reg_reg(fRegD dst, fRegD src1, fRegD src2) %{ + match(Set dst (DivD src1 src2)); + + ins_cost(FDIV_COST); + format %{ "fdiv.d $dst, $src1, $src2\t#@divD_reg_reg" %} + + ins_encode %{ + __ fdiv_d(as_FloatRegister($dst$$reg), + as_FloatRegister($src1$$reg), + as_FloatRegister($src2$$reg)); + %} + + ins_pipe(fp_div_d); +%} + +instruct negF_reg_reg(fRegF dst, fRegF src) %{ + match(Set dst (NegF src)); + + ins_cost(XFER_COST); + format %{ "fsgnjn.s $dst, $src, $src\t#@negF_reg_reg" %} + + ins_encode %{ + __ fneg_s(as_FloatRegister($dst$$reg), + as_FloatRegister($src$$reg)); + %} + + ins_pipe(fp_uop_s); +%} + +instruct negD_reg_reg(fRegD dst, fRegD src) %{ + match(Set dst (NegD src)); + + ins_cost(XFER_COST); + format %{ "fsgnjn.d $dst, $src, $src\t#@negD_reg_reg" %} + + ins_encode %{ + __ fneg_d(as_FloatRegister($dst$$reg), + as_FloatRegister($src$$reg)); + %} + + ins_pipe(fp_uop_d); +%} + +instruct absI_reg(iRegINoSp dst, iRegIorL2I src) %{ + match(Set dst (AbsI src)); + + ins_cost(ALU_COST * 3); + format %{ + "sraiw t0, $src, 0x1f\n\t" + "addw $dst, $src, t0\n\t" + "xorr $dst, $dst, t0\t#@absI_reg" + %} + + ins_encode %{ + __ sraiw(t0, as_Register($src$$reg), 0x1f); + __ addw(as_Register($dst$$reg), as_Register($src$$reg), t0); + __ xorr(as_Register($dst$$reg), as_Register($dst$$reg), t0); + %} + + ins_pipe(pipe_class_default); +%} + +instruct absL_reg(iRegLNoSp dst, iRegL src) %{ + match(Set dst (AbsL src)); + + ins_cost(ALU_COST * 3); + format %{ + "srai t0, $src, 0x3f\n\t" + "add $dst, $src, t0\n\t" + "xorr $dst, $dst, t0\t#@absL_reg" + %} + + ins_encode %{ + __ srai(t0, as_Register($src$$reg), 0x3f); + __ add(as_Register($dst$$reg), as_Register($src$$reg), t0); + __ xorr(as_Register($dst$$reg), as_Register($dst$$reg), t0); + %} + + ins_pipe(pipe_class_default); +%} + +instruct absF_reg(fRegF dst, fRegF src) %{ + match(Set dst (AbsF src)); + + ins_cost(XFER_COST); + format %{ "fsgnjx.s $dst, $src, $src\t#@absF_reg" %} + ins_encode %{ + __ fabs_s(as_FloatRegister($dst$$reg), + as_FloatRegister($src$$reg)); + %} + + ins_pipe(fp_uop_s); +%} + +instruct absD_reg(fRegD dst, fRegD src) %{ + match(Set dst (AbsD src)); + + ins_cost(XFER_COST); + format %{ "fsgnjx.d $dst, $src, $src\t#@absD_reg" %} + ins_encode %{ + __ fabs_d(as_FloatRegister($dst$$reg), + as_FloatRegister($src$$reg)); + %} + + ins_pipe(fp_uop_d); +%} + +instruct sqrtF_reg(fRegF dst, fRegF src) %{ + match(Set dst (SqrtF src)); + + ins_cost(FSQRT_COST); + format %{ "fsqrt.s $dst, $src\t#@sqrtF_reg" %} + ins_encode %{ + __ fsqrt_s(as_FloatRegister($dst$$reg), + as_FloatRegister($src$$reg)); + %} + + ins_pipe(fp_sqrt_s); +%} + +instruct sqrtD_reg(fRegD dst, fRegD src) %{ + match(Set dst (SqrtD src)); + + ins_cost(FSQRT_COST); + format %{ "fsqrt.d $dst, $src\t#@sqrtD_reg" %} + ins_encode %{ + __ fsqrt_d(as_FloatRegister($dst$$reg), + as_FloatRegister($src$$reg)); + %} + + ins_pipe(fp_sqrt_d); +%} + +// Arithmetic Instructions End + +// ============================================================================ +// Logical Instructions + +// Register And +instruct andI_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2) %{ + match(Set dst (AndI src1 src2)); + + format %{ "andr $dst, $src1, $src2\t#@andI_reg_reg" %} + + ins_cost(ALU_COST); + ins_encode %{ + __ andr(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg); +%} + +// Immediate And +instruct andI_reg_imm(iRegINoSp dst, iRegI src1, immIAdd src2) %{ + match(Set dst (AndI src1 src2)); + + format %{ "andi $dst, $src1, $src2\t#@andI_reg_imm" %} + + ins_cost(ALU_COST); + ins_encode %{ + __ andi(as_Register($dst$$reg), + as_Register($src1$$reg), + (int32_t)($src2$$constant)); + %} + + ins_pipe(ialu_reg_imm); +%} + +// Register Or +instruct orI_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2) %{ + match(Set dst (OrI src1 src2)); + + format %{ "orr $dst, $src1, $src2\t#@orI_reg_reg" %} + + ins_cost(ALU_COST); + ins_encode %{ + __ orr(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg); +%} + +// Immediate Or +instruct orI_reg_imm(iRegINoSp dst, iRegI src1, immIAdd src2) %{ + match(Set dst (OrI src1 src2)); + + format %{ "ori $dst, $src1, $src2\t#@orI_reg_imm" %} + + ins_cost(ALU_COST); + ins_encode %{ + __ ori(as_Register($dst$$reg), + as_Register($src1$$reg), + (int32_t)($src2$$constant)); + %} + + ins_pipe(ialu_reg_imm); +%} + +// Register Xor +instruct xorI_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2) %{ + match(Set dst (XorI src1 src2)); + + format %{ "xorr $dst, $src1, $src2\t#@xorI_reg_reg" %} + + ins_cost(ALU_COST); + ins_encode %{ + __ xorr(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg); +%} + +// Immediate Xor +instruct xorI_reg_imm(iRegINoSp dst, iRegI src1, immIAdd src2) %{ + match(Set dst (XorI src1 src2)); + + format %{ "xori $dst, $src1, $src2\t#@xorI_reg_imm" %} + + ins_cost(ALU_COST); + ins_encode %{ + __ xori(as_Register($dst$$reg), + as_Register($src1$$reg), + (int32_t)($src2$$constant)); + %} + + ins_pipe(ialu_reg_imm); +%} + +// Register And Long +instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{ + match(Set dst (AndL src1 src2)); + + format %{ "andr $dst, $src1, $src2\t#@andL_reg_reg" %} + + ins_cost(ALU_COST); + ins_encode %{ + __ andr(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg); +%} + +// Immediate And Long +instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLAdd src2) %{ + match(Set dst (AndL src1 src2)); + + format %{ "andi $dst, $src1, $src2\t#@andL_reg_imm" %} + + ins_cost(ALU_COST); + ins_encode %{ + __ andi(as_Register($dst$$reg), + as_Register($src1$$reg), + (int32_t)($src2$$constant)); + %} + + ins_pipe(ialu_reg_imm); +%} + +// Register Or Long +instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{ + match(Set dst (OrL src1 src2)); + + format %{ "orr $dst, $src1, $src2\t#@orL_reg_reg" %} + + ins_cost(ALU_COST); + ins_encode %{ + __ orr(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg); +%} + +// Immediate Or Long +instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLAdd src2) %{ + match(Set dst (OrL src1 src2)); + + format %{ "ori $dst, $src1, $src2\t#@orL_reg_imm" %} + + ins_cost(ALU_COST); + ins_encode %{ + __ ori(as_Register($dst$$reg), + as_Register($src1$$reg), + (int32_t)($src2$$constant)); + %} + + ins_pipe(ialu_reg_imm); +%} + +// Register Xor Long +instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{ + match(Set dst (XorL src1 src2)); + + format %{ "xorr $dst, $src1, $src2\t#@xorL_reg_reg" %} + + ins_cost(ALU_COST); + ins_encode %{ + __ xorr(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg); +%} + +// Immediate Xor Long +instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLAdd src2) %{ + match(Set dst (XorL src1 src2)); + + ins_cost(ALU_COST); + format %{ "xori $dst, $src1, $src2\t#@xorL_reg_imm" %} + + ins_encode %{ + __ xori(as_Register($dst$$reg), + as_Register($src1$$reg), + (int32_t)($src2$$constant)); + %} + + ins_pipe(ialu_reg_imm); +%} + +// ============================================================================ +// BSWAP Instructions + +instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr) %{ + match(Set dst (ReverseBytesI src)); + effect(KILL cr); + + ins_cost(ALU_COST * 13); + format %{ "revb_w_w $dst, $src\t#@bytes_reverse_int" %} + + ins_encode %{ + __ revb_w_w(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(pipe_class_default); +%} + +instruct bytes_reverse_long(iRegLNoSp dst, iRegL src, rFlagsReg cr) %{ + match(Set dst (ReverseBytesL src)); + effect(KILL cr); + + ins_cost(ALU_COST * 29); + format %{ "revb $dst, $src\t#@bytes_reverse_long" %} + + ins_encode %{ + __ revb(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(pipe_class_default); +%} + +instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{ + match(Set dst (ReverseBytesUS src)); + + ins_cost(ALU_COST * 5); + format %{ "revb_h_h_u $dst, $src\t#@bytes_reverse_unsigned_short" %} + + ins_encode %{ + __ revb_h_h_u(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(pipe_class_default); +%} + +instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{ + match(Set dst (ReverseBytesS src)); + + ins_cost(ALU_COST * 5); + format %{ "revb_h_h $dst, $src\t#@bytes_reverse_short" %} + + ins_encode %{ + __ revb_h_h(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(pipe_class_default); +%} + +// ============================================================================ +// MemBar Instruction + +instruct load_fence() %{ + match(LoadFence); + ins_cost(ALU_COST); + + format %{ "#@load_fence" %} + + ins_encode %{ + __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); + %} + ins_pipe(pipe_serial); +%} + +instruct membar_acquire() %{ + match(MemBarAcquire); + ins_cost(ALU_COST); + + format %{ "#@membar_acquire\n\t" + "fence ir iorw" %} + + ins_encode %{ + __ block_comment("membar_acquire"); + __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); + %} + + ins_pipe(pipe_serial); +%} + +instruct membar_acquire_lock() %{ + match(MemBarAcquireLock); + ins_cost(0); + + format %{ "#@membar_acquire_lock (elided)" %} + + ins_encode %{ + __ block_comment("membar_acquire_lock (elided)"); + %} + + ins_pipe(pipe_serial); +%} + +instruct store_fence() %{ + match(StoreFence); + ins_cost(ALU_COST); + + format %{ "#@store_fence" %} + + ins_encode %{ + __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore); + %} + ins_pipe(pipe_serial); +%} + +instruct membar_release() %{ + match(MemBarRelease); + ins_cost(ALU_COST); + + format %{ "#@membar_release\n\t" + "fence iorw ow" %} + + ins_encode %{ + __ block_comment("membar_release"); + __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore); + %} + ins_pipe(pipe_serial); +%} + +instruct membar_storestore() %{ + match(MemBarStoreStore); + match(StoreStoreFence); + ins_cost(ALU_COST); + + format %{ "MEMBAR-store-store\t#@membar_storestore" %} + + ins_encode %{ + __ membar(MacroAssembler::StoreStore); + %} + ins_pipe(pipe_serial); +%} + +instruct membar_release_lock() %{ + match(MemBarReleaseLock); + ins_cost(0); + + format %{ "#@membar_release_lock (elided)" %} + + ins_encode %{ + __ block_comment("membar_release_lock (elided)"); + %} + + ins_pipe(pipe_serial); +%} + +instruct membar_volatile() %{ + match(MemBarVolatile); + ins_cost(ALU_COST); + + format %{ "#@membar_volatile\n\t" + "fence iorw iorw"%} + + ins_encode %{ + __ block_comment("membar_volatile"); + __ membar(MacroAssembler::StoreLoad); + %} + + ins_pipe(pipe_serial); +%} + +// ============================================================================ +// Cast Instructions (Java-level type cast) + +instruct castX2P(iRegPNoSp dst, iRegL src) %{ + match(Set dst (CastX2P src)); + + ins_cost(ALU_COST); + format %{ "mv $dst, $src\t# long -> ptr, #@castX2P" %} + + ins_encode %{ + if ($dst$$reg != $src$$reg) { + __ mv(as_Register($dst$$reg), as_Register($src$$reg)); + } + %} + + ins_pipe(ialu_reg); +%} + +instruct castP2X(iRegLNoSp dst, iRegP src) %{ + match(Set dst (CastP2X src)); + + ins_cost(ALU_COST); + format %{ "mv $dst, $src\t# ptr -> long, #@castP2X" %} + + ins_encode %{ + if ($dst$$reg != $src$$reg) { + __ mv(as_Register($dst$$reg), as_Register($src$$reg)); + } + %} + + ins_pipe(ialu_reg); +%} + +instruct castPP(iRegPNoSp dst) +%{ + match(Set dst (CastPP dst)); + ins_cost(0); + + size(0); + format %{ "# castPP of $dst, #@castPP" %} + ins_encode(/* empty encoding */); + ins_pipe(pipe_class_empty); +%} + +instruct castLL(iRegL dst) +%{ + match(Set dst (CastLL dst)); + + size(0); + format %{ "# castLL of $dst, #@castLL" %} + ins_encode(/* empty encoding */); + ins_cost(0); + ins_pipe(pipe_class_empty); +%} + +instruct castII(iRegI dst) +%{ + match(Set dst (CastII dst)); + + size(0); + format %{ "# castII of $dst, #@castII" %} + ins_encode(/* empty encoding */); + ins_cost(0); + ins_pipe(pipe_class_empty); +%} + +instruct checkCastPP(iRegPNoSp dst) +%{ + match(Set dst (CheckCastPP dst)); + + size(0); + ins_cost(0); + format %{ "# checkcastPP of $dst, #@checkCastPP" %} + ins_encode(/* empty encoding */); + ins_pipe(pipe_class_empty); +%} + +instruct castFF(fRegF dst) +%{ + match(Set dst (CastFF dst)); + + size(0); + format %{ "# castFF of $dst" %} + ins_encode(/* empty encoding */); + ins_cost(0); + ins_pipe(pipe_class_empty); +%} + +instruct castDD(fRegD dst) +%{ + match(Set dst (CastDD dst)); + + size(0); + format %{ "# castDD of $dst" %} + ins_encode(/* empty encoding */); + ins_cost(0); + ins_pipe(pipe_class_empty); +%} + +instruct castVV(vReg dst) +%{ + match(Set dst (CastVV dst)); + + size(0); + format %{ "# castVV of $dst" %} + ins_encode(/* empty encoding */); + ins_cost(0); + ins_pipe(pipe_class_empty); +%} + +// ============================================================================ +// Convert Instructions + +// int to bool +instruct convI2Bool(iRegINoSp dst, iRegI src) +%{ + match(Set dst (Conv2B src)); + + ins_cost(ALU_COST); + format %{ "snez $dst, $src\t#@convI2Bool" %} + + ins_encode %{ + __ snez(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg); +%} + +// pointer to bool +instruct convP2Bool(iRegINoSp dst, iRegP src) +%{ + match(Set dst (Conv2B src)); + + ins_cost(ALU_COST); + format %{ "snez $dst, $src\t#@convP2Bool" %} + + ins_encode %{ + __ snez(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg); +%} + +// int <-> long + +instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src) +%{ + match(Set dst (ConvI2L src)); + + ins_cost(ALU_COST); + format %{ "addw $dst, $src, zr\t#@convI2L_reg_reg" %} + ins_encode %{ + __ sign_extend(as_Register($dst$$reg), as_Register($src$$reg), 32); + %} + ins_pipe(ialu_reg); +%} + +instruct convL2I_reg(iRegINoSp dst, iRegL src) %{ + match(Set dst (ConvL2I src)); + + ins_cost(ALU_COST); + format %{ "addw $dst, $src, zr\t#@convL2I_reg" %} + + ins_encode %{ + __ sign_extend(as_Register($dst$$reg), as_Register($src$$reg), 32); + %} + + ins_pipe(ialu_reg); +%} + +// int to unsigned long (Zero-extend) +instruct convI2UL_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask) +%{ + match(Set dst (AndL (ConvI2L src) mask)); + + ins_cost(ALU_COST * 2); + format %{ "zero_extend $dst, $src, 32\t# i2ul, #@convI2UL_reg_reg" %} + + ins_encode %{ + __ zero_extend(as_Register($dst$$reg), as_Register($src$$reg), 32); + %} + + ins_pipe(ialu_reg_shift); +%} + +// float <-> double + +instruct convF2D_reg(fRegD dst, fRegF src) %{ + match(Set dst (ConvF2D src)); + + ins_cost(XFER_COST); + format %{ "fcvt.d.s $dst, $src\t#@convF2D_reg" %} + + ins_encode %{ + __ fcvt_d_s(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg)); + %} + + ins_pipe(fp_f2d); +%} + +instruct convD2F_reg(fRegF dst, fRegD src) %{ + match(Set dst (ConvD2F src)); + + ins_cost(XFER_COST); + format %{ "fcvt.s.d $dst, $src\t#@convD2F_reg" %} + + ins_encode %{ + __ fcvt_s_d(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg)); + %} + + ins_pipe(fp_d2f); +%} + +// float <-> int + +instruct convF2I_reg_reg(iRegINoSp dst, fRegF src) %{ + match(Set dst (ConvF2I src)); + + ins_cost(XFER_COST); + format %{ "fcvt.w.s $dst, $src\t#@convF2I_reg_reg" %} + + ins_encode %{ + __ fcvt_w_s_safe($dst$$Register, $src$$FloatRegister); + %} + + ins_pipe(fp_f2i); +%} + +instruct convI2F_reg_reg(fRegF dst, iRegIorL2I src) %{ + match(Set dst (ConvI2F src)); + + ins_cost(XFER_COST); + format %{ "fcvt.s.w $dst, $src\t#@convI2F_reg_reg" %} + + ins_encode %{ + __ fcvt_s_w(as_FloatRegister($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(fp_i2f); +%} + +// float <-> long + +instruct convF2L_reg_reg(iRegLNoSp dst, fRegF src) %{ + match(Set dst (ConvF2L src)); + + ins_cost(XFER_COST); + format %{ "fcvt.l.s $dst, $src\t#@convF2L_reg_reg" %} + + ins_encode %{ + __ fcvt_l_s_safe($dst$$Register, $src$$FloatRegister); + %} + + ins_pipe(fp_f2l); +%} + +instruct convL2F_reg_reg(fRegF dst, iRegL src) %{ + match(Set dst (ConvL2F src)); + + ins_cost(XFER_COST); + format %{ "fcvt.s.l $dst, $src\t#@convL2F_reg_reg" %} + + ins_encode %{ + __ fcvt_s_l(as_FloatRegister($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(fp_l2f); +%} + +// double <-> int + +instruct convD2I_reg_reg(iRegINoSp dst, fRegD src) %{ + match(Set dst (ConvD2I src)); + + ins_cost(XFER_COST); + format %{ "fcvt.w.d $dst, $src\t#@convD2I_reg_reg" %} + + ins_encode %{ + __ fcvt_w_d_safe($dst$$Register, $src$$FloatRegister); + %} + + ins_pipe(fp_d2i); +%} + +instruct convI2D_reg_reg(fRegD dst, iRegIorL2I src) %{ + match(Set dst (ConvI2D src)); + + ins_cost(XFER_COST); + format %{ "fcvt.d.w $dst, $src\t#@convI2D_reg_reg" %} + + ins_encode %{ + __ fcvt_d_w(as_FloatRegister($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(fp_i2d); +%} + +// double <-> long + +instruct convD2L_reg_reg(iRegLNoSp dst, fRegD src) %{ + match(Set dst (ConvD2L src)); + + ins_cost(XFER_COST); + format %{ "fcvt.l.d $dst, $src\t#@convD2L_reg_reg" %} + + ins_encode %{ + __ fcvt_l_d_safe($dst$$Register, $src$$FloatRegister); + %} + + ins_pipe(fp_d2l); +%} + +instruct convL2D_reg_reg(fRegD dst, iRegL src) %{ + match(Set dst (ConvL2D src)); + + ins_cost(XFER_COST); + format %{ "fcvt.d.l $dst, $src\t#@convL2D_reg_reg" %} + + ins_encode %{ + __ fcvt_d_l(as_FloatRegister($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(fp_l2d); +%} + +// Convert oop into int for vectors alignment masking +instruct convP2I(iRegINoSp dst, iRegP src) %{ + match(Set dst (ConvL2I (CastP2X src))); + + ins_cost(ALU_COST * 2); + format %{ "zero_extend $dst, $src, 32\t# ptr -> int, #@convP2I" %} + + ins_encode %{ + __ zero_extend($dst$$Register, $src$$Register, 32); + %} + + ins_pipe(ialu_reg); +%} + +// Convert compressed oop into int for vectors alignment masking +// in case of 32bit oops (heap < 4Gb). +instruct convN2I(iRegINoSp dst, iRegN src) +%{ + predicate(CompressedOops::shift() == 0); + match(Set dst (ConvL2I (CastP2X (DecodeN src)))); + + ins_cost(ALU_COST); + format %{ "mv $dst, $src\t# compressed ptr -> int, #@convN2I" %} + + ins_encode %{ + __ mv($dst$$Register, $src$$Register); + %} + + ins_pipe(ialu_reg); +%} + +// Convert oop pointer into compressed form +instruct encodeHeapOop(iRegNNoSp dst, iRegP src) %{ + match(Set dst (EncodeP src)); + ins_cost(ALU_COST); + format %{ "encode_heap_oop $dst, $src\t#@encodeHeapOop" %} + ins_encode %{ + Register s = $src$$Register; + Register d = $dst$$Register; + __ encode_heap_oop(d, s); + %} + ins_pipe(pipe_class_default); +%} + +instruct decodeHeapOop(iRegPNoSp dst, iRegN src) %{ + predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull && + n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant); + match(Set dst (DecodeN src)); + + ins_cost(0); + format %{ "decode_heap_oop $dst, $src\t#@decodeHeapOop" %} + ins_encode %{ + Register s = $src$$Register; + Register d = $dst$$Register; + __ decode_heap_oop(d, s); + %} + ins_pipe(pipe_class_default); +%} + +instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src) %{ + predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull || + n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant); + match(Set dst (DecodeN src)); + + ins_cost(0); + format %{ "decode_heap_oop_not_null $dst, $src\t#@decodeHeapOop_not_null" %} + ins_encode %{ + Register s = $src$$Register; + Register d = $dst$$Register; + __ decode_heap_oop_not_null(d, s); + %} + ins_pipe(pipe_class_default); +%} + +// Convert klass pointer into compressed form. +instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{ + match(Set dst (EncodePKlass src)); + + ins_cost(ALU_COST); + format %{ "encode_klass_not_null $dst, $src\t#@encodeKlass_not_null" %} + + ins_encode %{ + Register src_reg = as_Register($src$$reg); + Register dst_reg = as_Register($dst$$reg); + __ encode_klass_not_null(dst_reg, src_reg, t0); + %} + + ins_pipe(pipe_class_default); +%} + +instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src, iRegPNoSp tmp) %{ + match(Set dst (DecodeNKlass src)); + + effect(TEMP tmp); + + ins_cost(ALU_COST); + format %{ "decode_klass_not_null $dst, $src\t#@decodeKlass_not_null" %} + + ins_encode %{ + Register src_reg = as_Register($src$$reg); + Register dst_reg = as_Register($dst$$reg); + Register tmp_reg = as_Register($tmp$$reg); + __ decode_klass_not_null(dst_reg, src_reg, tmp_reg); + %} + + ins_pipe(pipe_class_default); +%} + +// stack <-> reg and reg <-> reg shuffles with no conversion + +instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{ + + match(Set dst (MoveF2I src)); + + effect(DEF dst, USE src); + + ins_cost(LOAD_COST); + + format %{ "lw $dst, $src\t#@MoveF2I_stack_reg" %} + + ins_encode %{ + __ lw(as_Register($dst$$reg), Address(sp, $src$$disp)); + %} + + ins_pipe(iload_reg_reg); + +%} + +instruct MoveI2F_stack_reg(fRegF dst, stackSlotI src) %{ + + match(Set dst (MoveI2F src)); + + effect(DEF dst, USE src); + + ins_cost(LOAD_COST); + + format %{ "flw $dst, $src\t#@MoveI2F_stack_reg" %} + + ins_encode %{ + __ flw(as_FloatRegister($dst$$reg), Address(sp, $src$$disp)); + %} + + ins_pipe(fp_load_mem_s); + +%} + +instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{ + + match(Set dst (MoveD2L src)); + + effect(DEF dst, USE src); + + ins_cost(LOAD_COST); + + format %{ "ld $dst, $src\t#@MoveD2L_stack_reg" %} + + ins_encode %{ + __ ld(as_Register($dst$$reg), Address(sp, $src$$disp)); + %} + + ins_pipe(iload_reg_reg); + +%} + +instruct MoveL2D_stack_reg(fRegD dst, stackSlotL src) %{ + + match(Set dst (MoveL2D src)); + + effect(DEF dst, USE src); + + ins_cost(LOAD_COST); + + format %{ "fld $dst, $src\t#@MoveL2D_stack_reg" %} + + ins_encode %{ + __ fld(as_FloatRegister($dst$$reg), Address(sp, $src$$disp)); + %} + + ins_pipe(fp_load_mem_d); + +%} + +instruct MoveF2I_reg_stack(stackSlotI dst, fRegF src) %{ + + match(Set dst (MoveF2I src)); + + effect(DEF dst, USE src); + + ins_cost(STORE_COST); + + format %{ "fsw $src, $dst\t#@MoveF2I_reg_stack" %} + + ins_encode %{ + __ fsw(as_FloatRegister($src$$reg), Address(sp, $dst$$disp)); + %} + + ins_pipe(fp_store_reg_s); + +%} + +instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{ + + match(Set dst (MoveI2F src)); + + effect(DEF dst, USE src); + + ins_cost(STORE_COST); + + format %{ "sw $src, $dst\t#@MoveI2F_reg_stack" %} + + ins_encode %{ + __ sw(as_Register($src$$reg), Address(sp, $dst$$disp)); + %} + + ins_pipe(istore_reg_reg); + +%} + +instruct MoveD2L_reg_stack(stackSlotL dst, fRegD src) %{ + + match(Set dst (MoveD2L src)); + + effect(DEF dst, USE src); + + ins_cost(STORE_COST); + + format %{ "fsd $dst, $src\t#@MoveD2L_reg_stack" %} + + ins_encode %{ + __ fsd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp)); + %} + + ins_pipe(fp_store_reg_d); + +%} + +instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{ + + match(Set dst (MoveL2D src)); + + effect(DEF dst, USE src); + + ins_cost(STORE_COST); + + format %{ "sd $src, $dst\t#@MoveL2D_reg_stack" %} + + ins_encode %{ + __ sd(as_Register($src$$reg), Address(sp, $dst$$disp)); + %} + + ins_pipe(istore_reg_reg); + +%} + +instruct MoveF2I_reg_reg(iRegINoSp dst, fRegF src) %{ + + match(Set dst (MoveF2I src)); + + effect(DEF dst, USE src); + + ins_cost(XFER_COST); + + format %{ "fmv.x.w $dst, $src\t#@MoveL2D_reg_stack" %} + + ins_encode %{ + __ fmv_x_w(as_Register($dst$$reg), as_FloatRegister($src$$reg)); + %} + + ins_pipe(fp_f2i); + +%} + +instruct MoveI2F_reg_reg(fRegF dst, iRegI src) %{ + + match(Set dst (MoveI2F src)); + + effect(DEF dst, USE src); + + ins_cost(XFER_COST); + + format %{ "fmv.w.x $dst, $src\t#@MoveI2F_reg_reg" %} + + ins_encode %{ + __ fmv_w_x(as_FloatRegister($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(fp_i2f); + +%} + +instruct MoveD2L_reg_reg(iRegLNoSp dst, fRegD src) %{ + + match(Set dst (MoveD2L src)); + + effect(DEF dst, USE src); + + ins_cost(XFER_COST); + + format %{ "fmv.x.d $dst, $src\t#@MoveD2L_reg_reg" %} + + ins_encode %{ + __ fmv_x_d(as_Register($dst$$reg), as_FloatRegister($src$$reg)); + %} + + ins_pipe(fp_d2l); + +%} + +instruct MoveL2D_reg_reg(fRegD dst, iRegL src) %{ + + match(Set dst (MoveL2D src)); + + effect(DEF dst, USE src); + + ins_cost(XFER_COST); + + format %{ "fmv.d.x $dst, $src\t#@MoveD2L_reg_reg" %} + + ins_encode %{ + __ fmv_d_x(as_FloatRegister($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(fp_l2d); + +%} + +// ============================================================================ +// Compare Instructions which set the result float comparisons in dest register. + +instruct cmpF3_reg_reg(iRegINoSp dst, fRegF op1, fRegF op2) +%{ + match(Set dst (CmpF3 op1 op2)); + + ins_cost(XFER_COST * 2 + BRANCH_COST + ALU_COST); + format %{ "flt.s $dst, $op2, $op1\t#@cmpF3_reg_reg\n\t" + "bgtz $dst, done\n\t" + "feq.s $dst, $op1, $op2\n\t" + "addi $dst, $dst, -1\t#@cmpF3_reg_reg" + %} + + ins_encode %{ + // we want -1 for unordered or less than, 0 for equal and 1 for greater than. + __ float_compare(as_Register($dst$$reg), as_FloatRegister($op1$$reg), + as_FloatRegister($op2$$reg), -1 /*unordered_result < 0*/); + %} + + ins_pipe(pipe_class_default); +%} + +instruct cmpD3_reg_reg(iRegINoSp dst, fRegD op1, fRegD op2) +%{ + match(Set dst (CmpD3 op1 op2)); + + ins_cost(XFER_COST * 2 + BRANCH_COST + ALU_COST); + format %{ "flt.d $dst, $op2, $op1\t#@cmpD3_reg_reg\n\t" + "bgtz $dst, done\n\t" + "feq.d $dst, $op1, $op2\n\t" + "addi $dst, $dst, -1\t#@cmpD3_reg_reg" + %} + + ins_encode %{ + // we want -1 for unordered or less than, 0 for equal and 1 for greater than. + __ double_compare(as_Register($dst$$reg), as_FloatRegister($op1$$reg), as_FloatRegister($op2$$reg), -1 /*unordered_result < 0*/); + %} + + ins_pipe(pipe_class_default); +%} + +instruct cmpL3_reg_reg(iRegINoSp dst, iRegL op1, iRegL op2) +%{ + match(Set dst (CmpL3 op1 op2)); + + ins_cost(ALU_COST * 3 + BRANCH_COST); + format %{ "slt $dst, $op2, $op1\t#@cmpL3_reg_reg\n\t" + "bnez $dst, done\n\t" + "slt $dst, $op1, $op2\n\t" + "neg $dst, $dst\t#@cmpL3_reg_reg" + %} + ins_encode %{ + __ cmp_l2i(t0, as_Register($op1$$reg), as_Register($op2$$reg)); + __ mv(as_Register($dst$$reg), t0); + %} + + ins_pipe(pipe_class_default); +%} + +instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegI p, iRegI q) +%{ + match(Set dst (CmpLTMask p q)); + + ins_cost(2 * ALU_COST); + + format %{ "slt $dst, $p, $q\t#@cmpLTMask_reg_reg\n\t" + "subw $dst, zr, $dst\t#@cmpLTMask_reg_reg" + %} + + ins_encode %{ + __ slt(as_Register($dst$$reg), as_Register($p$$reg), as_Register($q$$reg)); + __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg)); + %} + + ins_pipe(ialu_reg_reg); +%} + +instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I op, immI0 zero) +%{ + match(Set dst (CmpLTMask op zero)); + + ins_cost(ALU_COST); + + format %{ "sraiw $dst, $dst, 31\t#@cmpLTMask_reg_reg" %} + + ins_encode %{ + __ sraiw(as_Register($dst$$reg), as_Register($op$$reg), 31); + %} + + ins_pipe(ialu_reg_shift); +%} + + +// ============================================================================ +// Max and Min + +instruct minI_reg_reg(iRegINoSp dst, iRegI src) +%{ + match(Set dst (MinI dst src)); + + ins_cost(BRANCH_COST + ALU_COST); + format %{ + "ble $dst, $src, skip\t#@minI_reg_reg\n\t" + "mv $dst, $src\n\t" + "skip:" + %} + + ins_encode %{ + Label Lskip; + __ ble(as_Register($dst$$reg), as_Register($src$$reg), Lskip); + __ mv(as_Register($dst$$reg), as_Register($src$$reg)); + __ bind(Lskip); + %} + + ins_pipe(pipe_class_compare); +%} + +instruct maxI_reg_reg(iRegINoSp dst, iRegI src) +%{ + match(Set dst (MaxI dst src)); + + ins_cost(BRANCH_COST + ALU_COST); + format %{ + "bge $dst, $src, skip\t#@maxI_reg_reg\n\t" + "mv $dst, $src\n\t" + "skip:" + %} + + ins_encode %{ + Label Lskip; + __ bge(as_Register($dst$$reg), as_Register($src$$reg), Lskip); + __ mv(as_Register($dst$$reg), as_Register($src$$reg)); + __ bind(Lskip); + %} + + ins_pipe(pipe_class_compare); +%} + +// special case for comparing with zero +// n.b. this is selected in preference to the rule above because it +// avoids loading constant 0 into a source register + +instruct minI_reg_zero(iRegINoSp dst, immI0 zero) +%{ + match(Set dst (MinI dst zero)); + match(Set dst (MinI zero dst)); + + ins_cost(BRANCH_COST + ALU_COST); + format %{ + "blez $dst, skip\t#@minI_reg_zero\n\t" + "mv $dst, zr\n\t" + "skip:" + %} + + ins_encode %{ + Label Lskip; + __ blez(as_Register($dst$$reg), Lskip); + __ mv(as_Register($dst$$reg), zr); + __ bind(Lskip); + %} + + ins_pipe(pipe_class_compare); +%} + +instruct maxI_reg_zero(iRegINoSp dst, immI0 zero) +%{ + match(Set dst (MaxI dst zero)); + match(Set dst (MaxI zero dst)); + + ins_cost(BRANCH_COST + ALU_COST); + format %{ + "bgez $dst, skip\t#@maxI_reg_zero\n\t" + "mv $dst, zr\n\t" + "skip:" + %} + + ins_encode %{ + Label Lskip; + __ bgez(as_Register($dst$$reg), Lskip); + __ mv(as_Register($dst$$reg), zr); + __ bind(Lskip); + %} + + ins_pipe(pipe_class_compare); +%} + +instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2) +%{ + match(Set dst (MinI src1 src2)); + + effect(DEF dst, USE src1, USE src2); + + ins_cost(BRANCH_COST + ALU_COST * 2); + format %{ + "ble $src1, $src2, Lsrc1.\t#@minI_rReg\n\t" + "mv $dst, $src2\n\t" + "j Ldone\n\t" + "bind Lsrc1\n\t" + "mv $dst, $src1\n\t" + "bind\t#@minI_rReg" + %} + + ins_encode %{ + Label Lsrc1, Ldone; + __ ble(as_Register($src1$$reg), as_Register($src2$$reg), Lsrc1); + __ mv(as_Register($dst$$reg), as_Register($src2$$reg)); + __ j(Ldone); + __ bind(Lsrc1); + __ mv(as_Register($dst$$reg), as_Register($src1$$reg)); + __ bind(Ldone); + %} + + ins_pipe(pipe_class_compare); +%} + +instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2) +%{ + match(Set dst (MaxI src1 src2)); + + effect(DEF dst, USE src1, USE src2); + + ins_cost(BRANCH_COST + ALU_COST * 2); + format %{ + "bge $src1, $src2, Lsrc1\t#@maxI_rReg\n\t" + "mv $dst, $src2\n\t" + "j Ldone\n\t" + "bind Lsrc1\n\t" + "mv $dst, $src1\n\t" + "bind\t#@maxI_rReg" + %} + + ins_encode %{ + Label Lsrc1, Ldone; + __ bge(as_Register($src1$$reg), as_Register($src2$$reg), Lsrc1); + __ mv(as_Register($dst$$reg), as_Register($src2$$reg)); + __ j(Ldone); + __ bind(Lsrc1); + __ mv(as_Register($dst$$reg), as_Register($src1$$reg)); + __ bind(Ldone); + + %} + + ins_pipe(pipe_class_compare); +%} + +// ============================================================================ +// Branch Instructions +// Direct Branch. +instruct branch(label lbl) +%{ + match(Goto); + + effect(USE lbl); + + ins_cost(BRANCH_COST); + format %{ "j $lbl\t#@branch" %} + + ins_encode(riscv_enc_j(lbl)); + + ins_pipe(pipe_branch); +%} + +// ============================================================================ +// Compare and Branch Instructions + +// Patterns for short (< 12KiB) variants + +// Compare flags and branch near instructions. +instruct cmpFlag_branch(cmpOpEqNe cmp, rFlagsReg cr, label lbl) %{ + match(If cmp cr); + effect(USE lbl); + + ins_cost(BRANCH_COST); + format %{ "b$cmp $cr, zr, $lbl\t#@cmpFlag_branch" %} + + ins_encode %{ + __ enc_cmpEqNe_imm0_branch($cmp$$cmpcode, as_Register($cr$$reg), *($lbl$$label)); + %} + ins_pipe(pipe_cmpz_branch); + ins_short_branch(1); +%} + +// Compare signed int and branch near instructions +instruct cmpI_branch(cmpOp cmp, iRegI op1, iRegI op2, label lbl) +%{ + // Same match rule as `far_cmpI_branch'. + match(If cmp (CmpI op1 op2)); + + effect(USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, $op2, $lbl\t#@cmpI_branch" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode, as_Register($op1$$reg), as_Register($op2$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmp_branch); + ins_short_branch(1); +%} + +instruct cmpI_loop(cmpOp cmp, iRegI op1, iRegI op2, label lbl) +%{ + // Same match rule as `far_cmpI_loop'. + match(CountedLoopEnd cmp (CmpI op1 op2)); + + effect(USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, $op2, $lbl\t#@cmpI_loop" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode, as_Register($op1$$reg), as_Register($op2$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmp_branch); + ins_short_branch(1); +%} + +// Compare unsigned int and branch near instructions +instruct cmpU_branch(cmpOpU cmp, iRegI op1, iRegI op2, label lbl) +%{ + // Same match rule as `far_cmpU_branch'. + match(If cmp (CmpU op1 op2)); + + effect(USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, $op2, $lbl\t#@cmpU_branch" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, as_Register($op1$$reg), + as_Register($op2$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmp_branch); + ins_short_branch(1); +%} + +instruct cmpU_loop(cmpOpU cmp, iRegI op1, iRegI op2, label lbl) +%{ + // Same match rule as `far_cmpU_loop'. + match(CountedLoopEnd cmp (CmpU op1 op2)); + + effect(USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, $op2, $lbl\t#@cmpU_loop" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, as_Register($op1$$reg), + as_Register($op2$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmp_branch); + ins_short_branch(1); +%} + +// Compare signed long and branch near instructions +instruct cmpL_branch(cmpOp cmp, iRegL op1, iRegL op2, label lbl) +%{ + // Same match rule as `far_cmpL_branch'. + match(If cmp (CmpL op1 op2)); + + effect(USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, $op2, $lbl\t#@cmpL_branch" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode, as_Register($op1$$reg), as_Register($op2$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmp_branch); + ins_short_branch(1); +%} + +instruct cmpL_loop(cmpOp cmp, iRegL op1, iRegL op2, label lbl) +%{ + // Same match rule as `far_cmpL_loop'. + match(CountedLoopEnd cmp (CmpL op1 op2)); + + effect(USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, $op2, $lbl\t#@cmpL_loop" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode, as_Register($op1$$reg), as_Register($op2$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmp_branch); + ins_short_branch(1); +%} + +// Compare unsigned long and branch near instructions +instruct cmpUL_branch(cmpOpU cmp, iRegL op1, iRegL op2, label lbl) +%{ + // Same match rule as `far_cmpUL_branch'. + match(If cmp (CmpUL op1 op2)); + + effect(USE lbl); + + ins_cost(BRANCH_COST); + format %{ "b$cmp $op1, $op2, $lbl\t#@cmpUL_branch" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, as_Register($op1$$reg), + as_Register($op2$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmp_branch); + ins_short_branch(1); +%} + +instruct cmpUL_loop(cmpOpU cmp, iRegL op1, iRegL op2, label lbl) +%{ + // Same match rule as `far_cmpUL_loop'. + match(CountedLoopEnd cmp (CmpUL op1 op2)); + + effect(USE lbl); + + ins_cost(BRANCH_COST); + format %{ "b$cmp $op1, $op2, $lbl\t#@cmpUL_loop" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, as_Register($op1$$reg), + as_Register($op2$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmp_branch); + ins_short_branch(1); +%} + +// Compare pointer and branch near instructions +instruct cmpP_branch(cmpOpU cmp, iRegP op1, iRegP op2, label lbl) +%{ + // Same match rule as `far_cmpP_branch'. + match(If cmp (CmpP op1 op2)); + + effect(USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, $op2, $lbl\t#@cmpP_branch" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, as_Register($op1$$reg), + as_Register($op2$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmp_branch); + ins_short_branch(1); +%} + +instruct cmpP_loop(cmpOpU cmp, iRegP op1, iRegP op2, label lbl) +%{ + // Same match rule as `far_cmpP_loop'. + match(CountedLoopEnd cmp (CmpP op1 op2)); + + effect(USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, $op2, $lbl\t#@cmpP_loop" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, as_Register($op1$$reg), + as_Register($op2$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmp_branch); + ins_short_branch(1); +%} + +// Compare narrow pointer and branch near instructions +instruct cmpN_branch(cmpOpU cmp, iRegN op1, iRegN op2, label lbl) +%{ + // Same match rule as `far_cmpN_branch'. + match(If cmp (CmpN op1 op2)); + + effect(USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, $op2, $lbl\t#@cmpN_branch" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, as_Register($op1$$reg), + as_Register($op2$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmp_branch); + ins_short_branch(1); +%} + +instruct cmpN_loop(cmpOpU cmp, iRegN op1, iRegN op2, label lbl) +%{ + // Same match rule as `far_cmpN_loop'. + match(CountedLoopEnd cmp (CmpN op1 op2)); + + effect(USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, $op2, $lbl\t#@cmpN_loop" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, as_Register($op1$$reg), + as_Register($op2$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmp_branch); + ins_short_branch(1); +%} + +// Compare float and branch near instructions +instruct cmpF_branch(cmpOp cmp, fRegF op1, fRegF op2, label lbl) +%{ + // Same match rule as `far_cmpF_branch'. + match(If cmp (CmpF op1 op2)); + + effect(USE lbl); + + ins_cost(XFER_COST + BRANCH_COST); + format %{ "float_b$cmp $op1, $op2, $lbl \t#@cmpF_branch"%} + + ins_encode %{ + __ float_cmp_branch($cmp$$cmpcode, as_FloatRegister($op1$$reg), as_FloatRegister($op2$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_class_compare); + ins_short_branch(1); +%} + +instruct cmpF_loop(cmpOp cmp, fRegF op1, fRegF op2, label lbl) +%{ + // Same match rule as `far_cmpF_loop'. + match(CountedLoopEnd cmp (CmpF op1 op2)); + effect(USE lbl); + + ins_cost(XFER_COST + BRANCH_COST); + format %{ "float_b$cmp $op1, $op2, $lbl\t#@cmpF_loop"%} + + ins_encode %{ + __ float_cmp_branch($cmp$$cmpcode, as_FloatRegister($op1$$reg), as_FloatRegister($op2$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_class_compare); + ins_short_branch(1); +%} + +// Compare double and branch near instructions +instruct cmpD_branch(cmpOp cmp, fRegD op1, fRegD op2, label lbl) +%{ + // Same match rule as `far_cmpD_branch'. + match(If cmp (CmpD op1 op2)); + effect(USE lbl); + + ins_cost(XFER_COST + BRANCH_COST); + format %{ "double_b$cmp $op1, $op2, $lbl\t#@cmpD_branch"%} + + ins_encode %{ + __ float_cmp_branch($cmp$$cmpcode | C2_MacroAssembler::double_branch_mask, as_FloatRegister($op1$$reg), + as_FloatRegister($op2$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_class_compare); + ins_short_branch(1); +%} + +instruct cmpD_loop(cmpOp cmp, fRegD op1, fRegD op2, label lbl) +%{ + // Same match rule as `far_cmpD_loop'. + match(CountedLoopEnd cmp (CmpD op1 op2)); + effect(USE lbl); + + ins_cost(XFER_COST + BRANCH_COST); + format %{ "double_b$cmp $op1, $op2, $lbl\t#@cmpD_loop"%} + + ins_encode %{ + __ float_cmp_branch($cmp$$cmpcode | C2_MacroAssembler::double_branch_mask, as_FloatRegister($op1$$reg), + as_FloatRegister($op2$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_class_compare); + ins_short_branch(1); +%} + +// Compare signed int with zero and branch near instructions +instruct cmpI_reg_imm0_branch(cmpOp cmp, iRegI op1, immI0 zero, label lbl) +%{ + // Same match rule as `far_cmpI_reg_imm0_branch'. + match(If cmp (CmpI op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST); + format %{ "b$cmp $op1, zr, $lbl\t#@cmpI_reg_imm0_branch" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode, as_Register($op1$$reg), zr, *($lbl$$label)); + %} + + ins_pipe(pipe_cmpz_branch); + ins_short_branch(1); +%} + +instruct cmpI_reg_imm0_loop(cmpOp cmp, iRegI op1, immI0 zero, label lbl) +%{ + // Same match rule as `far_cmpI_reg_imm0_loop'. + match(CountedLoopEnd cmp (CmpI op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, zr, $lbl\t#@cmpI_reg_imm0_loop" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode, as_Register($op1$$reg), zr, *($lbl$$label)); + %} + + ins_pipe(pipe_cmpz_branch); + ins_short_branch(1); +%} + +// Compare unsigned int with zero and branch near instructions +instruct cmpUEqNeLeGt_reg_imm0_branch(cmpOpUEqNeLeGt cmp, iRegI op1, immI0 zero, label lbl) +%{ + // Same match rule as `far_cmpUEqNeLeGt_reg_imm0_branch'. + match(If cmp (CmpU op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, zr, $lbl\t#@cmpUEqNeLeGt_reg_imm0_branch" %} + + ins_encode %{ + __ enc_cmpUEqNeLeGt_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmpz_branch); + ins_short_branch(1); +%} + +instruct cmpUEqNeLeGt_reg_imm0_loop(cmpOpUEqNeLeGt cmp, iRegI op1, immI0 zero, label lbl) +%{ + // Same match rule as `far_cmpUEqNeLeGt_reg_imm0_loop'. + match(CountedLoopEnd cmp (CmpU op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, zr, $lbl\t#@cmpUEqNeLeGt_reg_imm0_loop" %} + + + ins_encode %{ + __ enc_cmpUEqNeLeGt_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmpz_branch); + ins_short_branch(1); +%} + +// Compare signed long with zero and branch near instructions +instruct cmpL_reg_imm0_branch(cmpOp cmp, iRegL op1, immL0 zero, label lbl) +%{ + // Same match rule as `far_cmpL_reg_imm0_branch'. + match(If cmp (CmpL op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, zr, $lbl\t#@cmpL_reg_imm0_branch" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode, as_Register($op1$$reg), zr, *($lbl$$label)); + %} + + ins_pipe(pipe_cmpz_branch); + ins_short_branch(1); +%} + +instruct cmpL_reg_imm0_loop(cmpOp cmp, iRegL op1, immL0 zero, label lbl) +%{ + // Same match rule as `far_cmpL_reg_imm0_loop'. + match(CountedLoopEnd cmp (CmpL op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, zr, $lbl\t#@cmpL_reg_imm0_loop" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode, as_Register($op1$$reg), zr, *($lbl$$label)); + %} + + ins_pipe(pipe_cmpz_branch); + ins_short_branch(1); +%} + +// Compare unsigned long with zero and branch near instructions +instruct cmpULEqNeLeGt_reg_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 zero, label lbl) +%{ + // Same match rule as `far_cmpULEqNeLeGt_reg_imm0_branch'. + match(If cmp (CmpUL op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, zr, $lbl\t#@cmpULEqNeLeGt_reg_imm0_branch" %} + + ins_encode %{ + __ enc_cmpUEqNeLeGt_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmpz_branch); + ins_short_branch(1); +%} + +instruct cmpULEqNeLeGt_reg_imm0_loop(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 zero, label lbl) +%{ + // Same match rule as `far_cmpULEqNeLeGt_reg_imm0_loop'. + match(CountedLoopEnd cmp (CmpUL op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, zr, $lbl\t#@cmpULEqNeLeGt_reg_imm0_loop" %} + + ins_encode %{ + __ enc_cmpUEqNeLeGt_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmpz_branch); + ins_short_branch(1); +%} + +// Compare pointer with zero and branch near instructions +instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 zero, label lbl) %{ + // Same match rule as `far_cmpP_reg_imm0_branch'. + match(If cmp (CmpP op1 zero)); + effect(USE lbl); + + ins_cost(BRANCH_COST); + format %{ "b$cmp $op1, zr, $lbl\t#@cmpP_imm0_branch" %} + + ins_encode %{ + __ enc_cmpEqNe_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmpz_branch); + ins_short_branch(1); +%} + +instruct cmpP_imm0_loop(cmpOpEqNe cmp, iRegP op1, immP0 zero, label lbl) %{ + // Same match rule as `far_cmpP_reg_imm0_loop'. + match(CountedLoopEnd cmp (CmpP op1 zero)); + effect(USE lbl); + + ins_cost(BRANCH_COST); + format %{ "b$cmp $op1, zr, $lbl\t#@cmpP_imm0_loop" %} + + ins_encode %{ + __ enc_cmpEqNe_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmpz_branch); + ins_short_branch(1); +%} + +// Compare narrow pointer with zero and branch near instructions +instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 zero, label lbl) %{ + // Same match rule as `far_cmpN_reg_imm0_branch'. + match(If cmp (CmpN op1 zero)); + effect(USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, zr, $lbl\t#@cmpN_imm0_branch" %} + + ins_encode %{ + __ enc_cmpEqNe_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmpz_branch); + ins_short_branch(1); +%} + +instruct cmpN_imm0_loop(cmpOpEqNe cmp, iRegN op1, immN0 zero, label lbl) %{ + // Same match rule as `far_cmpN_reg_imm0_loop'. + match(CountedLoopEnd cmp (CmpN op1 zero)); + effect(USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "b$cmp $op1, zr, $lbl\t#@cmpN_imm0_loop" %} + + ins_encode %{ + __ enc_cmpEqNe_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmpz_branch); + ins_short_branch(1); +%} + +// Compare narrow pointer with pointer zero and branch near instructions +instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN op1, immP0 zero, label lbl) %{ + // Same match rule as `far_cmpP_narrowOop_imm0_branch'. + match(If cmp (CmpP (DecodeN op1) zero)); + effect(USE lbl); + + ins_cost(BRANCH_COST); + format %{ "b$cmp $op1, zr, $lbl\t#@cmpP_narrowOop_imm0_branch" %} + + ins_encode %{ + __ enc_cmpEqNe_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmpz_branch); + ins_short_branch(1); +%} + +instruct cmpP_narrowOop_imm0_loop(cmpOpEqNe cmp, iRegN op1, immP0 zero, label lbl) %{ + // Same match rule as `far_cmpP_narrowOop_imm0_loop'. + match(CountedLoopEnd cmp (CmpP (DecodeN op1) zero)); + effect(USE lbl); + + ins_cost(BRANCH_COST); + format %{ "b$cmp $op1, zr, $lbl\t#@cmpP_narrowOop_imm0_loop" %} + + ins_encode %{ + __ enc_cmpEqNe_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label)); + %} + + ins_pipe(pipe_cmpz_branch); + ins_short_branch(1); +%} + +// Patterns for far (20KiB) variants + +instruct far_cmpFlag_branch(cmpOp cmp, rFlagsReg cr, label lbl) %{ + match(If cmp cr); + effect(USE lbl); + + ins_cost(BRANCH_COST); + format %{ "far_b$cmp $cr, zr, $lbl\t#@far_cmpFlag_branch"%} + + ins_encode %{ + __ enc_cmpEqNe_imm0_branch($cmp$$cmpcode, as_Register($cr$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmpz_branch); +%} + +// Compare signed int and branch far instructions +instruct far_cmpI_branch(cmpOp cmp, iRegI op1, iRegI op2, label lbl) %{ + match(If cmp (CmpI op1 op2)); + effect(USE lbl); + + ins_cost(BRANCH_COST * 2); + + // the format instruction [far_b$cmp] here is be used as two insructions + // in macroassembler: b$not_cmp(op1, op2, done), j($lbl), bind(done) + format %{ "far_b$cmp $op1, $op2, $lbl\t#@far_cmpI_branch" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode, as_Register($op1$$reg), as_Register($op2$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmp_branch); +%} + +instruct far_cmpI_loop(cmpOp cmp, iRegI op1, iRegI op2, label lbl) %{ + match(CountedLoopEnd cmp (CmpI op1 op2)); + effect(USE lbl); + + ins_cost(BRANCH_COST * 2); + format %{ "far_b$cmp $op1, $op2, $lbl\t#@far_cmpI_loop" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode, as_Register($op1$$reg), as_Register($op2$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmp_branch); +%} + +instruct far_cmpU_branch(cmpOpU cmp, iRegI op1, iRegI op2, label lbl) %{ + match(If cmp (CmpU op1 op2)); + effect(USE lbl); + + ins_cost(BRANCH_COST * 2); + format %{ "far_b$cmp $op1, $op2, $lbl\t#@far_cmpU_branch" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, as_Register($op1$$reg), + as_Register($op2$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmp_branch); +%} + +instruct far_cmpU_loop(cmpOpU cmp, iRegI op1, iRegI op2, label lbl) %{ + match(CountedLoopEnd cmp (CmpU op1 op2)); + effect(USE lbl); + + ins_cost(BRANCH_COST * 2); + format %{ "far_b$cmp $op1, $op2, $lbl\t#@far_cmpU_loop" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, as_Register($op1$$reg), + as_Register($op2$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmp_branch); +%} + +instruct far_cmpL_branch(cmpOp cmp, iRegL op1, iRegL op2, label lbl) %{ + match(If cmp (CmpL op1 op2)); + effect(USE lbl); + + ins_cost(BRANCH_COST * 2); + format %{ "far_b$cmp $op1, $op2, $lbl\t#@far_cmpL_branch" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode, as_Register($op1$$reg), as_Register($op2$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmp_branch); +%} + +instruct far_cmpLloop(cmpOp cmp, iRegL op1, iRegL op2, label lbl) %{ + match(CountedLoopEnd cmp (CmpL op1 op2)); + effect(USE lbl); + + ins_cost(BRANCH_COST * 2); + format %{ "far_b$cmp $op1, $op2, $lbl\t#@far_cmpL_loop" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode, as_Register($op1$$reg), as_Register($op2$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmp_branch); +%} + +instruct far_cmpUL_branch(cmpOpU cmp, iRegL op1, iRegL op2, label lbl) %{ + match(If cmp (CmpUL op1 op2)); + effect(USE lbl); + + ins_cost(BRANCH_COST * 2); + format %{ "far_b$cmp $op1, $op2, $lbl\t#@far_cmpUL_branch" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, as_Register($op1$$reg), + as_Register($op2$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmp_branch); +%} + +instruct far_cmpUL_loop(cmpOpU cmp, iRegL op1, iRegL op2, label lbl) %{ + match(CountedLoopEnd cmp (CmpUL op1 op2)); + effect(USE lbl); + + ins_cost(BRANCH_COST * 2); + format %{ "far_b$cmp $op1, $op2, $lbl\t#@far_cmpUL_loop" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, as_Register($op1$$reg), + as_Register($op2$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmp_branch); +%} + +instruct far_cmpP_branch(cmpOpU cmp, iRegP op1, iRegP op2, label lbl) +%{ + match(If cmp (CmpP op1 op2)); + + effect(USE lbl); + + ins_cost(BRANCH_COST * 2); + + format %{ "far_b$cmp $op1, $op2, $lbl\t#@far_cmpP_branch" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, as_Register($op1$$reg), + as_Register($op2$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmp_branch); +%} + +instruct far_cmpP_loop(cmpOpU cmp, iRegP op1, iRegP op2, label lbl) +%{ + match(CountedLoopEnd cmp (CmpP op1 op2)); + + effect(USE lbl); + + ins_cost(BRANCH_COST * 2); + + format %{ "far_b$cmp $op1, $op2, $lbl\t#@far_cmpP_loop" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, as_Register($op1$$reg), + as_Register($op2$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmp_branch); +%} + +instruct far_cmpN_branch(cmpOpU cmp, iRegN op1, iRegN op2, label lbl) +%{ + match(If cmp (CmpN op1 op2)); + + effect(USE lbl); + + ins_cost(BRANCH_COST * 2); + + format %{ "far_b$cmp $op1, $op2, $lbl\t#@far_cmpN_branch" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, as_Register($op1$$reg), + as_Register($op2$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmp_branch); +%} + +instruct far_cmpN_loop(cmpOpU cmp, iRegN op1, iRegN op2, label lbl) +%{ + match(CountedLoopEnd cmp (CmpN op1 op2)); + + effect(USE lbl); + + ins_cost(BRANCH_COST * 2); + + format %{ "far_b$cmp $op1, $op2, $lbl\t#@far_cmpN_loop" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, as_Register($op1$$reg), + as_Register($op2$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmp_branch); +%} + +// Float compare and branch instructions +instruct far_cmpF_branch(cmpOp cmp, fRegF op1, fRegF op2, label lbl) +%{ + match(If cmp (CmpF op1 op2)); + + effect(USE lbl); + + ins_cost(XFER_COST + BRANCH_COST * 2); + format %{ "far_float_b$cmp $op1, $op2, $lbl\t#@far_cmpF_branch"%} + + ins_encode %{ + __ float_cmp_branch($cmp$$cmpcode, as_FloatRegister($op1$$reg), as_FloatRegister($op2$$reg), + *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_class_compare); +%} + +instruct far_cmpF_loop(cmpOp cmp, fRegF op1, fRegF op2, label lbl) +%{ + match(CountedLoopEnd cmp (CmpF op1 op2)); + effect(USE lbl); + + ins_cost(XFER_COST + BRANCH_COST * 2); + format %{ "far_float_b$cmp $op1, $op2, $lbl\t#@far_cmpF_loop"%} + + ins_encode %{ + __ float_cmp_branch($cmp$$cmpcode, as_FloatRegister($op1$$reg), as_FloatRegister($op2$$reg), + *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_class_compare); +%} + +// Double compare and branch instructions +instruct far_cmpD_branch(cmpOp cmp, fRegD op1, fRegD op2, label lbl) +%{ + match(If cmp (CmpD op1 op2)); + effect(USE lbl); + + ins_cost(XFER_COST + BRANCH_COST * 2); + format %{ "far_double_b$cmp $op1, $op2, $lbl\t#@far_cmpD_branch"%} + + ins_encode %{ + __ float_cmp_branch($cmp$$cmpcode | C2_MacroAssembler::double_branch_mask, as_FloatRegister($op1$$reg), + as_FloatRegister($op2$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_class_compare); +%} + +instruct far_cmpD_loop(cmpOp cmp, fRegD op1, fRegD op2, label lbl) +%{ + match(CountedLoopEnd cmp (CmpD op1 op2)); + effect(USE lbl); + + ins_cost(XFER_COST + BRANCH_COST * 2); + format %{ "far_double_b$cmp $op1, $op2, $lbl\t#@far_cmpD_loop"%} + + ins_encode %{ + __ float_cmp_branch($cmp$$cmpcode | C2_MacroAssembler::double_branch_mask, as_FloatRegister($op1$$reg), + as_FloatRegister($op2$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_class_compare); +%} + +instruct far_cmpI_reg_imm0_branch(cmpOp cmp, iRegI op1, immI0 zero, label lbl) +%{ + match(If cmp (CmpI op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST * 2); + + format %{ "far_b$cmp $op1, zr, $lbl\t#@far_cmpI_reg_imm0_branch" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode, as_Register($op1$$reg), zr, *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmpz_branch); +%} + +instruct far_cmpI_reg_imm0_loop(cmpOp cmp, iRegI op1, immI0 zero, label lbl) +%{ + match(CountedLoopEnd cmp (CmpI op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST * 2); + + format %{ "far_b$cmp $op1, zr, $lbl\t#@far_cmpI_reg_imm0_loop" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode, as_Register($op1$$reg), zr, *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmpz_branch); +%} + +instruct far_cmpUEqNeLeGt_imm0_branch(cmpOpUEqNeLeGt cmp, iRegI op1, immI0 zero, label lbl) +%{ + match(If cmp (CmpU op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST * 2); + + format %{ "far_b$cmp $op1, zr, $lbl\t#@far_cmpUEqNeLeGt_imm0_branch" %} + + ins_encode %{ + __ enc_cmpUEqNeLeGt_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmpz_branch); +%} + +instruct far_cmpUEqNeLeGt_reg_imm0_loop(cmpOpUEqNeLeGt cmp, iRegI op1, immI0 zero, label lbl) +%{ + match(CountedLoopEnd cmp (CmpU op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST * 2); + + format %{ "far_b$cmp $op1, zr, $lbl\t#@far_cmpUEqNeLeGt_reg_imm0_loop" %} + + + ins_encode %{ + __ enc_cmpUEqNeLeGt_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmpz_branch); +%} + +// compare lt/ge unsigned instructs has no short instruct with same match +instruct far_cmpULtGe_reg_imm0_branch(cmpOpULtGe cmp, iRegI op1, immI0 zero, label lbl) +%{ + match(If cmp (CmpU op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "j $lbl if $cmp == ge\t#@far_cmpULtGe_reg_imm0_branch" %} + + ins_encode(riscv_enc_far_cmpULtGe_imm0_branch(cmp, op1, lbl)); + + ins_pipe(pipe_cmpz_branch); +%} + +instruct far_cmpULtGe_reg_imm0_loop(cmpOpULtGe cmp, iRegI op1, immI0 zero, label lbl) +%{ + match(CountedLoopEnd cmp (CmpU op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "j $lbl if $cmp == ge\t#@far_cmpULtGe_reg_imm0_loop" %} + + ins_encode(riscv_enc_far_cmpULtGe_imm0_branch(cmp, op1, lbl)); + + ins_pipe(pipe_cmpz_branch); +%} + +instruct far_cmpL_reg_imm0_branch(cmpOp cmp, iRegL op1, immL0 zero, label lbl) +%{ + match(If cmp (CmpL op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST * 2); + + format %{ "far_b$cmp $op1, zr, $lbl\t#@far_cmpL_reg_imm0_branch" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode, as_Register($op1$$reg), zr, *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmpz_branch); +%} + +instruct far_cmpL_reg_imm0_loop(cmpOp cmp, iRegL op1, immL0 zero, label lbl) +%{ + match(CountedLoopEnd cmp (CmpL op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST * 2); + + format %{ "far_b$cmp $op1, zr, $lbl\t#@far_cmpL_reg_imm0_loop" %} + + ins_encode %{ + __ cmp_branch($cmp$$cmpcode, as_Register($op1$$reg), zr, *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmpz_branch); +%} + +instruct far_cmpULEqNeLeGt_reg_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 zero, label lbl) +%{ + match(If cmp (CmpUL op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST * 2); + + format %{ "far_b$cmp $op1, zr, $lbl\t#@far_cmpULEqNeLeGt_reg_imm0_branch" %} + + ins_encode %{ + __ enc_cmpUEqNeLeGt_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmpz_branch); +%} + +instruct far_cmpULEqNeLeGt_reg_imm0_loop(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 zero, label lbl) +%{ + match(CountedLoopEnd cmp (CmpUL op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST * 2); + + format %{ "far_b$cmp $op1, zr, $lbl\t#@far_cmpULEqNeLeGt_reg_imm0_loop" %} + + ins_encode %{ + __ enc_cmpUEqNeLeGt_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmpz_branch); +%} + +// compare lt/ge unsigned instructs has no short instruct with same match +instruct far_cmpULLtGe_reg_imm0_branch(cmpOpULtGe cmp, iRegL op1, immL0 zero, label lbl) +%{ + match(If cmp (CmpUL op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "j $lbl if $cmp == ge\t#@far_cmpULLtGe_reg_imm0_branch" %} + + ins_encode(riscv_enc_far_cmpULtGe_imm0_branch(cmp, op1, lbl)); + + ins_pipe(pipe_cmpz_branch); +%} + +instruct far_cmpULLtGe_reg_imm0_loop(cmpOpULtGe cmp, iRegL op1, immL0 zero, label lbl) +%{ + match(CountedLoopEnd cmp (CmpUL op1 zero)); + + effect(USE op1, USE lbl); + + ins_cost(BRANCH_COST); + + format %{ "j $lbl if $cmp == ge\t#@far_cmpULLtGe_reg_imm0_loop" %} + + ins_encode(riscv_enc_far_cmpULtGe_imm0_branch(cmp, op1, lbl)); + + ins_pipe(pipe_cmpz_branch); +%} + +instruct far_cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 zero, label lbl) %{ + match(If cmp (CmpP op1 zero)); + effect(USE lbl); + + ins_cost(BRANCH_COST * 2); + format %{ "far_b$cmp $op1, zr, $lbl\t#@far_cmpP_imm0_branch" %} + + ins_encode %{ + __ enc_cmpEqNe_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmpz_branch); +%} + +instruct far_cmpP_imm0_loop(cmpOpEqNe cmp, iRegP op1, immP0 zero, label lbl) %{ + match(CountedLoopEnd cmp (CmpP op1 zero)); + effect(USE lbl); + + ins_cost(BRANCH_COST * 2); + format %{ "far_b$cmp $op1, zr, $lbl\t#@far_cmpP_imm0_loop" %} + + ins_encode %{ + __ enc_cmpEqNe_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmpz_branch); +%} + +instruct far_cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 zero, label lbl) %{ + match(If cmp (CmpN op1 zero)); + effect(USE lbl); + + ins_cost(BRANCH_COST * 2); + + format %{ "far_b$cmp $op1, zr, $lbl\t#@far_cmpN_imm0_branch" %} + + ins_encode %{ + __ enc_cmpEqNe_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmpz_branch); +%} + +instruct far_cmpN_imm0_loop(cmpOpEqNe cmp, iRegN op1, immN0 zero, label lbl) %{ + match(CountedLoopEnd cmp (CmpN op1 zero)); + effect(USE lbl); + + ins_cost(BRANCH_COST * 2); + + format %{ "far_b$cmp $op1, zr, $lbl\t#@far_cmpN_imm0_loop" %} + + ins_encode %{ + __ enc_cmpEqNe_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmpz_branch); +%} + +instruct far_cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN op1, immP0 zero, label lbl) %{ + match(If cmp (CmpP (DecodeN op1) zero)); + effect(USE lbl); + + ins_cost(BRANCH_COST * 2); + format %{ "far_b$cmp $op1, zr, $lbl\t#@far_cmpP_narrowOop_imm0_branch" %} + + ins_encode %{ + __ enc_cmpEqNe_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmpz_branch); +%} + +instruct far_cmpP_narrowOop_imm0_loop(cmpOpEqNe cmp, iRegN op1, immP0 zero, label lbl) %{ + match(CountedLoopEnd cmp (CmpP (DecodeN op1) zero)); + effect(USE lbl); + + ins_cost(BRANCH_COST * 2); + format %{ "far_b$cmp $op1, zr, $lbl\t#@far_cmpP_narrowOop_imm0_loop" %} + + ins_encode %{ + __ enc_cmpEqNe_imm0_branch($cmp$$cmpcode, as_Register($op1$$reg), *($lbl$$label), /* is_far */ true); + %} + + ins_pipe(pipe_cmpz_branch); +%} + +// ============================================================================ +// Conditional Move Instructions +instruct cmovI_cmpI(iRegINoSp dst, iRegI src, iRegI op1, iRegI op2, cmpOp cop) %{ + match(Set dst (CMoveI (Binary cop (CmpI op1 op2)) (Binary dst src))); + ins_cost(ALU_COST + BRANCH_COST); + + format %{ + "CMove $dst, ($op1 $cop $op2), $dst, $src\t#@cmovI_cmpI\n\t" + %} + + ins_encode %{ + __ enc_cmove($cop$$cmpcode, + as_Register($op1$$reg), as_Register($op2$$reg), + as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(pipe_class_compare); +%} + +instruct cmovI_cmpU(iRegINoSp dst, iRegI src, iRegI op1, iRegI op2, cmpOpU cop) %{ + match(Set dst (CMoveI (Binary cop (CmpU op1 op2)) (Binary dst src))); + ins_cost(ALU_COST + BRANCH_COST); + + format %{ + "CMove $dst, ($op1 $cop $op2), $dst, $src\t#@cmovI_cmpU\n\t" + %} + + ins_encode %{ + __ enc_cmove($cop$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, + as_Register($op1$$reg), as_Register($op2$$reg), + as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(pipe_class_compare); +%} + +instruct cmovI_cmpL(iRegINoSp dst, iRegI src, iRegL op1, iRegL op2, cmpOp cop) %{ + match(Set dst (CMoveI (Binary cop (CmpL op1 op2)) (Binary dst src))); + ins_cost(ALU_COST + BRANCH_COST); + + format %{ + "CMove $dst, ($op1 $cop $op2), $dst, $src\t#@cmovI_cmpL\n\t" + %} + + ins_encode %{ + __ enc_cmove($cop$$cmpcode, + as_Register($op1$$reg), as_Register($op2$$reg), + as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(pipe_class_compare); +%} + +instruct cmovI_cmpUL(iRegINoSp dst, iRegI src, iRegL op1, iRegL op2, cmpOpU cop) %{ + match(Set dst (CMoveI (Binary cop (CmpUL op1 op2)) (Binary dst src))); + ins_cost(ALU_COST + BRANCH_COST); + + format %{ + "CMove $dst, ($op1 $cop $op2), $dst, $src\t#@cmovI_cmpUL\n\t" + %} + + ins_encode %{ + __ enc_cmove($cop$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, + as_Register($op1$$reg), as_Register($op2$$reg), + as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(pipe_class_compare); +%} + +instruct cmovL_cmpL(iRegLNoSp dst, iRegL src, iRegL op1, iRegL op2, cmpOp cop) %{ + match(Set dst (CMoveL (Binary cop (CmpL op1 op2)) (Binary dst src))); + ins_cost(ALU_COST + BRANCH_COST); + + format %{ + "CMove $dst, ($op1 $cop $op2), $dst, $src\t#@cmovL_cmpL\n\t" + %} + + ins_encode %{ + __ enc_cmove($cop$$cmpcode, + as_Register($op1$$reg), as_Register($op2$$reg), + as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(pipe_class_compare); +%} + +instruct cmovL_cmpUL(iRegLNoSp dst, iRegL src, iRegL op1, iRegL op2, cmpOpU cop) %{ + match(Set dst (CMoveL (Binary cop (CmpUL op1 op2)) (Binary dst src))); + ins_cost(ALU_COST + BRANCH_COST); + + format %{ + "CMove $dst, ($op1 $cop $op2), $dst, $src\t#@cmovL_cmpUL\n\t" + %} + + ins_encode %{ + __ enc_cmove($cop$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, + as_Register($op1$$reg), as_Register($op2$$reg), + as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(pipe_class_compare); +%} + +instruct cmovL_cmpI(iRegLNoSp dst, iRegL src, iRegI op1, iRegI op2, cmpOp cop) %{ + match(Set dst (CMoveL (Binary cop (CmpI op1 op2)) (Binary dst src))); + ins_cost(ALU_COST + BRANCH_COST); + + format %{ + "CMove $dst, ($op1 $cop $op2), $dst, $src\t#@cmovL_cmpI\n\t" + %} + + ins_encode %{ + __ enc_cmove($cop$$cmpcode, + as_Register($op1$$reg), as_Register($op2$$reg), + as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(pipe_class_compare); +%} + +instruct cmovL_cmpU(iRegLNoSp dst, iRegL src, iRegI op1, iRegI op2, cmpOpU cop) %{ + match(Set dst (CMoveL (Binary cop (CmpU op1 op2)) (Binary dst src))); + ins_cost(ALU_COST + BRANCH_COST); + + format %{ + "CMove $dst, ($op1 $cop $op2), $dst, $src\t#@cmovL_cmpU\n\t" + %} + + ins_encode %{ + __ enc_cmove($cop$$cmpcode | C2_MacroAssembler::unsigned_branch_mask, + as_Register($op1$$reg), as_Register($op2$$reg), + as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(pipe_class_compare); +%} + +// ============================================================================ +// Procedure Call/Return Instructions + +// Call Java Static Instruction +// Note: If this code changes, the corresponding ret_addr_offset() and +// compute_padding() functions will have to be adjusted. +instruct CallStaticJavaDirect(method meth) +%{ + match(CallStaticJava); + + effect(USE meth); + + ins_cost(BRANCH_COST); + + format %{ "CALL,static $meth\t#@CallStaticJavaDirect" %} + + ins_encode(riscv_enc_java_static_call(meth), + riscv_enc_call_epilog); + + ins_pipe(pipe_class_call); + ins_alignment(4); +%} + +// TO HERE + +// Call Java Dynamic Instruction +// Note: If this code changes, the corresponding ret_addr_offset() and +// compute_padding() functions will have to be adjusted. +instruct CallDynamicJavaDirect(method meth, rFlagsReg cr) +%{ + match(CallDynamicJava); + + effect(USE meth, KILL cr); + + ins_cost(BRANCH_COST + ALU_COST * 6); + + format %{ "CALL,dynamic $meth\t#@CallDynamicJavaDirect" %} + + ins_encode(riscv_enc_java_dynamic_call(meth), + riscv_enc_call_epilog); + + ins_pipe(pipe_class_call); + ins_alignment(4); +%} + +// Call Runtime Instruction + +instruct CallRuntimeDirect(method meth, rFlagsReg cr) +%{ + match(CallRuntime); + + effect(USE meth, KILL cr); + + ins_cost(BRANCH_COST); + + format %{ "CALL, runtime $meth\t#@CallRuntimeDirect" %} + + ins_encode(riscv_enc_java_to_runtime(meth)); + + ins_pipe(pipe_class_call); +%} + +// Call Runtime Instruction + +instruct CallLeafDirect(method meth, rFlagsReg cr) +%{ + match(CallLeaf); + + effect(USE meth, KILL cr); + + ins_cost(BRANCH_COST); + + format %{ "CALL, runtime leaf $meth\t#@CallLeafDirect" %} + + ins_encode(riscv_enc_java_to_runtime(meth)); + + ins_pipe(pipe_class_call); +%} + +// Call Runtime Instruction + +instruct CallLeafNoFPDirect(method meth, rFlagsReg cr) +%{ + match(CallLeafNoFP); + + effect(USE meth, KILL cr); + + ins_cost(BRANCH_COST); + + format %{ "CALL, runtime leaf nofp $meth\t#@CallLeafNoFPDirect" %} + + ins_encode(riscv_enc_java_to_runtime(meth)); + + ins_pipe(pipe_class_call); +%} + +// ============================================================================ +// Partial Subtype Check +// +// superklass array for an instance of the superklass. Set a hidden +// internal cache on a hit (cache is checked with exposed code in +// gen_subtype_check()). Return zero for a hit. The encoding +// ALSO sets flags. + +instruct partialSubtypeCheck(iRegP_R15 result, iRegP_R14 sub, iRegP_R10 super, iRegP_R12 tmp, rFlagsReg cr) +%{ + match(Set result (PartialSubtypeCheck sub super)); + effect(KILL tmp, KILL cr); + + ins_cost(2 * STORE_COST + 3 * LOAD_COST + 4 * ALU_COST + BRANCH_COST * 4); + format %{ "partialSubtypeCheck $result, $sub, $super\t#@partialSubtypeCheck" %} + + ins_encode(riscv_enc_partial_subtype_check(sub, super, tmp, result)); + + opcode(0x1); // Force zero of result reg on hit + + ins_pipe(pipe_class_memory); +%} + +instruct partialSubtypeCheckVsZero(iRegP_R15 result, iRegP_R14 sub, iRegP_R10 super, iRegP_R12 tmp, + immP0 zero, rFlagsReg cr) +%{ + match(Set cr (CmpP (PartialSubtypeCheck sub super) zero)); + effect(KILL tmp, KILL result); + + ins_cost(2 * STORE_COST + 3 * LOAD_COST + 4 * ALU_COST + BRANCH_COST * 4); + format %{ "partialSubtypeCheck $result, $sub, $super == 0\t#@partialSubtypeCheckVsZero" %} + + ins_encode(riscv_enc_partial_subtype_check(sub, super, tmp, result)); + + opcode(0x0); // Don't zero result reg on hit + + ins_pipe(pipe_class_memory); +%} + +instruct string_compareU(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, iRegI_R14 cnt2, + iRegI_R10 result, iRegP_R28 tmp1, iRegL_R29 tmp2, iRegL_R30 tmp3, rFlagsReg cr) +%{ + predicate(!UseRVV && ((StrCompNode *)n)->encoding() == StrIntrinsicNode::UU); + match(Set result (StrComp(Binary str1 cnt1)(Binary str2 cnt2))); + effect(KILL tmp1, KILL tmp2, KILL tmp3, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr); + + format %{ "String Compare $str1, $cnt1, $str2, $cnt2 -> $result\t#@string_compareU" %} + ins_encode %{ + // Count is in 8-bit bytes; non-Compact chars are 16 bits. + __ string_compare($str1$$Register, $str2$$Register, + $cnt1$$Register, $cnt2$$Register, $result$$Register, + $tmp1$$Register, $tmp2$$Register, $tmp3$$Register, + StrIntrinsicNode::UU); + %} + ins_pipe(pipe_class_memory); +%} + +instruct string_compareL(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, iRegI_R14 cnt2, + iRegI_R10 result, iRegP_R28 tmp1, iRegL_R29 tmp2, iRegL_R30 tmp3, rFlagsReg cr) +%{ + predicate(!UseRVV && ((StrCompNode *)n)->encoding() == StrIntrinsicNode::LL); + match(Set result (StrComp(Binary str1 cnt1)(Binary str2 cnt2))); + effect(KILL tmp1, KILL tmp2, KILL tmp3, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr); + + format %{ "String Compare $str1, $cnt1, $str2, $cnt2 -> $result\t#@string_compareL" %} + ins_encode %{ + __ string_compare($str1$$Register, $str2$$Register, + $cnt1$$Register, $cnt2$$Register, $result$$Register, + $tmp1$$Register, $tmp2$$Register, $tmp3$$Register, + StrIntrinsicNode::LL); + %} + ins_pipe(pipe_class_memory); +%} + +instruct string_compareUL(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, iRegI_R14 cnt2, + iRegI_R10 result, iRegP_R28 tmp1, iRegL_R29 tmp2, iRegL_R30 tmp3, rFlagsReg cr) +%{ + predicate(!UseRVV && ((StrCompNode *)n)->encoding() == StrIntrinsicNode::UL); + match(Set result (StrComp(Binary str1 cnt1)(Binary str2 cnt2))); + effect(KILL tmp1, KILL tmp2, KILL tmp3, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr); + + format %{"String Compare $str1, $cnt1, $str2, $cnt2 -> $result\t#@string_compareUL" %} + ins_encode %{ + __ string_compare($str1$$Register, $str2$$Register, + $cnt1$$Register, $cnt2$$Register, $result$$Register, + $tmp1$$Register, $tmp2$$Register, $tmp3$$Register, + StrIntrinsicNode::UL); + %} + ins_pipe(pipe_class_memory); +%} + +instruct string_compareLU(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, iRegI_R14 cnt2, + iRegI_R10 result, iRegP_R28 tmp1, iRegL_R29 tmp2, iRegL_R30 tmp3, + rFlagsReg cr) +%{ + predicate(!UseRVV && ((StrCompNode *)n)->encoding() == StrIntrinsicNode::LU); + match(Set result (StrComp(Binary str1 cnt1)(Binary str2 cnt2))); + effect(KILL tmp1, KILL tmp2, KILL tmp3, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr); + + format %{ "String Compare $str1, $cnt1, $str2, $cnt2 -> $result\t#@string_compareLU" %} + ins_encode %{ + __ string_compare($str1$$Register, $str2$$Register, + $cnt1$$Register, $cnt2$$Register, $result$$Register, + $tmp1$$Register, $tmp2$$Register, $tmp3$$Register, + StrIntrinsicNode::LU); + %} + ins_pipe(pipe_class_memory); +%} + +instruct string_indexofUU(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, iRegI_R14 cnt2, + iRegI_R10 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, + iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr) +%{ + predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU); + match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2))); + effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, TEMP_DEF result, + TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr); + + format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %} + ins_encode %{ + __ string_indexof($str1$$Register, $str2$$Register, + $cnt1$$Register, $cnt2$$Register, + $tmp1$$Register, $tmp2$$Register, + $tmp3$$Register, $tmp4$$Register, + $tmp5$$Register, $tmp6$$Register, + $result$$Register, StrIntrinsicNode::UU); + %} + ins_pipe(pipe_class_memory); +%} + +instruct string_indexofLL(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, iRegI_R14 cnt2, + iRegI_R10 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, + iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr) +%{ + predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL); + match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2))); + effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, TEMP_DEF result, + TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr); + + format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %} + ins_encode %{ + __ string_indexof($str1$$Register, $str2$$Register, + $cnt1$$Register, $cnt2$$Register, + $tmp1$$Register, $tmp2$$Register, + $tmp3$$Register, $tmp4$$Register, + $tmp5$$Register, $tmp6$$Register, + $result$$Register, StrIntrinsicNode::LL); + %} + ins_pipe(pipe_class_memory); +%} + +instruct string_indexofUL(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, iRegI_R14 cnt2, + iRegI_R10 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, + iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr) +%{ + predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL); + match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2))); + effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, TEMP_DEF result, + TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr); + format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %} + + ins_encode %{ + __ string_indexof($str1$$Register, $str2$$Register, + $cnt1$$Register, $cnt2$$Register, + $tmp1$$Register, $tmp2$$Register, + $tmp3$$Register, $tmp4$$Register, + $tmp5$$Register, $tmp6$$Register, + $result$$Register, StrIntrinsicNode::UL); + %} + ins_pipe(pipe_class_memory); +%} + +instruct string_indexof_conUU(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, + immI_le_4 int_cnt2, iRegI_R10 result, iRegINoSp tmp1, iRegINoSp tmp2, + iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr) +%{ + predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU); + match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2))); + effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, TEMP_DEF result, + TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr); + + format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %} + + ins_encode %{ + int icnt2 = (int)$int_cnt2$$constant; + __ string_indexof_linearscan($str1$$Register, $str2$$Register, + $cnt1$$Register, zr, + $tmp1$$Register, $tmp2$$Register, + $tmp3$$Register, $tmp4$$Register, + icnt2, $result$$Register, StrIntrinsicNode::UU); + %} + ins_pipe(pipe_class_memory); +%} + +instruct string_indexof_conLL(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, + immI_le_4 int_cnt2, iRegI_R10 result, iRegINoSp tmp1, iRegINoSp tmp2, + iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr) +%{ + predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL); + match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2))); + effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, TEMP_DEF result, + TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr); + + format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %} + ins_encode %{ + int icnt2 = (int)$int_cnt2$$constant; + __ string_indexof_linearscan($str1$$Register, $str2$$Register, + $cnt1$$Register, zr, + $tmp1$$Register, $tmp2$$Register, + $tmp3$$Register, $tmp4$$Register, + icnt2, $result$$Register, StrIntrinsicNode::LL); + %} + ins_pipe(pipe_class_memory); +%} + +instruct string_indexof_conUL(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, + immI_1 int_cnt2, iRegI_R10 result, iRegINoSp tmp1, iRegINoSp tmp2, + iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr) +%{ + predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL); + match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2))); + effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, TEMP_DEF result, + TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr); + + format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %} + ins_encode %{ + int icnt2 = (int)$int_cnt2$$constant; + __ string_indexof_linearscan($str1$$Register, $str2$$Register, + $cnt1$$Register, zr, + $tmp1$$Register, $tmp2$$Register, + $tmp3$$Register, $tmp4$$Register, + icnt2, $result$$Register, StrIntrinsicNode::UL); + %} + ins_pipe(pipe_class_memory); +%} + +instruct stringU_indexof_char(iRegP_R11 str1, iRegI_R12 cnt1, iRegI_R13 ch, + iRegI_R10 result, iRegINoSp tmp1, iRegINoSp tmp2, + iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr) +%{ + match(Set result (StrIndexOfChar (Binary str1 cnt1) ch)); + predicate(!UseRVV && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U)); + effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP_DEF result, + TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr); + + format %{ "StringUTF16 IndexOf char[] $str1, $cnt1, $ch -> $result" %} + ins_encode %{ + __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, + $result$$Register, $tmp1$$Register, $tmp2$$Register, + $tmp3$$Register, $tmp4$$Register, false /* isU */); + %} + ins_pipe(pipe_class_memory); +%} + + +instruct stringL_indexof_char(iRegP_R11 str1, iRegI_R12 cnt1, iRegI_R13 ch, + iRegI_R10 result, iRegINoSp tmp1, iRegINoSp tmp2, + iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr) +%{ + match(Set result (StrIndexOfChar (Binary str1 cnt1) ch)); + predicate(!UseRVV && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L)); + effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP_DEF result, + TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr); + + format %{ "StringLatin1 IndexOf char[] $str1, $cnt1, $ch -> $result" %} + ins_encode %{ + __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, + $result$$Register, $tmp1$$Register, $tmp2$$Register, + $tmp3$$Register, $tmp4$$Register, true /* isL */); + %} + ins_pipe(pipe_class_memory); +%} + +// clearing of an array +instruct clearArray_reg_reg(iRegL_R29 cnt, iRegP_R28 base, Universe dummy) +%{ + predicate(!UseRVV); + match(Set dummy (ClearArray cnt base)); + effect(USE_KILL cnt, USE_KILL base); + + ins_cost(4 * DEFAULT_COST); + format %{ "ClearArray $cnt, $base\t#@clearArray_reg_reg" %} + + ins_encode %{ + address tpc = __ zero_words($base$$Register, $cnt$$Register); + if (tpc == NULL) { + ciEnv::current()->record_failure("CodeCache is full"); + return; + } + %} + + ins_pipe(pipe_class_memory); +%} + +instruct clearArray_imm_reg(immL cnt, iRegP_R28 base, Universe dummy, rFlagsReg cr) +%{ + predicate(!UseRVV && (uint64_t)n->in(2)->get_long() + < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)); + match(Set dummy (ClearArray cnt base)); + effect(USE_KILL base, KILL cr); + + ins_cost(4 * DEFAULT_COST); + format %{ "ClearArray $cnt, $base\t#@clearArray_imm_reg" %} + + ins_encode %{ + __ zero_words($base$$Register, (uint64_t)$cnt$$constant); + %} + + ins_pipe(pipe_class_memory); +%} + +instruct string_equalsL(iRegP_R11 str1, iRegP_R13 str2, iRegI_R14 cnt, + iRegI_R10 result, rFlagsReg cr) +%{ + predicate(!UseRVV && ((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL); + match(Set result (StrEquals (Binary str1 str2) cnt)); + effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr); + + format %{ "String Equals $str1, $str2, $cnt -> $result\t#@string_equalsL" %} + ins_encode %{ + // Count is in 8-bit bytes; non-Compact chars are 16 bits. + __ string_equals($str1$$Register, $str2$$Register, + $result$$Register, $cnt$$Register, 1); + %} + ins_pipe(pipe_class_memory); +%} + +instruct string_equalsU(iRegP_R11 str1, iRegP_R13 str2, iRegI_R14 cnt, + iRegI_R10 result, rFlagsReg cr) +%{ + predicate(!UseRVV && ((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU); + match(Set result (StrEquals (Binary str1 str2) cnt)); + effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr); + + format %{ "String Equals $str1, $str2, $cnt -> $result\t#@string_equalsU" %} + ins_encode %{ + // Count is in 8-bit bytes; non-Compact chars are 16 bits. + __ string_equals($str1$$Register, $str2$$Register, + $result$$Register, $cnt$$Register, 2); + %} + ins_pipe(pipe_class_memory); +%} + +instruct array_equalsB(iRegP_R11 ary1, iRegP_R12 ary2, iRegI_R10 result, + iRegP_R13 tmp1, iRegP_R14 tmp2, iRegP_R15 tmp3, + iRegP_R16 tmp4, iRegP_R28 tmp5, rFlagsReg cr) +%{ + predicate(!UseRVV && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL); + match(Set result (AryEq ary1 ary2)); + effect(USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL tmp5, KILL cr); + + format %{ "Array Equals $ary1, ary2 -> $result\t#@array_equalsB // KILL $tmp5" %} + ins_encode %{ + __ arrays_equals($ary1$$Register, $ary2$$Register, + $tmp1$$Register, $tmp2$$Register, $tmp3$$Register, $tmp4$$Register, + $result$$Register, $tmp5$$Register, 1); + %} + ins_pipe(pipe_class_memory); +%} + +instruct array_equalsC(iRegP_R11 ary1, iRegP_R12 ary2, iRegI_R10 result, + iRegP_R13 tmp1, iRegP_R14 tmp2, iRegP_R15 tmp3, + iRegP_R16 tmp4, iRegP_R28 tmp5, rFlagsReg cr) +%{ + predicate(!UseRVV && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU); + match(Set result (AryEq ary1 ary2)); + effect(USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL tmp5, KILL cr); + + format %{ "Array Equals $ary1, ary2 -> $result\t#@array_equalsC // KILL $tmp5" %} + ins_encode %{ + __ arrays_equals($ary1$$Register, $ary2$$Register, + $tmp1$$Register, $tmp2$$Register, $tmp3$$Register, $tmp4$$Register, + $result$$Register, $tmp5$$Register, 2); + %} + ins_pipe(pipe_class_memory); +%} + +// ============================================================================ +// Safepoint Instructions + +instruct safePoint(iRegP poll) +%{ + match(SafePoint poll); + + ins_cost(2 * LOAD_COST); + format %{ + "lwu zr, [$poll]\t# Safepoint: poll for GC, #@safePoint" + %} + ins_encode %{ + __ read_polling_page(as_Register($poll$$reg), 0, relocInfo::poll_type); + %} + ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem); +%} + +// ============================================================================ +// This name is KNOWN by the ADLC and cannot be changed. +// The ADLC forces a 'TypeRawPtr::BOTTOM' output type +// for this guy. +instruct tlsLoadP(javaThread_RegP dst) +%{ + match(Set dst (ThreadLocal)); + + ins_cost(0); + + format %{ " -- \t// $dst=Thread::current(), empty, #@tlsLoadP" %} + + size(0); + + ins_encode( /*empty*/ ); + + ins_pipe(pipe_class_empty); +%} + +// inlined locking and unlocking +// using t1 as the 'flag' register to bridge the BoolNode producers and consumers +instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2) +%{ + match(Set cr (FastLock object box)); + effect(TEMP tmp1, TEMP tmp2); + + ins_cost(LOAD_COST * 2 + STORE_COST * 3 + ALU_COST * 6 + BRANCH_COST * 3); + format %{ "fastlock $object,$box\t! kills $tmp1,$tmp2, #@cmpFastLock" %} + + ins_encode(riscv_enc_fast_lock(object, box, tmp1, tmp2)); + + ins_pipe(pipe_serial); +%} + +// using t1 as the 'flag' register to bridge the BoolNode producers and consumers +instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2) +%{ + match(Set cr (FastUnlock object box)); + effect(TEMP tmp1, TEMP tmp2); + + ins_cost(LOAD_COST * 2 + STORE_COST + ALU_COST * 2 + BRANCH_COST * 4); + format %{ "fastunlock $object,$box\t! kills $tmp1, $tmp2, #@cmpFastUnlock" %} + + ins_encode(riscv_enc_fast_unlock(object, box, tmp1, tmp2)); + + ins_pipe(pipe_serial); +%} + +// Tail Call; Jump from runtime stub to Java code. +// Also known as an 'interprocedural jump'. +// Target of jump will eventually return to caller. +// TailJump below removes the return address. +instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_oop) +%{ + match(TailCall jump_target method_oop); + + ins_cost(BRANCH_COST); + + format %{ "jalr $jump_target\t# $method_oop holds method oop, #@TailCalljmpInd." %} + + ins_encode(riscv_enc_tail_call(jump_target)); + + ins_pipe(pipe_class_call); +%} + +instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R10 ex_oop) +%{ + match(TailJump jump_target ex_oop); + + ins_cost(ALU_COST + BRANCH_COST); + + format %{ "jalr $jump_target\t# $ex_oop holds exception oop, #@TailjmpInd." %} + + ins_encode(riscv_enc_tail_jmp(jump_target)); + + ins_pipe(pipe_class_call); +%} + +// Create exception oop: created by stack-crawling runtime code. +// Created exception is now available to this handler, and is setup +// just prior to jumping to this handler. No code emitted. +instruct CreateException(iRegP_R10 ex_oop) +%{ + match(Set ex_oop (CreateEx)); + + ins_cost(0); + format %{ " -- \t// exception oop; no code emitted, #@CreateException" %} + + size(0); + + ins_encode( /*empty*/ ); + + ins_pipe(pipe_class_empty); +%} + +// Rethrow exception: The exception oop will come in the first +// argument position. Then JUMP (not call) to the rethrow stub code. +instruct RethrowException() +%{ + match(Rethrow); + + ins_cost(BRANCH_COST); + + format %{ "j rethrow_stub\t#@RethrowException" %} + + ins_encode(riscv_enc_rethrow()); + + ins_pipe(pipe_class_call); +%} + +// Return Instruction +// epilog node loads ret address into ra as part of frame pop +instruct Ret() +%{ + match(Return); + + ins_cost(BRANCH_COST); + format %{ "ret\t// return register, #@Ret" %} + + ins_encode(riscv_enc_ret()); + + ins_pipe(pipe_branch); +%} + +// Die now. +instruct ShouldNotReachHere() %{ + match(Halt); + + ins_cost(BRANCH_COST); + + format %{ "#@ShouldNotReachHere" %} + + ins_encode %{ + if (is_reachable()) { + __ stop(_halt_reason); + } + %} + + ins_pipe(pipe_class_default); +%} + + +//----------PEEPHOLE RULES----------------------------------------------------- +// These must follow all instruction definitions as they use the names +// defined in the instructions definitions. +// +// peepmatch ( root_instr_name [preceding_instruction]* ); +// +// peepconstraint %{ +// (instruction_number.operand_name relational_op instruction_number.operand_name +// [, ...] ); +// // instruction numbers are zero-based using left to right order in peepmatch +// +// peepreplace ( instr_name ( [instruction_number.operand_name]* ) ); +// // provide an instruction_number.operand_name for each operand that appears +// // in the replacement instruction's match rule +// +// ---------VM FLAGS--------------------------------------------------------- +// +// All peephole optimizations can be turned off using -XX:-OptoPeephole +// +// Each peephole rule is given an identifying number starting with zero and +// increasing by one in the order seen by the parser. An individual peephole +// can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=# +// on the command-line. +// +// ---------CURRENT LIMITATIONS---------------------------------------------- +// +// Only match adjacent instructions in same basic block +// Only equality constraints +// Only constraints between operands, not (0.dest_reg == RAX_enc) +// Only one replacement instruction +// +//----------SMARTSPILL RULES--------------------------------------------------- +// These must follow all instruction definitions as they use the names +// defined in the instructions definitions. + +// Local Variables: +// mode: c++ +// End: diff --git a/src/hotspot/cpu/riscv/riscv_b.ad b/src/hotspot/cpu/riscv/riscv_b.ad new file mode 100644 index 0000000000000..f88ccfc7b5655 --- /dev/null +++ b/src/hotspot/cpu/riscv/riscv_b.ad @@ -0,0 +1,575 @@ +// +// Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2022, Huawei Technologies Co., Ltd. All rights reserved. +// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +// +// This code is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License version 2 only, as +// published by the Free Software Foundation. +// +// This code is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// version 2 for more details (a copy is included in the LICENSE file that +// accompanied this code). +// +// You should have received a copy of the GNU General Public License version +// 2 along with this work; if not, write to the Free Software Foundation, +// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +// +// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +// or visit www.oracle.com if you need additional information or have any +// questions. +// +// + +// RISCV Bit-Manipulation Extension Architecture Description File + +instruct rorI_imm_b(iRegINoSp dst, iRegI src, immI shift) %{ + predicate(UseZbb); + match(Set dst (RotateRight src shift)); + + format %{ "roriw $dst, $src, ($shift & 0x1f)\t#@rorI_imm_b" %} + + ins_cost(ALU_COST); + ins_encode %{ + __ roriw(as_Register($dst$$reg), as_Register($src$$reg), $shift$$constant & 0x1f); + %} + + ins_pipe(ialu_reg_shift); +%} + +instruct rorL_imm_b(iRegLNoSp dst, iRegL src, immI shift) %{ + predicate(UseZbb); + match(Set dst (RotateRight src shift)); + + format %{ "rori $dst, $src, ($shift & 0x3f)\t#@rorL_imm_b" %} + + ins_cost(ALU_COST); + ins_encode %{ + __ rori(as_Register($dst$$reg), as_Register($src$$reg), $shift$$constant & 0x3f); + %} + + ins_pipe(ialu_reg_shift); +%} + +instruct rorI_reg_b(iRegINoSp dst, iRegI src, iRegI shift) %{ + predicate(UseZbb); + match(Set dst (RotateRight src shift)); + + format %{ "rorw $dst, $src, $shift\t#@rorI_reg_b" %} + ins_cost(ALU_COST); + ins_encode %{ + __ rorw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg)); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct rorL_reg_b(iRegLNoSp dst, iRegL src, iRegI shift) %{ + predicate(UseZbb); + match(Set dst (RotateRight src shift)); + + format %{ "ror $dst, $src, $shift\t#@rorL_reg_b" %} + ins_cost(ALU_COST); + ins_encode %{ + __ ror(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg)); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct rolI_reg_b(iRegINoSp dst, iRegI src, iRegI shift) %{ + predicate(UseZbb); + match(Set dst (RotateLeft src shift)); + + format %{ "rolw $dst, $src, $shift\t#@rolI_reg_b" %} + ins_cost(ALU_COST); + ins_encode %{ + __ rolw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg)); + %} + ins_pipe(ialu_reg_reg); +%} + +instruct rolL_reg_b(iRegLNoSp dst, iRegL src, iRegI shift) %{ + predicate(UseZbb); + match(Set dst (RotateLeft src shift)); + + format %{ "rol $dst, $src, $shift\t#@rolL_reg_b" %} + ins_cost(ALU_COST); + ins_encode %{ + __ rol(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg)); + %} + ins_pipe(ialu_reg_reg); +%} + +// Convert oop into int for vectors alignment masking +instruct convP2I_b(iRegINoSp dst, iRegP src) %{ + predicate(UseZba); + match(Set dst (ConvL2I (CastP2X src))); + + format %{ "zext.w $dst, $src\t# ptr -> int @convP2I_b" %} + + ins_cost(ALU_COST); + ins_encode %{ + __ zext_w(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg); +%} + +// byte to int +instruct convB2I_reg_reg_b(iRegINoSp dst, iRegIorL2I src, immI_24 lshift, immI_24 rshift) %{ + predicate(UseZbb); + match(Set dst (RShiftI (LShiftI src lshift) rshift)); + + format %{ "sext.b $dst, $src\t# b2i, #@convB2I_reg_reg_b" %} + + ins_cost(ALU_COST); + ins_encode %{ + __ sext_b(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg); +%} + +// int to short +instruct convI2S_reg_reg_b(iRegINoSp dst, iRegIorL2I src, immI_16 lshift, immI_16 rshift) %{ + predicate(UseZbb); + match(Set dst (RShiftI (LShiftI src lshift) rshift)); + + format %{ "sext.h $dst, $src\t# i2s, #@convI2S_reg_reg_b" %} + + ins_cost(ALU_COST); + ins_encode %{ + __ sext_h(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg); +%} + +// short to unsigned int +instruct convS2UI_reg_reg_b(iRegINoSp dst, iRegIorL2I src, immI_16bits mask) %{ + predicate(UseZbb); + match(Set dst (AndI src mask)); + + format %{ "zext.h $dst, $src\t# s2ui, #@convS2UI_reg_reg_b" %} + + ins_cost(ALU_COST); + ins_encode %{ + __ zext_h(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg); +%} + +// int to unsigned long (zero extend) +instruct convI2UL_reg_reg_b(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask) %{ + predicate(UseZba); + match(Set dst (AndL (ConvI2L src) mask)); + + format %{ "zext.w $dst, $src\t# i2ul, #@convI2UL_reg_reg_b" %} + + ins_cost(ALU_COST); + ins_encode %{ + __ zext_w(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg_shift); +%} + +// BSWAP instructions +instruct bytes_reverse_int_b(iRegINoSp dst, iRegIorL2I src) %{ + predicate(UseZbb); + match(Set dst (ReverseBytesI src)); + + ins_cost(ALU_COST * 2); + format %{ "revb_w_w $dst, $src\t#@bytes_reverse_int_b" %} + + ins_encode %{ + __ revb_w_w(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg); +%} + +instruct bytes_reverse_long_b(iRegLNoSp dst, iRegL src) %{ + predicate(UseZbb); + match(Set dst (ReverseBytesL src)); + + ins_cost(ALU_COST); + format %{ "rev8 $dst, $src\t#@bytes_reverse_long_b" %} + + ins_encode %{ + __ rev8(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg); +%} + +instruct bytes_reverse_unsigned_short_b(iRegINoSp dst, iRegIorL2I src) %{ + predicate(UseZbb); + match(Set dst (ReverseBytesUS src)); + + ins_cost(ALU_COST * 2); + format %{ "revb_h_h_u $dst, $src\t#@bytes_reverse_unsigned_short_b" %} + + ins_encode %{ + __ revb_h_h_u(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg); +%} + +instruct bytes_reverse_short_b(iRegINoSp dst, iRegIorL2I src) %{ + predicate(UseZbb); + match(Set dst (ReverseBytesS src)); + + ins_cost(ALU_COST * 2); + format %{ "revb_h_h $dst, $src\t#@bytes_reverse_short_b" %} + + ins_encode %{ + __ revb_h_h(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg); +%} + +// Shift Add Pointer +instruct shaddP_reg_reg_b(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale imm) %{ + predicate(UseZba); + match(Set dst (AddP src1 (LShiftL src2 imm))); + + ins_cost(ALU_COST); + format %{ "shadd $dst, $src2, $src1, $imm\t# ptr, #@shaddP_reg_reg_b" %} + + ins_encode %{ + __ shadd(as_Register($dst$$reg), + as_Register($src2$$reg), + as_Register($src1$$reg), + t0, + $imm$$constant); + %} + + ins_pipe(ialu_reg_reg); +%} + +instruct shaddP_reg_reg_ext_b(iRegPNoSp dst, iRegP src1, iRegI src2, immIScale imm) %{ + predicate(UseZba); + match(Set dst (AddP src1 (LShiftL (ConvI2L src2) imm))); + + ins_cost(ALU_COST); + format %{ "shadd $dst, $src2, $src1, $imm\t# ptr, #@shaddP_reg_reg_ext_b" %} + + ins_encode %{ + __ shadd(as_Register($dst$$reg), + as_Register($src2$$reg), + as_Register($src1$$reg), + t0, + $imm$$constant); + %} + + ins_pipe(ialu_reg_reg); +%} + +// Shift Add Long +instruct shaddL_reg_reg_b(iRegLNoSp dst, iRegL src1, iRegL src2, immIScale imm) %{ + predicate(UseZba); + match(Set dst (AddL src1 (LShiftL src2 imm))); + + ins_cost(ALU_COST); + format %{ "shadd $dst, $src2, $src1, $imm\t#@shaddL_reg_reg_b" %} + + ins_encode %{ + __ shadd(as_Register($dst$$reg), + as_Register($src2$$reg), + as_Register($src1$$reg), + t0, + $imm$$constant); + %} + + ins_pipe(ialu_reg_reg); +%} + +instruct shaddL_reg_reg_ext_b(iRegLNoSp dst, iRegL src1, iRegI src2, immIScale imm) %{ + predicate(UseZba); + match(Set dst (AddL src1 (LShiftL (ConvI2L src2) imm))); + + ins_cost(ALU_COST); + format %{ "shadd $dst, $src2, $src1, $imm\t#@shaddL_reg_reg_ext_b" %} + + ins_encode %{ + __ shadd(as_Register($dst$$reg), + as_Register($src2$$reg), + as_Register($src1$$reg), + t0, + $imm$$constant); + %} + + ins_pipe(ialu_reg_reg); +%} + +// Zeros Count instructions +instruct countLeadingZerosI_b(iRegINoSp dst, iRegIorL2I src) %{ + predicate(UseZbb); + match(Set dst (CountLeadingZerosI src)); + + ins_cost(ALU_COST); + format %{ "clzw $dst, $src\t#@countLeadingZerosI_b" %} + + ins_encode %{ + __ clzw(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg); +%} + +instruct countLeadingZerosL_b(iRegINoSp dst, iRegL src) %{ + predicate(UseZbb); + match(Set dst (CountLeadingZerosL src)); + + ins_cost(ALU_COST); + format %{ "clz $dst, $src\t#@countLeadingZerosL_b" %} + + ins_encode %{ + __ clz(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg); +%} + +instruct countTrailingZerosI_b(iRegINoSp dst, iRegIorL2I src) %{ + predicate(UseZbb); + match(Set dst (CountTrailingZerosI src)); + + ins_cost(ALU_COST); + format %{ "ctzw $dst, $src\t#@countTrailingZerosI_b" %} + + ins_encode %{ + __ ctzw(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg); +%} + +instruct countTrailingZerosL_b(iRegINoSp dst, iRegL src) %{ + predicate(UseZbb); + match(Set dst (CountTrailingZerosL src)); + + ins_cost(ALU_COST); + format %{ "ctz $dst, $src\t#@countTrailingZerosL_b" %} + + ins_encode %{ + __ ctz(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg); +%} + +// Population Count instructions +instruct popCountI_b(iRegINoSp dst, iRegIorL2I src) %{ + predicate(UsePopCountInstruction); + match(Set dst (PopCountI src)); + + ins_cost(ALU_COST); + format %{ "cpopw $dst, $src\t#@popCountI_b" %} + + ins_encode %{ + __ cpopw(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg); +%} + +// Note: Long/bitCount(long) returns an int. +instruct popCountL_b(iRegINoSp dst, iRegL src) %{ + predicate(UsePopCountInstruction); + match(Set dst (PopCountL src)); + + ins_cost(ALU_COST); + format %{ "cpop $dst, $src\t#@popCountL_b" %} + + ins_encode %{ + __ cpop(as_Register($dst$$reg), as_Register($src$$reg)); + %} + + ins_pipe(ialu_reg); +%} + +// Max and Min +instruct minI_reg_reg_b(iRegINoSp dst, iRegI src1, iRegI src2) %{ + predicate(UseZbb); + match(Set dst (MinI src1 src2)); + + ins_cost(ALU_COST); + format %{ "min $dst, $src1, $src2\t#@minI_reg_reg_b" %} + + ins_encode %{ + __ min(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg); +%} + +instruct maxI_reg_reg_b(iRegINoSp dst, iRegI src1, iRegI src2) %{ + predicate(UseZbb); + match(Set dst (MaxI src1 src2)); + + ins_cost(ALU_COST); + format %{ "max $dst, $src1, $src2\t#@maxI_reg_reg_b" %} + + ins_encode %{ + __ max(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg); +%} + +// special case for comparing with zero +// n.b. this is selected in preference to the rule above because it +// avoids loading constant 0 into a source register + +instruct minI_reg_zero_b(iRegINoSp dst, iRegI src1, immI0 zero) %{ + predicate(UseZbb); + match(Set dst (MinI src1 zero)); + match(Set dst (MinI zero src1)); + + ins_cost(ALU_COST); + format %{ "min $dst, $src1, zr\t#@minI_reg_zero_b" %} + + ins_encode %{ + __ min(as_Register($dst$$reg), as_Register($src1$$reg), zr); + %} + + ins_pipe(ialu_reg_reg); +%} + +instruct maxI_reg_zero_b(iRegINoSp dst, iRegI src1, immI0 zero) %{ + predicate(UseZbb); + match(Set dst (MaxI src1 zero)); + match(Set dst (MaxI zero src1)); + + ins_cost(ALU_COST); + format %{ "max $dst, $src1, zr\t#@maxI_reg_zero_b" %} + + ins_encode %{ + __ max(as_Register($dst$$reg), as_Register($src1$$reg), zr); + %} + + ins_pipe(ialu_reg_reg); +%} + +// Abs +instruct absI_reg_b(iRegINoSp dst, iRegI src) %{ + predicate(UseZbb); + match(Set dst (AbsI src)); + + ins_cost(ALU_COST * 2); + format %{ + "negw t0, $src\n\t" + "max $dst, $src, t0\t#@absI_reg_b" + %} + + ins_encode %{ + __ negw(t0, as_Register($src$$reg)); + __ max(as_Register($dst$$reg), as_Register($src$$reg), t0); + %} + + ins_pipe(ialu_reg_reg); +%} + +instruct absL_reg_b(iRegLNoSp dst, iRegL src) %{ + predicate(UseZbb); + match(Set dst (AbsL src)); + + ins_cost(ALU_COST * 2); + format %{ + "neg t0, $src\n\t" + "max $dst, $src, t0\t#@absL_reg_b" + %} + + ins_encode %{ + __ neg(t0, as_Register($src$$reg)); + __ max(as_Register($dst$$reg), as_Register($src$$reg), t0); + %} + + ins_pipe(ialu_reg); +%} + +// And Not +instruct andnI_reg_reg_b(iRegINoSp dst, iRegI src1, iRegI src2, immI_M1 m1) %{ + predicate(UseZbb); + match(Set dst (AndI src1 (XorI src2 m1))); + + ins_cost(ALU_COST); + format %{ "andn $dst, $src1, $src2\t#@andnI_reg_reg_b" %} + + ins_encode %{ + __ andn(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg); +%} + +instruct andnL_reg_reg_b(iRegLNoSp dst, iRegL src1, iRegL src2, immL_M1 m1) %{ + predicate(UseZbb); + match(Set dst (AndL src1 (XorL src2 m1))); + + ins_cost(ALU_COST); + format %{ "andn $dst, $src1, $src2\t#@andnL_reg_reg_b" %} + + ins_encode %{ + __ andn(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg); +%} + +// Or Not +instruct ornI_reg_reg_b(iRegINoSp dst, iRegI src1, iRegI src2, immI_M1 m1) %{ + predicate(UseZbb); + match(Set dst (OrI src1 (XorI src2 m1))); + + ins_cost(ALU_COST); + format %{ "orn $dst, $src1, $src2\t#@ornI_reg_reg_b" %} + + ins_encode %{ + __ orn(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg); +%} + +instruct ornL_reg_reg_b(iRegLNoSp dst, iRegL src1, iRegL src2, immL_M1 m1) %{ + predicate(UseZbb); + match(Set dst (OrL src1 (XorL src2 m1))); + + ins_cost(ALU_COST); + format %{ "orn $dst, $src1, $src2\t#@ornL_reg_reg_b" %} + + ins_encode %{ + __ orn(as_Register($dst$$reg), + as_Register($src1$$reg), + as_Register($src2$$reg)); + %} + + ins_pipe(ialu_reg_reg); +%} + +// AndI 0b0..010..0 + ConvI2B +instruct convI2Bool_andI_reg_immIpowerOf2(iRegINoSp dst, iRegIorL2I src, immIpowerOf2 mask) %{ + predicate(UseZbs); + match(Set dst (Conv2B (AndI src mask))); + ins_cost(ALU_COST); + + format %{ "bexti $dst, $src, $mask\t#@convI2Bool_andI_reg_immIpowerOf2" %} + ins_encode %{ + __ bexti($dst$$Register, $src$$Register, exact_log2((juint)($mask$$constant))); + %} + + ins_pipe(ialu_reg_reg); +%} \ No newline at end of file diff --git a/src/hotspot/cpu/riscv/riscv_v.ad b/src/hotspot/cpu/riscv/riscv_v.ad new file mode 100644 index 0000000000000..4359ea7c34bac --- /dev/null +++ b/src/hotspot/cpu/riscv/riscv_v.ad @@ -0,0 +1,2038 @@ +// +// Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2020, Arm Limited. All rights reserved. +// Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. +// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +// +// This code is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License version 2 only, as +// published by the Free Software Foundation. +// +// This code is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// version 2 for more details (a copy is included in the LICENSE file that +// accompanied this code). +// +// You should have received a copy of the GNU General Public License version +// 2 along with this work; if not, write to the Free Software Foundation, +// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +// +// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +// or visit www.oracle.com if you need additional information or have any +// questions. +// +// + +// RISCV Vector Extension Architecture Description File + +opclass vmemA(indirect); + +source_hpp %{ + bool op_vec_supported(int opcode); +%} + +source %{ + static inline BasicType vector_element_basic_type(const MachNode* n) { + const TypeVect* vt = n->bottom_type()->is_vect(); + return vt->element_basic_type(); + } + + static inline BasicType vector_element_basic_type(const MachNode* use, const MachOper* opnd) { + int def_idx = use->operand_index(opnd); + Node* def = use->in(def_idx); + const TypeVect* vt = def->bottom_type()->is_vect(); + return vt->element_basic_type(); + } + + static void loadStore(C2_MacroAssembler masm, bool is_store, + VectorRegister reg, BasicType bt, Register base) { + Assembler::SEW sew = Assembler::elemtype_to_sew(bt); + masm.vsetvli(t0, x0, sew); + if (is_store) { + masm.vsex_v(reg, base, sew); + } else { + masm.vlex_v(reg, base, sew); + } + } + + bool op_vec_supported(int opcode) { + switch (opcode) { + // No multiply reduction instructions + case Op_MulReductionVD: + case Op_MulReductionVF: + case Op_MulReductionVI: + case Op_MulReductionVL: + // Others + case Op_Extract: + case Op_ExtractB: + case Op_ExtractC: + case Op_ExtractD: + case Op_ExtractF: + case Op_ExtractI: + case Op_ExtractL: + case Op_ExtractS: + case Op_ExtractUB: + // Vector API specific + case Op_AndReductionV: + case Op_OrReductionV: + case Op_XorReductionV: + case Op_LoadVectorGather: + case Op_StoreVectorScatter: + case Op_VectorBlend: + case Op_VectorCast: + case Op_VectorCastB2X: + case Op_VectorCastD2X: + case Op_VectorCastF2X: + case Op_VectorCastI2X: + case Op_VectorCastL2X: + case Op_VectorCastS2X: + case Op_VectorInsert: + case Op_VectorLoadConst: + case Op_VectorLoadMask: + case Op_VectorLoadShuffle: + case Op_VectorMaskCmp: + case Op_VectorRearrange: + case Op_VectorReinterpret: + case Op_VectorStoreMask: + case Op_VectorTest: + case Op_PopCountVI: + return false; + default: + return UseRVV; + } + } + +%} + +definitions %{ + int_def VEC_COST (200, 200); +%} + +// All VEC instructions + +// vector load/store +instruct loadV(vReg dst, vmemA mem) %{ + match(Set dst (LoadVector mem)); + ins_cost(VEC_COST); + format %{ "vle $dst, $mem\t#@loadV" %} + ins_encode %{ + VectorRegister dst_reg = as_VectorRegister($dst$$reg); + loadStore(C2_MacroAssembler(&cbuf), false, dst_reg, + vector_element_basic_type(this), as_Register($mem$$base)); + %} + ins_pipe(pipe_slow); +%} + +instruct storeV(vReg src, vmemA mem) %{ + match(Set mem (StoreVector mem src)); + ins_cost(VEC_COST); + format %{ "vse $src, $mem\t#@storeV" %} + ins_encode %{ + VectorRegister src_reg = as_VectorRegister($src$$reg); + loadStore(C2_MacroAssembler(&cbuf), true, src_reg, + vector_element_basic_type(this, $src), as_Register($mem$$base)); + %} + ins_pipe(pipe_slow); +%} + +// vector abs + +instruct vabsB(vReg dst, vReg src, vReg tmp) %{ + match(Set dst (AbsVB src)); + ins_cost(VEC_COST); + effect(TEMP tmp); + format %{ "vrsub.vi $tmp, 0, $src\t#@vabsB\n\t" + "vmax.vv $dst, $tmp, $src" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e8); + __ vrsub_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($src$$reg), 0); + __ vmax_vv(as_VectorRegister($dst$$reg), as_VectorRegister($tmp$$reg), as_VectorRegister($src$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vabsS(vReg dst, vReg src, vReg tmp) %{ + match(Set dst (AbsVS src)); + ins_cost(VEC_COST); + effect(TEMP tmp); + format %{ "vrsub.vi $tmp, 0, $src\t#@vabsS\n\t" + "vmax.vv $dst, $tmp, $src" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e16); + __ vrsub_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($src$$reg), 0); + __ vmax_vv(as_VectorRegister($dst$$reg), as_VectorRegister($tmp$$reg), as_VectorRegister($src$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vabsI(vReg dst, vReg src, vReg tmp) %{ + match(Set dst (AbsVI src)); + ins_cost(VEC_COST); + effect(TEMP tmp); + format %{ "vrsub.vi $tmp, 0, $src\t#@vabsI\n\t" + "vmax.vv $dst, $tmp, $src" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vrsub_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($src$$reg), 0); + __ vmax_vv(as_VectorRegister($dst$$reg), as_VectorRegister($tmp$$reg), as_VectorRegister($src$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vabsL(vReg dst, vReg src, vReg tmp) %{ + match(Set dst (AbsVL src)); + ins_cost(VEC_COST); + effect(TEMP tmp); + format %{ "vrsub.vi $tmp, 0, $src\t#@vabsL\n\t" + "vmax.vv $dst, $tmp, $src" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vrsub_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($src$$reg), 0); + __ vmax_vv(as_VectorRegister($dst$$reg), as_VectorRegister($tmp$$reg), as_VectorRegister($src$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vabsF(vReg dst, vReg src) %{ + match(Set dst (AbsVF src)); + ins_cost(VEC_COST); + format %{ "vfsgnjx.vv $dst, $src, $src, vm\t#@vabsF" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vfsgnjx_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), as_VectorRegister($src$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vabsD(vReg dst, vReg src) %{ + match(Set dst (AbsVD src)); + ins_cost(VEC_COST); + format %{ "vfsgnjx.vv $dst, $src, $src, vm\t#@vabsD" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vfsgnjx_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), as_VectorRegister($src$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector add + +instruct vaddB(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (AddVB src1 src2)); + ins_cost(VEC_COST); + format %{ "vadd.vv $dst, $src1, $src2\t#@vaddB" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e8); + __ vadd_vv(as_VectorRegister($dst$$reg), + as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vaddS(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (AddVS src1 src2)); + ins_cost(VEC_COST); + format %{ "vadd.vv $dst, $src1, $src2\t#@vaddS" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e16); + __ vadd_vv(as_VectorRegister($dst$$reg), + as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vaddI(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (AddVI src1 src2)); + ins_cost(VEC_COST); + format %{ "vadd.vv $dst, $src1, $src2\t#@vaddI" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vadd_vv(as_VectorRegister($dst$$reg), + as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vaddL(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (AddVL src1 src2)); + ins_cost(VEC_COST); + format %{ "vadd.vv $dst, $src1, $src2\t#@vaddL" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vadd_vv(as_VectorRegister($dst$$reg), + as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vaddF(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (AddVF src1 src2)); + ins_cost(VEC_COST); + format %{ "vfadd.vv $dst, $src1, $src2\t#@vaddF" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vfadd_vv(as_VectorRegister($dst$$reg), + as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vaddD(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (AddVD src1 src2)); + ins_cost(VEC_COST); + format %{ "vfadd.vv $dst, $src1, $src2\t#@vaddD" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vfadd_vv(as_VectorRegister($dst$$reg), + as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector and + +instruct vand(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (AndV src1 src2)); + ins_cost(VEC_COST); + format %{ "vand.vv $dst, $src1, $src2\t#@vand" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vand_vv(as_VectorRegister($dst$$reg), + as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector or + +instruct vor(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (OrV src1 src2)); + ins_cost(VEC_COST); + format %{ "vor.vv $dst, $src1, $src2\t#@vor" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vor_vv(as_VectorRegister($dst$$reg), + as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector xor + +instruct vxor(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (XorV src1 src2)); + ins_cost(VEC_COST); + format %{ "vxor.vv $dst, $src1, $src2\t#@vxor" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vxor_vv(as_VectorRegister($dst$$reg), + as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector float div + +instruct vdivF(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (DivVF src1 src2)); + ins_cost(VEC_COST); + format %{ "vfdiv.vv $dst, $src1, $src2\t#@vdivF" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vfdiv_vv(as_VectorRegister($dst$$reg), + as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vdivD(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (DivVD src1 src2)); + ins_cost(VEC_COST); + format %{ "vfdiv.vv $dst, $src1, $src2\t#@vdivD" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vfdiv_vv(as_VectorRegister($dst$$reg), + as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector integer max/min + +instruct vmax(vReg dst, vReg src1, vReg src2) %{ + predicate(n->bottom_type()->is_vect()->element_basic_type() != T_FLOAT && + n->bottom_type()->is_vect()->element_basic_type() != T_DOUBLE); + match(Set dst (MaxV src1 src2)); + ins_cost(VEC_COST); + format %{ "vmax.vv $dst, $src1, $src2\t#@vmax" %} + ins_encode %{ + BasicType bt = vector_element_basic_type(this); + Assembler::SEW sew = Assembler::elemtype_to_sew(bt); + __ vsetvli(t0, x0, sew); + __ vmax_vv(as_VectorRegister($dst$$reg), + as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vmin(vReg dst, vReg src1, vReg src2) %{ + predicate(n->bottom_type()->is_vect()->element_basic_type() != T_FLOAT && + n->bottom_type()->is_vect()->element_basic_type() != T_DOUBLE); + match(Set dst (MinV src1 src2)); + ins_cost(VEC_COST); + format %{ "vmin.vv $dst, $src1, $src2\t#@vmin" %} + ins_encode %{ + BasicType bt = vector_element_basic_type(this); + Assembler::SEW sew = Assembler::elemtype_to_sew(bt); + __ vsetvli(t0, x0, sew); + __ vmin_vv(as_VectorRegister($dst$$reg), + as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector float-point max/min + +instruct vmaxF(vReg dst, vReg src1, vReg src2) %{ + predicate(n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT); + match(Set dst (MaxV src1 src2)); + effect(TEMP_DEF dst); + ins_cost(VEC_COST); + format %{ "vmaxF $dst, $src1, $src2\t#@vmaxF" %} + ins_encode %{ + __ minmax_FD_v(as_VectorRegister($dst$$reg), + as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg), + false /* is_double */, false /* is_min */); + %} + ins_pipe(pipe_slow); +%} + +instruct vmaxD(vReg dst, vReg src1, vReg src2) %{ + predicate(n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE); + match(Set dst (MaxV src1 src2)); + effect(TEMP_DEF dst); + ins_cost(VEC_COST); + format %{ "vmaxD $dst, $src1, $src2\t#@vmaxD" %} + ins_encode %{ + __ minmax_FD_v(as_VectorRegister($dst$$reg), + as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg), + true /* is_double */, false /* is_min */); + %} + ins_pipe(pipe_slow); +%} + +instruct vminF(vReg dst, vReg src1, vReg src2) %{ + predicate(n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT); + match(Set dst (MinV src1 src2)); + effect(TEMP_DEF dst); + ins_cost(VEC_COST); + format %{ "vminF $dst, $src1, $src2\t#@vminF" %} + ins_encode %{ + __ minmax_FD_v(as_VectorRegister($dst$$reg), + as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg), + false /* is_double */, true /* is_min */); + %} + ins_pipe(pipe_slow); +%} + +instruct vminD(vReg dst, vReg src1, vReg src2) %{ + predicate(n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE); + match(Set dst (MinV src1 src2)); + effect(TEMP_DEF dst); + ins_cost(VEC_COST); + format %{ "vminD $dst, $src1, $src2\t#@vminD" %} + ins_encode %{ + __ minmax_FD_v(as_VectorRegister($dst$$reg), + as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg), + true /* is_double */, true /* is_min */); + %} + ins_pipe(pipe_slow); +%} + +// vector fmla + +// dst_src1 = dst_src1 + src2 * src3 +instruct vfmlaF(vReg dst_src1, vReg src2, vReg src3) %{ + predicate(UseFMA); + match(Set dst_src1 (FmaVF dst_src1 (Binary src2 src3))); + ins_cost(VEC_COST); + format %{ "vfmacc.vv $dst_src1, $src2, $src3\t#@vfmlaF" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vfmacc_vv(as_VectorRegister($dst_src1$$reg), + as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// dst_src1 = dst_src1 + src2 * src3 +instruct vfmlaD(vReg dst_src1, vReg src2, vReg src3) %{ + predicate(UseFMA); + match(Set dst_src1 (FmaVD dst_src1 (Binary src2 src3))); + ins_cost(VEC_COST); + format %{ "vfmacc.vv $dst_src1, $src2, $src3\t#@vfmlaD" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vfmacc_vv(as_VectorRegister($dst_src1$$reg), + as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector fmls + +// dst_src1 = dst_src1 + -src2 * src3 +// dst_src1 = dst_src1 + src2 * -src3 +instruct vfmlsF(vReg dst_src1, vReg src2, vReg src3) %{ + predicate(UseFMA); + match(Set dst_src1 (FmaVF dst_src1 (Binary (NegVF src2) src3))); + match(Set dst_src1 (FmaVF dst_src1 (Binary src2 (NegVF src3)))); + ins_cost(VEC_COST); + format %{ "vfnmsac.vv $dst_src1, $src2, $src3\t#@vfmlsF" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vfnmsac_vv(as_VectorRegister($dst_src1$$reg), + as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// dst_src1 = dst_src1 + -src2 * src3 +// dst_src1 = dst_src1 + src2 * -src3 +instruct vfmlsD(vReg dst_src1, vReg src2, vReg src3) %{ + predicate(UseFMA); + match(Set dst_src1 (FmaVD dst_src1 (Binary (NegVD src2) src3))); + match(Set dst_src1 (FmaVD dst_src1 (Binary src2 (NegVD src3)))); + ins_cost(VEC_COST); + format %{ "vfnmsac.vv $dst_src1, $src2, $src3\t#@vfmlsD" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vfnmsac_vv(as_VectorRegister($dst_src1$$reg), + as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector fnmla + +// dst_src1 = -dst_src1 + -src2 * src3 +// dst_src1 = -dst_src1 + src2 * -src3 +instruct vfnmlaF(vReg dst_src1, vReg src2, vReg src3) %{ + predicate(UseFMA); + match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary (NegVF src2) src3))); + match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 (NegVF src3)))); + ins_cost(VEC_COST); + format %{ "vfnmacc.vv $dst_src1, $src2, $src3\t#@vfnmlaF" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vfnmacc_vv(as_VectorRegister($dst_src1$$reg), + as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// dst_src1 = -dst_src1 + -src2 * src3 +// dst_src1 = -dst_src1 + src2 * -src3 +instruct vfnmlaD(vReg dst_src1, vReg src2, vReg src3) %{ + predicate(UseFMA); + match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary (NegVD src2) src3))); + match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 (NegVD src3)))); + ins_cost(VEC_COST); + format %{ "vfnmacc.vv $dst_src1, $src2, $src3\t#@vfnmlaD" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vfnmacc_vv(as_VectorRegister($dst_src1$$reg), + as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector fnmls + +// dst_src1 = -dst_src1 + src2 * src3 +instruct vfnmlsF(vReg dst_src1, vReg src2, vReg src3) %{ + predicate(UseFMA); + match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 src3))); + ins_cost(VEC_COST); + format %{ "vfmsac.vv $dst_src1, $src2, $src3\t#@vfnmlsF" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vfmsac_vv(as_VectorRegister($dst_src1$$reg), + as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// dst_src1 = -dst_src1 + src2 * src3 +instruct vfnmlsD(vReg dst_src1, vReg src2, vReg src3) %{ + predicate(UseFMA); + match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 src3))); + ins_cost(VEC_COST); + format %{ "vfmsac.vv $dst_src1, $src2, $src3\t#@vfnmlsD" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vfmsac_vv(as_VectorRegister($dst_src1$$reg), + as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector mla + +// dst_src1 = dst_src1 + src2 * src3 +instruct vmlaB(vReg dst_src1, vReg src2, vReg src3) %{ + match(Set dst_src1 (AddVB dst_src1 (MulVB src2 src3))); + ins_cost(VEC_COST); + format %{ "vmacc.vv $dst_src1, src2, src3\t#@vmlaB" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e8); + __ vmacc_vv(as_VectorRegister($dst_src1$$reg), + as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// dst_src1 = dst_src1 + src2 * src3 +instruct vmlaS(vReg dst_src1, vReg src2, vReg src3) %{ + match(Set dst_src1 (AddVS dst_src1 (MulVS src2 src3))); + ins_cost(VEC_COST); + format %{ "vmacc.vv $dst_src1, src2, src3\t#@vmlaS" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e16); + __ vmacc_vv(as_VectorRegister($dst_src1$$reg), + as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// dst_src1 = dst_src1 + src2 * src3 +instruct vmlaI(vReg dst_src1, vReg src2, vReg src3) %{ + match(Set dst_src1 (AddVI dst_src1 (MulVI src2 src3))); + ins_cost(VEC_COST); + format %{ "vmacc.vv $dst_src1, src2, src3\t#@vmlaI" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vmacc_vv(as_VectorRegister($dst_src1$$reg), + as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// dst_src1 = dst_src1 + src2 * src3 +instruct vmlaL(vReg dst_src1, vReg src2, vReg src3) %{ + match(Set dst_src1 (AddVL dst_src1 (MulVL src2 src3))); + ins_cost(VEC_COST); + format %{ "vmacc.vv $dst_src1, src2, src3\t#@vmlaL" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vmacc_vv(as_VectorRegister($dst_src1$$reg), + as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector mls + +// dst_src1 = dst_src1 - src2 * src3 +instruct vmlsB(vReg dst_src1, vReg src2, vReg src3) %{ + match(Set dst_src1 (SubVB dst_src1 (MulVB src2 src3))); + ins_cost(VEC_COST); + format %{ "vnmsac.vv $dst_src1, src2, src3\t#@vmlsB" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e8); + __ vnmsac_vv(as_VectorRegister($dst_src1$$reg), + as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// dst_src1 = dst_src1 - src2 * src3 +instruct vmlsS(vReg dst_src1, vReg src2, vReg src3) %{ + match(Set dst_src1 (SubVS dst_src1 (MulVS src2 src3))); + ins_cost(VEC_COST); + format %{ "vnmsac.vv $dst_src1, src2, src3\t#@vmlsS" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e16); + __ vnmsac_vv(as_VectorRegister($dst_src1$$reg), + as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// dst_src1 = dst_src1 - src2 * src3 +instruct vmlsI(vReg dst_src1, vReg src2, vReg src3) %{ + match(Set dst_src1 (SubVI dst_src1 (MulVI src2 src3))); + ins_cost(VEC_COST); + format %{ "vnmsac.vv $dst_src1, src2, src3\t#@vmlsI" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vnmsac_vv(as_VectorRegister($dst_src1$$reg), + as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// dst_src1 = dst_src1 - src2 * src3 +instruct vmlsL(vReg dst_src1, vReg src2, vReg src3) %{ + match(Set dst_src1 (SubVL dst_src1 (MulVL src2 src3))); + ins_cost(VEC_COST); + format %{ "vnmsac.vv $dst_src1, src2, src3\t#@vmlsL" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vnmsac_vv(as_VectorRegister($dst_src1$$reg), + as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector mul + +instruct vmulB(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (MulVB src1 src2)); + ins_cost(VEC_COST); + format %{ "vmul.vv $dst, $src1, $src2\t#@vmulB" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e8); + __ vmul_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vmulS(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (MulVS src1 src2)); + ins_cost(VEC_COST); + format %{ "vmul.vv $dst, $src1, $src2\t#@vmulS" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e16); + __ vmul_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vmulI(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (MulVI src1 src2)); + ins_cost(VEC_COST); + format %{ "vmul.vv $dst, $src1, $src2\t#@vmulI" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vmul_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vmulL(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (MulVL src1 src2)); + ins_cost(VEC_COST); + format %{ "vmul.vv $dst, $src1, $src2\t#@vmulL" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vmul_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vmulF(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (MulVF src1 src2)); + ins_cost(VEC_COST); + format %{ "vfmul.vv $dst, $src1, $src2\t#@vmulF" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vfmul_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vmulD(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (MulVD src1 src2)); + ins_cost(VEC_COST); + format %{ "vfmul.vv $dst, $src1, $src2\t#@vmulD" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vfmul_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector fneg + +instruct vnegF(vReg dst, vReg src) %{ + match(Set dst (NegVF src)); + ins_cost(VEC_COST); + format %{ "vfsgnjn.vv $dst, $src, $src\t#@vnegF" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vfneg_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vnegD(vReg dst, vReg src) %{ + match(Set dst (NegVD src)); + ins_cost(VEC_COST); + format %{ "vfsgnjn.vv $dst, $src, $src\t#@vnegD" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vfneg_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector add reduction + +instruct reduce_addB(iRegINoSp dst, iRegIorL2I src1, vReg src2, vReg tmp) %{ + predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE); + match(Set dst (AddReductionVI src1 src2)); + effect(TEMP tmp); + ins_cost(VEC_COST); + format %{ "vmv.s.x $tmp, $src1\t#@reduce_addB\n\t" + "vredsum.vs $tmp, $src2, $tmp\n\t" + "vmv.x.s $dst, $tmp" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e8); + __ vmv_s_x(as_VectorRegister($tmp$$reg), $src1$$Register); + __ vredsum_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg), + as_VectorRegister($tmp$$reg)); + __ vmv_x_s($dst$$Register, as_VectorRegister($tmp$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct reduce_addS(iRegINoSp dst, iRegIorL2I src1, vReg src2, vReg tmp) %{ + predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT); + match(Set dst (AddReductionVI src1 src2)); + effect(TEMP tmp); + ins_cost(VEC_COST); + format %{ "vmv.s.x $tmp, $src1\t#@reduce_addS\n\t" + "vredsum.vs $tmp, $src2, $tmp\n\t" + "vmv.x.s $dst, $tmp" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e16); + __ vmv_s_x(as_VectorRegister($tmp$$reg), $src1$$Register); + __ vredsum_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg), + as_VectorRegister($tmp$$reg)); + __ vmv_x_s($dst$$Register, as_VectorRegister($tmp$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct reduce_addI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vReg tmp) %{ + predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT); + match(Set dst (AddReductionVI src1 src2)); + effect(TEMP tmp); + ins_cost(VEC_COST); + format %{ "vmv.s.x $tmp, $src1\t#@reduce_addI\n\t" + "vredsum.vs $tmp, $src2, $tmp\n\t" + "vmv.x.s $dst, $tmp" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vmv_s_x(as_VectorRegister($tmp$$reg), $src1$$Register); + __ vredsum_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg), + as_VectorRegister($tmp$$reg)); + __ vmv_x_s($dst$$Register, as_VectorRegister($tmp$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct reduce_addL(iRegLNoSp dst, iRegL src1, vReg src2, vReg tmp) %{ + predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG); + match(Set dst (AddReductionVL src1 src2)); + effect(TEMP tmp); + ins_cost(VEC_COST); + format %{ "vmv.s.x $tmp, $src1\t#@reduce_addL\n\t" + "vredsum.vs $tmp, $src2, $tmp\n\t" + "vmv.x.s $dst, $tmp" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vmv_s_x(as_VectorRegister($tmp$$reg), $src1$$Register); + __ vredsum_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg), + as_VectorRegister($tmp$$reg)); + __ vmv_x_s($dst$$Register, as_VectorRegister($tmp$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct reduce_addF(fRegF src1_dst, vReg src2, vReg tmp) %{ + match(Set src1_dst (AddReductionVF src1_dst src2)); + effect(TEMP tmp); + ins_cost(VEC_COST); + format %{ "vfmv.s.f $tmp, $src1_dst\t#@reduce_addF\n\t" + "vfredosum.vs $tmp, $src2, $tmp\n\t" + "vfmv.f.s $src1_dst, $tmp" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vfmv_s_f(as_VectorRegister($tmp$$reg), $src1_dst$$FloatRegister); + __ vfredosum_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg), + as_VectorRegister($tmp$$reg)); + __ vfmv_f_s($src1_dst$$FloatRegister, as_VectorRegister($tmp$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct reduce_addD(fRegD src1_dst, vReg src2, vReg tmp) %{ + match(Set src1_dst (AddReductionVD src1_dst src2)); + effect(TEMP tmp); + ins_cost(VEC_COST); + format %{ "vfmv.s.f $tmp, $src1_dst\t#@reduce_addD\n\t" + "vfredosum.vs $tmp, $src2, $tmp\n\t" + "vfmv.f.s $src1_dst, $tmp" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vfmv_s_f(as_VectorRegister($tmp$$reg), $src1_dst$$FloatRegister); + __ vfredosum_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg), + as_VectorRegister($tmp$$reg)); + __ vfmv_f_s($src1_dst$$FloatRegister, as_VectorRegister($tmp$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector integer max reduction +instruct vreduce_maxB(iRegINoSp dst, iRegI src1, vReg src2, vReg tmp) %{ + predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE); + match(Set dst (MaxReductionV src1 src2)); + ins_cost(VEC_COST); + effect(TEMP tmp); + format %{ "vreduce_maxB $dst, $src1, $src2, $tmp" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e8); + __ vredmax_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg), as_VectorRegister($src2$$reg)); + __ vmv_x_s($dst$$Register, as_VectorRegister($tmp$$reg)); + Label Ldone; + __ ble(as_Register($src1$$reg), as_Register($dst$$reg), Ldone); + __ mv(as_Register($dst$$reg), as_Register($src1$$reg)); + __ bind(Ldone); + %} + ins_pipe(pipe_slow); +%} + +instruct vreduce_maxS(iRegINoSp dst, iRegI src1, vReg src2, vReg tmp) %{ + predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT); + match(Set dst (MaxReductionV src1 src2)); + ins_cost(VEC_COST); + effect(TEMP tmp); + format %{ "vreduce_maxS $dst, $src1, $src2, $tmp" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e16); + __ vredmax_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg), as_VectorRegister($src2$$reg)); + __ vmv_x_s($dst$$Register, as_VectorRegister($tmp$$reg)); + Label Ldone; + __ ble(as_Register($src1$$reg), as_Register($dst$$reg), Ldone); + __ mv(as_Register($dst$$reg), as_Register($src1$$reg)); + __ bind(Ldone); + %} + ins_pipe(pipe_slow); +%} + +instruct vreduce_maxI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vReg tmp) %{ + predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT); + match(Set dst (MaxReductionV src1 src2)); + ins_cost(VEC_COST); + effect(TEMP tmp); + format %{ "vreduce_maxI $dst, $src1, $src2, $tmp" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vmv_s_x(as_VectorRegister($tmp$$reg), $src1$$Register); + __ vredmax_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg)); + __ vmv_x_s($dst$$Register, as_VectorRegister($tmp$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vreduce_maxL(iRegLNoSp dst, iRegL src1, vReg src2, vReg tmp) %{ + predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG); + match(Set dst (MaxReductionV src1 src2)); + ins_cost(VEC_COST); + effect(TEMP tmp); + format %{ "vreduce_maxL $dst, $src1, $src2, $tmp" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vmv_s_x(as_VectorRegister($tmp$$reg), $src1$$Register); + __ vredmax_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg)); + __ vmv_x_s($dst$$Register, as_VectorRegister($tmp$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector integer min reduction +instruct vreduce_minB(iRegINoSp dst, iRegI src1, vReg src2, vReg tmp) %{ + predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE); + match(Set dst (MinReductionV src1 src2)); + ins_cost(VEC_COST); + effect(TEMP tmp); + format %{ "vreduce_minB $dst, $src1, $src2, $tmp" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e8); + __ vredmin_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg), as_VectorRegister($src2$$reg)); + __ vmv_x_s($dst$$Register, as_VectorRegister($tmp$$reg)); + Label Ldone; + __ bge(as_Register($src1$$reg), as_Register($dst$$reg), Ldone); + __ mv(as_Register($dst$$reg), as_Register($src1$$reg)); + __ bind(Ldone); + %} + ins_pipe(pipe_slow); +%} + +instruct vreduce_minS(iRegINoSp dst, iRegI src1, vReg src2, vReg tmp) %{ + predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT); + match(Set dst (MinReductionV src1 src2)); + ins_cost(VEC_COST); + effect(TEMP tmp); + format %{ "vreduce_minS $dst, $src1, $src2, $tmp" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e16); + __ vredmin_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg), as_VectorRegister($src2$$reg)); + __ vmv_x_s($dst$$Register, as_VectorRegister($tmp$$reg)); + Label Ldone; + __ bge(as_Register($src1$$reg), as_Register($dst$$reg), Ldone); + __ mv(as_Register($dst$$reg), as_Register($src1$$reg)); + __ bind(Ldone); + %} + ins_pipe(pipe_slow); +%} + +instruct vreduce_minI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vReg tmp) %{ + predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT); + match(Set dst (MinReductionV src1 src2)); + ins_cost(VEC_COST); + effect(TEMP tmp); + format %{ "vreduce_minI $dst, $src1, $src2, $tmp" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vmv_s_x(as_VectorRegister($tmp$$reg), $src1$$Register); + __ vredmin_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg)); + __ vmv_x_s($dst$$Register, as_VectorRegister($tmp$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vreduce_minL(iRegLNoSp dst, iRegL src1, vReg src2, vReg tmp) %{ + predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG); + match(Set dst (MinReductionV src1 src2)); + ins_cost(VEC_COST); + effect(TEMP tmp); + format %{ "vreduce_minL $dst, $src1, $src2, $tmp" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vmv_s_x(as_VectorRegister($tmp$$reg), $src1$$Register); + __ vredmin_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg)); + __ vmv_x_s($dst$$Register, as_VectorRegister($tmp$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector float max reduction + +instruct vreduce_maxF(fRegF dst, fRegF src1, vReg src2, vReg tmp1, vReg tmp2) %{ + predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT); + match(Set dst (MaxReductionV src1 src2)); + ins_cost(VEC_COST); + effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2); + format %{ "reduce_maxF $dst, $src1, $src2, $tmp1, $tmp2" %} + ins_encode %{ + __ reduce_minmax_FD_v($dst$$FloatRegister, + $src1$$FloatRegister, as_VectorRegister($src2$$reg), + as_VectorRegister($tmp1$$reg), as_VectorRegister($tmp2$$reg), + false /* is_double */, false /* is_min */); + %} + ins_pipe(pipe_slow); +%} + +instruct vreduce_maxD(fRegD dst, fRegD src1, vReg src2, vReg tmp1, vReg tmp2) %{ + predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE); + match(Set dst (MaxReductionV src1 src2)); + ins_cost(VEC_COST); + effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2); + format %{ "reduce_maxD $dst, $src1, $src2, $tmp1, $tmp2" %} + ins_encode %{ + __ reduce_minmax_FD_v($dst$$FloatRegister, + $src1$$FloatRegister, as_VectorRegister($src2$$reg), + as_VectorRegister($tmp1$$reg), as_VectorRegister($tmp2$$reg), + true /* is_double */, false /* is_min */); + %} + ins_pipe(pipe_slow); +%} + +// vector float min reduction + +instruct vreduce_minF(fRegF dst, fRegF src1, vReg src2, vReg tmp1, vReg tmp2) %{ + predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT); + match(Set dst (MinReductionV src1 src2)); + ins_cost(VEC_COST); + effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2); + format %{ "reduce_minF $dst, $src1, $src2, $tmp1, $tmp2" %} + ins_encode %{ + __ reduce_minmax_FD_v($dst$$FloatRegister, + $src1$$FloatRegister, as_VectorRegister($src2$$reg), + as_VectorRegister($tmp1$$reg), as_VectorRegister($tmp2$$reg), + false /* is_double */, true /* is_min */); + %} + ins_pipe(pipe_slow); +%} + +instruct vreduce_minD(fRegD dst, fRegD src1, vReg src2, vReg tmp1, vReg tmp2) %{ + predicate(n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE); + match(Set dst (MinReductionV src1 src2)); + ins_cost(VEC_COST); + effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2); + format %{ "reduce_minD $dst, $src1, $src2, $tmp1, $tmp2" %} + ins_encode %{ + __ reduce_minmax_FD_v($dst$$FloatRegister, + $src1$$FloatRegister, as_VectorRegister($src2$$reg), + as_VectorRegister($tmp1$$reg), as_VectorRegister($tmp2$$reg), + true /* is_double */, true /* is_min */); + %} + ins_pipe(pipe_slow); +%} + +// vector replicate + +instruct replicateB(vReg dst, iRegIorL2I src) %{ + match(Set dst (ReplicateB src)); + ins_cost(VEC_COST); + format %{ "vmv.v.x $dst, $src\t#@replicateB" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e8); + __ vmv_v_x(as_VectorRegister($dst$$reg), as_Register($src$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct replicateS(vReg dst, iRegIorL2I src) %{ + match(Set dst (ReplicateS src)); + ins_cost(VEC_COST); + format %{ "vmv.v.x $dst, $src\t#@replicateS" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e16); + __ vmv_v_x(as_VectorRegister($dst$$reg), as_Register($src$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct replicateI(vReg dst, iRegIorL2I src) %{ + match(Set dst (ReplicateI src)); + ins_cost(VEC_COST); + format %{ "vmv.v.x $dst, $src\t#@replicateI" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vmv_v_x(as_VectorRegister($dst$$reg), as_Register($src$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct replicateL(vReg dst, iRegL src) %{ + match(Set dst (ReplicateL src)); + ins_cost(VEC_COST); + format %{ "vmv.v.x $dst, $src\t#@replicateL" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vmv_v_x(as_VectorRegister($dst$$reg), as_Register($src$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct replicateB_imm5(vReg dst, immI5 con) %{ + match(Set dst (ReplicateB con)); + ins_cost(VEC_COST); + format %{ "vmv.v.i $dst, $con\t#@replicateB_imm5" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e8); + __ vmv_v_i(as_VectorRegister($dst$$reg), $con$$constant); + %} + ins_pipe(pipe_slow); +%} + +instruct replicateS_imm5(vReg dst, immI5 con) %{ + match(Set dst (ReplicateS con)); + ins_cost(VEC_COST); + format %{ "vmv.v.i $dst, $con\t#@replicateS_imm5" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e16); + __ vmv_v_i(as_VectorRegister($dst$$reg), $con$$constant); + %} + ins_pipe(pipe_slow); +%} + +instruct replicateI_imm5(vReg dst, immI5 con) %{ + match(Set dst (ReplicateI con)); + ins_cost(VEC_COST); + format %{ "vmv.v.i $dst, $con\t#@replicateI_imm5" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vmv_v_i(as_VectorRegister($dst$$reg), $con$$constant); + %} + ins_pipe(pipe_slow); +%} + +instruct replicateL_imm5(vReg dst, immL5 con) %{ + match(Set dst (ReplicateL con)); + ins_cost(VEC_COST); + format %{ "vmv.v.i $dst, $con\t#@replicateL_imm5" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vmv_v_i(as_VectorRegister($dst$$reg), $con$$constant); + %} + ins_pipe(pipe_slow); +%} + +instruct replicateF(vReg dst, fRegF src) %{ + match(Set dst (ReplicateF src)); + ins_cost(VEC_COST); + format %{ "vfmv.v.f $dst, $src\t#@replicateF" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vfmv_v_f(as_VectorRegister($dst$$reg), $src$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} + +instruct replicateD(vReg dst, fRegD src) %{ + match(Set dst (ReplicateD src)); + ins_cost(VEC_COST); + format %{ "vfmv.v.f $dst, $src\t#@replicateD" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vfmv_v_f(as_VectorRegister($dst$$reg), $src$$FloatRegister); + %} + ins_pipe(pipe_slow); +%} + +// vector shift + +instruct vasrB(vReg dst, vReg src, vReg shift) %{ + match(Set dst (RShiftVB src shift)); + ins_cost(VEC_COST); + effect(TEMP_DEF dst); + format %{ "vmsgtu.vi v0, $shift 7\t#@vasrB\n\t" + "vsra.vi $dst, $src, 7, Assembler::v0_t\n\t" + "vmnot.m v0, v0\n\t" + "vsra.vv $dst, $src, $shift, Assembler::v0_t" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e8); + // if shift > BitsPerByte - 1, clear the low BitsPerByte - 1 bits + __ vmsgtu_vi(v0, as_VectorRegister($shift$$reg), BitsPerByte - 1); + __ vsra_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + BitsPerByte - 1, Assembler::v0_t); + // otherwise, shift + __ vmnot_m(v0, v0); + __ vsra_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($shift$$reg), Assembler::v0_t); + %} + ins_pipe(pipe_slow); +%} + +instruct vasrS(vReg dst, vReg src, vReg shift) %{ + match(Set dst (RShiftVS src shift)); + ins_cost(VEC_COST); + effect(TEMP_DEF dst); + format %{ "vmsgtu.vi v0, $shift, 15\t#@vasrS\n\t" + "vsra.vi $dst, $src, 15, Assembler::v0_t\n\t" + "vmnot.m v0, v0\n\t" + "vsra.vv $dst, $src, $shift, Assembler::v0_t" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e16); + // if shift > BitsPerShort - 1, clear the low BitsPerShort - 1 bits + __ vmsgtu_vi(v0, as_VectorRegister($shift$$reg), BitsPerShort - 1); + __ vsra_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + BitsPerShort - 1, Assembler::v0_t); + // otherwise, shift + __ vmnot_m(v0, v0); + __ vsra_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($shift$$reg), Assembler::v0_t); + %} + ins_pipe(pipe_slow); +%} + +instruct vasrI(vReg dst, vReg src, vReg shift) %{ + match(Set dst (RShiftVI src shift)); + ins_cost(VEC_COST); + format %{ "vsra.vv $dst, $src, $shift\t#@vasrI" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vsra_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($shift$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vasrL(vReg dst, vReg src, vReg shift) %{ + match(Set dst (RShiftVL src shift)); + ins_cost(VEC_COST); + format %{ "vsra.vv $dst, $src, $shift\t#@vasrL" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vsra_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($shift$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vlslB(vReg dst, vReg src, vReg shift) %{ + match(Set dst (LShiftVB src shift)); + ins_cost(VEC_COST); + effect( TEMP_DEF dst); + format %{ "vmsgtu.vi v0, $shift, 7\t#@vlslB\n\t" + "vxor.vv $dst, $src, $src, Assembler::v0_t\n\t" + "vmnot.m v0, v0\n\t" + "vsll.vv $dst, $src, $shift, Assembler::v0_t" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e8); + // if shift > BitsPerByte - 1, clear the element + __ vmsgtu_vi(v0, as_VectorRegister($shift$$reg), BitsPerByte - 1); + __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($src$$reg), Assembler::v0_t); + // otherwise, shift + __ vmnot_m(v0, v0); + __ vsll_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($shift$$reg), Assembler::v0_t); + %} + ins_pipe(pipe_slow); +%} + +instruct vlslS(vReg dst, vReg src, vReg shift) %{ + match(Set dst (LShiftVS src shift)); + ins_cost(VEC_COST); + effect(TEMP_DEF dst); + format %{ "vmsgtu.vi v0, $shift, 15\t#@vlslS\n\t" + "vxor.vv $dst, $src, $src, Assembler::v0_t\n\t" + "vmnot.m v0, v0\n\t" + "vsll.vv $dst, $src, $shift, Assembler::v0_t" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e16); + // if shift > BitsPerShort - 1, clear the element + __ vmsgtu_vi(v0, as_VectorRegister($shift$$reg), BitsPerShort - 1); + __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($src$$reg), Assembler::v0_t); + // otherwise, shift + __ vmnot_m(v0, v0); + __ vsll_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($shift$$reg), Assembler::v0_t); + %} + ins_pipe(pipe_slow); +%} + +instruct vlslI(vReg dst, vReg src, vReg shift) %{ + match(Set dst (LShiftVI src shift)); + ins_cost(VEC_COST); + format %{ "vsll.vv $dst, $src, $shift\t#@vlslI" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vsll_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($shift$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vlslL(vReg dst, vReg src, vReg shift) %{ + match(Set dst (LShiftVL src shift)); + ins_cost(VEC_COST); + format %{ "vsll.vv $dst, $src, $shift\t# vector (D)" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vsll_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($shift$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vlsrB(vReg dst, vReg src, vReg shift) %{ + match(Set dst (URShiftVB src shift)); + ins_cost(VEC_COST); + effect(TEMP_DEF dst); + format %{ "vmsgtu.vi v0, $shift, 7\t#@vlsrB\n\t" + "vxor.vv $dst, $src, $src, Assembler::v0_t\n\t" + "vmnot.m v0, v0, v0\n\t" + "vsll.vv $dst, $src, $shift, Assembler::v0_t" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e8); + // if shift > BitsPerByte - 1, clear the element + __ vmsgtu_vi(v0, as_VectorRegister($shift$$reg), BitsPerByte - 1); + __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($src$$reg), Assembler::v0_t); + // otherwise, shift + __ vmnot_m(v0, v0); + __ vsrl_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($shift$$reg), Assembler::v0_t); + %} + ins_pipe(pipe_slow); +%} + +instruct vlsrS(vReg dst, vReg src, vReg shift) %{ + match(Set dst (URShiftVS src shift)); + ins_cost(VEC_COST); + effect(TEMP_DEF dst); + format %{ "vmsgtu.vi v0, $shift, 15\t#@vlsrS\n\t" + "vxor.vv $dst, $src, $src, Assembler::v0_t\n\t" + "vmnot.m v0, v0\n\t" + "vsll.vv $dst, $src, $shift, Assembler::v0_t" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e16); + // if shift > BitsPerShort - 1, clear the element + __ vmsgtu_vi(v0, as_VectorRegister($shift$$reg), BitsPerShort - 1); + __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($src$$reg), Assembler::v0_t); + // otherwise, shift + __ vmnot_m(v0, v0); + __ vsrl_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($shift$$reg), Assembler::v0_t); + %} + ins_pipe(pipe_slow); +%} + + +instruct vlsrI(vReg dst, vReg src, vReg shift) %{ + match(Set dst (URShiftVI src shift)); + ins_cost(VEC_COST); + format %{ "vsrl.vv $dst, $src, $shift\t#@vlsrI" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vsrl_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($shift$$reg)); + %} + ins_pipe(pipe_slow); +%} + + +instruct vlsrL(vReg dst, vReg src, vReg shift) %{ + match(Set dst (URShiftVL src shift)); + ins_cost(VEC_COST); + format %{ "vsrl.vv $dst, $src, $shift\t#@vlsrL" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vsrl_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($shift$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vasrB_imm(vReg dst, vReg src, immI shift) %{ + match(Set dst (RShiftVB src (RShiftCntV shift))); + ins_cost(VEC_COST); + format %{ "vsra.vi $dst, $src, $shift\t#@vasrB_imm" %} + ins_encode %{ + uint32_t con = (unsigned)$shift$$constant & 0x1f; + __ vsetvli(t0, x0, Assembler::e8); + if (con == 0) { + __ vor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($src$$reg)); + return; + } + if (con >= BitsPerByte) con = BitsPerByte - 1; + __ vsra_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con); + %} + ins_pipe(pipe_slow); +%} + +instruct vasrS_imm(vReg dst, vReg src, immI shift) %{ + match(Set dst (RShiftVS src (RShiftCntV shift))); + ins_cost(VEC_COST); + format %{ "vsra.vi $dst, $src, $shift\t#@vasrS_imm" %} + ins_encode %{ + uint32_t con = (unsigned)$shift$$constant & 0x1f; + __ vsetvli(t0, x0, Assembler::e16); + if (con == 0) { + __ vor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($src$$reg)); + return; + } + if (con >= BitsPerShort) con = BitsPerShort - 1; + __ vsra_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con); + %} + ins_pipe(pipe_slow); +%} + +instruct vasrI_imm(vReg dst, vReg src, immI shift) %{ + match(Set dst (RShiftVI src (RShiftCntV shift))); + ins_cost(VEC_COST); + format %{ "vsrl.vi $dst, $src, $shift\t#@vasrI_imm" %} + ins_encode %{ + uint32_t con = (unsigned)$shift$$constant & 0x1f; + __ vsetvli(t0, x0, Assembler::e32); + if (con == 0) { + __ vor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($src$$reg)); + return; + } + __ vsra_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con); + %} + ins_pipe(pipe_slow); +%} + +instruct vasrL_imm(vReg dst, vReg src, immI shift) %{ + predicate((n->in(2)->in(1)->get_int() & 0x3f) < 32); + match(Set dst (RShiftVL src (RShiftCntV shift))); + ins_cost(VEC_COST); + format %{ "vsrl.vi $dst, $src, $shift\t#@vasrL_imm" %} + ins_encode %{ + uint32_t con = (unsigned)$shift$$constant & 0x1f; + __ vsetvli(t0, x0, Assembler::e64); + if (con == 0) { + __ vor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($src$$reg)); + return; + } + __ vsra_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con); + %} + ins_pipe(pipe_slow); +%} + +instruct vlsrB_imm(vReg dst, vReg src, immI shift) %{ + match(Set dst (URShiftVB src (RShiftCntV shift))); + ins_cost(VEC_COST); + format %{ "vsrl.vi $dst, $src, $shift\t#@vlsrB_imm" %} + ins_encode %{ + uint32_t con = (unsigned)$shift$$constant & 0x1f; + __ vsetvli(t0, x0, Assembler::e8); + if (con == 0) { + __ vor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($src$$reg)); + return; + } + if (con >= BitsPerByte) { + __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($src$$reg)); + return; + } + __ vsrl_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con); + %} + ins_pipe(pipe_slow); +%} + +instruct vlsrS_imm(vReg dst, vReg src, immI shift) %{ + match(Set dst (URShiftVS src (RShiftCntV shift))); + ins_cost(VEC_COST); + format %{ "vsrl.vi $dst, $src, $shift\t#@vlsrS_imm" %} + ins_encode %{ + uint32_t con = (unsigned)$shift$$constant & 0x1f; + __ vsetvli(t0, x0, Assembler::e16); + if (con == 0) { + __ vor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($src$$reg)); + return; + } + if (con >= BitsPerShort) { + __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($src$$reg)); + return; + } + __ vsrl_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con); + %} + ins_pipe(pipe_slow); +%} + +instruct vlsrI_imm(vReg dst, vReg src, immI shift) %{ + match(Set dst (URShiftVI src (RShiftCntV shift))); + ins_cost(VEC_COST); + format %{ "vsrl.vi $dst, $src, $shift\t#@vlsrI_imm" %} + ins_encode %{ + uint32_t con = (unsigned)$shift$$constant & 0x1f; + __ vsetvli(t0, x0, Assembler::e32); + if (con == 0) { + __ vor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($src$$reg)); + return; + } + __ vsrl_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con); + %} + ins_pipe(pipe_slow); +%} + +instruct vlsrL_imm(vReg dst, vReg src, immI shift) %{ + predicate((n->in(2)->in(1)->get_int() & 0x3f) < 32); + match(Set dst (URShiftVL src (RShiftCntV shift))); + ins_cost(VEC_COST); + format %{ "vsrl.vi $dst, $src, $shift\t#@vlsrL_imm" %} + ins_encode %{ + uint32_t con = (unsigned)$shift$$constant & 0x1f; + __ vsetvli(t0, x0, Assembler::e64); + if (con == 0) { + __ vor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($src$$reg)); + return; + } + __ vsrl_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con); + %} + ins_pipe(pipe_slow); +%} + +instruct vlslB_imm(vReg dst, vReg src, immI shift) %{ + match(Set dst (LShiftVB src (LShiftCntV shift))); + ins_cost(VEC_COST); + format %{ "vsll.vi $dst, $src, $shift\t#@vlslB_imm" %} + ins_encode %{ + uint32_t con = (unsigned)$shift$$constant & 0x1f; + __ vsetvli(t0, x0, Assembler::e8); + if (con >= BitsPerByte) { + __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($src$$reg)); + return; + } + __ vsll_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con); + %} + ins_pipe(pipe_slow); +%} + +instruct vlslS_imm(vReg dst, vReg src, immI shift) %{ + match(Set dst (LShiftVS src (LShiftCntV shift))); + ins_cost(VEC_COST); + format %{ "vsll.vi $dst, $src, $shift\t#@vlslS_imm" %} + ins_encode %{ + uint32_t con = (unsigned)$shift$$constant & 0x1f; + __ vsetvli(t0, x0, Assembler::e16); + if (con >= BitsPerShort) { + __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), + as_VectorRegister($src$$reg)); + return; + } + __ vsll_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con); + %} + ins_pipe(pipe_slow); +%} + +instruct vlslI_imm(vReg dst, vReg src, immI shift) %{ + match(Set dst (LShiftVI src (LShiftCntV shift))); + ins_cost(VEC_COST); + format %{ "vsll.vi $dst, $src, $shift\t#@vlslI_imm" %} + ins_encode %{ + uint32_t con = (unsigned)$shift$$constant & 0x1f; + __ vsetvli(t0, x0, Assembler::e32); + __ vsll_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con); + %} + ins_pipe(pipe_slow); +%} + +instruct vlslL_imm(vReg dst, vReg src, immI shift) %{ + predicate((n->in(2)->in(1)->get_int() & 0x3f) < 32); + match(Set dst (LShiftVL src (LShiftCntV shift))); + ins_cost(VEC_COST); + format %{ "vsll.vi $dst, $src, $shift\t#@vlslL_imm" %} + ins_encode %{ + uint32_t con = (unsigned)$shift$$constant & 0x1f; + __ vsetvli(t0, x0, Assembler::e64); + __ vsll_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con); + %} + ins_pipe(pipe_slow); +%} + +instruct vshiftcntB(vReg dst, iRegIorL2I cnt) %{ + predicate(n->bottom_type()->is_vect()->element_basic_type() == T_BYTE); + match(Set dst (LShiftCntV cnt)); + match(Set dst (RShiftCntV cnt)); + format %{ "vmv.v.x $dst, $cnt\t#@vshiftcntB" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e8); + __ vmv_v_x(as_VectorRegister($dst$$reg), as_Register($cnt$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vshiftcntS(vReg dst, iRegIorL2I cnt) %{ + predicate(n->bottom_type()->is_vect()->element_basic_type() == T_SHORT || + n->bottom_type()->is_vect()->element_basic_type() == T_CHAR); + match(Set dst (LShiftCntV cnt)); + match(Set dst (RShiftCntV cnt)); + format %{ "vmv.v.x $dst, $cnt\t#@vshiftcntS" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e16); + __ vmv_v_x(as_VectorRegister($dst$$reg), as_Register($cnt$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vshiftcntI(vReg dst, iRegIorL2I cnt) %{ + predicate(n->bottom_type()->is_vect()->element_basic_type() == T_INT); + match(Set dst (LShiftCntV cnt)); + match(Set dst (RShiftCntV cnt)); + format %{ "vmv.v.x $dst, $cnt\t#@vshiftcntI" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vmv_v_x(as_VectorRegister($dst$$reg), as_Register($cnt$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vshiftcntL(vReg dst, iRegIorL2I cnt) %{ + predicate(n->bottom_type()->is_vect()->element_basic_type() == T_LONG); + match(Set dst (LShiftCntV cnt)); + match(Set dst (RShiftCntV cnt)); + format %{ "vmv.v.x $dst, $cnt\t#@vshiftcntL" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vmv_v_x(as_VectorRegister($dst$$reg), as_Register($cnt$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector sqrt + +instruct vsqrtF(vReg dst, vReg src) %{ + match(Set dst (SqrtVF src)); + ins_cost(VEC_COST); + format %{ "vfsqrt.v $dst, $src\t#@vsqrtF" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vfsqrt_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vsqrtD(vReg dst, vReg src) %{ + match(Set dst (SqrtVD src)); + ins_cost(VEC_COST); + format %{ "vfsqrt.v $dst, $src\t#@vsqrtD" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vfsqrt_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg)); + %} + ins_pipe(pipe_slow); +%} + +// vector sub + +instruct vsubB(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (SubVB src1 src2)); + ins_cost(VEC_COST); + format %{ "vsub.vv $dst, $src1, $src2\t#@vsubB" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e8); + __ vsub_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vsubS(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (SubVS src1 src2)); + ins_cost(VEC_COST); + format %{ "vsub.vv $dst, $src1, $src2\t#@vsubS" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e16); + __ vsub_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vsubI(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (SubVI src1 src2)); + ins_cost(VEC_COST); + format %{ "vsub.vv $dst, $src1, $src2\t#@vsubI" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vsub_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vsubL(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (SubVL src1 src2)); + ins_cost(VEC_COST); + format %{ "vsub.vv $dst, $src1, $src2\t#@vsubL" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vsub_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vsubF(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (SubVF src1 src2)); + ins_cost(VEC_COST); + format %{ "vfsub.vv $dst, $src1, $src2\t@vsubF" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e32); + __ vfsub_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vsubD(vReg dst, vReg src1, vReg src2) %{ + match(Set dst (SubVD src1 src2)); + ins_cost(VEC_COST); + format %{ "vfsub.vv $dst, $src1, $src2\t#@vsubD" %} + ins_encode %{ + __ vsetvli(t0, x0, Assembler::e64); + __ vfsub_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg), + as_VectorRegister($src2$$reg)); + %} + ins_pipe(pipe_slow); +%} + +instruct vstring_equalsL(iRegP_R11 str1, iRegP_R13 str2, iRegI_R14 cnt, + iRegI_R10 result, vReg_V1 v1, + vReg_V2 v2, vReg_V3 v3, rFlagsReg cr) +%{ + predicate(UseRVV && ((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL); + match(Set result (StrEquals (Binary str1 str2) cnt)); + effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, TEMP v1, TEMP v2, TEMP v3, KILL cr); + + format %{ "String Equals $str1, $str2, $cnt -> $result\t#@string_equalsL" %} + ins_encode %{ + // Count is in 8-bit bytes; non-Compact chars are 16 bits. + __ string_equals_v($str1$$Register, $str2$$Register, + $result$$Register, $cnt$$Register, 1); + %} + ins_pipe(pipe_class_memory); +%} + +instruct vstring_equalsU(iRegP_R11 str1, iRegP_R13 str2, iRegI_R14 cnt, + iRegI_R10 result, vReg_V1 v1, + vReg_V2 v2, vReg_V3 v3, rFlagsReg cr) +%{ + predicate(UseRVV && ((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU); + match(Set result (StrEquals (Binary str1 str2) cnt)); + effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, TEMP v1, TEMP v2, TEMP v3, KILL cr); + + format %{ "String Equals $str1, $str2, $cnt -> $result\t#@string_equalsU" %} + ins_encode %{ + // Count is in 8-bit bytes; non-Compact chars are 16 bits. + __ string_equals_v($str1$$Register, $str2$$Register, + $result$$Register, $cnt$$Register, 2); + %} + ins_pipe(pipe_class_memory); +%} + +instruct varray_equalsB(iRegP_R11 ary1, iRegP_R12 ary2, iRegI_R10 result, + vReg_V1 v1, vReg_V2 v2, vReg_V3 v3, iRegP_R28 tmp, rFlagsReg cr) +%{ + predicate(UseRVV && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL); + match(Set result (AryEq ary1 ary2)); + effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP v1, TEMP v2, TEMP v3, KILL cr); + + format %{ "Array Equals $ary1, ary2 -> $result\t#@array_equalsB // KILL $tmp" %} + ins_encode %{ + __ arrays_equals_v($ary1$$Register, $ary2$$Register, + $result$$Register, $tmp$$Register, 1); + %} + ins_pipe(pipe_class_memory); +%} + +instruct varray_equalsC(iRegP_R11 ary1, iRegP_R12 ary2, iRegI_R10 result, + vReg_V1 v1, vReg_V2 v2, vReg_V3 v3, iRegP_R28 tmp, rFlagsReg cr) +%{ + predicate(UseRVV && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU); + match(Set result (AryEq ary1 ary2)); + effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP v1, TEMP v2, TEMP v3, KILL cr); + + format %{ "Array Equals $ary1, ary2 -> $result\t#@array_equalsC // KILL $tmp" %} + ins_encode %{ + __ arrays_equals_v($ary1$$Register, $ary2$$Register, + $result$$Register, $tmp$$Register, 2); + %} + ins_pipe(pipe_class_memory); +%} + +instruct vstring_compareU(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, iRegI_R14 cnt2, + iRegI_R10 result, vReg_V1 v1, vReg_V2 v2, vReg_V3 v3, vReg_V4 v4, vReg_V5 v5, + iRegP_R28 tmp1, iRegL_R29 tmp2) +%{ + predicate(UseRVV && ((StrCompNode *)n)->encoding() == StrIntrinsicNode::UU); + match(Set result(StrComp(Binary str1 cnt1)(Binary str2 cnt2))); + effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, + TEMP v1, TEMP v2, TEMP v3, TEMP v4, TEMP v5); + + format %{ "String Compare $str1, $cnt1, $str2, $cnt2 -> $result\t#@string_compareU" %} + ins_encode %{ + // Count is in 8-bit bytes; non-Compact chars are 16 bits. + __ string_compare_v($str1$$Register, $str2$$Register, + $cnt1$$Register, $cnt2$$Register, $result$$Register, + $tmp1$$Register, $tmp2$$Register, + StrIntrinsicNode::UU); + %} + ins_pipe(pipe_class_memory); +%} +instruct vstring_compareL(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, iRegI_R14 cnt2, + iRegI_R10 result, vReg_V1 v1, vReg_V2 v2, vReg_V3 v3, vReg_V4 v4, vReg_V5 v5, + iRegP_R28 tmp1, iRegL_R29 tmp2) +%{ + predicate(UseRVV && ((StrCompNode *)n)->encoding() == StrIntrinsicNode::LL); + match(Set result(StrComp(Binary str1 cnt1)(Binary str2 cnt2))); + effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, + TEMP v1, TEMP v2, TEMP v3, TEMP v4, TEMP v5); + + format %{ "String Compare $str1, $cnt1, $str2, $cnt2 -> $result\t#@string_compareL" %} + ins_encode %{ + __ string_compare_v($str1$$Register, $str2$$Register, + $cnt1$$Register, $cnt2$$Register, $result$$Register, + $tmp1$$Register, $tmp2$$Register, + StrIntrinsicNode::LL); + %} + ins_pipe(pipe_class_memory); +%} + +instruct vstring_compareUL(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, iRegI_R14 cnt2, + iRegI_R10 result, vReg_V1 v1, vReg_V2 v2, vReg_V3 v3, vReg_V4 v4, vReg_V5 v5, + iRegP_R28 tmp1, iRegL_R29 tmp2) +%{ + predicate(UseRVV && ((StrCompNode *)n)->encoding() == StrIntrinsicNode::UL); + match(Set result(StrComp(Binary str1 cnt1)(Binary str2 cnt2))); + effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, + TEMP v1, TEMP v2, TEMP v3, TEMP v4, TEMP v5); + + format %{"String Compare $str1, $cnt1, $str2, $cnt2 -> $result\t#@string_compareUL" %} + ins_encode %{ + __ string_compare_v($str1$$Register, $str2$$Register, + $cnt1$$Register, $cnt2$$Register, $result$$Register, + $tmp1$$Register, $tmp2$$Register, + StrIntrinsicNode::UL); + %} + ins_pipe(pipe_class_memory); +%} +instruct vstring_compareLU(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, iRegI_R14 cnt2, + iRegI_R10 result, vReg_V1 v1, vReg_V2 v2, vReg_V3 v3, vReg_V4 v4, vReg_V5 v5, + iRegP_R28 tmp1, iRegL_R29 tmp2) +%{ + predicate(UseRVV && ((StrCompNode *)n)->encoding() == StrIntrinsicNode::LU); + match(Set result(StrComp(Binary str1 cnt1)(Binary str2 cnt2))); + effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, + TEMP v1, TEMP v2, TEMP v3, TEMP v4, TEMP v5); + + format %{ "String Compare $str1, $cnt1, $str2, $cnt2 -> $result\t#@string_compareLU" %} + ins_encode %{ + __ string_compare_v($str1$$Register, $str2$$Register, + $cnt1$$Register, $cnt2$$Register, $result$$Register, + $tmp1$$Register, $tmp2$$Register, + StrIntrinsicNode::LU); + %} + ins_pipe(pipe_class_memory); +%} + +// fast byte[] to char[] inflation +instruct vstring_inflate(Universe dummy, iRegP_R10 src, iRegP_R11 dst, iRegI_R12 len, + vReg_V1 v1, vReg_V2 v2, vReg_V3 v3, iRegLNoSp tmp) +%{ + predicate(UseRVV); + match(Set dummy (StrInflatedCopy src (Binary dst len))); + effect(TEMP v1, TEMP v2, TEMP v3, TEMP tmp, USE_KILL src, USE_KILL dst, USE_KILL len); + + format %{ "String Inflate $src,$dst" %} + ins_encode %{ + __ byte_array_inflate_v($src$$Register, $dst$$Register, $len$$Register, $tmp$$Register); + %} + ins_pipe(pipe_class_memory); +%} + +// encode char[] to byte[] in ISO_8859_1 +instruct vencode_iso_array(iRegP_R12 src, iRegP_R11 dst, iRegI_R13 len, iRegI_R10 result, + vReg_V1 v1, vReg_V2 v2, vReg_V3 v3, iRegLNoSp tmp) +%{ + predicate(UseRVV); + match(Set result (EncodeISOArray src (Binary dst len))); + effect(TEMP_DEF result, USE_KILL src, USE_KILL dst, USE_KILL len, + TEMP v1, TEMP v2, TEMP v3, TEMP tmp); + + format %{ "Encode array $src,$dst,$len -> $result" %} + ins_encode %{ + __ encode_iso_array_v($src$$Register, $dst$$Register, $len$$Register, + $result$$Register, $tmp$$Register); + %} + ins_pipe( pipe_class_memory ); +%} + +// fast char[] to byte[] compression +instruct vstring_compress(iRegP_R12 src, iRegP_R11 dst, iRegI_R13 len, iRegI_R10 result, + vReg_V1 v1, vReg_V2 v2, vReg_V3 v3, iRegLNoSp tmp) +%{ + predicate(UseRVV); + match(Set result (StrCompressedCopy src (Binary dst len))); + effect(TEMP_DEF result, USE_KILL src, USE_KILL dst, USE_KILL len, + TEMP v1, TEMP v2, TEMP v3, TEMP tmp); + + format %{ "String Compress $src,$dst -> $result // KILL R11, R12, R13" %} + ins_encode %{ + __ char_array_compress_v($src$$Register, $dst$$Register, $len$$Register, + $result$$Register, $tmp$$Register); + %} + ins_pipe( pipe_slow ); +%} + +instruct vhas_negatives(iRegP_R11 ary, iRegI_R12 len, iRegI_R10 result, + vReg_V1 v1, vReg_V2 v2, vReg_V3 v3, iRegLNoSp tmp) +%{ + predicate(UseRVV); + match(Set result (HasNegatives ary len)); + effect(TEMP_DEF result, USE_KILL ary, USE_KILL len, TEMP v1, TEMP v2, TEMP v3, TEMP tmp); + + format %{ "has negatives byte[] $ary, $len -> $result" %} + ins_encode %{ + __ has_negatives_v($ary$$Register, $len$$Register, $result$$Register, $tmp$$Register); + %} + + ins_pipe(pipe_slow); +%} + +instruct vstringU_indexof_char(iRegP_R11 str1, iRegI_R12 cnt1, iRegI_R13 ch, + iRegI_R10 result, iRegINoSp tmp1, iRegINoSp tmp2, + vReg_V1 v1, vReg_V2 v2, vReg_V3 v3) +%{ + predicate(UseRVV && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U)); + match(Set result (StrIndexOfChar (Binary str1 cnt1) ch)); + effect(TEMP_DEF result, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, + TEMP tmp1, TEMP tmp2, TEMP v1, TEMP v2, TEMP v3); + + format %{ "StringUTF16 IndexOf char[] $str1, $cnt1, $ch -> $result" %} + + ins_encode %{ + __ string_indexof_char_v($str1$$Register, $cnt1$$Register, $ch$$Register, + $result$$Register, $tmp1$$Register, $tmp2$$Register, + false /* isL */); + %} + + ins_pipe(pipe_class_memory); +%} + +instruct vstringL_indexof_char(iRegP_R11 str1, iRegI_R12 cnt1, iRegI_R13 ch, + iRegI_R10 result, iRegINoSp tmp1, iRegINoSp tmp2, + vReg_V1 v1, vReg_V2 v2, vReg_V3 v3) +%{ + predicate(UseRVV && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L)); + match(Set result (StrIndexOfChar (Binary str1 cnt1) ch)); + effect(TEMP_DEF result, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, + TEMP tmp1, TEMP tmp2, TEMP v1, TEMP v2, TEMP v3); + + format %{ "StringLatin1 IndexOf char[] $str1, $cnt1, $ch -> $result" %} + + ins_encode %{ + __ string_indexof_char_v($str1$$Register, $cnt1$$Register, $ch$$Register, + $result$$Register, $tmp1$$Register, $tmp2$$Register, + true /* isL */); + %} + + ins_pipe(pipe_class_memory); +%} + +// clearing of an array +instruct vclearArray_reg_reg(iRegL_R29 cnt, iRegP_R28 base, Universe dummy, + vReg_V1 vReg1, vReg_V2 vReg2, vReg_V3 vReg3) +%{ + predicate(UseRVV); + match(Set dummy (ClearArray cnt base)); + effect(USE_KILL cnt, USE_KILL base, TEMP vReg1, TEMP vReg2, TEMP vReg3); + + format %{ "ClearArray $cnt, $base\t#@clearArray_reg_reg" %} + + ins_encode %{ + __ clear_array_v($base$$Register, $cnt$$Register); + %} + + ins_pipe(pipe_class_memory); +%} diff --git a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp new file mode 100644 index 0000000000000..92d9f8b4b1459 --- /dev/null +++ b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp @@ -0,0 +1,2965 @@ +/* + * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "code/debugInfoRec.hpp" +#include "code/icBuffer.hpp" +#include "code/vtableStubs.hpp" +#include "compiler/oopMap.hpp" +#include "gc/shared/barrierSetAssembler.hpp" +#include "interpreter/interp_masm.hpp" +#include "interpreter/interpreter.hpp" +#include "logging/log.hpp" +#include "memory/resourceArea.hpp" +#include "nativeInst_riscv.hpp" +#include "oops/compiledICHolder.hpp" +#include "oops/klass.inline.hpp" +#include "prims/methodHandles.hpp" +#include "runtime/jniHandles.hpp" +#include "runtime/safepointMechanism.hpp" +#include "runtime/sharedRuntime.hpp" +#include "runtime/signature.hpp" +#include "runtime/stubRoutines.hpp" +#include "runtime/vframeArray.hpp" +#include "utilities/align.hpp" +#include "utilities/formatBuffer.hpp" +#include "vmreg_riscv.inline.hpp" +#ifdef COMPILER1 +#include "c1/c1_Runtime1.hpp" +#endif +#ifdef COMPILER2 +#include "adfiles/ad_riscv.hpp" +#include "opto/runtime.hpp" +#endif + +#define __ masm-> + +const int StackAlignmentInSlots = StackAlignmentInBytes / VMRegImpl::stack_slot_size; + +class SimpleRuntimeFrame { +public: + + // Most of the runtime stubs have this simple frame layout. + // This class exists to make the layout shared in one place. + // Offsets are for compiler stack slots, which are jints. + enum layout { + // The frame sender code expects that fp will be in the "natural" place and + // will override any oopMap setting for it. We must therefore force the layout + // so that it agrees with the frame sender code. + // we don't expect any arg reg save area so riscv asserts that + // frame::arg_reg_save_area_bytes == 0 + fp_off = 0, fp_off2, + return_off, return_off2, + framesize + }; +}; + +class RegisterSaver { + const bool _save_vectors; + public: + RegisterSaver(bool save_vectors) : _save_vectors(UseRVV && save_vectors) {} + ~RegisterSaver() {} + OopMap* save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words); + void restore_live_registers(MacroAssembler* masm); + + // Offsets into the register save area + // Used by deoptimization when it is managing result register + // values on its own + // gregs:28, float_register:32; except: x1(ra) & x2(sp) & gp(x3) & tp(x4) + // |---v0---|<---SP + // |---v1---|save vectors only in generate_handler_blob + // |-- .. --| + // |---v31--|----- + // |---f0---| + // |---f1---| + // | .. | + // |---f31--| + // |---reserved slot for stack alignment---| + // |---x5---| + // | x6 | + // |---.. --| + // |---x31--| + // |---fp---| + // |---ra---| + int v0_offset_in_bytes(void) { return 0; } + int f0_offset_in_bytes(void) { + int f0_offset = 0; +#ifdef COMPILER2 + if (_save_vectors) { + f0_offset += Matcher::scalable_vector_reg_size(T_INT) * VectorRegisterImpl::number_of_registers * + BytesPerInt; + } +#endif + return f0_offset; + } + int reserved_slot_offset_in_bytes(void) { + return f0_offset_in_bytes() + + FloatRegisterImpl::max_slots_per_register * + FloatRegisterImpl::number_of_registers * + BytesPerInt; + } + + int reg_offset_in_bytes(Register r) { + assert (r->encoding() > 4, "ra, sp, gp and tp not saved"); + return reserved_slot_offset_in_bytes() + (r->encoding() - 4 /* x1, x2, x3, x4 */) * wordSize; + } + + int freg_offset_in_bytes(FloatRegister f) { + return f0_offset_in_bytes() + f->encoding() * wordSize; + } + + int ra_offset_in_bytes(void) { + return reserved_slot_offset_in_bytes() + + (RegisterImpl::number_of_registers - 3) * + RegisterImpl::max_slots_per_register * + BytesPerInt; + } +}; + +OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words) { + int vector_size_in_bytes = 0; + int vector_size_in_slots = 0; +#ifdef COMPILER2 + if (_save_vectors) { + vector_size_in_bytes += Matcher::scalable_vector_reg_size(T_BYTE); + vector_size_in_slots += Matcher::scalable_vector_reg_size(T_INT); + } +#endif + + int frame_size_in_bytes = align_up(additional_frame_words * wordSize + ra_offset_in_bytes() + wordSize, 16); + // OopMap frame size is in compiler stack slots (jint's) not bytes or words + int frame_size_in_slots = frame_size_in_bytes / BytesPerInt; + // The caller will allocate additional_frame_words + int additional_frame_slots = additional_frame_words * wordSize / BytesPerInt; + // CodeBlob frame size is in words. + int frame_size_in_words = frame_size_in_bytes / wordSize; + *total_frame_words = frame_size_in_words; + + // Save Integer, Float and Vector registers. + __ enter(); + __ push_CPU_state(_save_vectors, vector_size_in_bytes); + + // Set an oopmap for the call site. This oopmap will map all + // oop-registers and debug-info registers as callee-saved. This + // will allow deoptimization at this safepoint to find all possible + // debug-info recordings, as well as let GC find all oops. + + OopMapSet *oop_maps = new OopMapSet(); + OopMap* oop_map = new OopMap(frame_size_in_slots, 0); + assert_cond(oop_maps != NULL && oop_map != NULL); + + int sp_offset_in_slots = 0; + int step_in_slots = 0; + if (_save_vectors) { + step_in_slots = vector_size_in_slots; + for (int i = 0; i < VectorRegisterImpl::number_of_registers; i++, sp_offset_in_slots += step_in_slots) { + VectorRegister r = as_VectorRegister(i); + oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset_in_slots), r->as_VMReg()); + } + } + + step_in_slots = FloatRegisterImpl::max_slots_per_register; + for (int i = 0; i < FloatRegisterImpl::number_of_registers; i++, sp_offset_in_slots += step_in_slots) { + FloatRegister r = as_FloatRegister(i); + oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset_in_slots), r->as_VMReg()); + } + + step_in_slots = RegisterImpl::max_slots_per_register; + // skip the slot reserved for alignment, see MacroAssembler::push_reg; + // also skip x5 ~ x6 on the stack because they are caller-saved registers. + sp_offset_in_slots += RegisterImpl::max_slots_per_register * 3; + // besides, we ignore x0 ~ x4 because push_CPU_state won't push them on the stack. + for (int i = 7; i < RegisterImpl::number_of_registers; i++, sp_offset_in_slots += step_in_slots) { + Register r = as_Register(i); + if (r != xthread) { + oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset_in_slots + additional_frame_slots), r->as_VMReg()); + } + } + + return oop_map; +} + +void RegisterSaver::restore_live_registers(MacroAssembler* masm) { +#ifdef COMPILER2 + __ pop_CPU_state(_save_vectors, Matcher::scalable_vector_reg_size(T_BYTE)); +#else + __ pop_CPU_state(_save_vectors); +#endif + __ leave(); +} + +// Is vector's size (in bytes) bigger than a size saved by default? +// riscv does not ovlerlay the floating-point registers on vector registers like aarch64. +bool SharedRuntime::is_wide_vector(int size) { + return UseRVV; +} + +// The java_calling_convention describes stack locations as ideal slots on +// a frame with no abi restrictions. Since we must observe abi restrictions +// (like the placement of the register window) the slots must be biased by +// the following value. +static int reg2offset_in(VMReg r) { + // Account for saved fp and ra + // This should really be in_preserve_stack_slots + return r->reg2stack() * VMRegImpl::stack_slot_size; +} + +static int reg2offset_out(VMReg r) { + return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size; +} + +// --------------------------------------------------------------------------- +// Read the array of BasicTypes from a signature, and compute where the +// arguments should go. Values in the VMRegPair regs array refer to 4-byte +// quantities. Values less than VMRegImpl::stack0 are registers, those above +// refer to 4-byte stack slots. All stack slots are based off of the stack pointer +// as framesizes are fixed. +// VMRegImpl::stack0 refers to the first slot 0(sp). +// and VMRegImpl::stack0+1 refers to the memory word 4-byes higher. Register +// up to RegisterImpl::number_of_registers) are the 64-bit +// integer registers. + +// Note: the INPUTS in sig_bt are in units of Java argument words, +// which are 64-bit. The OUTPUTS are in 32-bit units. + +// The Java calling convention is a "shifted" version of the C ABI. +// By skipping the first C ABI register we can call non-static jni +// methods with small numbers of arguments without having to shuffle +// the arguments at all. Since we control the java ABI we ought to at +// least get some advantage out of it. + +int SharedRuntime::java_calling_convention(const BasicType *sig_bt, + VMRegPair *regs, + int total_args_passed) { + // Create the mapping between argument positions and + // registers. + static const Register INT_ArgReg[Argument::n_int_register_parameters_j] = { + j_rarg0, j_rarg1, j_rarg2, j_rarg3, + j_rarg4, j_rarg5, j_rarg6, j_rarg7 + }; + static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_j] = { + j_farg0, j_farg1, j_farg2, j_farg3, + j_farg4, j_farg5, j_farg6, j_farg7 + }; + + uint int_args = 0; + uint fp_args = 0; + uint stk_args = 0; // inc by 2 each time + + for (int i = 0; i < total_args_passed; i++) { + switch (sig_bt[i]) { + case T_BOOLEAN: // fall through + case T_CHAR: // fall through + case T_BYTE: // fall through + case T_SHORT: // fall through + case T_INT: + if (int_args < Argument::n_int_register_parameters_j) { + regs[i].set1(INT_ArgReg[int_args++]->as_VMReg()); + } else { + regs[i].set1(VMRegImpl::stack2reg(stk_args)); + stk_args += 2; + } + break; + case T_VOID: + // halves of T_LONG or T_DOUBLE + assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half"); + regs[i].set_bad(); + break; + case T_LONG: // fall through + assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half"); + case T_OBJECT: // fall through + case T_ARRAY: // fall through + case T_ADDRESS: + if (int_args < Argument::n_int_register_parameters_j) { + regs[i].set2(INT_ArgReg[int_args++]->as_VMReg()); + } else { + regs[i].set2(VMRegImpl::stack2reg(stk_args)); + stk_args += 2; + } + break; + case T_FLOAT: + if (fp_args < Argument::n_float_register_parameters_j) { + regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg()); + } else { + regs[i].set1(VMRegImpl::stack2reg(stk_args)); + stk_args += 2; + } + break; + case T_DOUBLE: + assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half"); + if (fp_args < Argument::n_float_register_parameters_j) { + regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg()); + } else { + regs[i].set2(VMRegImpl::stack2reg(stk_args)); + stk_args += 2; + } + break; + default: + ShouldNotReachHere(); + } + } + + return align_up(stk_args, 2); +} + +// Patch the callers callsite with entry to compiled code if it exists. +static void patch_callers_callsite(MacroAssembler *masm) { + Label L; + __ ld(t0, Address(xmethod, in_bytes(Method::code_offset()))); + __ beqz(t0, L); + + __ enter(); + __ push_CPU_state(); + + // VM needs caller's callsite + // VM needs target method + // This needs to be a long call since we will relocate this adapter to + // the codeBuffer and it may not reach + +#ifndef PRODUCT + assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area"); +#endif + + __ mv(c_rarg0, xmethod); + __ mv(c_rarg1, ra); + RuntimeAddress target(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ jalr(x1, t0, offset); + }); + + __ pop_CPU_state(); + // restore sp + __ leave(); + __ bind(L); +} + +static void gen_c2i_adapter(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + Label& skip_fixup) { + // Before we get into the guts of the C2I adapter, see if we should be here + // at all. We've come from compiled code and are attempting to jump to the + // interpreter, which means the caller made a static call to get here + // (vcalls always get a compiled target if there is one). Check for a + // compiled target. If there is one, we need to patch the caller's call. + patch_callers_callsite(masm); + + __ bind(skip_fixup); + + int words_pushed = 0; + + // Since all args are passed on the stack, total_args_passed * + // Interpreter::stackElementSize is the space we need. + + int extraspace = total_args_passed * Interpreter::stackElementSize; + + __ mv(x30, sp); + + // stack is aligned, keep it that way + extraspace = align_up(extraspace, 2 * wordSize); + + if (extraspace) { + __ sub(sp, sp, extraspace); + } + + // Now write the args into the outgoing interpreter space + for (int i = 0; i < total_args_passed; i++) { + if (sig_bt[i] == T_VOID) { + assert(i > 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "missing half"); + continue; + } + + // offset to start parameters + int st_off = (total_args_passed - i - 1) * Interpreter::stackElementSize; + int next_off = st_off - Interpreter::stackElementSize; + + // Say 4 args: + // i st_off + // 0 32 T_LONG + // 1 24 T_VOID + // 2 16 T_OBJECT + // 3 8 T_BOOL + // - 0 return address + // + // However to make thing extra confusing. Because we can fit a Java long/double in + // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter + // leaves one slot empty and only stores to a single slot. In this case the + // slot that is occupied is the T_VOID slot. See I said it was confusing. + + VMReg r_1 = regs[i].first(); + VMReg r_2 = regs[i].second(); + if (!r_1->is_valid()) { + assert(!r_2->is_valid(), ""); + continue; + } + if (r_1->is_stack()) { + // memory to memory use t0 + int ld_off = (r_1->reg2stack() * VMRegImpl::stack_slot_size + + extraspace + + words_pushed * wordSize); + if (!r_2->is_valid()) { + __ lwu(t0, Address(sp, ld_off)); + __ sd(t0, Address(sp, st_off), /*temp register*/esp); + } else { + __ ld(t0, Address(sp, ld_off), /*temp register*/esp); + + // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG + // T_DOUBLE and T_LONG use two slots in the interpreter + if (sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) { + // ld_off == LSW, ld_off+wordSize == MSW + // st_off == MSW, next_off == LSW + __ sd(t0, Address(sp, next_off), /*temp register*/esp); +#ifdef ASSERT + // Overwrite the unused slot with known junk + __ mv(t0, 0xdeadffffdeadaaaaul); + __ sd(t0, Address(sp, st_off), /*temp register*/esp); +#endif /* ASSERT */ + } else { + __ sd(t0, Address(sp, st_off), /*temp register*/esp); + } + } + } else if (r_1->is_Register()) { + Register r = r_1->as_Register(); + if (!r_2->is_valid()) { + // must be only an int (or less ) so move only 32bits to slot + __ sd(r, Address(sp, st_off)); + } else { + // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG + // T_DOUBLE and T_LONG use two slots in the interpreter + if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) { + // long/double in gpr +#ifdef ASSERT + // Overwrite the unused slot with known junk + __ mv(t0, 0xdeadffffdeadaaabul); + __ sd(t0, Address(sp, st_off), /*temp register*/esp); +#endif /* ASSERT */ + __ sd(r, Address(sp, next_off)); + } else { + __ sd(r, Address(sp, st_off)); + } + } + } else { + assert(r_1->is_FloatRegister(), ""); + if (!r_2->is_valid()) { + // only a float use just part of the slot + __ fsw(r_1->as_FloatRegister(), Address(sp, st_off)); + } else { +#ifdef ASSERT + // Overwrite the unused slot with known junk + __ mv(t0, 0xdeadffffdeadaaacul); + __ sd(t0, Address(sp, st_off), /*temp register*/esp); +#endif /* ASSERT */ + __ fsd(r_1->as_FloatRegister(), Address(sp, next_off)); + } + } + } + + __ mv(esp, sp); // Interp expects args on caller's expression stack + + __ ld(t0, Address(xmethod, in_bytes(Method::interpreter_entry_offset()))); + __ jr(t0); +} + +void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs) { + // Cut-out for having no stack args. + int comp_words_on_stack = align_up(comp_args_on_stack * VMRegImpl::stack_slot_size, wordSize) >> LogBytesPerWord; + if (comp_args_on_stack != 0) { + __ sub(t0, sp, comp_words_on_stack * wordSize); + __ andi(sp, t0, -16); + } + + // Will jump to the compiled code just as if compiled code was doing it. + // Pre-load the register-jump target early, to schedule it better. + __ ld(t1, Address(xmethod, in_bytes(Method::from_compiled_offset()))); + + // Now generate the shuffle code. + for (int i = 0; i < total_args_passed; i++) { + if (sig_bt[i] == T_VOID) { + assert(i > 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "missing half"); + continue; + } + + // Pick up 0, 1 or 2 words from SP+offset. + + assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(), + "scrambled load targets?"); + // Load in argument order going down. + int ld_off = (total_args_passed - i - 1) * Interpreter::stackElementSize; + // Point to interpreter value (vs. tag) + int next_off = ld_off - Interpreter::stackElementSize; + + VMReg r_1 = regs[i].first(); + VMReg r_2 = regs[i].second(); + if (!r_1->is_valid()) { + assert(!r_2->is_valid(), ""); + continue; + } + if (r_1->is_stack()) { + // Convert stack slot to an SP offset (+ wordSize to account for return address ) + int st_off = regs[i].first()->reg2stack() * VMRegImpl::stack_slot_size; + if (!r_2->is_valid()) { + __ lw(t0, Address(esp, ld_off)); + __ sd(t0, Address(sp, st_off), /*temp register*/t2); + } else { + // + // We are using two optoregs. This can be either T_OBJECT, + // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates + // two slots but only uses one for thr T_LONG or T_DOUBLE case + // So we must adjust where to pick up the data to match the + // interpreter. + // + // Interpreter local[n] == MSW, local[n+1] == LSW however locals + // are accessed as negative so LSW is at LOW address + + // ld_off is MSW so get LSW + const int offset = (sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) ? + next_off : ld_off; + __ ld(t0, Address(esp, offset)); + // st_off is LSW (i.e. reg.first()) + __ sd(t0, Address(sp, st_off), /*temp register*/t2); + } + } else if (r_1->is_Register()) { // Register argument + Register r = r_1->as_Register(); + if (r_2->is_valid()) { + // + // We are using two VMRegs. This can be either T_OBJECT, + // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates + // two slots but only uses one for thr T_LONG or T_DOUBLE case + // So we must adjust where to pick up the data to match the + // interpreter. + + const int offset = (sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) ? + next_off : ld_off; + + // this can be a misaligned move + __ ld(r, Address(esp, offset)); + } else { + // sign extend and use a full word? + __ lw(r, Address(esp, ld_off)); + } + } else { + if (!r_2->is_valid()) { + __ flw(r_1->as_FloatRegister(), Address(esp, ld_off)); + } else { + __ fld(r_1->as_FloatRegister(), Address(esp, next_off)); + } + } + } + + // 6243940 We might end up in handle_wrong_method if + // the callee is deoptimized as we race thru here. If that + // happens we don't want to take a safepoint because the + // caller frame will look interpreted and arguments are now + // "compiled" so it is much better to make this transition + // invisible to the stack walking code. Unfortunately if + // we try and find the callee by normal means a safepoint + // is possible. So we stash the desired callee in the thread + // and the vm will find there should this case occur. + + __ sd(xmethod, Address(xthread, JavaThread::callee_target_offset())); + + __ jr(t1); +} + +// --------------------------------------------------------------- +AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm, + int total_args_passed, + int comp_args_on_stack, + const BasicType *sig_bt, + const VMRegPair *regs, + AdapterFingerPrint* fingerprint) { + address i2c_entry = __ pc(); + gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs); + + address c2i_unverified_entry = __ pc(); + Label skip_fixup; + + Label ok; + + const Register holder = t1; + const Register receiver = j_rarg0; + const Register tmp = t2; // A call-clobbered register not used for arg passing + + // ------------------------------------------------------------------------- + // Generate a C2I adapter. On entry we know xmethod holds the Method* during calls + // to the interpreter. The args start out packed in the compiled layout. They + // need to be unpacked into the interpreter layout. This will almost always + // require some stack space. We grow the current (compiled) stack, then repack + // the args. We finally end in a jump to the generic interpreter entry point. + // On exit from the interpreter, the interpreter will restore our SP (lest the + // compiled code, which relys solely on SP and not FP, get sick). + + { + __ block_comment("c2i_unverified_entry {"); + __ load_klass(t0, receiver, tmp); + __ ld(tmp, Address(holder, CompiledICHolder::holder_klass_offset())); + __ ld(xmethod, Address(holder, CompiledICHolder::holder_metadata_offset())); + __ beq(t0, tmp, ok); + __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub())); + + __ bind(ok); + // Method might have been compiled since the call site was patched to + // interpreted; if that is the case treat it as a miss so we can get + // the call site corrected. + __ ld(t0, Address(xmethod, in_bytes(Method::code_offset()))); + __ beqz(t0, skip_fixup); + __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub())); + __ block_comment("} c2i_unverified_entry"); + } + + address c2i_entry = __ pc(); + + // Class initialization barrier for static methods + address c2i_no_clinit_check_entry = NULL; + if (VM_Version::supports_fast_class_init_checks()) { + Label L_skip_barrier; + + { // Bypass the barrier for non-static methods + __ lwu(t0, Address(xmethod, Method::access_flags_offset())); + __ test_bit(t1, t0, exact_log2(JVM_ACC_STATIC)); + __ beqz(t1, L_skip_barrier); // non-static + } + + __ load_method_holder(t1, xmethod); + __ clinit_barrier(t1, t0, &L_skip_barrier); + __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); + + __ bind(L_skip_barrier); + c2i_no_clinit_check_entry = __ pc(); + } + + BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->c2i_entry_barrier(masm); + + gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); + + return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); +} + +int SharedRuntime::vector_calling_convention(VMRegPair *regs, + uint num_bits, + uint total_args_passed) { + Unimplemented(); + return 0; +} + +int SharedRuntime::c_calling_convention(const BasicType *sig_bt, + VMRegPair *regs, + VMRegPair *regs2, + int total_args_passed) { + assert(regs2 == NULL, "not needed on riscv"); + + // We return the amount of VMRegImpl stack slots we need to reserve for all + // the arguments NOT counting out_preserve_stack_slots. + + static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = { + c_rarg0, c_rarg1, c_rarg2, c_rarg3, + c_rarg4, c_rarg5, c_rarg6, c_rarg7 + }; + static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_c] = { + c_farg0, c_farg1, c_farg2, c_farg3, + c_farg4, c_farg5, c_farg6, c_farg7 + }; + + uint int_args = 0; + uint fp_args = 0; + uint stk_args = 0; // inc by 2 each time + + for (int i = 0; i < total_args_passed; i++) { + switch (sig_bt[i]) { + case T_BOOLEAN: // fall through + case T_CHAR: // fall through + case T_BYTE: // fall through + case T_SHORT: // fall through + case T_INT: + if (int_args < Argument::n_int_register_parameters_c) { + regs[i].set1(INT_ArgReg[int_args++]->as_VMReg()); + } else { + regs[i].set1(VMRegImpl::stack2reg(stk_args)); + stk_args += 2; + } + break; + case T_LONG: // fall through + assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half"); + case T_OBJECT: // fall through + case T_ARRAY: // fall through + case T_ADDRESS: // fall through + case T_METADATA: + if (int_args < Argument::n_int_register_parameters_c) { + regs[i].set2(INT_ArgReg[int_args++]->as_VMReg()); + } else { + regs[i].set2(VMRegImpl::stack2reg(stk_args)); + stk_args += 2; + } + break; + case T_FLOAT: + if (fp_args < Argument::n_float_register_parameters_c) { + regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg()); + } else if (int_args < Argument::n_int_register_parameters_c) { + regs[i].set1(INT_ArgReg[int_args++]->as_VMReg()); + } else { + regs[i].set1(VMRegImpl::stack2reg(stk_args)); + stk_args += 2; + } + break; + case T_DOUBLE: + assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half"); + if (fp_args < Argument::n_float_register_parameters_c) { + regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg()); + } else if (int_args < Argument::n_int_register_parameters_c) { + regs[i].set2(INT_ArgReg[int_args++]->as_VMReg()); + } else { + regs[i].set2(VMRegImpl::stack2reg(stk_args)); + stk_args += 2; + } + break; + case T_VOID: // Halves of longs and doubles + assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half"); + regs[i].set_bad(); + break; + default: + ShouldNotReachHere(); + } + } + + return stk_args; +} + +// On 64 bit we will store integer like items to the stack as +// 64 bits items (riscv64 abi) even though java would only store +// 32bits for a parameter. On 32bit it will simply be 32 bits +// So this routine will do 32->32 on 32bit and 32->64 on 64bit +static void move32_64(MacroAssembler* masm, VMRegPair src, VMRegPair dst) { + if (src.first()->is_stack()) { + if (dst.first()->is_stack()) { + // stack to stack + __ ld(t0, Address(fp, reg2offset_in(src.first()))); + __ sd(t0, Address(sp, reg2offset_out(dst.first()))); + } else { + // stack to reg + __ lw(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first()))); + } + } else if (dst.first()->is_stack()) { + // reg to stack + __ sd(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first()))); + } else { + if (dst.first() != src.first()) { + // 32bits extend sign + __ sign_extend(dst.first()->as_Register(), src.first()->as_Register(), 32); + } + } +} + +// An oop arg. Must pass a handle not the oop itself +static void object_move(MacroAssembler* masm, + OopMap* map, + int oop_handle_offset, + int framesize_in_slots, + VMRegPair src, + VMRegPair dst, + bool is_receiver, + int* receiver_offset) { + // must pass a handle. First figure out the location we use as a handle + Register rHandle = dst.first()->is_stack() ? t1 : dst.first()->as_Register(); + + // See if oop is NULL if it is we need no handle + + if (src.first()->is_stack()) { + + // Oop is already on the stack as an argument + int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots(); + map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots)); + if (is_receiver) { + *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size; + } + + __ ld(t0, Address(fp, reg2offset_in(src.first()))); + __ la(rHandle, Address(fp, reg2offset_in(src.first()))); + // conditionally move a NULL + Label notZero1; + __ bnez(t0, notZero1); + __ mv(rHandle, zr); + __ bind(notZero1); + } else { + + // Oop is in an a register we must store it to the space we reserve + // on the stack for oop_handles and pass a handle if oop is non-NULL + + const Register rOop = src.first()->as_Register(); + int oop_slot = -1; + if (rOop == j_rarg0) { + oop_slot = 0; + } else if (rOop == j_rarg1) { + oop_slot = 1; + } else if (rOop == j_rarg2) { + oop_slot = 2; + } else if (rOop == j_rarg3) { + oop_slot = 3; + } else if (rOop == j_rarg4) { + oop_slot = 4; + } else if (rOop == j_rarg5) { + oop_slot = 5; + } else if (rOop == j_rarg6) { + oop_slot = 6; + } else { + assert(rOop == j_rarg7, "wrong register"); + oop_slot = 7; + } + + oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset; + int offset = oop_slot * VMRegImpl::stack_slot_size; + + map->set_oop(VMRegImpl::stack2reg(oop_slot)); + // Store oop in handle area, may be NULL + __ sd(rOop, Address(sp, offset)); + if (is_receiver) { + *receiver_offset = offset; + } + + //rOop maybe the same as rHandle + if (rOop == rHandle) { + Label isZero; + __ beqz(rOop, isZero); + __ la(rHandle, Address(sp, offset)); + __ bind(isZero); + } else { + Label notZero2; + __ la(rHandle, Address(sp, offset)); + __ bnez(rOop, notZero2); + __ mv(rHandle, zr); + __ bind(notZero2); + } + } + + // If arg is on the stack then place it otherwise it is already in correct reg. + if (dst.first()->is_stack()) { + __ sd(rHandle, Address(sp, reg2offset_out(dst.first()))); + } +} + +// A float arg may have to do float reg int reg conversion +static void float_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) { + assert(src.first()->is_stack() && dst.first()->is_stack() || + src.first()->is_reg() && dst.first()->is_reg() || src.first()->is_stack() && dst.first()->is_reg(), "Unexpected error"); + if (src.first()->is_stack()) { + if (dst.first()->is_stack()) { + __ lwu(t0, Address(fp, reg2offset_in(src.first()))); + __ sw(t0, Address(sp, reg2offset_out(dst.first()))); + } else if (dst.first()->is_Register()) { + __ lwu(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first()))); + } else { + ShouldNotReachHere(); + } + } else if (src.first() != dst.first()) { + if (src.is_single_phys_reg() && dst.is_single_phys_reg()) { + __ fmv_s(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister()); + } else { + ShouldNotReachHere(); + } + } +} + +// A long move +static void long_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) { + if (src.first()->is_stack()) { + if (dst.first()->is_stack()) { + // stack to stack + __ ld(t0, Address(fp, reg2offset_in(src.first()))); + __ sd(t0, Address(sp, reg2offset_out(dst.first()))); + } else { + // stack to reg + __ ld(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first()))); + } + } else if (dst.first()->is_stack()) { + // reg to stack + __ sd(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first()))); + } else { + if (dst.first() != src.first()) { + __ mv(dst.first()->as_Register(), src.first()->as_Register()); + } + } +} + +// A double move +static void double_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) { + assert(src.first()->is_stack() && dst.first()->is_stack() || + src.first()->is_reg() && dst.first()->is_reg() || src.first()->is_stack() && dst.first()->is_reg(), "Unexpected error"); + if (src.first()->is_stack()) { + if (dst.first()->is_stack()) { + __ ld(t0, Address(fp, reg2offset_in(src.first()))); + __ sd(t0, Address(sp, reg2offset_out(dst.first()))); + } else if (dst.first()-> is_Register()) { + __ ld(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first()))); + } else { + ShouldNotReachHere(); + } + } else if (src.first() != dst.first()) { + if (src.is_single_phys_reg() && dst.is_single_phys_reg()) { + __ fmv_d(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister()); + } else { + ShouldNotReachHere(); + } + } +} + +void SharedRuntime::save_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) { + // We always ignore the frame_slots arg and just use the space just below frame pointer + // which by this time is free to use + switch (ret_type) { + case T_FLOAT: + __ fsw(f10, Address(fp, -3 * wordSize)); + break; + case T_DOUBLE: + __ fsd(f10, Address(fp, -3 * wordSize)); + break; + case T_VOID: break; + default: { + __ sd(x10, Address(fp, -3 * wordSize)); + } + } +} + +void SharedRuntime::restore_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) { + // We always ignore the frame_slots arg and just use the space just below frame pointer + // which by this time is free to use + switch (ret_type) { + case T_FLOAT: + __ flw(f10, Address(fp, -3 * wordSize)); + break; + case T_DOUBLE: + __ fld(f10, Address(fp, -3 * wordSize)); + break; + case T_VOID: break; + default: { + __ ld(x10, Address(fp, -3 * wordSize)); + } + } +} + +static void save_args(MacroAssembler *masm, int arg_count, int first_arg, VMRegPair *args) { + RegSet x; + for ( int i = first_arg ; i < arg_count ; i++ ) { + if (args[i].first()->is_Register()) { + x = x + args[i].first()->as_Register(); + } else if (args[i].first()->is_FloatRegister()) { + __ addi(sp, sp, -2 * wordSize); + __ fsd(args[i].first()->as_FloatRegister(), Address(sp, 0)); + } + } + __ push_reg(x, sp); +} + +static void restore_args(MacroAssembler *masm, int arg_count, int first_arg, VMRegPair *args) { + RegSet x; + for ( int i = first_arg ; i < arg_count ; i++ ) { + if (args[i].first()->is_Register()) { + x = x + args[i].first()->as_Register(); + } else { + ; + } + } + __ pop_reg(x, sp); + for ( int i = arg_count - 1 ; i >= first_arg ; i-- ) { + if (args[i].first()->is_Register()) { + ; + } else if (args[i].first()->is_FloatRegister()) { + __ fld(args[i].first()->as_FloatRegister(), Address(sp, 0)); + __ add(sp, sp, 2 * wordSize); + } + } +} + +// Unpack an array argument into a pointer to the body and the length +// if the array is non-null, otherwise pass 0 for both. +static void unpack_array_argument(MacroAssembler* masm, VMRegPair reg, BasicType in_elem_type, VMRegPair body_arg, VMRegPair length_arg) { Unimplemented(); } + +class ComputeMoveOrder: public StackObj { + class MoveOperation: public ResourceObj { + friend class ComputeMoveOrder; + private: + VMRegPair _src; + VMRegPair _dst; + int _src_index; + int _dst_index; + bool _processed; + MoveOperation* _next; + MoveOperation* _prev; + + static int get_id(VMRegPair r) { Unimplemented(); return 0; } + + public: + MoveOperation(int src_index, VMRegPair src, int dst_index, VMRegPair dst): + _src(src) + , _dst(dst) + , _src_index(src_index) + , _dst_index(dst_index) + , _processed(false) + , _next(NULL) + , _prev(NULL) { Unimplemented(); } + + VMRegPair src() const { Unimplemented(); return _src; } + int src_id() const { Unimplemented(); return 0; } + int src_index() const { Unimplemented(); return 0; } + VMRegPair dst() const { Unimplemented(); return _src; } + void set_dst(int i, VMRegPair dst) { Unimplemented(); } + int dst_index() const { Unimplemented(); return 0; } + int dst_id() const { Unimplemented(); return 0; } + MoveOperation* next() const { Unimplemented(); return 0; } + MoveOperation* prev() const { Unimplemented(); return 0; } + void set_processed() { Unimplemented(); } + bool is_processed() const { Unimplemented(); return 0; } + + // insert + void break_cycle(VMRegPair temp_register) { Unimplemented(); } + + void link(GrowableArray& killer) { Unimplemented(); } + }; + + private: + GrowableArray edges; + + public: + ComputeMoveOrder(int total_in_args, VMRegPair* in_regs, int total_c_args, VMRegPair* out_regs, + BasicType* in_sig_bt, GrowableArray& arg_order, VMRegPair tmp_vmreg) { Unimplemented(); } + + // Collected all the move operations + void add_edge(int src_index, VMRegPair src, int dst_index, VMRegPair dst) { Unimplemented(); } + + // Walk the edges breaking cycles between moves. The result list + // can be walked in order to produce the proper set of loads + GrowableArray* get_store_order(VMRegPair temp_register) { Unimplemented(); return 0; } +}; + +static void rt_call(MacroAssembler* masm, address dest) { + CodeBlob *cb = CodeCache::find_blob(dest); + RuntimeAddress target(dest); + if (cb) { + __ far_call(target); + } else { + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ jalr(x1, t0, offset); + }); + } +} + +static void verify_oop_args(MacroAssembler* masm, + const methodHandle& method, + const BasicType* sig_bt, + const VMRegPair* regs) { + const Register temp_reg = x9; // not part of any compiled calling seq + if (VerifyOops) { + for (int i = 0; i < method->size_of_parameters(); i++) { + if (sig_bt[i] == T_OBJECT || + sig_bt[i] == T_ARRAY) { + VMReg r = regs[i].first(); + assert(r->is_valid(), "bad oop arg"); + if (r->is_stack()) { + __ ld(temp_reg, Address(sp, r->reg2stack() * VMRegImpl::stack_slot_size)); + __ verify_oop(temp_reg); + } else { + __ verify_oop(r->as_Register()); + } + } + } + } +} + +static void gen_special_dispatch(MacroAssembler* masm, + const methodHandle& method, + const BasicType* sig_bt, + const VMRegPair* regs) { + verify_oop_args(masm, method, sig_bt, regs); + vmIntrinsics::ID iid = method->intrinsic_id(); + + // Now write the args into the outgoing interpreter space + bool has_receiver = false; + Register receiver_reg = noreg; + int member_arg_pos = -1; + Register member_reg = noreg; + int ref_kind = MethodHandles::signature_polymorphic_intrinsic_ref_kind(iid); + if (ref_kind != 0) { + member_arg_pos = method->size_of_parameters() - 1; // trailing MemberName argument + member_reg = x9; // known to be free at this point + has_receiver = MethodHandles::ref_kind_has_receiver(ref_kind); + } else if (iid == vmIntrinsics::_invokeBasic || iid == vmIntrinsics::_linkToNative) { + has_receiver = true; + } else { + fatal("unexpected intrinsic id %d", vmIntrinsics::as_int(iid)); + } + + if (member_reg != noreg) { + // Load the member_arg into register, if necessary. + SharedRuntime::check_member_name_argument_is_last_argument(method, sig_bt, regs); + VMReg r = regs[member_arg_pos].first(); + if (r->is_stack()) { + __ ld(member_reg, Address(sp, r->reg2stack() * VMRegImpl::stack_slot_size)); + } else { + // no data motion is needed + member_reg = r->as_Register(); + } + } + + if (has_receiver) { + // Make sure the receiver is loaded into a register. + assert(method->size_of_parameters() > 0, "oob"); + assert(sig_bt[0] == T_OBJECT, "receiver argument must be an object"); + VMReg r = regs[0].first(); + assert(r->is_valid(), "bad receiver arg"); + if (r->is_stack()) { + // Porting note: This assumes that compiled calling conventions always + // pass the receiver oop in a register. If this is not true on some + // platform, pick a temp and load the receiver from stack. + fatal("receiver always in a register"); + receiver_reg = x12; // known to be free at this point + __ ld(receiver_reg, Address(sp, r->reg2stack() * VMRegImpl::stack_slot_size)); + } else { + // no data motion is needed + receiver_reg = r->as_Register(); + } + } + + // Figure out which address we are really jumping to: + MethodHandles::generate_method_handle_dispatch(masm, iid, + receiver_reg, member_reg, /*for_compiler_entry:*/ true); +} + +// --------------------------------------------------------------------------- +// Generate a native wrapper for a given method. The method takes arguments +// in the Java compiled code convention, marshals them to the native +// convention (handlizes oops, etc), transitions to native, makes the call, +// returns to java state (possibly blocking), unhandlizes any result and +// returns. +// +// Critical native functions are a shorthand for the use of +// GetPrimtiveArrayCritical and disallow the use of any other JNI +// functions. The wrapper is expected to unpack the arguments before +// passing them to the callee and perform checks before and after the +// native call to ensure that they GCLocker +// lock_critical/unlock_critical semantics are followed. Some other +// parts of JNI setup are skipped like the tear down of the JNI handle +// block and the check for pending exceptions it's impossible for them +// to be thrown. +// +// They are roughly structured like this: +// if (GCLocker::needs_gc()) SharedRuntime::block_for_jni_critical() +// tranistion to thread_in_native +// unpack arrray arguments and call native entry point +// check for safepoint in progress +// check if any thread suspend flags are set +// call into JVM and possible unlock the JNI critical +// if a GC was suppressed while in the critical native. +// transition back to thread_in_Java +// return to caller +// +nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, + const methodHandle& method, + int compile_id, + BasicType* in_sig_bt, + VMRegPair* in_regs, + BasicType ret_type, + address critical_entry) { + if (method->is_method_handle_intrinsic()) { + vmIntrinsics::ID iid = method->intrinsic_id(); + intptr_t start = (intptr_t)__ pc(); + int vep_offset = ((intptr_t)__ pc()) - start; + + // First instruction must be a nop as it may need to be patched on deoptimisation + { + Assembler::IncompressibleRegion ir(masm); // keep the nop as 4 bytes for patching. + MacroAssembler::assert_alignment(__ pc()); + __ nop(); // 4 bytes + } + gen_special_dispatch(masm, + method, + in_sig_bt, + in_regs); + int frame_complete = ((intptr_t)__ pc()) - start; // not complete, period + __ flush(); + int stack_slots = SharedRuntime::out_preserve_stack_slots(); // no out slots at all, actually + return nmethod::new_native_nmethod(method, + compile_id, + masm->code(), + vep_offset, + frame_complete, + stack_slots / VMRegImpl::slots_per_word, + in_ByteSize(-1), + in_ByteSize(-1), + (OopMapSet*)NULL); + } + bool is_critical_native = true; + address native_func = critical_entry; + if (native_func == NULL) { + native_func = method->native_function(); + is_critical_native = false; + } + assert(native_func != NULL, "must have function"); + + // An OopMap for lock (and class if static) + OopMapSet *oop_maps = new OopMapSet(); + assert_cond(oop_maps != NULL); + intptr_t start = (intptr_t)__ pc(); + + // We have received a description of where all the java arg are located + // on entry to the wrapper. We need to convert these args to where + // the jni function will expect them. To figure out where they go + // we convert the java signature to a C signature by inserting + // the hidden arguments as arg[0] and possibly arg[1] (static method) + + const int total_in_args = method->size_of_parameters(); + int total_c_args = total_in_args; + if (!is_critical_native) { + total_c_args += 1; + if (method->is_static()) { + total_c_args++; + } + } else { + for (int i = 0; i < total_in_args; i++) { + if (in_sig_bt[i] == T_ARRAY) { + total_c_args++; + } + } + } + + BasicType* out_sig_bt = NEW_RESOURCE_ARRAY(BasicType, total_c_args); + VMRegPair* out_regs = NEW_RESOURCE_ARRAY(VMRegPair, total_c_args); + BasicType* in_elem_bt = NULL; + + int argc = 0; + if (!is_critical_native) { + out_sig_bt[argc++] = T_ADDRESS; + if (method->is_static()) { + out_sig_bt[argc++] = T_OBJECT; + } + + for (int i = 0; i < total_in_args ; i++ ) { + out_sig_bt[argc++] = in_sig_bt[i]; + } + } else { + in_elem_bt = NEW_RESOURCE_ARRAY(BasicType, total_in_args); + SignatureStream ss(method->signature()); + for (int i = 0; i < total_in_args ; i++ ) { + if (in_sig_bt[i] == T_ARRAY) { + // Arrays are passed as int, elem* pair + out_sig_bt[argc++] = T_INT; + out_sig_bt[argc++] = T_ADDRESS; + ss.skip_array_prefix(1); // skip one '[' + assert(ss.is_primitive(), "primitive type expected"); + in_elem_bt[i] = ss.type(); + } else { + out_sig_bt[argc++] = in_sig_bt[i]; + in_elem_bt[i] = T_VOID; + } + if (in_sig_bt[i] != T_VOID) { + assert(in_sig_bt[i] == ss.type() || + in_sig_bt[i] == T_ARRAY, "must match"); + ss.next(); + } + } + } + + // Now figure out where the args must be stored and how much stack space + // they require. + int out_arg_slots = c_calling_convention(out_sig_bt, out_regs, NULL, total_c_args); + + // Compute framesize for the wrapper. We need to handlize all oops in + // incoming registers + + // Calculate the total number of stack slots we will need. + + // First count the abi requirement plus all of the outgoing args + int stack_slots = SharedRuntime::out_preserve_stack_slots() + out_arg_slots; + + // Now the space for the inbound oop handle area + int total_save_slots = 8 * VMRegImpl::slots_per_word; // 8 arguments passed in registers + if (is_critical_native) { + // Critical natives may have to call out so they need a save area + // for register arguments. + int double_slots = 0; + int single_slots = 0; + for ( int i = 0; i < total_in_args; i++) { + if (in_regs[i].first()->is_Register()) { + const Register reg = in_regs[i].first()->as_Register(); + switch (in_sig_bt[i]) { + case T_BOOLEAN: + case T_BYTE: + case T_SHORT: + case T_CHAR: + case T_INT: single_slots++; break; + case T_ARRAY: // specific to LP64 (7145024) + case T_LONG: double_slots++; break; + default: ShouldNotReachHere(); + } + } else if (in_regs[i].first()->is_FloatRegister()) { + ShouldNotReachHere(); + } + } + total_save_slots = double_slots * 2 + single_slots; + // align the save area + if (double_slots != 0) { + stack_slots = align_up(stack_slots, 2); + } + } + + int oop_handle_offset = stack_slots; + stack_slots += total_save_slots; + + // Now any space we need for handlizing a klass if static method + + int klass_slot_offset = 0; + int klass_offset = -1; + int lock_slot_offset = 0; + bool is_static = false; + + if (method->is_static()) { + klass_slot_offset = stack_slots; + stack_slots += VMRegImpl::slots_per_word; + klass_offset = klass_slot_offset * VMRegImpl::stack_slot_size; + is_static = true; + } + + // Plus a lock if needed + + if (method->is_synchronized()) { + lock_slot_offset = stack_slots; + stack_slots += VMRegImpl::slots_per_word; + } + + // Now a place (+2) to save return values or temp during shuffling + // + 4 for return address (which we own) and saved fp + stack_slots += 6; + + // Ok The space we have allocated will look like: + // + // + // FP-> | | + // | 2 slots (ra) | + // | 2 slots (fp) | + // |---------------------| + // | 2 slots for moves | + // |---------------------| + // | lock box (if sync) | + // |---------------------| <- lock_slot_offset + // | klass (if static) | + // |---------------------| <- klass_slot_offset + // | oopHandle area | + // |---------------------| <- oop_handle_offset (8 java arg registers) + // | outbound memory | + // | based arguments | + // | | + // |---------------------| + // | | + // SP-> | out_preserved_slots | + // + // + + + // Now compute actual number of stack words we need rounding to make + // stack properly aligned. + stack_slots = align_up(stack_slots, StackAlignmentInSlots); + + int stack_size = stack_slots * VMRegImpl::stack_slot_size; + + // First thing make an ic check to see if we should even be here + + // We are free to use all registers as temps without saving them and + // restoring them except fp. fp is the only callee save register + // as far as the interpreter and the compiler(s) are concerned. + + + const Register ic_reg = t1; + const Register receiver = j_rarg0; + + Label hit; + Label exception_pending; + + __ verify_oop(receiver); + assert_different_registers(ic_reg, receiver, t0, t2); + __ cmp_klass(receiver, ic_reg, t0, t2 /* call-clobbered t2 as a tmp */, hit); + + __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub())); + + // Verified entry point must be aligned + __ align(8); + + __ bind(hit); + + int vep_offset = ((intptr_t)__ pc()) - start; + + // If we have to make this method not-entrant we'll overwrite its + // first instruction with a jump. + { + Assembler::IncompressibleRegion ir(masm); // keep the nop as 4 bytes for patching. + MacroAssembler::assert_alignment(__ pc()); + __ nop(); // 4 bytes + } + + if (VM_Version::supports_fast_class_init_checks() && method->needs_clinit_barrier()) { + Label L_skip_barrier; + __ mov_metadata(t1, method->method_holder()); // InstanceKlass* + __ clinit_barrier(t1, t0, &L_skip_barrier); + __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); + + __ bind(L_skip_barrier); + } + + // Generate stack overflow check + __ bang_stack_with_offset(checked_cast(StackOverflow::stack_shadow_zone_size())); + + // Generate a new frame for the wrapper. + __ enter(); + // -2 because return address is already present and so is saved fp + __ sub(sp, sp, stack_size - 2 * wordSize); + + BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); + assert_cond(bs != NULL); + bs->nmethod_entry_barrier(masm); + + // Frame is now completed as far as size and linkage. + int frame_complete = ((intptr_t)__ pc()) - start; + + // We use x18 as the oop handle for the receiver/klass + // It is callee save so it survives the call to native + + const Register oop_handle_reg = x18; + + // + // We immediately shuffle the arguments so that any vm call we have to + // make from here on out (sync slow path, jvmti, etc.) we will have + // captured the oops from our caller and have a valid oopMap for + // them. + + // ----------------- + // The Grand Shuffle + + // The Java calling convention is either equal (linux) or denser (win64) than the + // c calling convention. However the because of the jni_env argument the c calling + // convention always has at least one more (and two for static) arguments than Java. + // Therefore if we move the args from java -> c backwards then we will never have + // a register->register conflict and we don't have to build a dependency graph + // and figure out how to break any cycles. + // + + // Record esp-based slot for receiver on stack for non-static methods + int receiver_offset = -1; + + // This is a trick. We double the stack slots so we can claim + // the oops in the caller's frame. Since we are sure to have + // more args than the caller doubling is enough to make + // sure we can capture all the incoming oop args from the + // caller. + // + OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/); + assert_cond(map != NULL); + + int float_args = 0; + int int_args = 0; + +#ifdef ASSERT + bool reg_destroyed[RegisterImpl::number_of_registers]; + bool freg_destroyed[FloatRegisterImpl::number_of_registers]; + for ( int r = 0 ; r < RegisterImpl::number_of_registers ; r++ ) { + reg_destroyed[r] = false; + } + for ( int f = 0 ; f < FloatRegisterImpl::number_of_registers ; f++ ) { + freg_destroyed[f] = false; + } + +#endif /* ASSERT */ + + // This may iterate in two different directions depending on the + // kind of native it is. The reason is that for regular JNI natives + // the incoming and outgoing registers are offset upwards and for + // critical natives they are offset down. + GrowableArray arg_order(2 * total_in_args); + VMRegPair tmp_vmreg; + tmp_vmreg.set2(x9->as_VMReg()); + + if (!is_critical_native) { + for (int i = total_in_args - 1, c_arg = total_c_args - 1; i >= 0; i--, c_arg--) { + arg_order.push(i); + arg_order.push(c_arg); + } + } else { + // Compute a valid move order, using tmp_vmreg to break any cycles + ComputeMoveOrder cmo(total_in_args, in_regs, total_c_args, out_regs, in_sig_bt, arg_order, tmp_vmreg); + } + + int temploc = -1; + for (int ai = 0; ai < arg_order.length(); ai += 2) { + int i = arg_order.at(ai); + int c_arg = arg_order.at(ai + 1); + __ block_comment(err_msg("mv %d -> %d", i, c_arg)); + if (c_arg == -1) { + assert(is_critical_native, "should only be required for critical natives"); + // This arg needs to be moved to a temporary + __ mv(tmp_vmreg.first()->as_Register(), in_regs[i].first()->as_Register()); + in_regs[i] = tmp_vmreg; + temploc = i; + continue; + } else if (i == -1) { + assert(is_critical_native, "should only be required for critical natives"); + // Read from the temporary location + assert(temploc != -1, "must be valid"); + i = temploc; + temploc = -1; + } +#ifdef ASSERT + if (in_regs[i].first()->is_Register()) { + assert(!reg_destroyed[in_regs[i].first()->as_Register()->encoding()], "destroyed reg!"); + } else if (in_regs[i].first()->is_FloatRegister()) { + assert(!freg_destroyed[in_regs[i].first()->as_FloatRegister()->encoding()], "destroyed reg!"); + } + if (out_regs[c_arg].first()->is_Register()) { + reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true; + } else if (out_regs[c_arg].first()->is_FloatRegister()) { + freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding()] = true; + } +#endif /* ASSERT */ + switch (in_sig_bt[i]) { + case T_ARRAY: + if (is_critical_native) { + unpack_array_argument(masm, in_regs[i], in_elem_bt[i], out_regs[c_arg + 1], out_regs[c_arg]); + c_arg++; +#ifdef ASSERT + if (out_regs[c_arg].first()->is_Register()) { + reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true; + } else if (out_regs[c_arg].first()->is_FloatRegister()) { + freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding()] = true; + } +#endif + int_args++; + break; + } + case T_OBJECT: + assert(!is_critical_native, "no oop arguments"); + object_move(masm, map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg], + ((i == 0) && (!is_static)), + &receiver_offset); + int_args++; + break; + case T_VOID: + break; + + case T_FLOAT: + float_move(masm, in_regs[i], out_regs[c_arg]); + float_args++; + break; + + case T_DOUBLE: + assert( i + 1 < total_in_args && + in_sig_bt[i + 1] == T_VOID && + out_sig_bt[c_arg + 1] == T_VOID, "bad arg list"); + double_move(masm, in_regs[i], out_regs[c_arg]); + float_args++; + break; + + case T_LONG : + long_move(masm, in_regs[i], out_regs[c_arg]); + int_args++; + break; + + case T_ADDRESS: + assert(false, "found T_ADDRESS in java args"); + break; + + default: + move32_64(masm, in_regs[i], out_regs[c_arg]); + int_args++; + } + } + + // point c_arg at the first arg that is already loaded in case we + // need to spill before we call out + int c_arg = total_c_args - total_in_args; + + // Pre-load a static method's oop into c_rarg1. + if (method->is_static() && !is_critical_native) { + + // load oop into a register + __ movoop(c_rarg1, + JNIHandles::make_local(method->method_holder()->java_mirror()), + /*immediate*/true); + + // Now handlize the static class mirror it's known not-null. + __ sd(c_rarg1, Address(sp, klass_offset)); + map->set_oop(VMRegImpl::stack2reg(klass_slot_offset)); + + // Now get the handle + __ la(c_rarg1, Address(sp, klass_offset)); + // and protect the arg if we must spill + c_arg--; + } + + // Change state to native (we save the return address in the thread, since it might not + // be pushed on the stack when we do a stack traversal). + // We use the same pc/oopMap repeatedly when we call out + + Label native_return; + __ set_last_Java_frame(sp, noreg, native_return, t0); + + Label dtrace_method_entry, dtrace_method_entry_done; + { + ExternalAddress target((address)&DTraceMethodProbes); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ lbu(t0, Address(t0, offset)); + }); + __ bnez(t0, dtrace_method_entry); + __ bind(dtrace_method_entry_done); + } + + // RedefineClasses() tracing support for obsolete method entry + if (log_is_enabled(Trace, redefine, class, obsolete)) { + // protect the args we've loaded + save_args(masm, total_c_args, c_arg, out_regs); + __ mov_metadata(c_rarg1, method()); + __ call_VM_leaf( + CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry), + xthread, c_rarg1); + restore_args(masm, total_c_args, c_arg, out_regs); + } + + // Lock a synchronized method + + // Register definitions used by locking and unlocking + + const Register swap_reg = x10; + const Register obj_reg = x9; // Will contain the oop + const Register lock_reg = x30; // Address of compiler lock object (BasicLock) + const Register old_hdr = x30; // value of old header at unlock time + const Register tmp = ra; + + Label slow_path_lock; + Label lock_done; + + if (method->is_synchronized()) { + + const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes(); + + // Get the handle (the 2nd argument) + __ mv(oop_handle_reg, c_rarg1); + + // Get address of the box + __ la(lock_reg, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size)); + + // Load the oop from the handle + __ ld(obj_reg, Address(oop_handle_reg, 0)); + + if (UseBiasedLocking) { + __ biased_locking_enter(lock_reg, obj_reg, swap_reg, tmp, false, lock_done, &slow_path_lock); + } + + // Load (object->mark() | 1) into swap_reg % x10 + __ ld(t0, Address(obj_reg, oopDesc::mark_offset_in_bytes())); + __ ori(swap_reg, t0, 1); + + // Save (object->mark() | 1) into BasicLock's displaced header + __ sd(swap_reg, Address(lock_reg, mark_word_offset)); + + // src -> dest if dest == x10 else x10 <- dest + { + Label here; + __ cmpxchg_obj_header(x10, lock_reg, obj_reg, t0, lock_done, /*fallthrough*/NULL); + } + + // Test if the oopMark is an obvious stack pointer, i.e., + // 1) (mark & 3) == 0, and + // 2) sp <= mark < mark + os::pagesize() + // These 3 tests can be done by evaluating the following + // expression: ((mark - sp) & (3 - os::vm_page_size())), + // assuming both stack pointer and pagesize have their + // least significant 2 bits clear. + // NOTE: the oopMark is in swap_reg % 10 as the result of cmpxchg + + __ sub(swap_reg, swap_reg, sp); + __ andi(swap_reg, swap_reg, 3 - os::vm_page_size()); + + // Save the test result, for recursive case, the result is zero + __ sd(swap_reg, Address(lock_reg, mark_word_offset)); + __ bnez(swap_reg, slow_path_lock); + + // Slow path will re-enter here + __ bind(lock_done); + } + + + // Finally just about ready to make the JNI call + + // get JNIEnv* which is first argument to native + if (!is_critical_native) { + __ la(c_rarg0, Address(xthread, in_bytes(JavaThread::jni_environment_offset()))); + + // Now set thread in native + __ la(t1, Address(xthread, JavaThread::thread_state_offset())); + __ mv(t0, _thread_in_native); + __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore); + __ sw(t0, Address(t1)); + } + + rt_call(masm, native_func); + + __ bind(native_return); + + intptr_t return_pc = (intptr_t) __ pc(); + oop_maps->add_gc_map(return_pc - start, map); + + // Unpack native results. + if (ret_type != T_OBJECT && ret_type != T_ARRAY) { + __ cast_primitive_type(ret_type, x10); + } + + Label safepoint_in_progress, safepoint_in_progress_done; + Label after_transition; + + // If this is a critical native, check for a safepoint or suspend request after the call. + // If a safepoint is needed, transition to native, then to native_trans to handle + // safepoint like the native methods that are not critical natives. + if (is_critical_native) { + Label needs_safepoint; + __ safepoint_poll(needs_safepoint, false /* as_return */, true /* acquire */, false /* in_nmethod */); + __ lwu(t0, Address(xthread, JavaThread::suspend_flags_offset())); + __ bnez(t0, needs_safepoint); + __ j(after_transition); + __ bind(needs_safepoint); + } + + // Switch thread to "native transition" state before reading the synchronization state. + // This additional state is necessary because reading and testing the synchronization + // state is not atomic w.r.t. GC, as this scenario demonstrates: + // Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted. + // VM thread changes sync state to synchronizing and suspends threads for GC. + // Thread A is resumed to finish this native method, but doesn't block here since it + // didn't see any synchronization is progress, and escapes. + __ mv(t0, _thread_in_native_trans); + + __ sw(t0, Address(xthread, JavaThread::thread_state_offset())); + + // Force this write out before the read below + __ membar(MacroAssembler::AnyAny); + + // check for safepoint operation in progress and/or pending suspend requests + { + // We need an acquire here to ensure that any subsequent load of the + // global SafepointSynchronize::_state flag is ordered after this load + // of the thread-local polling word. We don't want this poll to + // return false (i.e. not safepointing) and a later poll of the global + // SafepointSynchronize::_state spuriously to return true. + // This is to avoid a race when we're in a native->Java transition + // racing the code which wakes up from a safepoint. + + __ safepoint_poll(safepoint_in_progress, true /* at_return */, true /* acquire */, false /* in_nmethod */); + __ lwu(t0, Address(xthread, JavaThread::suspend_flags_offset())); + __ bnez(t0, safepoint_in_progress); + __ bind(safepoint_in_progress_done); + } + + // change thread state + __ la(t1, Address(xthread, JavaThread::thread_state_offset())); + __ mv(t0, _thread_in_Java); + __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore); + __ sw(t0, Address(t1)); + __ bind(after_transition); + + Label reguard; + Label reguard_done; + __ lbu(t0, Address(xthread, JavaThread::stack_guard_state_offset())); + __ mv(t1, StackOverflow::stack_guard_yellow_reserved_disabled); + __ beq(t0, t1, reguard); + __ bind(reguard_done); + + // native result if any is live + + // Unlock + Label unlock_done; + Label slow_path_unlock; + if (method->is_synchronized()) { + + // Get locked oop from the handle we passed to jni + __ ld(obj_reg, Address(oop_handle_reg, 0)); + + Label done; + + if (UseBiasedLocking) { + __ biased_locking_exit(obj_reg, old_hdr, done); + } + + // Simple recursive lock? + __ ld(t0, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size)); + __ beqz(t0, done); + + // Must save x10 if it is live now because cmpxchg must use it + if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) { + save_native_result(masm, ret_type, stack_slots); + } + + // get address of the stack lock + __ la(x10, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size)); + // get old displaced header + __ ld(old_hdr, Address(x10, 0)); + + // Atomic swap old header if oop still contains the stack lock + Label succeed; + __ cmpxchg_obj_header(x10, old_hdr, obj_reg, t0, succeed, &slow_path_unlock); + __ bind(succeed); + + // slow path re-enters here + __ bind(unlock_done); + if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) { + restore_native_result(masm, ret_type, stack_slots); + } + + __ bind(done); + } + + Label dtrace_method_exit, dtrace_method_exit_done; + { + ExternalAddress target((address)&DTraceMethodProbes); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ lbu(t0, Address(t0, offset)); + }); + __ bnez(t0, dtrace_method_exit); + __ bind(dtrace_method_exit_done); + } + + __ reset_last_Java_frame(false); + + // Unbox oop result, e.g. JNIHandles::resolve result. + if (is_reference_type(ret_type)) { + __ resolve_jobject(x10, xthread, t1); + } + + if (CheckJNICalls) { + // clear_pending_jni_exception_check + __ sd(zr, Address(xthread, JavaThread::pending_jni_exception_check_fn_offset())); + } + + if (!is_critical_native) { + // reset handle block + __ ld(x12, Address(xthread, JavaThread::active_handles_offset())); + __ sd(zr, Address(x12, JNIHandleBlock::top_offset_in_bytes())); + } + + __ leave(); + + if (!is_critical_native) { + // Any exception pending? + __ ld(t0, Address(xthread, in_bytes(Thread::pending_exception_offset()))); + __ bnez(t0, exception_pending); + } + + // We're done + __ ret(); + + // Unexpected paths are out of line and go here + + if (!is_critical_native) { + // forward the exception + __ bind(exception_pending); + + // and forward the exception + __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry())); + } + + // Slow path locking & unlocking + if (method->is_synchronized()) { + + __ block_comment("Slow path lock {"); + __ bind(slow_path_lock); + + // has last_Java_frame setup. No exceptions so do vanilla call not call_VM + // args are (oop obj, BasicLock* lock, JavaThread* thread) + + // protect the args we've loaded + save_args(masm, total_c_args, c_arg, out_regs); + + __ mv(c_rarg0, obj_reg); + __ mv(c_rarg1, lock_reg); + __ mv(c_rarg2, xthread); + + // Not a leaf but we have last_Java_frame setup as we want + __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), 3); + restore_args(masm, total_c_args, c_arg, out_regs); + +#ifdef ASSERT + { Label L; + __ ld(t0, Address(xthread, in_bytes(Thread::pending_exception_offset()))); + __ beqz(t0, L); + __ stop("no pending exception allowed on exit from monitorenter"); + __ bind(L); + } +#endif + __ j(lock_done); + + __ block_comment("} Slow path lock"); + + __ block_comment("Slow path unlock {"); + __ bind(slow_path_unlock); + + if (ret_type == T_FLOAT || ret_type == T_DOUBLE) { + save_native_result(masm, ret_type, stack_slots); + } + + __ mv(c_rarg2, xthread); + __ la(c_rarg1, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size)); + __ mv(c_rarg0, obj_reg); + + // Save pending exception around call to VM (which contains an EXCEPTION_MARK) + // NOTE that obj_reg == x9 currently + __ ld(x9, Address(xthread, in_bytes(Thread::pending_exception_offset()))); + __ sd(zr, Address(xthread, in_bytes(Thread::pending_exception_offset()))); + + rt_call(masm, CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C)); + +#ifdef ASSERT + { + Label L; + __ ld(t0, Address(xthread, in_bytes(Thread::pending_exception_offset()))); + __ beqz(t0, L); + __ stop("no pending exception allowed on exit complete_monitor_unlocking_C"); + __ bind(L); + } +#endif /* ASSERT */ + + __ sd(x9, Address(xthread, in_bytes(Thread::pending_exception_offset()))); + + if (ret_type == T_FLOAT || ret_type == T_DOUBLE) { + restore_native_result(masm, ret_type, stack_slots); + } + __ j(unlock_done); + + __ block_comment("} Slow path unlock"); + + } // synchronized + + // SLOW PATH Reguard the stack if needed + + __ bind(reguard); + save_native_result(masm, ret_type, stack_slots); + rt_call(masm, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages)); + restore_native_result(masm, ret_type, stack_slots); + // and continue + __ j(reguard_done); + + // SLOW PATH safepoint + { + __ block_comment("safepoint {"); + __ bind(safepoint_in_progress); + + // Don't use call_VM as it will see a possible pending exception and forward it + // and never return here preventing us from clearing _last_native_pc down below. + // + save_native_result(masm, ret_type, stack_slots); + __ mv(c_rarg0, xthread); +#ifndef PRODUCT + assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area"); +#endif + RuntimeAddress target(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ jalr(x1, t0, offset); + }); + + // Restore any method result value + restore_native_result(masm, ret_type, stack_slots); + + __ j(safepoint_in_progress_done); + __ block_comment("} safepoint"); + } + + // SLOW PATH dtrace support + { + __ block_comment("dtrace entry {"); + __ bind(dtrace_method_entry); + + // We have all of the arguments setup at this point. We must not touch any register + // argument registers at this point (what if we save/restore them there are no oop? + + save_args(masm, total_c_args, c_arg, out_regs); + __ mov_metadata(c_rarg1, method()); + __ call_VM_leaf( + CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry), + xthread, c_rarg1); + restore_args(masm, total_c_args, c_arg, out_regs); + __ j(dtrace_method_entry_done); + __ block_comment("} dtrace entry"); + } + + { + __ block_comment("dtrace exit {"); + __ bind(dtrace_method_exit); + save_native_result(masm, ret_type, stack_slots); + __ mov_metadata(c_rarg1, method()); + __ call_VM_leaf( + CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), + xthread, c_rarg1); + restore_native_result(masm, ret_type, stack_slots); + __ j(dtrace_method_exit_done); + __ block_comment("} dtrace exit"); + } + + __ flush(); + + nmethod *nm = nmethod::new_native_nmethod(method, + compile_id, + masm->code(), + vep_offset, + frame_complete, + stack_slots / VMRegImpl::slots_per_word, + (is_static ? in_ByteSize(klass_offset) : in_ByteSize(receiver_offset)), + in_ByteSize(lock_slot_offset*VMRegImpl::stack_slot_size), + oop_maps); + assert(nm != NULL, "create native nmethod fail!"); + return nm; +} + +// this function returns the adjust size (in number of words) to a c2i adapter +// activation for use during deoptimization +int Deoptimization::last_frame_adjust(int callee_parameters, int callee_locals) { + assert(callee_locals >= callee_parameters, + "test and remove; got more parms than locals"); + if (callee_locals < callee_parameters) { + return 0; // No adjustment for negative locals + } + int diff = (callee_locals - callee_parameters) * Interpreter::stackElementWords; + // diff is counted in stack words + return align_up(diff, 2); +} + +//------------------------------generate_deopt_blob---------------------------- +void SharedRuntime::generate_deopt_blob() { + // Allocate space for the code + ResourceMark rm; + // Setup code generation tools + int pad = 0; + CodeBuffer buffer("deopt_blob", 2048 + pad, 1024); + MacroAssembler* masm = new MacroAssembler(&buffer); + int frame_size_in_words = -1; + OopMap* map = NULL; + OopMapSet *oop_maps = new OopMapSet(); + assert_cond(masm != NULL && oop_maps != NULL); + RegisterSaver reg_saver(COMPILER2_OR_JVMCI != 0); + + // ------------- + // This code enters when returning to a de-optimized nmethod. A return + // address has been pushed on the the stack, and return values are in + // registers. + // If we are doing a normal deopt then we were called from the patched + // nmethod from the point we returned to the nmethod. So the return + // address on the stack is wrong by NativeCall::instruction_size + // We will adjust the value so it looks like we have the original return + // address on the stack (like when we eagerly deoptimized). + // In the case of an exception pending when deoptimizing, we enter + // with a return address on the stack that points after the call we patched + // into the exception handler. We have the following register state from, + // e.g., the forward exception stub (see stubGenerator_riscv.cpp). + // x10: exception oop + // x9: exception handler + // x13: throwing pc + // So in this case we simply jam x13 into the useless return address and + // the stack looks just like we want. + // + // At this point we need to de-opt. We save the argument return + // registers. We call the first C routine, fetch_unroll_info(). This + // routine captures the return values and returns a structure which + // describes the current frame size and the sizes of all replacement frames. + // The current frame is compiled code and may contain many inlined + // functions, each with their own JVM state. We pop the current frame, then + // push all the new frames. Then we call the C routine unpack_frames() to + // populate these frames. Finally unpack_frames() returns us the new target + // address. Notice that callee-save registers are BLOWN here; they have + // already been captured in the vframeArray at the time the return PC was + // patched. + address start = __ pc(); + Label cont; + + // Prolog for non exception case! + + // Save everything in sight. + map = reg_saver.save_live_registers(masm, 0, &frame_size_in_words); + + // Normal deoptimization. Save exec mode for unpack_frames. + __ mv(xcpool, Deoptimization::Unpack_deopt); // callee-saved + __ j(cont); + + int reexecute_offset = __ pc() - start; + + // Reexecute case + // return address is the pc describes what bci to do re-execute at + + // No need to update map as each call to save_live_registers will produce identical oopmap + (void) reg_saver.save_live_registers(masm, 0, &frame_size_in_words); + + __ mv(xcpool, Deoptimization::Unpack_reexecute); // callee-saved + __ j(cont); + + int exception_offset = __ pc() - start; + + // Prolog for exception case + + // all registers are dead at this entry point, except for x10, and + // x13 which contain the exception oop and exception pc + // respectively. Set them in TLS and fall thru to the + // unpack_with_exception_in_tls entry point. + + __ sd(x13, Address(xthread, JavaThread::exception_pc_offset())); + __ sd(x10, Address(xthread, JavaThread::exception_oop_offset())); + + int exception_in_tls_offset = __ pc() - start; + + // new implementation because exception oop is now passed in JavaThread + + // Prolog for exception case + // All registers must be preserved because they might be used by LinearScan + // Exceptiop oop and throwing PC are passed in JavaThread + // tos: stack at point of call to method that threw the exception (i.e. only + // args are on the stack, no return address) + + // The return address pushed by save_live_registers will be patched + // later with the throwing pc. The correct value is not available + // now because loading it from memory would destroy registers. + + // NB: The SP at this point must be the SP of the method that is + // being deoptimized. Deoptimization assumes that the frame created + // here by save_live_registers is immediately below the method's SP. + // This is a somewhat fragile mechanism. + + // Save everything in sight. + map = reg_saver.save_live_registers(masm, 0, &frame_size_in_words); + + // Now it is safe to overwrite any register + + // Deopt during an exception. Save exec mode for unpack_frames. + __ mv(xcpool, Deoptimization::Unpack_exception); // callee-saved + + // load throwing pc from JavaThread and patch it as the return address + // of the current frame. Then clear the field in JavaThread + + __ ld(x13, Address(xthread, JavaThread::exception_pc_offset())); + __ sd(x13, Address(fp, frame::return_addr_offset * wordSize)); + __ sd(zr, Address(xthread, JavaThread::exception_pc_offset())); + +#ifdef ASSERT + // verify that there is really an exception oop in JavaThread + __ ld(x10, Address(xthread, JavaThread::exception_oop_offset())); + __ verify_oop(x10); + + // verify that there is no pending exception + Label no_pending_exception; + __ ld(t0, Address(xthread, Thread::pending_exception_offset())); + __ beqz(t0, no_pending_exception); + __ stop("must not have pending exception here"); + __ bind(no_pending_exception); +#endif + + __ bind(cont); + + // Call C code. Need thread and this frame, but NOT official VM entry + // crud. We cannot block on this call, no GC can happen. + // + // UnrollBlock* fetch_unroll_info(JavaThread* thread) + + // fetch_unroll_info needs to call last_java_frame(). + + Label retaddr; + __ set_last_Java_frame(sp, noreg, retaddr, t0); +#ifdef ASSERT + { + Label L; + __ ld(t0, Address(xthread, + JavaThread::last_Java_fp_offset())); + __ beqz(t0, L); + __ stop("SharedRuntime::generate_deopt_blob: last_Java_fp not cleared"); + __ bind(L); + } +#endif // ASSERT + __ mv(c_rarg0, xthread); + __ mv(c_rarg1, xcpool); + RuntimeAddress target(CAST_FROM_FN_PTR(address, Deoptimization::fetch_unroll_info)); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ jalr(x1, t0, offset); + }); + __ bind(retaddr); + + // Need to have an oopmap that tells fetch_unroll_info where to + // find any register it might need. + oop_maps->add_gc_map(__ pc() - start, map); + + __ reset_last_Java_frame(false); + + // Load UnrollBlock* into x15 + __ mv(x15, x10); + + __ lwu(xcpool, Address(x15, Deoptimization::UnrollBlock::unpack_kind_offset_in_bytes())); + Label noException; + __ mv(t0, Deoptimization::Unpack_exception); + __ bne(xcpool, t0, noException); // Was exception pending? + __ ld(x10, Address(xthread, JavaThread::exception_oop_offset())); + __ ld(x13, Address(xthread, JavaThread::exception_pc_offset())); + __ sd(zr, Address(xthread, JavaThread::exception_oop_offset())); + __ sd(zr, Address(xthread, JavaThread::exception_pc_offset())); + + __ verify_oop(x10); + + // Overwrite the result registers with the exception results. + __ sd(x10, Address(sp, reg_saver.reg_offset_in_bytes(x10))); + + __ bind(noException); + + // Only register save data is on the stack. + // Now restore the result registers. Everything else is either dead + // or captured in the vframeArray. + + // Restore fp result register + __ fld(f10, Address(sp, reg_saver.freg_offset_in_bytes(f10))); + // Restore integer result register + __ ld(x10, Address(sp, reg_saver.reg_offset_in_bytes(x10))); + + // Pop all of the register save area off the stack + __ add(sp, sp, frame_size_in_words * wordSize); + + // All of the register save area has been popped of the stack. Only the + // return address remains. + + // Pop all the frames we must move/replace. + // + // Frame picture (youngest to oldest) + // 1: self-frame (no frame link) + // 2: deopting frame (no frame link) + // 3: caller of deopting frame (could be compiled/interpreted). + // + // Note: by leaving the return address of self-frame on the stack + // and using the size of frame 2 to adjust the stack + // when we are done the return to frame 3 will still be on the stack. + + // Pop deoptimized frame + __ lwu(x12, Address(x15, Deoptimization::UnrollBlock::size_of_deoptimized_frame_offset_in_bytes())); + __ sub(x12, x12, 2 * wordSize); + __ add(sp, sp, x12); + __ ld(fp, Address(sp, 0)); + __ ld(ra, Address(sp, wordSize)); + __ addi(sp, sp, 2 * wordSize); + // RA should now be the return address to the caller (3) + +#ifdef ASSERT + // Compilers generate code that bang the stack by as much as the + // interpreter would need. So this stack banging should never + // trigger a fault. Verify that it does not on non product builds. + __ lwu(x9, Address(x15, Deoptimization::UnrollBlock::total_frame_sizes_offset_in_bytes())); + __ bang_stack_size(x9, x12); +#endif + // Load address of array of frame pcs into x12 + __ ld(x12, Address(x15, Deoptimization::UnrollBlock::frame_pcs_offset_in_bytes())); + + // Load address of array of frame sizes into x14 + __ ld(x14, Address(x15, Deoptimization::UnrollBlock::frame_sizes_offset_in_bytes())); + + // Load counter into x13 + __ lwu(x13, Address(x15, Deoptimization::UnrollBlock::number_of_frames_offset_in_bytes())); + + // Now adjust the caller's stack to make up for the extra locals + // but record the original sp so that we can save it in the skeletal interpreter + // frame and the stack walking of interpreter_sender will get the unextended sp + // value and not the "real" sp value. + + const Register sender_sp = x16; + + __ mv(sender_sp, sp); + __ lwu(x9, Address(x15, + Deoptimization::UnrollBlock:: + caller_adjustment_offset_in_bytes())); + __ sub(sp, sp, x9); + + // Push interpreter frames in a loop + __ mv(t0, 0xDEADDEAD); // Make a recognizable pattern + __ mv(t1, t0); + Label loop; + __ bind(loop); + __ ld(x9, Address(x14, 0)); // Load frame size + __ addi(x14, x14, wordSize); + __ sub(x9, x9, 2 * wordSize); // We'll push pc and fp by hand + __ ld(ra, Address(x12, 0)); // Load pc + __ addi(x12, x12, wordSize); + __ enter(); // Save old & set new fp + __ sub(sp, sp, x9); // Prolog + // This value is corrected by layout_activation_impl + __ sd(zr, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize)); + __ sd(sender_sp, Address(fp, frame::interpreter_frame_sender_sp_offset * wordSize)); // Make it walkable + __ mv(sender_sp, sp); // Pass sender_sp to next frame + __ addi(x13, x13, -1); // Decrement counter + __ bnez(x13, loop); + + // Re-push self-frame + __ ld(ra, Address(x12)); + __ enter(); + + // Allocate a full sized register save area. We subtract 2 because + // enter() just pushed 2 words + __ sub(sp, sp, (frame_size_in_words - 2) * wordSize); + + // Restore frame locals after moving the frame + __ fsd(f10, Address(sp, reg_saver.freg_offset_in_bytes(f10))); + __ sd(x10, Address(sp, reg_saver.reg_offset_in_bytes(x10))); + + // Call C code. Need thread but NOT official VM entry + // crud. We cannot block on this call, no GC can happen. Call should + // restore return values to their stack-slots with the new SP. + // + // void Deoptimization::unpack_frames(JavaThread* thread, int exec_mode) + + // Use fp because the frames look interpreted now + // Don't need the precise return PC here, just precise enough to point into this code blob. + address the_pc = __ pc(); + __ set_last_Java_frame(sp, fp, the_pc, t0); + + __ mv(c_rarg0, xthread); + __ mv(c_rarg1, xcpool); // second arg: exec_mode + target = RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames)); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ jalr(x1, t0, offset); + }); + + // Set an oopmap for the call site + // Use the same PC we used for the last java frame + oop_maps->add_gc_map(the_pc - start, + new OopMap(frame_size_in_words, 0)); + + // Clear fp AND pc + __ reset_last_Java_frame(true); + + // Collect return values + __ fld(f10, Address(sp, reg_saver.freg_offset_in_bytes(f10))); + __ ld(x10, Address(sp, reg_saver.reg_offset_in_bytes(x10))); + + // Pop self-frame. + __ leave(); // Epilog + + // Jump to interpreter + __ ret(); + + // Make sure all code is generated + masm->flush(); + + _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, frame_size_in_words); + assert(_deopt_blob != NULL, "create deoptimization blob fail!"); + _deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset); +} + +// Number of stack slots between incoming argument block and the start of +// a new frame. The PROLOG must add this many slots to the stack. The +// EPILOG must remove this many slots. +// RISCV needs two words for RA (return address) and FP (frame pointer). +uint SharedRuntime::in_preserve_stack_slots() { + return 2 * VMRegImpl::slots_per_word; +} + +uint SharedRuntime::out_preserve_stack_slots() { + return 0; +} + +#ifdef COMPILER2 +//------------------------------generate_uncommon_trap_blob-------------------- +void SharedRuntime::generate_uncommon_trap_blob() { + // Allocate space for the code + ResourceMark rm; + // Setup code generation tools + CodeBuffer buffer("uncommon_trap_blob", 2048, 1024); + MacroAssembler* masm = new MacroAssembler(&buffer); + assert_cond(masm != NULL); + + assert(SimpleRuntimeFrame::framesize % 4 == 0, "sp not 16-byte aligned"); + + address start = __ pc(); + + // Push self-frame. We get here with a return address in RA + // and sp should be 16 byte aligned + // push fp and retaddr by hand + __ addi(sp, sp, -2 * wordSize); + __ sd(ra, Address(sp, wordSize)); + __ sd(fp, Address(sp, 0)); + // we don't expect an arg reg save area +#ifndef PRODUCT + assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area"); +#endif + // compiler left unloaded_class_index in j_rarg0 move to where the + // runtime expects it. + __ sign_extend(c_rarg1, j_rarg0, 32); + + // we need to set the past SP to the stack pointer of the stub frame + // and the pc to the address where this runtime call will return + // although actually any pc in this code blob will do). + Label retaddr; + __ set_last_Java_frame(sp, noreg, retaddr, t0); + + // Call C code. Need thread but NOT official VM entry + // crud. We cannot block on this call, no GC can happen. Call should + // capture callee-saved registers as well as return values. + // + // UnrollBlock* uncommon_trap(JavaThread* thread, jint unloaded_class_index, jint exec_mode) + // + // n.b. 3 gp args, 0 fp args, integral return type + + __ mv(c_rarg0, xthread); + __ mv(c_rarg2, Deoptimization::Unpack_uncommon_trap); + RuntimeAddress target(CAST_FROM_FN_PTR(address, Deoptimization::uncommon_trap)); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ jalr(x1, t0, offset); + }); + __ bind(retaddr); + + // Set an oopmap for the call site + OopMapSet* oop_maps = new OopMapSet(); + OopMap* map = new OopMap(SimpleRuntimeFrame::framesize, 0); + assert_cond(oop_maps != NULL && map != NULL); + + // location of fp is known implicitly by the frame sender code + + oop_maps->add_gc_map(__ pc() - start, map); + + __ reset_last_Java_frame(false); + + // move UnrollBlock* into x14 + __ mv(x14, x10); + +#ifdef ASSERT + { Label L; + __ lwu(t0, Address(x14, Deoptimization::UnrollBlock::unpack_kind_offset_in_bytes())); + __ mv(t1, Deoptimization::Unpack_uncommon_trap); + __ beq(t0, t1, L); + __ stop("SharedRuntime::generate_deopt_blob: last_Java_fp not cleared"); + __ bind(L); + } +#endif + + // Pop all the frames we must move/replace. + // + // Frame picture (youngest to oldest) + // 1: self-frame (no frame link) + // 2: deopting frame (no frame link) + // 3: caller of deopting frame (could be compiled/interpreted). + + __ add(sp, sp, (SimpleRuntimeFrame::framesize) << LogBytesPerInt); // Epilog! + + // Pop deoptimized frame (int) + __ lwu(x12, Address(x14, + Deoptimization::UnrollBlock:: + size_of_deoptimized_frame_offset_in_bytes())); + __ sub(x12, x12, 2 * wordSize); + __ add(sp, sp, x12); + __ ld(fp, Address(sp, 0)); + __ ld(ra, Address(sp, wordSize)); + __ addi(sp, sp, 2 * wordSize); + // RA should now be the return address to the caller (3) frame + +#ifdef ASSERT + // Compilers generate code that bang the stack by as much as the + // interpreter would need. So this stack banging should never + // trigger a fault. Verify that it does not on non product builds. + __ lwu(x11, Address(x14, + Deoptimization::UnrollBlock:: + total_frame_sizes_offset_in_bytes())); + __ bang_stack_size(x11, x12); +#endif + + // Load address of array of frame pcs into x12 (address*) + __ ld(x12, Address(x14, + Deoptimization::UnrollBlock::frame_pcs_offset_in_bytes())); + + // Load address of array of frame sizes into x15 (intptr_t*) + __ ld(x15, Address(x14, + Deoptimization::UnrollBlock:: + frame_sizes_offset_in_bytes())); + + // Counter + __ lwu(x13, Address(x14, + Deoptimization::UnrollBlock:: + number_of_frames_offset_in_bytes())); // (int) + + // Now adjust the caller's stack to make up for the extra locals but + // record the original sp so that we can save it in the skeletal + // interpreter frame and the stack walking of interpreter_sender + // will get the unextended sp value and not the "real" sp value. + + const Register sender_sp = t1; // temporary register + + __ lwu(x11, Address(x14, + Deoptimization::UnrollBlock:: + caller_adjustment_offset_in_bytes())); // (int) + __ mv(sender_sp, sp); + __ sub(sp, sp, x11); + + // Push interpreter frames in a loop + Label loop; + __ bind(loop); + __ ld(x11, Address(x15, 0)); // Load frame size + __ sub(x11, x11, 2 * wordSize); // We'll push pc and fp by hand + __ ld(ra, Address(x12, 0)); // Save return address + __ enter(); // and old fp & set new fp + __ sub(sp, sp, x11); // Prolog + __ sd(sender_sp, Address(fp, frame::interpreter_frame_sender_sp_offset * wordSize)); // Make it walkable + // This value is corrected by layout_activation_impl + __ sd(zr, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize)); + __ mv(sender_sp, sp); // Pass sender_sp to next frame + __ add(x15, x15, wordSize); // Bump array pointer (sizes) + __ add(x12, x12, wordSize); // Bump array pointer (pcs) + __ subw(x13, x13, 1); // Decrement counter + __ bgtz(x13, loop); + __ ld(ra, Address(x12, 0)); // save final return address + // Re-push self-frame + __ enter(); // & old fp & set new fp + + // Use fp because the frames look interpreted now + // Save "the_pc" since it cannot easily be retrieved using the last_java_SP after we aligned SP. + // Don't need the precise return PC here, just precise enough to point into this code blob. + address the_pc = __ pc(); + __ set_last_Java_frame(sp, fp, the_pc, t0); + + // Call C code. Need thread but NOT official VM entry + // crud. We cannot block on this call, no GC can happen. Call should + // restore return values to their stack-slots with the new SP. + // + // BasicType unpack_frames(JavaThread* thread, int exec_mode) + // + + // n.b. 2 gp args, 0 fp args, integral return type + + // sp should already be aligned + __ mv(c_rarg0, xthread); + __ mv(c_rarg1, Deoptimization::Unpack_uncommon_trap); + target = RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames)); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ jalr(x1, t0, offset); + }); + + // Set an oopmap for the call site + // Use the same PC we used for the last java frame + oop_maps->add_gc_map(the_pc - start, new OopMap(SimpleRuntimeFrame::framesize, 0)); + + // Clear fp AND pc + __ reset_last_Java_frame(true); + + // Pop self-frame. + __ leave(); // Epilog + + // Jump to interpreter + __ ret(); + + // Make sure all code is generated + masm->flush(); + + _uncommon_trap_blob = UncommonTrapBlob::create(&buffer, oop_maps, + SimpleRuntimeFrame::framesize >> 1); +} +#endif // COMPILER2 + +//------------------------------generate_handler_blob------ +// +// Generate a special Compile2Runtime blob that saves all registers, +// and setup oopmap. +// +SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_type) { + ResourceMark rm; + OopMapSet *oop_maps = new OopMapSet(); + assert_cond(oop_maps != NULL); + OopMap* map = NULL; + + // Allocate space for the code. Setup code generation tools. + CodeBuffer buffer("handler_blob", 2048, 1024); + MacroAssembler* masm = new MacroAssembler(&buffer); + assert_cond(masm != NULL); + + address start = __ pc(); + address call_pc = NULL; + int frame_size_in_words = -1; + bool cause_return = (poll_type == POLL_AT_RETURN); + RegisterSaver reg_saver(poll_type == POLL_AT_VECTOR_LOOP /* save_vectors */); + + // Save Integer and Float registers. + map = reg_saver.save_live_registers(masm, 0, &frame_size_in_words); + + // The following is basically a call_VM. However, we need the precise + // address of the call in order to generate an oopmap. Hence, we do all the + // work outselves. + + Label retaddr; + __ set_last_Java_frame(sp, noreg, retaddr, t0); + + // The return address must always be correct so that frame constructor never + // sees an invalid pc. + + if (!cause_return) { + // overwrite the return address pushed by save_live_registers + // Additionally, x18 is a callee-saved register so we can look at + // it later to determine if someone changed the return address for + // us! + __ ld(x18, Address(xthread, JavaThread::saved_exception_pc_offset())); + __ sd(x18, Address(fp, frame::return_addr_offset * wordSize)); + } + + // Do the call + __ mv(c_rarg0, xthread); + RuntimeAddress target(call_ptr); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ jalr(x1, t0, offset); + }); + __ bind(retaddr); + + // Set an oopmap for the call site. This oopmap will map all + // oop-registers and debug-info registers as callee-saved. This + // will allow deoptimization at this safepoint to find all possible + // debug-info recordings, as well as let GC find all oops. + + oop_maps->add_gc_map( __ pc() - start, map); + + Label noException; + + __ reset_last_Java_frame(false); + + __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); + + __ ld(t0, Address(xthread, Thread::pending_exception_offset())); + __ beqz(t0, noException); + + // Exception pending + + reg_saver.restore_live_registers(masm); + + __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry())); + + // No exception case + __ bind(noException); + + Label no_adjust, bail; + if (!cause_return) { + // If our stashed return pc was modified by the runtime we avoid touching it + __ ld(t0, Address(fp, frame::return_addr_offset * wordSize)); + __ bne(x18, t0, no_adjust); + +#ifdef ASSERT + // Verify the correct encoding of the poll we're about to skip. + // See NativeInstruction::is_lwu_to_zr() + __ lwu(t0, Address(x18)); + __ andi(t1, t0, 0b0000011); + __ mv(t2, 0b0000011); + __ bne(t1, t2, bail); // 0-6:0b0000011 + __ srli(t1, t0, 7); + __ andi(t1, t1, 0b00000); + __ bnez(t1, bail); // 7-11:0b00000 + __ srli(t1, t0, 12); + __ andi(t1, t1, 0b110); + __ mv(t2, 0b110); + __ bne(t1, t2, bail); // 12-14:0b110 +#endif + // Adjust return pc forward to step over the safepoint poll instruction + __ add(x18, x18, NativeInstruction::instruction_size); + __ sd(x18, Address(fp, frame::return_addr_offset * wordSize)); + } + + __ bind(no_adjust); + // Normal exit, restore registers and exit. + + reg_saver.restore_live_registers(masm); + __ ret(); + +#ifdef ASSERT + __ bind(bail); + __ stop("Attempting to adjust pc to skip safepoint poll but the return point is not what we expected"); +#endif + + // Make sure all code is generated + masm->flush(); + + // Fill-out other meta info + return SafepointBlob::create(&buffer, oop_maps, frame_size_in_words); +} + +// +// generate_resolve_blob - call resolution (static/virtual/opt-virtual/ic-miss +// +// Generate a stub that calls into vm to find out the proper destination +// of a java call. All the argument registers are live at this point +// but since this is generic code we don't know what they are and the caller +// must do any gc of the args. +// +RuntimeStub* SharedRuntime::generate_resolve_blob(address destination, const char* name) { + assert(StubRoutines::forward_exception_entry() != NULL, "must be generated before"); + + // allocate space for the code + ResourceMark rm; + + CodeBuffer buffer(name, 1000, 512); + MacroAssembler* masm = new MacroAssembler(&buffer); + assert_cond(masm != NULL); + + int frame_size_in_words = -1; + RegisterSaver reg_saver(false /* save_vectors */); + + OopMapSet *oop_maps = new OopMapSet(); + assert_cond(oop_maps != NULL); + OopMap* map = NULL; + + int start = __ offset(); + + map = reg_saver.save_live_registers(masm, 0, &frame_size_in_words); + + int frame_complete = __ offset(); + + { + Label retaddr; + __ set_last_Java_frame(sp, noreg, retaddr, t0); + + __ mv(c_rarg0, xthread); + RuntimeAddress target(destination); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ jalr(x1, t0, offset); + }); + __ bind(retaddr); + } + + // Set an oopmap for the call site. + // We need this not only for callee-saved registers, but also for volatile + // registers that the compiler might be keeping live across a safepoint. + + oop_maps->add_gc_map( __ offset() - start, map); + + // x10 contains the address we are going to jump to assuming no exception got installed + + // clear last_Java_sp + __ reset_last_Java_frame(false); + // check for pending exceptions + Label pending; + __ ld(t0, Address(xthread, Thread::pending_exception_offset())); + __ bnez(t0, pending); + + // get the returned Method* + __ get_vm_result_2(xmethod, xthread); + __ sd(xmethod, Address(sp, reg_saver.reg_offset_in_bytes(xmethod))); + + // x10 is where we want to jump, overwrite t0 which is saved and temporary + __ sd(x10, Address(sp, reg_saver.reg_offset_in_bytes(t0))); + reg_saver.restore_live_registers(masm); + + // We are back the the original state on entry and ready to go. + + __ jr(t0); + + // Pending exception after the safepoint + + __ bind(pending); + + reg_saver.restore_live_registers(masm); + + // exception pending => remove activation and forward to exception handler + + __ sd(zr, Address(xthread, JavaThread::vm_result_offset())); + + __ ld(x10, Address(xthread, Thread::pending_exception_offset())); + __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry())); + + // ------------- + // make sure all code is generated + masm->flush(); + + // return the blob + return RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, true); +} + +#ifdef COMPILER2 +RuntimeStub* SharedRuntime::make_native_invoker(address call_target, + int shadow_space_bytes, + const GrowableArray& input_registers, + const GrowableArray& output_registers) { + Unimplemented(); + return nullptr; +} + +//------------------------------generate_exception_blob--------------------------- +// creates exception blob at the end +// Using exception blob, this code is jumped from a compiled method. +// (see emit_exception_handler in riscv.ad file) +// +// Given an exception pc at a call we call into the runtime for the +// handler in this method. This handler might merely restore state +// (i.e. callee save registers) unwind the frame and jump to the +// exception handler for the nmethod if there is no Java level handler +// for the nmethod. +// +// This code is entered with a jmp. +// +// Arguments: +// x10: exception oop +// x13: exception pc +// +// Results: +// x10: exception oop +// x13: exception pc in caller +// destination: exception handler of caller +// +// Note: the exception pc MUST be at a call (precise debug information) +// Registers x10, x13, x12, x14, x15, t0 are not callee saved. +// + +void OptoRuntime::generate_exception_blob() { + assert(!OptoRuntime::is_callee_saved_register(R13_num), ""); + assert(!OptoRuntime::is_callee_saved_register(R10_num), ""); + assert(!OptoRuntime::is_callee_saved_register(R12_num), ""); + + assert(SimpleRuntimeFrame::framesize % 4 == 0, "sp not 16-byte aligned"); + + // Allocate space for the code + ResourceMark rm; + // Setup code generation tools + CodeBuffer buffer("exception_blob", 2048, 1024); + MacroAssembler* masm = new MacroAssembler(&buffer); + assert_cond(masm != NULL); + + // TODO check various assumptions made here + // + // make sure we do so before running this + + address start = __ pc(); + + // push fp and retaddr by hand + // Exception pc is 'return address' for stack walker + __ addi(sp, sp, -2 * wordSize); + __ sd(ra, Address(sp, wordSize)); + __ sd(fp, Address(sp)); + // there are no callee save registers and we don't expect an + // arg reg save area +#ifndef PRODUCT + assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area"); +#endif + // Store exception in Thread object. We cannot pass any arguments to the + // handle_exception call, since we do not want to make any assumption + // about the size of the frame where the exception happened in. + __ sd(x10, Address(xthread, JavaThread::exception_oop_offset())); + __ sd(x13, Address(xthread, JavaThread::exception_pc_offset())); + + // This call does all the hard work. It checks if an exception handler + // exists in the method. + // If so, it returns the handler address. + // If not, it prepares for stack-unwinding, restoring the callee-save + // registers of the frame being removed. + // + // address OptoRuntime::handle_exception_C(JavaThread* thread) + // + // n.b. 1 gp arg, 0 fp args, integral return type + + // the stack should always be aligned + address the_pc = __ pc(); + __ set_last_Java_frame(sp, noreg, the_pc, t0); + __ mv(c_rarg0, xthread); + RuntimeAddress target(CAST_FROM_FN_PTR(address, OptoRuntime::handle_exception_C)); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ jalr(x1, t0, offset); + }); + + + // handle_exception_C is a special VM call which does not require an explicit + // instruction sync afterwards. + + // Set an oopmap for the call site. This oopmap will only be used if we + // are unwinding the stack. Hence, all locations will be dead. + // Callee-saved registers will be the same as the frame above (i.e., + // handle_exception_stub), since they were restored when we got the + // exception. + + OopMapSet* oop_maps = new OopMapSet(); + assert_cond(oop_maps != NULL); + + oop_maps->add_gc_map(the_pc - start, new OopMap(SimpleRuntimeFrame::framesize, 0)); + + __ reset_last_Java_frame(false); + + // Restore callee-saved registers + + // fp is an implicitly saved callee saved register (i.e. the calling + // convention will save restore it in prolog/epilog) Other than that + // there are no callee save registers now that adapter frames are gone. + // and we dont' expect an arg reg save area + __ ld(fp, Address(sp)); + __ ld(x13, Address(sp, wordSize)); + __ addi(sp, sp , 2 * wordSize); + + // x10: exception handler + + // We have a handler in x10 (could be deopt blob). + __ mv(t0, x10); + + // Get the exception oop + __ ld(x10, Address(xthread, JavaThread::exception_oop_offset())); + // Get the exception pc in case we are deoptimized + __ ld(x14, Address(xthread, JavaThread::exception_pc_offset())); +#ifdef ASSERT + __ sd(zr, Address(xthread, JavaThread::exception_handler_pc_offset())); + __ sd(zr, Address(xthread, JavaThread::exception_pc_offset())); +#endif + // Clear the exception oop so GC no longer processes it as a root. + __ sd(zr, Address(xthread, JavaThread::exception_oop_offset())); + + // x10: exception oop + // t0: exception handler + // x14: exception pc + // Jump to handler + + __ jr(t0); + + // Make sure all code is generated + masm->flush(); + + // Set exception blob + _exception_blob = ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1); +} +#endif // COMPILER2 diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp new file mode 100644 index 0000000000000..a008d1d8bef55 --- /dev/null +++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp @@ -0,0 +1,3912 @@ +/* + * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "compiler/oopMap.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/barrierSetAssembler.hpp" +#include "interpreter/interpreter.hpp" +#include "memory/universe.hpp" +#include "nativeInst_riscv.hpp" +#include "oops/instanceOop.hpp" +#include "oops/method.hpp" +#include "oops/objArrayKlass.hpp" +#include "oops/oop.inline.hpp" +#include "prims/methodHandles.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/sharedRuntime.hpp" +#include "runtime/stubCodeGenerator.hpp" +#include "runtime/stubRoutines.hpp" +#include "runtime/thread.inline.hpp" +#include "utilities/align.hpp" +#include "utilities/powerOfTwo.hpp" +#ifdef COMPILER2 +#include "opto/runtime.hpp" +#endif +#if INCLUDE_ZGC +#include "gc/z/zThreadLocalData.hpp" +#endif + +// Declaration and definition of StubGenerator (no .hpp file). +// For a more detailed description of the stub routine structure +// see the comment in stubRoutines.hpp + +#undef __ +#define __ _masm-> + +#ifdef PRODUCT +#define BLOCK_COMMENT(str) /* nothing */ +#else +#define BLOCK_COMMENT(str) __ block_comment(str) +#endif + +#define BIND(label) bind(label); BLOCK_COMMENT(#label ":") + +// Stub Code definitions + +class StubGenerator: public StubCodeGenerator { + private: + +#ifdef PRODUCT +#define inc_counter_np(counter) ((void)0) +#else + void inc_counter_np_(int& counter) { + __ la(t1, ExternalAddress((address)&counter)); + __ lwu(t0, Address(t1, 0)); + __ addiw(t0, t0, 1); + __ sw(t0, Address(t1, 0)); + } +#define inc_counter_np(counter) \ + BLOCK_COMMENT("inc_counter " #counter); \ + inc_counter_np_(counter); +#endif + + // Call stubs are used to call Java from C + // + // Arguments: + // c_rarg0: call wrapper address address + // c_rarg1: result address + // c_rarg2: result type BasicType + // c_rarg3: method Method* + // c_rarg4: (interpreter) entry point address + // c_rarg5: parameters intptr_t* + // c_rarg6: parameter size (in words) int + // c_rarg7: thread Thread* + // + // There is no return from the stub itself as any Java result + // is written to result + // + // we save x1 (ra) as the return PC at the base of the frame and + // link x8 (fp) below it as the frame pointer installing sp (x2) + // into fp. + // + // we save x10-x17, which accounts for all the c arguments. + // + // TODO: strictly do we need to save them all? they are treated as + // volatile by C so could we omit saving the ones we are going to + // place in global registers (thread? method?) or those we only use + // during setup of the Java call? + // + // we don't need to save x5 which C uses as an indirect result location + // return register. + // + // we don't need to save x6-x7 and x28-x31 which both C and Java treat as + // volatile + // + // we save x9, x18-x27, f8-f9, and f18-f27 which Java uses as temporary + // registers and C expects to be callee-save + // + // so the stub frame looks like this when we enter Java code + // + // [ return_from_Java ] <--- sp + // [ argument word n ] + // ... + // -34 [ argument word 1 ] + // -33 [ saved f27 ] <--- sp_after_call + // -32 [ saved f26 ] + // -31 [ saved f25 ] + // -30 [ saved f24 ] + // -29 [ saved f23 ] + // -28 [ saved f22 ] + // -27 [ saved f21 ] + // -26 [ saved f20 ] + // -25 [ saved f19 ] + // -24 [ saved f18 ] + // -23 [ saved f9 ] + // -22 [ saved f8 ] + // -21 [ saved x27 ] + // -20 [ saved x26 ] + // -19 [ saved x25 ] + // -18 [ saved x24 ] + // -17 [ saved x23 ] + // -16 [ saved x22 ] + // -15 [ saved x21 ] + // -14 [ saved x20 ] + // -13 [ saved x19 ] + // -12 [ saved x18 ] + // -11 [ saved x9 ] + // -10 [ call wrapper (x10) ] + // -9 [ result (x11) ] + // -8 [ result type (x12) ] + // -7 [ method (x13) ] + // -6 [ entry point (x14) ] + // -5 [ parameters (x15) ] + // -4 [ parameter size (x16) ] + // -3 [ thread (x17) ] + // -2 [ saved fp (x8) ] + // -1 [ saved ra (x1) ] + // 0 [ ] <--- fp == saved sp (x2) + + // Call stub stack layout word offsets from fp + enum call_stub_layout { + sp_after_call_off = -33, + + f27_off = -33, + f26_off = -32, + f25_off = -31, + f24_off = -30, + f23_off = -29, + f22_off = -28, + f21_off = -27, + f20_off = -26, + f19_off = -25, + f18_off = -24, + f9_off = -23, + f8_off = -22, + + x27_off = -21, + x26_off = -20, + x25_off = -19, + x24_off = -18, + x23_off = -17, + x22_off = -16, + x21_off = -15, + x20_off = -14, + x19_off = -13, + x18_off = -12, + x9_off = -11, + + call_wrapper_off = -10, + result_off = -9, + result_type_off = -8, + method_off = -7, + entry_point_off = -6, + parameters_off = -5, + parameter_size_off = -4, + thread_off = -3, + fp_f = -2, + retaddr_off = -1, + }; + + address generate_call_stub(address& return_address) { + assert((int)frame::entry_frame_after_call_words == -(int)sp_after_call_off + 1 && + (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off, + "adjust this code"); + + StubCodeMark mark(this, "StubRoutines", "call_stub"); + address start = __ pc(); + + const Address sp_after_call (fp, sp_after_call_off * wordSize); + + const Address call_wrapper (fp, call_wrapper_off * wordSize); + const Address result (fp, result_off * wordSize); + const Address result_type (fp, result_type_off * wordSize); + const Address method (fp, method_off * wordSize); + const Address entry_point (fp, entry_point_off * wordSize); + const Address parameters (fp, parameters_off * wordSize); + const Address parameter_size(fp, parameter_size_off * wordSize); + + const Address thread (fp, thread_off * wordSize); + + const Address f27_save (fp, f27_off * wordSize); + const Address f26_save (fp, f26_off * wordSize); + const Address f25_save (fp, f25_off * wordSize); + const Address f24_save (fp, f24_off * wordSize); + const Address f23_save (fp, f23_off * wordSize); + const Address f22_save (fp, f22_off * wordSize); + const Address f21_save (fp, f21_off * wordSize); + const Address f20_save (fp, f20_off * wordSize); + const Address f19_save (fp, f19_off * wordSize); + const Address f18_save (fp, f18_off * wordSize); + const Address f9_save (fp, f9_off * wordSize); + const Address f8_save (fp, f8_off * wordSize); + + const Address x27_save (fp, x27_off * wordSize); + const Address x26_save (fp, x26_off * wordSize); + const Address x25_save (fp, x25_off * wordSize); + const Address x24_save (fp, x24_off * wordSize); + const Address x23_save (fp, x23_off * wordSize); + const Address x22_save (fp, x22_off * wordSize); + const Address x21_save (fp, x21_off * wordSize); + const Address x20_save (fp, x20_off * wordSize); + const Address x19_save (fp, x19_off * wordSize); + const Address x18_save (fp, x18_off * wordSize); + + const Address x9_save (fp, x9_off * wordSize); + + // stub code + + address riscv_entry = __ pc(); + + // set up frame and move sp to end of save area + __ enter(); + __ addi(sp, fp, sp_after_call_off * wordSize); + + // save register parameters and Java temporary/global registers + // n.b. we save thread even though it gets installed in + // xthread because we want to sanity check tp later + __ sd(c_rarg7, thread); + __ sw(c_rarg6, parameter_size); + __ sd(c_rarg5, parameters); + __ sd(c_rarg4, entry_point); + __ sd(c_rarg3, method); + __ sd(c_rarg2, result_type); + __ sd(c_rarg1, result); + __ sd(c_rarg0, call_wrapper); + + __ sd(x9, x9_save); + + __ sd(x18, x18_save); + __ sd(x19, x19_save); + __ sd(x20, x20_save); + __ sd(x21, x21_save); + __ sd(x22, x22_save); + __ sd(x23, x23_save); + __ sd(x24, x24_save); + __ sd(x25, x25_save); + __ sd(x26, x26_save); + __ sd(x27, x27_save); + + __ fsd(f8, f8_save); + __ fsd(f9, f9_save); + __ fsd(f18, f18_save); + __ fsd(f19, f19_save); + __ fsd(f20, f20_save); + __ fsd(f21, f21_save); + __ fsd(f22, f22_save); + __ fsd(f23, f23_save); + __ fsd(f24, f24_save); + __ fsd(f25, f25_save); + __ fsd(f26, f26_save); + __ fsd(f27, f27_save); + + // install Java thread in global register now we have saved + // whatever value it held + __ mv(xthread, c_rarg7); + + // And method + __ mv(xmethod, c_rarg3); + + // set up the heapbase register + __ reinit_heapbase(); + +#ifdef ASSERT + // make sure we have no pending exceptions + { + Label L; + __ ld(t0, Address(xthread, in_bytes(Thread::pending_exception_offset()))); + __ beqz(t0, L); + __ stop("StubRoutines::call_stub: entered with pending exception"); + __ BIND(L); + } +#endif + // pass parameters if any + __ mv(esp, sp); + __ slli(t0, c_rarg6, LogBytesPerWord); + __ sub(t0, sp, t0); // Move SP out of the way + __ andi(sp, t0, -2 * wordSize); + + BLOCK_COMMENT("pass parameters if any"); + Label parameters_done; + // parameter count is still in c_rarg6 + // and parameter pointer identifying param 1 is in c_rarg5 + __ beqz(c_rarg6, parameters_done); + + address loop = __ pc(); + __ ld(t0, Address(c_rarg5, 0)); + __ addi(c_rarg5, c_rarg5, wordSize); + __ addi(c_rarg6, c_rarg6, -1); + __ push_reg(t0); + __ bgtz(c_rarg6, loop); + + __ BIND(parameters_done); + + // call Java entry -- passing methdoOop, and current sp + // xmethod: Method* + // x30: sender sp + BLOCK_COMMENT("call Java function"); + __ mv(x30, sp); + __ jalr(c_rarg4); + + // save current address for use by exception handling code + + return_address = __ pc(); + + // store result depending on type (everything that is not + // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT) + // n.b. this assumes Java returns an integral result in x10 + // and a floating result in j_farg0 + __ ld(j_rarg2, result); + Label is_long, is_float, is_double, exit; + __ ld(j_rarg1, result_type); + __ mv(t0, (u1)T_OBJECT); + __ beq(j_rarg1, t0, is_long); + __ mv(t0, (u1)T_LONG); + __ beq(j_rarg1, t0, is_long); + __ mv(t0, (u1)T_FLOAT); + __ beq(j_rarg1, t0, is_float); + __ mv(t0, (u1)T_DOUBLE); + __ beq(j_rarg1, t0, is_double); + + // handle T_INT case + __ sw(x10, Address(j_rarg2)); + + __ BIND(exit); + + // pop parameters + __ addi(esp, fp, sp_after_call_off * wordSize); + +#ifdef ASSERT + // verify that threads correspond + { + Label L, S; + __ ld(t0, thread); + __ bne(xthread, t0, S); + __ get_thread(t0); + __ beq(xthread, t0, L); + __ BIND(S); + __ stop("StubRoutines::call_stub: threads must correspond"); + __ BIND(L); + } +#endif + + // restore callee-save registers + __ fld(f27, f27_save); + __ fld(f26, f26_save); + __ fld(f25, f25_save); + __ fld(f24, f24_save); + __ fld(f23, f23_save); + __ fld(f22, f22_save); + __ fld(f21, f21_save); + __ fld(f20, f20_save); + __ fld(f19, f19_save); + __ fld(f18, f18_save); + __ fld(f9, f9_save); + __ fld(f8, f8_save); + + __ ld(x27, x27_save); + __ ld(x26, x26_save); + __ ld(x25, x25_save); + __ ld(x24, x24_save); + __ ld(x23, x23_save); + __ ld(x22, x22_save); + __ ld(x21, x21_save); + __ ld(x20, x20_save); + __ ld(x19, x19_save); + __ ld(x18, x18_save); + + __ ld(x9, x9_save); + + __ ld(c_rarg0, call_wrapper); + __ ld(c_rarg1, result); + __ ld(c_rarg2, result_type); + __ ld(c_rarg3, method); + __ ld(c_rarg4, entry_point); + __ ld(c_rarg5, parameters); + __ ld(c_rarg6, parameter_size); + __ ld(c_rarg7, thread); + + // leave frame and return to caller + __ leave(); + __ ret(); + + // handle return types different from T_INT + + __ BIND(is_long); + __ sd(x10, Address(j_rarg2, 0)); + __ j(exit); + + __ BIND(is_float); + __ fsw(j_farg0, Address(j_rarg2, 0), t0); + __ j(exit); + + __ BIND(is_double); + __ fsd(j_farg0, Address(j_rarg2, 0), t0); + __ j(exit); + + return start; + } + + // Return point for a Java call if there's an exception thrown in + // Java code. The exception is caught and transformed into a + // pending exception stored in JavaThread that can be tested from + // within the VM. + // + // Note: Usually the parameters are removed by the callee. In case + // of an exception crossing an activation frame boundary, that is + // not the case if the callee is compiled code => need to setup the + // sp. + // + // x10: exception oop + + address generate_catch_exception() { + StubCodeMark mark(this, "StubRoutines", "catch_exception"); + address start = __ pc(); + + // same as in generate_call_stub(): + const Address thread(fp, thread_off * wordSize); + +#ifdef ASSERT + // verify that threads correspond + { + Label L, S; + __ ld(t0, thread); + __ bne(xthread, t0, S); + __ get_thread(t0); + __ beq(xthread, t0, L); + __ bind(S); + __ stop("StubRoutines::catch_exception: threads must correspond"); + __ bind(L); + } +#endif + + // set pending exception + __ verify_oop(x10); + + __ sd(x10, Address(xthread, Thread::pending_exception_offset())); + __ mv(t0, (address)__FILE__); + __ sd(t0, Address(xthread, Thread::exception_file_offset())); + __ mv(t0, (int)__LINE__); + __ sw(t0, Address(xthread, Thread::exception_line_offset())); + + // complete return to VM + assert(StubRoutines::_call_stub_return_address != NULL, + "_call_stub_return_address must have been generated before"); + __ j(StubRoutines::_call_stub_return_address); + + return start; + } + + // Continuation point for runtime calls returning with a pending + // exception. The pending exception check happened in the runtime + // or native call stub. The pending exception in Thread is + // converted into a Java-level exception. + // + // Contract with Java-level exception handlers: + // x10: exception + // x13: throwing pc + // + // NOTE: At entry of this stub, exception-pc must be in RA !! + + // NOTE: this is always used as a jump target within generated code + // so it just needs to be generated code with no x86 prolog + + address generate_forward_exception() { + StubCodeMark mark(this, "StubRoutines", "forward exception"); + address start = __ pc(); + + // Upon entry, RA points to the return address returning into + // Java (interpreted or compiled) code; i.e., the return address + // becomes the throwing pc. + // + // Arguments pushed before the runtime call are still on the stack + // but the exception handler will reset the stack pointer -> + // ignore them. A potential result in registers can be ignored as + // well. + +#ifdef ASSERT + // make sure this code is only executed if there is a pending exception + { + Label L; + __ ld(t0, Address(xthread, Thread::pending_exception_offset())); + __ bnez(t0, L); + __ stop("StubRoutines::forward exception: no pending exception (1)"); + __ bind(L); + } +#endif + + // compute exception handler into x9 + + // call the VM to find the handler address associated with the + // caller address. pass thread in x10 and caller pc (ret address) + // in x11. n.b. the caller pc is in ra, unlike x86 where it is on + // the stack. + __ mv(c_rarg1, ra); + // ra will be trashed by the VM call so we move it to x9 + // (callee-saved) because we also need to pass it to the handler + // returned by this call. + __ mv(x9, ra); + BLOCK_COMMENT("call exception_handler_for_return_address"); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, + SharedRuntime::exception_handler_for_return_address), + xthread, c_rarg1); + // we should not really care that ra is no longer the callee + // address. we saved the value the handler needs in x9 so we can + // just copy it to x13. however, the C2 handler will push its own + // frame and then calls into the VM and the VM code asserts that + // the PC for the frame above the handler belongs to a compiled + // Java method. So, we restore ra here to satisfy that assert. + __ mv(ra, x9); + // setup x10 & x13 & clear pending exception + __ mv(x13, x9); + __ mv(x9, x10); + __ ld(x10, Address(xthread, Thread::pending_exception_offset())); + __ sd(zr, Address(xthread, Thread::pending_exception_offset())); + +#ifdef ASSERT + // make sure exception is set + { + Label L; + __ bnez(x10, L); + __ stop("StubRoutines::forward exception: no pending exception (2)"); + __ bind(L); + } +#endif + + // continue at exception handler + // x10: exception + // x13: throwing pc + // x9: exception handler + __ verify_oop(x10); + __ jr(x9); + + return start; + } + + // Non-destructive plausibility checks for oops + // + // Arguments: + // x10: oop to verify + // t0: error message + // + // Stack after saving c_rarg3: + // [tos + 0]: saved c_rarg3 + // [tos + 1]: saved c_rarg2 + // [tos + 2]: saved ra + // [tos + 3]: saved t1 + // [tos + 4]: saved x10 + // [tos + 5]: saved t0 + address generate_verify_oop() { + + StubCodeMark mark(this, "StubRoutines", "verify_oop"); + address start = __ pc(); + + Label exit, error; + + __ push_reg(RegSet::of(c_rarg2, c_rarg3), sp); // save c_rarg2 and c_rarg3 + + __ la(c_rarg2, ExternalAddress((address) StubRoutines::verify_oop_count_addr())); + __ ld(c_rarg3, Address(c_rarg2)); + __ add(c_rarg3, c_rarg3, 1); + __ sd(c_rarg3, Address(c_rarg2)); + + // object is in x10 + // make sure object is 'reasonable' + __ beqz(x10, exit); // if obj is NULL it is OK + +#if INCLUDE_ZGC + if (UseZGC) { + // Check if mask is good. + // verifies that ZAddressBadMask & x10 == 0 + __ ld(c_rarg3, Address(xthread, ZThreadLocalData::address_bad_mask_offset())); + __ andr(c_rarg2, x10, c_rarg3); + __ bnez(c_rarg2, error); + } +#endif + + // Check if the oop is in the right area of memory + __ mv(c_rarg3, (intptr_t) Universe::verify_oop_mask()); + __ andr(c_rarg2, x10, c_rarg3); + __ mv(c_rarg3, (intptr_t) Universe::verify_oop_bits()); + + // Compare c_rarg2 and c_rarg3. + __ bne(c_rarg2, c_rarg3, error); + + // make sure klass is 'reasonable', which is not zero. + __ load_klass(x10, x10); // get klass + __ beqz(x10, error); // if klass is NULL it is broken + + // return if everything seems ok + __ bind(exit); + + __ pop_reg(RegSet::of(c_rarg2, c_rarg3), sp); // pop c_rarg2 and c_rarg3 + __ ret(); + + // handle errors + __ bind(error); + __ pop_reg(RegSet::of(c_rarg2, c_rarg3), sp); // pop c_rarg2 and c_rarg3 + + __ push_reg(RegSet::range(x0, x31), sp); + // debug(char* msg, int64_t pc, int64_t regs[]) + __ mv(c_rarg0, t0); // pass address of error message + __ mv(c_rarg1, ra); // pass return address + __ mv(c_rarg2, sp); // pass address of regs on stack +#ifndef PRODUCT + assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area"); +#endif + BLOCK_COMMENT("call MacroAssembler::debug"); + __ call(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)); + __ ebreak(); + + return start; + } + + // The inner part of zero_words(). + // + // Inputs: + // x28: the HeapWord-aligned base address of an array to zero. + // x29: the count in HeapWords, x29 > 0. + // + // Returns x28 and x29, adjusted for the caller to clear. + // x28: the base address of the tail of words left to clear. + // x29: the number of words in the tail. + // x29 < MacroAssembler::zero_words_block_size. + + address generate_zero_blocks() { + Label done; + + const Register base = x28, cnt = x29; + + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", "zero_blocks"); + address start = __ pc(); + + { + // Clear the remaining blocks. + Label loop; + __ sub(cnt, cnt, MacroAssembler::zero_words_block_size); + __ bltz(cnt, done); + __ bind(loop); + for (int i = 0; i < MacroAssembler::zero_words_block_size; i++) { + __ sd(zr, Address(base, 0)); + __ add(base, base, 8); + } + __ sub(cnt, cnt, MacroAssembler::zero_words_block_size); + __ bgez(cnt, loop); + __ bind(done); + __ add(cnt, cnt, MacroAssembler::zero_words_block_size); + } + + __ ret(); + + return start; + } + + typedef enum { + copy_forwards = 1, + copy_backwards = -1 + } copy_direction; + + // Bulk copy of blocks of 8 words. + // + // count is a count of words. + // + // Precondition: count >= 8 + // + // Postconditions: + // + // The least significant bit of count contains the remaining count + // of words to copy. The rest of count is trash. + // + // s and d are adjusted to point to the remaining words to copy + // + void generate_copy_longs(Label &start, Register s, Register d, Register count, + copy_direction direction) { + int unit = wordSize * direction; + int bias = wordSize; + + const Register tmp_reg0 = x13, tmp_reg1 = x14, tmp_reg2 = x15, tmp_reg3 = x16, + tmp_reg4 = x17, tmp_reg5 = x7, tmp_reg6 = x28, tmp_reg7 = x29; + + const Register stride = x30; + + assert_different_registers(t0, tmp_reg0, tmp_reg1, tmp_reg2, tmp_reg3, + tmp_reg4, tmp_reg5, tmp_reg6, tmp_reg7); + assert_different_registers(s, d, count, t0); + + Label again, drain; + const char* stub_name = NULL; + if (direction == copy_forwards) { + stub_name = "forward_copy_longs"; + } else { + stub_name = "backward_copy_longs"; + } + StubCodeMark mark(this, "StubRoutines", stub_name); + __ align(CodeEntryAlignment); + __ bind(start); + + if (direction == copy_forwards) { + __ sub(s, s, bias); + __ sub(d, d, bias); + } + +#ifdef ASSERT + // Make sure we are never given < 8 words + { + Label L; + + __ mv(t0, 8); + __ bge(count, t0, L); + __ stop("genrate_copy_longs called with < 8 words"); + __ bind(L); + } +#endif + + __ ld(tmp_reg0, Address(s, 1 * unit)); + __ ld(tmp_reg1, Address(s, 2 * unit)); + __ ld(tmp_reg2, Address(s, 3 * unit)); + __ ld(tmp_reg3, Address(s, 4 * unit)); + __ ld(tmp_reg4, Address(s, 5 * unit)); + __ ld(tmp_reg5, Address(s, 6 * unit)); + __ ld(tmp_reg6, Address(s, 7 * unit)); + __ ld(tmp_reg7, Address(s, 8 * unit)); + __ addi(s, s, 8 * unit); + + __ sub(count, count, 16); + __ bltz(count, drain); + + __ bind(again); + + __ sd(tmp_reg0, Address(d, 1 * unit)); + __ sd(tmp_reg1, Address(d, 2 * unit)); + __ sd(tmp_reg2, Address(d, 3 * unit)); + __ sd(tmp_reg3, Address(d, 4 * unit)); + __ sd(tmp_reg4, Address(d, 5 * unit)); + __ sd(tmp_reg5, Address(d, 6 * unit)); + __ sd(tmp_reg6, Address(d, 7 * unit)); + __ sd(tmp_reg7, Address(d, 8 * unit)); + + __ ld(tmp_reg0, Address(s, 1 * unit)); + __ ld(tmp_reg1, Address(s, 2 * unit)); + __ ld(tmp_reg2, Address(s, 3 * unit)); + __ ld(tmp_reg3, Address(s, 4 * unit)); + __ ld(tmp_reg4, Address(s, 5 * unit)); + __ ld(tmp_reg5, Address(s, 6 * unit)); + __ ld(tmp_reg6, Address(s, 7 * unit)); + __ ld(tmp_reg7, Address(s, 8 * unit)); + + __ addi(s, s, 8 * unit); + __ addi(d, d, 8 * unit); + + __ sub(count, count, 8); + __ bgez(count, again); + + // Drain + __ bind(drain); + + __ sd(tmp_reg0, Address(d, 1 * unit)); + __ sd(tmp_reg1, Address(d, 2 * unit)); + __ sd(tmp_reg2, Address(d, 3 * unit)); + __ sd(tmp_reg3, Address(d, 4 * unit)); + __ sd(tmp_reg4, Address(d, 5 * unit)); + __ sd(tmp_reg5, Address(d, 6 * unit)); + __ sd(tmp_reg6, Address(d, 7 * unit)); + __ sd(tmp_reg7, Address(d, 8 * unit)); + __ addi(d, d, 8 * unit); + + { + Label L1, L2; + __ test_bit(t0, count, 2); + __ beqz(t0, L1); + + __ ld(tmp_reg0, Address(s, 1 * unit)); + __ ld(tmp_reg1, Address(s, 2 * unit)); + __ ld(tmp_reg2, Address(s, 3 * unit)); + __ ld(tmp_reg3, Address(s, 4 * unit)); + __ addi(s, s, 4 * unit); + + __ sd(tmp_reg0, Address(d, 1 * unit)); + __ sd(tmp_reg1, Address(d, 2 * unit)); + __ sd(tmp_reg2, Address(d, 3 * unit)); + __ sd(tmp_reg3, Address(d, 4 * unit)); + __ addi(d, d, 4 * unit); + + __ bind(L1); + + if (direction == copy_forwards) { + __ addi(s, s, bias); + __ addi(d, d, bias); + } + + __ test_bit(t0, count, 1); + __ beqz(t0, L2); + if (direction == copy_backwards) { + __ addi(s, s, 2 * unit); + __ ld(tmp_reg0, Address(s)); + __ ld(tmp_reg1, Address(s, wordSize)); + __ addi(d, d, 2 * unit); + __ sd(tmp_reg0, Address(d)); + __ sd(tmp_reg1, Address(d, wordSize)); + } else { + __ ld(tmp_reg0, Address(s)); + __ ld(tmp_reg1, Address(s, wordSize)); + __ addi(s, s, 2 * unit); + __ sd(tmp_reg0, Address(d)); + __ sd(tmp_reg1, Address(d, wordSize)); + __ addi(d, d, 2 * unit); + } + __ bind(L2); + } + + __ ret(); + } + + Label copy_f, copy_b; + + // All-singing all-dancing memory copy. + // + // Copy count units of memory from s to d. The size of a unit is + // step, which can be positive or negative depending on the direction + // of copy. If is_aligned is false, we align the source address. + // + /* + * if (is_aligned) { + * if (count >= 32) + * goto copy32_loop; + * if (count >= 8) + * goto copy8_loop; + * goto copy_small; + * } + * bool is_backwards = step < 0; + * int granularity = uabs(step); + * count = count * granularity; * count bytes + * + * if (is_backwards) { + * s += count; + * d += count; + * } + * + * count limit maybe greater than 16, for better performance + * if (count < 16) { + * goto copy_small; + * } + * + * if ((dst % 8) == (src % 8)) { + * aligned; + * goto copy_big; + * } + * + * copy_big: + * if the amount to copy is more than (or equal to) 32 bytes goto copy32_loop + * else goto copy8_loop + * copy_small: + * load element one by one; + * done; + */ + + typedef void (MacroAssembler::*copy_insn)(Register Rd, const Address &adr, Register temp); + + void copy_memory_v(Register s, Register d, Register count, Register tmp, int step) { + bool is_backward = step < 0; + int granularity = uabs(step); + + const Register src = x30, dst = x31, vl = x14, cnt = x15, tmp1 = x16, tmp2 = x17; + assert_different_registers(s, d, cnt, vl, tmp, tmp1, tmp2); + Assembler::SEW sew = Assembler::elembytes_to_sew(granularity); + Label loop_forward, loop_backward, done; + + __ mv(dst, d); + __ mv(src, s); + __ mv(cnt, count); + + __ bind(loop_forward); + __ vsetvli(vl, cnt, sew, Assembler::m8); + if (is_backward) { + __ bne(vl, cnt, loop_backward); + } + + __ vlex_v(v0, src, sew); + __ sub(cnt, cnt, vl); + __ slli(vl, vl, (int)sew); + __ add(src, src, vl); + + __ vsex_v(v0, dst, sew); + __ add(dst, dst, vl); + __ bnez(cnt, loop_forward); + + if (is_backward) { + __ j(done); + + __ bind(loop_backward); + __ sub(tmp, cnt, vl); + __ slli(tmp, tmp, sew); + __ add(tmp1, s, tmp); + __ vlex_v(v0, tmp1, sew); + __ add(tmp2, d, tmp); + __ vsex_v(v0, tmp2, sew); + __ sub(cnt, cnt, vl); + __ bnez(cnt, loop_forward); + __ bind(done); + } + } + + void copy_memory(bool is_aligned, Register s, Register d, + Register count, Register tmp, int step) { + if (UseRVV) { + return copy_memory_v(s, d, count, tmp, step); + } + + bool is_backwards = step < 0; + int granularity = uabs(step); + + const Register src = x30, dst = x31, cnt = x15, tmp3 = x16, tmp4 = x17, tmp5 = x14, tmp6 = x13; + + Label same_aligned; + Label copy_big, copy32_loop, copy8_loop, copy_small, done; + + copy_insn ld_arr = NULL, st_arr = NULL; + switch (granularity) { + case 1 : + ld_arr = (copy_insn)&MacroAssembler::lbu; + st_arr = (copy_insn)&MacroAssembler::sb; + break; + case 2 : + ld_arr = (copy_insn)&MacroAssembler::lhu; + st_arr = (copy_insn)&MacroAssembler::sh; + break; + case 4 : + ld_arr = (copy_insn)&MacroAssembler::lwu; + st_arr = (copy_insn)&MacroAssembler::sw; + break; + case 8 : + ld_arr = (copy_insn)&MacroAssembler::ld; + st_arr = (copy_insn)&MacroAssembler::sd; + break; + default : + ShouldNotReachHere(); + } + + __ beqz(count, done); + __ slli(cnt, count, exact_log2(granularity)); + if (is_backwards) { + __ add(src, s, cnt); + __ add(dst, d, cnt); + } else { + __ mv(src, s); + __ mv(dst, d); + } + + if (is_aligned) { + __ addi(tmp, cnt, -32); + __ bgez(tmp, copy32_loop); + __ addi(tmp, cnt, -8); + __ bgez(tmp, copy8_loop); + __ j(copy_small); + } else { + __ mv(tmp, 16); + __ blt(cnt, tmp, copy_small); + + __ xorr(tmp, src, dst); + __ andi(tmp, tmp, 0b111); + __ bnez(tmp, copy_small); + + __ bind(same_aligned); + __ andi(tmp, src, 0b111); + __ beqz(tmp, copy_big); + if (is_backwards) { + __ addi(src, src, step); + __ addi(dst, dst, step); + } + (_masm->*ld_arr)(tmp3, Address(src), t0); + (_masm->*st_arr)(tmp3, Address(dst), t0); + if (!is_backwards) { + __ addi(src, src, step); + __ addi(dst, dst, step); + } + __ addi(cnt, cnt, -granularity); + __ beqz(cnt, done); + __ j(same_aligned); + + __ bind(copy_big); + __ mv(tmp, 32); + __ blt(cnt, tmp, copy8_loop); + } + __ bind(copy32_loop); + if (is_backwards) { + __ addi(src, src, -wordSize * 4); + __ addi(dst, dst, -wordSize * 4); + } + // we first load 32 bytes, then write it, so the direction here doesn't matter + __ ld(tmp3, Address(src)); + __ ld(tmp4, Address(src, 8)); + __ ld(tmp5, Address(src, 16)); + __ ld(tmp6, Address(src, 24)); + __ sd(tmp3, Address(dst)); + __ sd(tmp4, Address(dst, 8)); + __ sd(tmp5, Address(dst, 16)); + __ sd(tmp6, Address(dst, 24)); + + if (!is_backwards) { + __ addi(src, src, wordSize * 4); + __ addi(dst, dst, wordSize * 4); + } + __ addi(tmp, cnt, -(32 + wordSize * 4)); + __ addi(cnt, cnt, -wordSize * 4); + __ bgez(tmp, copy32_loop); // cnt >= 32, do next loop + + __ beqz(cnt, done); // if that's all - done + + __ addi(tmp, cnt, -8); // if not - copy the reminder + __ bltz(tmp, copy_small); // cnt < 8, go to copy_small, else fall throught to copy8_loop + + __ bind(copy8_loop); + if (is_backwards) { + __ addi(src, src, -wordSize); + __ addi(dst, dst, -wordSize); + } + __ ld(tmp3, Address(src)); + __ sd(tmp3, Address(dst)); + if (!is_backwards) { + __ addi(src, src, wordSize); + __ addi(dst, dst, wordSize); + } + __ addi(tmp, cnt, -(8 + wordSize)); + __ addi(cnt, cnt, -wordSize); + __ bgez(tmp, copy8_loop); // cnt >= 8, do next loop + + __ beqz(cnt, done); // if that's all - done + + __ bind(copy_small); + if (is_backwards) { + __ addi(src, src, step); + __ addi(dst, dst, step); + } + (_masm->*ld_arr)(tmp3, Address(src), t0); + (_masm->*st_arr)(tmp3, Address(dst), t0); + if (!is_backwards) { + __ addi(src, src, step); + __ addi(dst, dst, step); + } + __ addi(cnt, cnt, -granularity); + __ bgtz(cnt, copy_small); + + __ bind(done); + } + + // Scan over array at a for count oops, verifying each one. + // Preserves a and count, clobbers t0 and t1. + void verify_oop_array(size_t size, Register a, Register count, Register temp) { + Label loop, end; + __ mv(t1, zr); + __ slli(t0, count, exact_log2(size)); + __ bind(loop); + __ bgeu(t1, t0, end); + + __ add(temp, a, t1); + if (size == (size_t)wordSize) { + __ ld(temp, Address(temp, 0)); + __ verify_oop(temp); + } else { + __ lwu(temp, Address(temp, 0)); + __ decode_heap_oop(temp); // calls verify_oop + } + __ add(t1, t1, size); + __ j(loop); + __ bind(end); + } + + // Arguments: + // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary + // ignored + // is_oop - true => oop array, so generate store check code + // name - stub name string + // + // Inputs: + // c_rarg0 - source array address + // c_rarg1 - destination array address + // c_rarg2 - element count, treated as ssize_t, can be zero + // + // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let + // the hardware handle it. The two dwords within qwords that span + // cache line boundaries will still be loaded and stored atomicly. + // + // Side Effects: + // disjoint_int_copy_entry is set to the no-overlap entry point + // used by generate_conjoint_int_oop_copy(). + // + address generate_disjoint_copy(size_t size, bool aligned, bool is_oop, address* entry, + const char* name, bool dest_uninitialized = false) { + const Register s = c_rarg0, d = c_rarg1, count = c_rarg2; + RegSet saved_reg = RegSet::of(s, d, count); + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", name); + address start = __ pc(); + __ enter(); + + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Entry:"); + } + + DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT; + if (dest_uninitialized) { + decorators |= IS_DEST_UNINITIALIZED; + } + if (aligned) { + decorators |= ARRAYCOPY_ALIGNED; + } + + BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->arraycopy_prologue(_masm, decorators, is_oop, s, d, count, saved_reg); + + if (is_oop) { + // save regs before copy_memory + __ push_reg(RegSet::of(d, count), sp); + } + + { + // UnsafeCopyMemory page error: continue after ucm + bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size); + UnsafeCopyMemoryMark ucmm(this, add_entry, true); + copy_memory(aligned, s, d, count, t0, size); + } + + if (is_oop) { + __ pop_reg(RegSet::of(d, count), sp); + if (VerifyOops) { + verify_oop_array(size, d, count, t2); + } + } + + bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, t0, RegSet()); + + __ leave(); + __ mv(x10, zr); // return 0 + __ ret(); + return start; + } + + // Arguments: + // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary + // ignored + // is_oop - true => oop array, so generate store check code + // name - stub name string + // + // Inputs: + // c_rarg0 - source array address + // c_rarg1 - destination array address + // c_rarg2 - element count, treated as ssize_t, can be zero + // + // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let + // the hardware handle it. The two dwords within qwords that span + // cache line boundaries will still be loaded and stored atomicly. + // + address generate_conjoint_copy(size_t size, bool aligned, bool is_oop, address nooverlap_target, + address* entry, const char* name, + bool dest_uninitialized = false) { + const Register s = c_rarg0, d = c_rarg1, count = c_rarg2; + RegSet saved_regs = RegSet::of(s, d, count); + StubCodeMark mark(this, "StubRoutines", name); + address start = __ pc(); + __ enter(); + + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Entry:"); + } + + // use fwd copy when (d-s) above_equal (count*size) + __ sub(t0, d, s); + __ slli(t1, count, exact_log2(size)); + __ bgeu(t0, t1, nooverlap_target); + + DecoratorSet decorators = IN_HEAP | IS_ARRAY; + if (dest_uninitialized) { + decorators |= IS_DEST_UNINITIALIZED; + } + if (aligned) { + decorators |= ARRAYCOPY_ALIGNED; + } + + BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->arraycopy_prologue(_masm, decorators, is_oop, s, d, count, saved_regs); + + if (is_oop) { + // save regs before copy_memory + __ push_reg(RegSet::of(d, count), sp); + } + + { + // UnsafeCopyMemory page error: continue after ucm + bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size); + UnsafeCopyMemoryMark ucmm(this, add_entry, true); + copy_memory(aligned, s, d, count, t0, -size); + } + + if (is_oop) { + __ pop_reg(RegSet::of(d, count), sp); + if (VerifyOops) { + verify_oop_array(size, d, count, t2); + } + } + bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, t0, RegSet()); + __ leave(); + __ mv(x10, zr); // return 0 + __ ret(); + return start; + } + + // Arguments: + // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary + // ignored + // name - stub name string + // + // Inputs: + // c_rarg0 - source array address + // c_rarg1 - destination array address + // c_rarg2 - element count, treated as ssize_t, can be zero + // + // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries, + // we let the hardware handle it. The one to eight bytes within words, + // dwords or qwords that span cache line boundaries will still be loaded + // and stored atomically. + // + // Side Effects: + // disjoint_byte_copy_entry is set to the no-overlap entry point // + // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries, + // we let the hardware handle it. The one to eight bytes within words, + // dwords or qwords that span cache line boundaries will still be loaded + // and stored atomically. + // + // Side Effects: + // disjoint_byte_copy_entry is set to the no-overlap entry point + // used by generate_conjoint_byte_copy(). + // + address generate_disjoint_byte_copy(bool aligned, address* entry, const char* name) { + const bool not_oop = false; + return generate_disjoint_copy(sizeof (jbyte), aligned, not_oop, entry, name); + } + + // Arguments: + // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary + // ignored + // name - stub name string + // + // Inputs: + // c_rarg0 - source array address + // c_rarg1 - destination array address + // c_rarg2 - element count, treated as ssize_t, can be zero + // + // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries, + // we let the hardware handle it. The one to eight bytes within words, + // dwords or qwords that span cache line boundaries will still be loaded + // and stored atomically. + // + address generate_conjoint_byte_copy(bool aligned, address nooverlap_target, + address* entry, const char* name) { + const bool not_oop = false; + return generate_conjoint_copy(sizeof (jbyte), aligned, not_oop, nooverlap_target, entry, name); + } + + // Arguments: + // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary + // ignored + // name - stub name string + // + // Inputs: + // c_rarg0 - source array address + // c_rarg1 - destination array address + // c_rarg2 - element count, treated as ssize_t, can be zero + // + // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we + // let the hardware handle it. The two or four words within dwords + // or qwords that span cache line boundaries will still be loaded + // and stored atomically. + // + // Side Effects: + // disjoint_short_copy_entry is set to the no-overlap entry point + // used by generate_conjoint_short_copy(). + // + address generate_disjoint_short_copy(bool aligned, + address* entry, const char* name) { + const bool not_oop = false; + return generate_disjoint_copy(sizeof (jshort), aligned, not_oop, entry, name); + } + + // Arguments: + // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary + // ignored + // name - stub name string + // + // Inputs: + // c_rarg0 - source array address + // c_rarg1 - destination array address + // c_rarg2 - element count, treated as ssize_t, can be zero + // + // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we + // let the hardware handle it. The two or four words within dwords + // or qwords that span cache line boundaries will still be loaded + // and stored atomically. + // + address generate_conjoint_short_copy(bool aligned, address nooverlap_target, + address* entry, const char* name) { + const bool not_oop = false; + return generate_conjoint_copy(sizeof (jshort), aligned, not_oop, nooverlap_target, entry, name); + } + + // Arguments: + // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary + // ignored + // name - stub name string + // + // Inputs: + // c_rarg0 - source array address + // c_rarg1 - destination array address + // c_rarg2 - element count, treated as ssize_t, can be zero + // + // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let + // the hardware handle it. The two dwords within qwords that span + // cache line boundaries will still be loaded and stored atomicly. + // + // Side Effects: + // disjoint_int_copy_entry is set to the no-overlap entry point + // used by generate_conjoint_int_oop_copy(). + // + address generate_disjoint_int_copy(bool aligned, address* entry, + const char* name, bool dest_uninitialized = false) { + const bool not_oop = false; + return generate_disjoint_copy(sizeof (jint), aligned, not_oop, entry, name); + } + + // Arguments: + // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary + // ignored + // name - stub name string + // + // Inputs: + // c_rarg0 - source array address + // c_rarg1 - destination array address + // c_rarg2 - element count, treated as ssize_t, can be zero + // + // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let + // the hardware handle it. The two dwords within qwords that span + // cache line boundaries will still be loaded and stored atomicly. + // + address generate_conjoint_int_copy(bool aligned, address nooverlap_target, + address* entry, const char* name, + bool dest_uninitialized = false) { + const bool not_oop = false; + return generate_conjoint_copy(sizeof (jint), aligned, not_oop, nooverlap_target, entry, name); + } + + + // Arguments: + // aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes + // ignored + // name - stub name string + // + // Inputs: + // c_rarg0 - source array address + // c_rarg1 - destination array address + // c_rarg2 - element count, treated as size_t, can be zero + // + // Side Effects: + // disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the + // no-overlap entry point used by generate_conjoint_long_oop_copy(). + // + address generate_disjoint_long_copy(bool aligned, address* entry, + const char* name, bool dest_uninitialized = false) { + const bool not_oop = false; + return generate_disjoint_copy(sizeof (jlong), aligned, not_oop, entry, name); + } + + // Arguments: + // aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes + // ignored + // name - stub name string + // + // Inputs: + // c_rarg0 - source array address + // c_rarg1 - destination array address + // c_rarg2 - element count, treated as size_t, can be zero + // + address generate_conjoint_long_copy(bool aligned, + address nooverlap_target, address* entry, + const char* name, bool dest_uninitialized = false) { + const bool not_oop = false; + return generate_conjoint_copy(sizeof (jlong), aligned, not_oop, nooverlap_target, entry, name); + } + + // Arguments: + // aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes + // ignored + // name - stub name string + // + // Inputs: + // c_rarg0 - source array address + // c_rarg1 - destination array address + // c_rarg2 - element count, treated as size_t, can be zero + // + // Side Effects: + // disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the + // no-overlap entry point used by generate_conjoint_long_oop_copy(). + // + address generate_disjoint_oop_copy(bool aligned, address* entry, + const char* name, bool dest_uninitialized) { + const bool is_oop = true; + const size_t size = UseCompressedOops ? sizeof (jint) : sizeof (jlong); + return generate_disjoint_copy(size, aligned, is_oop, entry, name, dest_uninitialized); + } + + // Arguments: + // aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes + // ignored + // name - stub name string + // + // Inputs: + // c_rarg0 - source array address + // c_rarg1 - destination array address + // c_rarg2 - element count, treated as size_t, can be zero + // + address generate_conjoint_oop_copy(bool aligned, + address nooverlap_target, address* entry, + const char* name, bool dest_uninitialized) { + const bool is_oop = true; + const size_t size = UseCompressedOops ? sizeof (jint) : sizeof (jlong); + return generate_conjoint_copy(size, aligned, is_oop, nooverlap_target, entry, + name, dest_uninitialized); + } + + // Helper for generating a dynamic type check. + // Smashes t0, t1. + void generate_type_check(Register sub_klass, + Register super_check_offset, + Register super_klass, + Label& L_success) { + assert_different_registers(sub_klass, super_check_offset, super_klass); + + BLOCK_COMMENT("type_check:"); + + Label L_miss; + + __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg, &L_success, &L_miss, NULL, super_check_offset); + __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg, &L_success, NULL); + + // Fall through on failure! + __ BIND(L_miss); + } + + // + // Generate checkcasting array copy stub + // + // Input: + // c_rarg0 - source array address + // c_rarg1 - destination array address + // c_rarg2 - element count, treated as ssize_t, can be zero + // c_rarg3 - size_t ckoff (super_check_offset) + // c_rarg4 - oop ckval (super_klass) + // + // Output: + // x10 == 0 - success + // x10 == -1^K - failure, where K is partial transfer count + // + address generate_checkcast_copy(const char* name, address* entry, + bool dest_uninitialized = false) { + Label L_load_element, L_store_element, L_do_card_marks, L_done, L_done_pop; + + // Input registers (after setup_arg_regs) + const Register from = c_rarg0; // source array address + const Register to = c_rarg1; // destination array address + const Register count = c_rarg2; // elementscount + const Register ckoff = c_rarg3; // super_check_offset + const Register ckval = c_rarg4; // super_klass + + RegSet wb_pre_saved_regs = RegSet::range(c_rarg0, c_rarg4); + RegSet wb_post_saved_regs = RegSet::of(count); + + // Registers used as temps (x7, x9, x18 are save-on-entry) + const Register count_save = x19; // orig elementscount + const Register start_to = x18; // destination array start address + const Register copied_oop = x7; // actual oop copied + const Register r9_klass = x9; // oop._klass + + //--------------------------------------------------------------- + // Assembler stub will be used for this call to arraycopy + // if the two arrays are subtypes of Object[] but the + // destination array type is not equal to or a supertype + // of the source type. Each element must be separately + // checked. + + assert_different_registers(from, to, count, ckoff, ckval, start_to, + copied_oop, r9_klass, count_save); + + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", name); + address start = __ pc(); + + __ enter(); // required for proper stackwalking of RuntimeStub frame + + // Caller of this entry point must set up the argument registers. + if (entry != NULL) { + *entry = __ pc(); + BLOCK_COMMENT("Entry:"); + } + + // Empty array: Nothing to do + __ beqz(count, L_done); + + __ push_reg(RegSet::of(x7, x9, x18, x19), sp); + +#ifdef ASSERT + BLOCK_COMMENT("assert consistent ckoff/ckval"); + // The ckoff and ckval must be mutually consistent, + // even though caller generates both. + { Label L; + int sco_offset = in_bytes(Klass::super_check_offset_offset()); + __ lwu(start_to, Address(ckval, sco_offset)); + __ beq(ckoff, start_to, L); + __ stop("super_check_offset inconsistent"); + __ bind(L); + } +#endif //ASSERT + + DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST | ARRAYCOPY_DISJOINT; + bool is_oop = true; + if (dest_uninitialized) { + decorators |= IS_DEST_UNINITIALIZED; + } + + BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->arraycopy_prologue(_masm, decorators, is_oop, from, to, count, wb_pre_saved_regs); + + // save the original count + __ mv(count_save, count); + + // Copy from low to high addresses + __ mv(start_to, to); // Save destination array start address + __ j(L_load_element); + + // ======== begin loop ======== + // (Loop is rotated; its entry is L_load_element.) + // Loop control: + // for count to 0 do + // copied_oop = load_heap_oop(from++) + // ... generate_type_check ... + // store_heap_oop(to++, copied_oop) + // end + + __ align(OptoLoopAlignment); + + __ BIND(L_store_element); + __ store_heap_oop(Address(to, 0), copied_oop, noreg, noreg, AS_RAW); // store the oop + __ add(to, to, UseCompressedOops ? 4 : 8); + __ sub(count, count, 1); + __ beqz(count, L_do_card_marks); + + // ======== loop entry is here ======== + __ BIND(L_load_element); + __ load_heap_oop(copied_oop, Address(from, 0), noreg, noreg, AS_RAW); // load the oop + __ add(from, from, UseCompressedOops ? 4 : 8); + __ beqz(copied_oop, L_store_element); + + __ load_klass(r9_klass, copied_oop);// query the object klass + generate_type_check(r9_klass, ckoff, ckval, L_store_element); + // ======== end loop ======== + + // It was a real error; we must depend on the caller to finish the job. + // Register count = remaining oops, count_orig = total oops. + // Emit GC store barriers for the oops we have copied and report + // their number to the caller. + + __ sub(count, count_save, count); // K = partially copied oop count + __ xori(count, count, -1); // report (-1^K) to caller + __ beqz(count, L_done_pop); + + __ BIND(L_do_card_marks); + bs->arraycopy_epilogue(_masm, decorators, is_oop, start_to, count_save, t0, wb_post_saved_regs); + + __ bind(L_done_pop); + __ pop_reg(RegSet::of(x7, x9, x18, x19), sp); + inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); + + __ bind(L_done); + __ mv(x10, count); + __ leave(); + __ ret(); + + return start; + } + + // Perform range checks on the proposed arraycopy. + // Kills temp, but nothing else. + // Also, clean the sign bits of src_pos and dst_pos. + void arraycopy_range_checks(Register src, // source array oop (c_rarg0) + Register src_pos, // source position (c_rarg1) + Register dst, // destination array oo (c_rarg2) + Register dst_pos, // destination position (c_rarg3) + Register length, + Register temp, + Label& L_failed) { + BLOCK_COMMENT("arraycopy_range_checks:"); + + assert_different_registers(t0, temp); + + // if [src_pos + length > arrayOop(src)->length()] then FAIL + __ lwu(t0, Address(src, arrayOopDesc::length_offset_in_bytes())); + __ addw(temp, length, src_pos); + __ bgtu(temp, t0, L_failed); + + // if [dst_pos + length > arrayOop(dst)->length()] then FAIL + __ lwu(t0, Address(dst, arrayOopDesc::length_offset_in_bytes())); + __ addw(temp, length, dst_pos); + __ bgtu(temp, t0, L_failed); + + // Have to clean up high 32 bits of 'src_pos' and 'dst_pos'. + __ zero_extend(src_pos, src_pos, 32); + __ zero_extend(dst_pos, dst_pos, 32); + + BLOCK_COMMENT("arraycopy_range_checks done"); + } + + // + // Generate 'unsafe' array copy stub + // Though just as safe as the other stubs, it takes an unscaled + // size_t argument instead of an element count. + // + // Input: + // c_rarg0 - source array address + // c_rarg1 - destination array address + // c_rarg2 - byte count, treated as ssize_t, can be zero + // + // Examines the alignment of the operands and dispatches + // to a long, int, short, or byte copy loop. + // + address generate_unsafe_copy(const char* name, + address byte_copy_entry, + address short_copy_entry, + address int_copy_entry, + address long_copy_entry) { + assert_cond(byte_copy_entry != NULL && short_copy_entry != NULL && + int_copy_entry != NULL && long_copy_entry != NULL); + Label L_long_aligned, L_int_aligned, L_short_aligned; + const Register s = c_rarg0, d = c_rarg1, count = c_rarg2; + + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", name); + address start = __ pc(); + __ enter(); // required for proper stackwalking of RuntimeStub frame + + // bump this on entry, not on exit: + inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr); + + __ orr(t0, s, d); + __ orr(t0, t0, count); + + __ andi(t0, t0, BytesPerLong - 1); + __ beqz(t0, L_long_aligned); + __ andi(t0, t0, BytesPerInt - 1); + __ beqz(t0, L_int_aligned); + __ test_bit(t0, t0, 0); + __ beqz(t0, L_short_aligned); + __ j(RuntimeAddress(byte_copy_entry)); + + __ BIND(L_short_aligned); + __ srli(count, count, LogBytesPerShort); // size => short_count + __ j(RuntimeAddress(short_copy_entry)); + __ BIND(L_int_aligned); + __ srli(count, count, LogBytesPerInt); // size => int_count + __ j(RuntimeAddress(int_copy_entry)); + __ BIND(L_long_aligned); + __ srli(count, count, LogBytesPerLong); // size => long_count + __ j(RuntimeAddress(long_copy_entry)); + + return start; + } + + // + // Generate generic array copy stubs + // + // Input: + // c_rarg0 - src oop + // c_rarg1 - src_pos (32-bits) + // c_rarg2 - dst oop + // c_rarg3 - dst_pos (32-bits) + // c_rarg4 - element count (32-bits) + // + // Output: + // x10 == 0 - success + // x10 == -1^K - failure, where K is partial transfer count + // + address generate_generic_copy(const char* name, + address byte_copy_entry, address short_copy_entry, + address int_copy_entry, address oop_copy_entry, + address long_copy_entry, address checkcast_copy_entry) { + assert_cond(byte_copy_entry != NULL && short_copy_entry != NULL && + int_copy_entry != NULL && oop_copy_entry != NULL && + long_copy_entry != NULL && checkcast_copy_entry != NULL); + Label L_failed, L_failed_0, L_objArray; + Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs; + + // Input registers + const Register src = c_rarg0; // source array oop + const Register src_pos = c_rarg1; // source position + const Register dst = c_rarg2; // destination array oop + const Register dst_pos = c_rarg3; // destination position + const Register length = c_rarg4; + + // Registers used as temps + const Register dst_klass = c_rarg5; + + __ align(CodeEntryAlignment); + + StubCodeMark mark(this, "StubRoutines", name); + + address start = __ pc(); + + __ enter(); // required for proper stackwalking of RuntimeStub frame + + // bump this on entry, not on exit: + inc_counter_np(SharedRuntime::_generic_array_copy_ctr); + + //----------------------------------------------------------------------- + // Assembler stub will be used for this call to arraycopy + // if the following conditions are met: + // + // (1) src and dst must not be null. + // (2) src_pos must not be negative. + // (3) dst_pos must not be negative. + // (4) length must not be negative. + // (5) src klass and dst klass should be the same and not NULL. + // (6) src and dst should be arrays. + // (7) src_pos + length must not exceed length of src. + // (8) dst_pos + length must not exceed length of dst. + // + + // if [src == NULL] then return -1 + __ beqz(src, L_failed); + + // if [src_pos < 0] then return -1 + __ sign_extend(t0, src_pos, 32); + __ bltz(t0, L_failed); + + // if [dst == NULL] then return -1 + __ beqz(dst, L_failed); + + // if [dst_pos < 0] then return -1 + __ sign_extend(t0, dst_pos, 32); + __ bltz(t0, L_failed); + + // registers used as temp + const Register scratch_length = x28; // elements count to copy + const Register scratch_src_klass = x29; // array klass + const Register lh = x30; // layout helper + + // if [length < 0] then return -1 + __ sign_extend(scratch_length, length, 32); // length (elements count, 32-bits value) + __ bltz(scratch_length, L_failed); + + __ load_klass(scratch_src_klass, src); +#ifdef ASSERT + { + BLOCK_COMMENT("assert klasses not null {"); + Label L1, L2; + __ bnez(scratch_src_klass, L2); // it is broken if klass is NULL + __ bind(L1); + __ stop("broken null klass"); + __ bind(L2); + __ load_klass(t0, dst, t1); + __ beqz(t0, L1); // this would be broken also + BLOCK_COMMENT("} assert klasses not null done"); + } +#endif + + // Load layout helper (32-bits) + // + // |array_tag| | header_size | element_type | |log2_element_size| + // 32 30 24 16 8 2 0 + // + // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0 + // + + const int lh_offset = in_bytes(Klass::layout_helper_offset()); + + // Handle objArrays completely differently... + const jint objArray_lh = Klass::array_layout_helper(T_OBJECT); + __ lw(lh, Address(scratch_src_klass, lh_offset)); + __ mv(t0, objArray_lh); + __ beq(lh, t0, L_objArray); + + // if [src->klass() != dst->klass()] then return -1 + __ load_klass(t1, dst); + __ bne(t1, scratch_src_klass, L_failed); + + // if [src->is_Array() != NULL] then return -1 + // i.e. (lh >= 0) + __ bgez(lh, L_failed); + + // At this point, it is known to be a typeArray (array_tag 0x3). +#ifdef ASSERT + { + BLOCK_COMMENT("assert primitive array {"); + Label L; + __ mv(t1, (int32_t)(Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift)); + __ bge(lh, t1, L); + __ stop("must be a primitive array"); + __ bind(L); + BLOCK_COMMENT("} assert primitive array done"); + } +#endif + + arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length, + t1, L_failed); + + // TypeArrayKlass + // + // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize) + // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize) + // + + const Register t0_offset = t0; // array offset + const Register x30_elsize = lh; // element size + + // Get array_header_in_bytes() + int lh_header_size_width = exact_log2(Klass::_lh_header_size_mask + 1); + int lh_header_size_msb = Klass::_lh_header_size_shift + lh_header_size_width; + __ slli(t0_offset, lh, XLEN - lh_header_size_msb); // left shift to remove 24 ~ 32; + __ srli(t0_offset, t0_offset, XLEN - lh_header_size_width); // array_offset + + __ add(src, src, t0_offset); // src array offset + __ add(dst, dst, t0_offset); // dst array offset + BLOCK_COMMENT("choose copy loop based on element size"); + + // next registers should be set before the jump to corresponding stub + const Register from = c_rarg0; // source array address + const Register to = c_rarg1; // destination array address + const Register count = c_rarg2; // elements count + + // 'from', 'to', 'count' registers should be set in such order + // since they are the same as 'src', 'src_pos', 'dst'. + + assert(Klass::_lh_log2_element_size_shift == 0, "fix this code"); + + // The possible values of elsize are 0-3, i.e. exact_log2(element + // size in bytes). We do a simple bitwise binary search. + __ BIND(L_copy_bytes); + __ test_bit(t0, x30_elsize, 1); + __ bnez(t0, L_copy_ints); + __ test_bit(t0, x30_elsize, 0); + __ bnez(t0, L_copy_shorts); + __ add(from, src, src_pos); // src_addr + __ add(to, dst, dst_pos); // dst_addr + __ sign_extend(count, scratch_length, 32); // length + __ j(RuntimeAddress(byte_copy_entry)); + + __ BIND(L_copy_shorts); + __ shadd(from, src_pos, src, t0, 1); // src_addr + __ shadd(to, dst_pos, dst, t0, 1); // dst_addr + __ sign_extend(count, scratch_length, 32); // length + __ j(RuntimeAddress(short_copy_entry)); + + __ BIND(L_copy_ints); + __ test_bit(t0, x30_elsize, 0); + __ bnez(t0, L_copy_longs); + __ shadd(from, src_pos, src, t0, 2); // src_addr + __ shadd(to, dst_pos, dst, t0, 2); // dst_addr + __ sign_extend(count, scratch_length, 32); // length + __ j(RuntimeAddress(int_copy_entry)); + + __ BIND(L_copy_longs); +#ifdef ASSERT + { + BLOCK_COMMENT("assert long copy {"); + Label L; + __ andi(lh, lh, Klass::_lh_log2_element_size_mask); // lh -> x30_elsize + __ sign_extend(lh, lh, 32); + __ mv(t0, LogBytesPerLong); + __ beq(x30_elsize, t0, L); + __ stop("must be long copy, but elsize is wrong"); + __ bind(L); + BLOCK_COMMENT("} assert long copy done"); + } +#endif + __ shadd(from, src_pos, src, t0, 3); // src_addr + __ shadd(to, dst_pos, dst, t0, 3); // dst_addr + __ sign_extend(count, scratch_length, 32); // length + __ j(RuntimeAddress(long_copy_entry)); + + // ObjArrayKlass + __ BIND(L_objArray); + // live at this point: scratch_src_klass, scratch_length, src[_pos], dst[_pos] + + Label L_plain_copy, L_checkcast_copy; + // test array classes for subtyping + __ load_klass(t2, dst); + __ bne(scratch_src_klass, t2, L_checkcast_copy); // usual case is exact equality + + // Identically typed arrays can be copied without element-wise checks. + arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length, + t1, L_failed); + + __ shadd(from, src_pos, src, t0, LogBytesPerHeapOop); + __ add(from, from, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); + __ shadd(to, dst_pos, dst, t0, LogBytesPerHeapOop); + __ add(to, to, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); + __ sign_extend(count, scratch_length, 32); // length + __ BIND(L_plain_copy); + __ j(RuntimeAddress(oop_copy_entry)); + + __ BIND(L_checkcast_copy); + // live at this point: scratch_src_klass, scratch_length, t2 (dst_klass) + { + // Before looking at dst.length, make sure dst is also an objArray. + __ lwu(t0, Address(t2, lh_offset)); + __ mv(t1, objArray_lh); + __ bne(t0, t1, L_failed); + + // It is safe to examine both src.length and dst.length. + arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length, + t2, L_failed); + + __ load_klass(dst_klass, dst); // reload + + // Marshal the base address arguments now, freeing registers. + __ shadd(from, src_pos, src, t0, LogBytesPerHeapOop); + __ add(from, from, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); + __ shadd(to, dst_pos, dst, t0, LogBytesPerHeapOop); + __ add(to, to, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); + __ sign_extend(count, length, 32); // length (reloaded) + const Register sco_temp = c_rarg3; // this register is free now + assert_different_registers(from, to, count, sco_temp, + dst_klass, scratch_src_klass); + + // Generate the type check. + const int sco_offset = in_bytes(Klass::super_check_offset_offset()); + __ lwu(sco_temp, Address(dst_klass, sco_offset)); + + // Smashes t0, t1 + generate_type_check(scratch_src_klass, sco_temp, dst_klass, L_plain_copy); + + // Fetch destination element klass from the ObjArrayKlass header. + int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset()); + __ ld(dst_klass, Address(dst_klass, ek_offset)); + __ lwu(sco_temp, Address(dst_klass, sco_offset)); + + // the checkcast_copy loop needs two extra arguments: + assert(c_rarg3 == sco_temp, "#3 already in place"); + // Set up arguments for checkcast_copy_entry. + __ mv(c_rarg4, dst_klass); // dst.klass.element_klass + __ j(RuntimeAddress(checkcast_copy_entry)); + } + + __ BIND(L_failed); + __ mv(x10, -1); + __ leave(); // required for proper stackwalking of RuntimeStub frame + __ ret(); + + return start; + } + + // + // Generate stub for array fill. If "aligned" is true, the + // "to" address is assumed to be heapword aligned. + // + // Arguments for generated stub: + // to: c_rarg0 + // value: c_rarg1 + // count: c_rarg2 treated as signed + // + address generate_fill(BasicType t, bool aligned, const char* name) { + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", name); + address start = __ pc(); + + BLOCK_COMMENT("Entry:"); + + const Register to = c_rarg0; // source array address + const Register value = c_rarg1; // value + const Register count = c_rarg2; // elements count + + const Register bz_base = x28; // base for block_zero routine + const Register cnt_words = x29; // temp register + const Register tmp_reg = t1; + + __ enter(); + + Label L_fill_elements, L_exit1; + + int shift = -1; + switch (t) { + case T_BYTE: + shift = 0; + + // Zero extend value + // 8 bit -> 16 bit + __ andi(value, value, 0xff); + __ mv(tmp_reg, value); + __ slli(tmp_reg, tmp_reg, 8); + __ orr(value, value, tmp_reg); + + // 16 bit -> 32 bit + __ mv(tmp_reg, value); + __ slli(tmp_reg, tmp_reg, 16); + __ orr(value, value, tmp_reg); + + __ mv(tmp_reg, 8 >> shift); // Short arrays (< 8 bytes) fill by element + __ bltu(count, tmp_reg, L_fill_elements); + break; + case T_SHORT: + shift = 1; + // Zero extend value + // 16 bit -> 32 bit + __ andi(value, value, 0xffff); + __ mv(tmp_reg, value); + __ slli(tmp_reg, tmp_reg, 16); + __ orr(value, value, tmp_reg); + + // Short arrays (< 8 bytes) fill by element + __ mv(tmp_reg, 8 >> shift); + __ bltu(count, tmp_reg, L_fill_elements); + break; + case T_INT: + shift = 2; + + // Short arrays (< 8 bytes) fill by element + __ mv(tmp_reg, 8 >> shift); + __ bltu(count, tmp_reg, L_fill_elements); + break; + default: ShouldNotReachHere(); + } + + // Align source address at 8 bytes address boundary. + Label L_skip_align1, L_skip_align2, L_skip_align4; + if (!aligned) { + switch (t) { + case T_BYTE: + // One byte misalignment happens only for byte arrays. + __ test_bit(t0, to, 0); + __ beqz(t0, L_skip_align1); + __ sb(value, Address(to, 0)); + __ addi(to, to, 1); + __ addiw(count, count, -1); + __ bind(L_skip_align1); + // Fallthrough + case T_SHORT: + // Two bytes misalignment happens only for byte and short (char) arrays. + __ test_bit(t0, to, 1); + __ beqz(t0, L_skip_align2); + __ sh(value, Address(to, 0)); + __ addi(to, to, 2); + __ addiw(count, count, -(2 >> shift)); + __ bind(L_skip_align2); + // Fallthrough + case T_INT: + // Align to 8 bytes, we know we are 4 byte aligned to start. + __ test_bit(t0, to, 2); + __ beqz(t0, L_skip_align4); + __ sw(value, Address(to, 0)); + __ addi(to, to, 4); + __ addiw(count, count, -(4 >> shift)); + __ bind(L_skip_align4); + break; + default: ShouldNotReachHere(); + } + } + + // + // Fill large chunks + // + __ srliw(cnt_words, count, 3 - shift); // number of words + + // 32 bit -> 64 bit + __ andi(value, value, 0xffffffff); + __ mv(tmp_reg, value); + __ slli(tmp_reg, tmp_reg, 32); + __ orr(value, value, tmp_reg); + + __ slli(tmp_reg, cnt_words, 3 - shift); + __ subw(count, count, tmp_reg); + { + __ fill_words(to, cnt_words, value); + } + + // Remaining count is less than 8 bytes. Fill it by a single store. + // Note that the total length is no less than 8 bytes. + if (t == T_BYTE || t == T_SHORT) { + __ beqz(count, L_exit1); + __ shadd(to, count, to, tmp_reg, shift); // points to the end + __ sd(value, Address(to, -8)); // overwrite some elements + __ bind(L_exit1); + __ leave(); + __ ret(); + } + + // Handle copies less than 8 bytes. + Label L_fill_2, L_fill_4, L_exit2; + __ bind(L_fill_elements); + switch (t) { + case T_BYTE: + __ test_bit(t0, count, 0); + __ beqz(t0, L_fill_2); + __ sb(value, Address(to, 0)); + __ addi(to, to, 1); + __ bind(L_fill_2); + __ test_bit(t0, count, 1); + __ beqz(t0, L_fill_4); + __ sh(value, Address(to, 0)); + __ addi(to, to, 2); + __ bind(L_fill_4); + __ test_bit(t0, count, 2); + __ beqz(t0, L_exit2); + __ sw(value, Address(to, 0)); + break; + case T_SHORT: + __ test_bit(t0, count, 0); + __ beqz(t0, L_fill_4); + __ sh(value, Address(to, 0)); + __ addi(to, to, 2); + __ bind(L_fill_4); + __ test_bit(t0, count, 1); + __ beqz(t0, L_exit2); + __ sw(value, Address(to, 0)); + break; + case T_INT: + __ beqz(count, L_exit2); + __ sw(value, Address(to, 0)); + break; + default: ShouldNotReachHere(); + } + __ bind(L_exit2); + __ leave(); + __ ret(); + return start; + } + + void generate_arraycopy_stubs() { + address entry = NULL; + address entry_jbyte_arraycopy = NULL; + address entry_jshort_arraycopy = NULL; + address entry_jint_arraycopy = NULL; + address entry_oop_arraycopy = NULL; + address entry_jlong_arraycopy = NULL; + address entry_checkcast_arraycopy = NULL; + + generate_copy_longs(copy_f, c_rarg0, c_rarg1, t1, copy_forwards); + generate_copy_longs(copy_b, c_rarg0, c_rarg1, t1, copy_backwards); + + StubRoutines::riscv::_zero_blocks = generate_zero_blocks(); + + //*** jbyte + // Always need aligned and unaligned versions + StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(false, &entry, + "jbyte_disjoint_arraycopy"); + StubRoutines::_jbyte_arraycopy = generate_conjoint_byte_copy(false, entry, + &entry_jbyte_arraycopy, + "jbyte_arraycopy"); + StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(true, &entry, + "arrayof_jbyte_disjoint_arraycopy"); + StubRoutines::_arrayof_jbyte_arraycopy = generate_conjoint_byte_copy(true, entry, NULL, + "arrayof_jbyte_arraycopy"); + + //*** jshort + // Always need aligned and unaligned versions + StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, &entry, + "jshort_disjoint_arraycopy"); + StubRoutines::_jshort_arraycopy = generate_conjoint_short_copy(false, entry, + &entry_jshort_arraycopy, + "jshort_arraycopy"); + StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_short_copy(true, &entry, + "arrayof_jshort_disjoint_arraycopy"); + StubRoutines::_arrayof_jshort_arraycopy = generate_conjoint_short_copy(true, entry, NULL, + "arrayof_jshort_arraycopy"); + + //*** jint + // Aligned versions + StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_int_copy(true, &entry, + "arrayof_jint_disjoint_arraycopy"); + StubRoutines::_arrayof_jint_arraycopy = generate_conjoint_int_copy(true, entry, &entry_jint_arraycopy, + "arrayof_jint_arraycopy"); + // In 64 bit we need both aligned and unaligned versions of jint arraycopy. + // entry_jint_arraycopy always points to the unaligned version + StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_int_copy(false, &entry, + "jint_disjoint_arraycopy"); + StubRoutines::_jint_arraycopy = generate_conjoint_int_copy(false, entry, + &entry_jint_arraycopy, + "jint_arraycopy"); + + //*** jlong + // It is always aligned + StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_long_copy(true, &entry, + "arrayof_jlong_disjoint_arraycopy"); + StubRoutines::_arrayof_jlong_arraycopy = generate_conjoint_long_copy(true, entry, &entry_jlong_arraycopy, + "arrayof_jlong_arraycopy"); + StubRoutines::_jlong_disjoint_arraycopy = StubRoutines::_arrayof_jlong_disjoint_arraycopy; + StubRoutines::_jlong_arraycopy = StubRoutines::_arrayof_jlong_arraycopy; + + //*** oops + { + // With compressed oops we need unaligned versions; notice that + // we overwrite entry_oop_arraycopy. + bool aligned = !UseCompressedOops; + + StubRoutines::_arrayof_oop_disjoint_arraycopy + = generate_disjoint_oop_copy(aligned, &entry, "arrayof_oop_disjoint_arraycopy", + /*dest_uninitialized*/false); + StubRoutines::_arrayof_oop_arraycopy + = generate_conjoint_oop_copy(aligned, entry, &entry_oop_arraycopy, "arrayof_oop_arraycopy", + /*dest_uninitialized*/false); + // Aligned versions without pre-barriers + StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit + = generate_disjoint_oop_copy(aligned, &entry, "arrayof_oop_disjoint_arraycopy_uninit", + /*dest_uninitialized*/true); + StubRoutines::_arrayof_oop_arraycopy_uninit + = generate_conjoint_oop_copy(aligned, entry, NULL, "arrayof_oop_arraycopy_uninit", + /*dest_uninitialized*/true); + } + + StubRoutines::_oop_disjoint_arraycopy = StubRoutines::_arrayof_oop_disjoint_arraycopy; + StubRoutines::_oop_arraycopy = StubRoutines::_arrayof_oop_arraycopy; + StubRoutines::_oop_disjoint_arraycopy_uninit = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit; + StubRoutines::_oop_arraycopy_uninit = StubRoutines::_arrayof_oop_arraycopy_uninit; + + StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy); + StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy("checkcast_arraycopy_uninit", NULL, + /*dest_uninitialized*/true); + + + StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy", + entry_jbyte_arraycopy, + entry_jshort_arraycopy, + entry_jint_arraycopy, + entry_jlong_arraycopy); + + StubRoutines::_generic_arraycopy = generate_generic_copy("generic_arraycopy", + entry_jbyte_arraycopy, + entry_jshort_arraycopy, + entry_jint_arraycopy, + entry_oop_arraycopy, + entry_jlong_arraycopy, + entry_checkcast_arraycopy); + + StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill"); + StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill"); + StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill"); + StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill"); + StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill"); + StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill"); + } + + // code for comparing 16 bytes of strings with same encoding + void compare_string_16_bytes_same(Label &DIFF1, Label &DIFF2) { + const Register result = x10, str1 = x11, cnt1 = x12, str2 = x13, tmp1 = x28, tmp2 = x29, tmp4 = x7, tmp5 = x31; + __ ld(tmp5, Address(str1)); + __ addi(str1, str1, 8); + __ xorr(tmp4, tmp1, tmp2); + __ ld(cnt1, Address(str2)); + __ addi(str2, str2, 8); + __ bnez(tmp4, DIFF1); + __ ld(tmp1, Address(str1)); + __ addi(str1, str1, 8); + __ xorr(tmp4, tmp5, cnt1); + __ ld(tmp2, Address(str2)); + __ addi(str2, str2, 8); + __ bnez(tmp4, DIFF2); + } + + // code for comparing 8 characters of strings with Latin1 and Utf16 encoding + void compare_string_8_x_LU(Register tmpL, Register tmpU, Label &DIFF1, + Label &DIFF2) { + const Register strU = x12, curU = x7, strL = x29, tmp = x30; + __ ld(tmpL, Address(strL)); + __ addi(strL, strL, 8); + __ ld(tmpU, Address(strU)); + __ addi(strU, strU, 8); + __ inflate_lo32(tmp, tmpL); + __ mv(t0, tmp); + __ xorr(tmp, curU, t0); + __ bnez(tmp, DIFF2); + + __ ld(curU, Address(strU)); + __ addi(strU, strU, 8); + __ inflate_hi32(tmp, tmpL); + __ mv(t0, tmp); + __ xorr(tmp, tmpU, t0); + __ bnez(tmp, DIFF1); + } + + // x10 = result + // x11 = str1 + // x12 = cnt1 + // x13 = str2 + // x14 = cnt2 + // x28 = tmp1 + // x29 = tmp2 + // x30 = tmp3 + address generate_compare_long_string_different_encoding(bool isLU) { + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", isLU ? "compare_long_string_different_encoding LU" : "compare_long_string_different_encoding UL"); + address entry = __ pc(); + Label SMALL_LOOP, TAIL, TAIL_LOAD_16, LOAD_LAST, DIFF1, DIFF2, + DONE, CALCULATE_DIFFERENCE; + const Register result = x10, str1 = x11, cnt1 = x12, str2 = x13, cnt2 = x14, + tmp1 = x28, tmp2 = x29, tmp3 = x30, tmp4 = x7, tmp5 = x31; + RegSet spilled_regs = RegSet::of(tmp4, tmp5); + + // cnt2 == amount of characters left to compare + // Check already loaded first 4 symbols + __ inflate_lo32(tmp3, isLU ? tmp1 : tmp2); + __ mv(isLU ? tmp1 : tmp2, tmp3); + __ addi(str1, str1, isLU ? wordSize / 2 : wordSize); + __ addi(str2, str2, isLU ? wordSize : wordSize / 2); + __ sub(cnt2, cnt2, 8); // Already loaded 4 symbols. Last 4 is special case. + __ push_reg(spilled_regs, sp); + + if (isLU) { + __ add(str1, str1, cnt2); + __ shadd(str2, cnt2, str2, t0, 1); + } else { + __ shadd(str1, cnt2, str1, t0, 1); + __ add(str2, str2, cnt2); + } + __ xorr(tmp3, tmp1, tmp2); + __ mv(tmp5, tmp2); + __ bnez(tmp3, CALCULATE_DIFFERENCE); + + Register strU = isLU ? str2 : str1, + strL = isLU ? str1 : str2, + tmpU = isLU ? tmp5 : tmp1, // where to keep U for comparison + tmpL = isLU ? tmp1 : tmp5; // where to keep L for comparison + + __ sub(tmp2, strL, cnt2); // strL pointer to load from + __ slli(t0, cnt2, 1); + __ sub(cnt1, strU, t0); // strU pointer to load from + + __ ld(tmp4, Address(cnt1)); + __ addi(cnt1, cnt1, 8); + __ beqz(cnt2, LOAD_LAST); // no characters left except last load + __ sub(cnt2, cnt2, 16); + __ bltz(cnt2, TAIL); + __ bind(SMALL_LOOP); // smaller loop + __ sub(cnt2, cnt2, 16); + compare_string_8_x_LU(tmpL, tmpU, DIFF1, DIFF2); + compare_string_8_x_LU(tmpL, tmpU, DIFF1, DIFF2); + __ bgez(cnt2, SMALL_LOOP); + __ addi(t0, cnt2, 16); + __ beqz(t0, LOAD_LAST); + __ bind(TAIL); // 1..15 characters left until last load (last 4 characters) + // Address of 8 bytes before last 4 characters in UTF-16 string + __ shadd(cnt1, cnt2, cnt1, t0, 1); + // Address of 16 bytes before last 4 characters in Latin1 string + __ add(tmp2, tmp2, cnt2); + __ ld(tmp4, Address(cnt1, -8)); + // last 16 characters before last load + compare_string_8_x_LU(tmpL, tmpU, DIFF1, DIFF2); + compare_string_8_x_LU(tmpL, tmpU, DIFF1, DIFF2); + __ j(LOAD_LAST); + __ bind(DIFF2); + __ mv(tmpU, tmp4); + __ bind(DIFF1); + __ mv(tmpL, t0); + __ j(CALCULATE_DIFFERENCE); + __ bind(LOAD_LAST); + // Last 4 UTF-16 characters are already pre-loaded into tmp4 by compare_string_8_x_LU. + // No need to load it again + __ mv(tmpU, tmp4); + __ ld(tmpL, Address(strL)); + __ inflate_lo32(tmp3, tmpL); + __ mv(tmpL, tmp3); + __ xorr(tmp3, tmpU, tmpL); + __ beqz(tmp3, DONE); + + // Find the first different characters in the longwords and + // compute their difference. + __ bind(CALCULATE_DIFFERENCE); + __ ctzc_bit(tmp4, tmp3); + __ srl(tmp1, tmp1, tmp4); + __ srl(tmp5, tmp5, tmp4); + __ andi(tmp1, tmp1, 0xFFFF); + __ andi(tmp5, tmp5, 0xFFFF); + __ sub(result, tmp1, tmp5); + __ bind(DONE); + __ pop_reg(spilled_regs, sp); + __ ret(); + return entry; + } + + address generate_method_entry_barrier() { + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", "nmethod_entry_barrier"); + + Label deoptimize_label; + + address start = __ pc(); + + __ set_last_Java_frame(sp, fp, ra, t0); + + __ enter(); + __ add(t1, sp, wordSize); + + __ sub(sp, sp, 4 * wordSize); + + __ push_call_clobbered_registers(); + + __ mv(c_rarg0, t1); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetNMethod::nmethod_stub_entry_barrier), 1); + + __ reset_last_Java_frame(true); + + __ mv(t0, x10); + + __ pop_call_clobbered_registers(); + + __ bnez(t0, deoptimize_label); + + __ leave(); + __ ret(); + + __ BIND(deoptimize_label); + + __ ld(t0, Address(sp, 0)); + __ ld(fp, Address(sp, wordSize)); + __ ld(ra, Address(sp, wordSize * 2)); + __ ld(t1, Address(sp, wordSize * 3)); + + __ mv(sp, t0); + __ jr(t1); + + return start; + } + + // x10 = result + // x11 = str1 + // x12 = cnt1 + // x13 = str2 + // x14 = cnt2 + // x28 = tmp1 + // x29 = tmp2 + // x30 = tmp3 + // x31 = tmp4 + address generate_compare_long_string_same_encoding(bool isLL) { + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", isLL ? + "compare_long_string_same_encoding LL" : "compare_long_string_same_encoding UU"); + address entry = __ pc(); + Label SMALL_LOOP, CHECK_LAST, DIFF2, TAIL, + LENGTH_DIFF, DIFF, LAST_CHECK_AND_LENGTH_DIFF; + const Register result = x10, str1 = x11, cnt1 = x12, str2 = x13, cnt2 = x14, + tmp1 = x28, tmp2 = x29, tmp3 = x30, tmp4 = x7, tmp5 = x31; + RegSet spilled_regs = RegSet::of(tmp4, tmp5); + + // cnt1/cnt2 contains amount of characters to compare. cnt1 can be re-used + // update cnt2 counter with already loaded 8 bytes + __ sub(cnt2, cnt2, wordSize / (isLL ? 1 : 2)); + // update pointers, because of previous read + __ add(str1, str1, wordSize); + __ add(str2, str2, wordSize); + // less than 16 bytes left? + __ sub(cnt2, cnt2, isLL ? 16 : 8); + __ push_reg(spilled_regs, sp); + __ bltz(cnt2, TAIL); + __ bind(SMALL_LOOP); + compare_string_16_bytes_same(DIFF, DIFF2); + __ sub(cnt2, cnt2, isLL ? 16 : 8); + __ bgez(cnt2, SMALL_LOOP); + __ bind(TAIL); + __ addi(cnt2, cnt2, isLL ? 16 : 8); + __ beqz(cnt2, LAST_CHECK_AND_LENGTH_DIFF); + __ sub(cnt2, cnt2, isLL ? 8 : 4); + __ blez(cnt2, CHECK_LAST); + __ xorr(tmp4, tmp1, tmp2); + __ bnez(tmp4, DIFF); + __ ld(tmp1, Address(str1)); + __ addi(str1, str1, 8); + __ ld(tmp2, Address(str2)); + __ addi(str2, str2, 8); + __ sub(cnt2, cnt2, isLL ? 8 : 4); + __ bind(CHECK_LAST); + if (!isLL) { + __ add(cnt2, cnt2, cnt2); // now in bytes + } + __ xorr(tmp4, tmp1, tmp2); + __ bnez(tmp4, DIFF); + __ add(str1, str1, cnt2); + __ ld(tmp5, Address(str1)); + __ add(str2, str2, cnt2); + __ ld(cnt1, Address(str2)); + __ xorr(tmp4, tmp5, cnt1); + __ beqz(tmp4, LENGTH_DIFF); + // Find the first different characters in the longwords and + // compute their difference. + __ bind(DIFF2); + __ ctzc_bit(tmp3, tmp4, isLL); // count zero from lsb to msb + __ srl(tmp5, tmp5, tmp3); + __ srl(cnt1, cnt1, tmp3); + if (isLL) { + __ andi(tmp5, tmp5, 0xFF); + __ andi(cnt1, cnt1, 0xFF); + } else { + __ andi(tmp5, tmp5, 0xFFFF); + __ andi(cnt1, cnt1, 0xFFFF); + } + __ sub(result, tmp5, cnt1); + __ j(LENGTH_DIFF); + __ bind(DIFF); + __ ctzc_bit(tmp3, tmp4, isLL); // count zero from lsb to msb + __ srl(tmp1, tmp1, tmp3); + __ srl(tmp2, tmp2, tmp3); + if (isLL) { + __ andi(tmp1, tmp1, 0xFF); + __ andi(tmp2, tmp2, 0xFF); + } else { + __ andi(tmp1, tmp1, 0xFFFF); + __ andi(tmp2, tmp2, 0xFFFF); + } + __ sub(result, tmp1, tmp2); + __ j(LENGTH_DIFF); + __ bind(LAST_CHECK_AND_LENGTH_DIFF); + __ xorr(tmp4, tmp1, tmp2); + __ bnez(tmp4, DIFF); + __ bind(LENGTH_DIFF); + __ pop_reg(spilled_regs, sp); + __ ret(); + return entry; + } + + void generate_compare_long_strings() { + StubRoutines::riscv::_compare_long_string_LL = generate_compare_long_string_same_encoding(true); + StubRoutines::riscv::_compare_long_string_UU = generate_compare_long_string_same_encoding(false); + StubRoutines::riscv::_compare_long_string_LU = generate_compare_long_string_different_encoding(true); + StubRoutines::riscv::_compare_long_string_UL = generate_compare_long_string_different_encoding(false); + } + + // x10 result + // x11 src + // x12 src count + // x13 pattern + // x14 pattern count + address generate_string_indexof_linear(bool needle_isL, bool haystack_isL) + { + const char* stubName = needle_isL + ? (haystack_isL ? "indexof_linear_ll" : "indexof_linear_ul") + : "indexof_linear_uu"; + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", stubName); + address entry = __ pc(); + + int needle_chr_size = needle_isL ? 1 : 2; + int haystack_chr_size = haystack_isL ? 1 : 2; + int needle_chr_shift = needle_isL ? 0 : 1; + int haystack_chr_shift = haystack_isL ? 0 : 1; + bool isL = needle_isL && haystack_isL; + // parameters + Register result = x10, haystack = x11, haystack_len = x12, needle = x13, needle_len = x14; + // temporary registers + Register mask1 = x20, match_mask = x21, first = x22, trailing_zeros = x23, mask2 = x24, tmp = x25; + // redefinitions + Register ch1 = x28, ch2 = x29; + RegSet spilled_regs = RegSet::range(x20, x25) + RegSet::range(x28, x29); + + __ push_reg(spilled_regs, sp); + + Label L_LOOP, L_LOOP_PROCEED, L_SMALL, L_HAS_ZERO, + L_HAS_ZERO_LOOP, L_CMP_LOOP, L_CMP_LOOP_NOMATCH, L_SMALL_PROCEED, + L_SMALL_HAS_ZERO_LOOP, L_SMALL_CMP_LOOP_NOMATCH, L_SMALL_CMP_LOOP, + L_POST_LOOP, L_CMP_LOOP_LAST_CMP, L_HAS_ZERO_LOOP_NOMATCH, + L_SMALL_CMP_LOOP_LAST_CMP, L_SMALL_CMP_LOOP_LAST_CMP2, + L_CMP_LOOP_LAST_CMP2, DONE, NOMATCH; + + __ ld(ch1, Address(needle)); + __ ld(ch2, Address(haystack)); + // src.length - pattern.length + __ sub(haystack_len, haystack_len, needle_len); + + // first is needle[0] + __ andi(first, ch1, needle_isL ? 0xFF : 0xFFFF, first); + uint64_t mask0101 = UCONST64(0x0101010101010101); + uint64_t mask0001 = UCONST64(0x0001000100010001); + __ mv(mask1, haystack_isL ? mask0101 : mask0001); + __ mul(first, first, mask1); + uint64_t mask7f7f = UCONST64(0x7f7f7f7f7f7f7f7f); + uint64_t mask7fff = UCONST64(0x7fff7fff7fff7fff); + __ mv(mask2, haystack_isL ? mask7f7f : mask7fff); + if (needle_isL != haystack_isL) { + __ mv(tmp, ch1); + } + __ sub(haystack_len, haystack_len, wordSize / haystack_chr_size - 1); + __ blez(haystack_len, L_SMALL); + + if (needle_isL != haystack_isL) { + __ inflate_lo32(ch1, tmp, match_mask, trailing_zeros); + } + // xorr, sub, orr, notr, andr + // compare and set match_mask[i] with 0x80/0x8000 (Latin1/UTF16) if ch2[i] == first[i] + // eg: + // first: aa aa aa aa aa aa aa aa + // ch2: aa aa li nx jd ka aa aa + // match_mask: 80 80 00 00 00 00 80 80 + __ compute_match_mask(ch2, first, match_mask, mask1, mask2); + + // search first char of needle, if success, goto L_HAS_ZERO; + __ bnez(match_mask, L_HAS_ZERO); + __ sub(haystack_len, haystack_len, wordSize / haystack_chr_size); + __ add(result, result, wordSize / haystack_chr_size); + __ add(haystack, haystack, wordSize); + __ bltz(haystack_len, L_POST_LOOP); + + __ bind(L_LOOP); + __ ld(ch2, Address(haystack)); + __ compute_match_mask(ch2, first, match_mask, mask1, mask2); + __ bnez(match_mask, L_HAS_ZERO); + + __ bind(L_LOOP_PROCEED); + __ sub(haystack_len, haystack_len, wordSize / haystack_chr_size); + __ add(haystack, haystack, wordSize); + __ add(result, result, wordSize / haystack_chr_size); + __ bgez(haystack_len, L_LOOP); + + __ bind(L_POST_LOOP); + __ mv(ch2, -wordSize / haystack_chr_size); + __ ble(haystack_len, ch2, NOMATCH); // no extra characters to check + __ ld(ch2, Address(haystack)); + __ slli(haystack_len, haystack_len, LogBitsPerByte + haystack_chr_shift); + __ neg(haystack_len, haystack_len); + __ xorr(ch2, first, ch2); + __ sub(match_mask, ch2, mask1); + __ orr(ch2, ch2, mask2); + __ mv(trailing_zeros, -1); // all bits set + __ j(L_SMALL_PROCEED); + + __ align(OptoLoopAlignment); + __ bind(L_SMALL); + __ slli(haystack_len, haystack_len, LogBitsPerByte + haystack_chr_shift); + __ neg(haystack_len, haystack_len); + if (needle_isL != haystack_isL) { + __ inflate_lo32(ch1, tmp, match_mask, trailing_zeros); + } + __ xorr(ch2, first, ch2); + __ sub(match_mask, ch2, mask1); + __ orr(ch2, ch2, mask2); + __ mv(trailing_zeros, -1); // all bits set + + __ bind(L_SMALL_PROCEED); + __ srl(trailing_zeros, trailing_zeros, haystack_len); // mask. zeroes on useless bits. + __ notr(ch2, ch2); + __ andr(match_mask, match_mask, ch2); + __ andr(match_mask, match_mask, trailing_zeros); // clear useless bits and check + __ beqz(match_mask, NOMATCH); + + __ bind(L_SMALL_HAS_ZERO_LOOP); + __ ctzc_bit(trailing_zeros, match_mask, haystack_isL, ch2, tmp); // count trailing zeros + __ addi(trailing_zeros, trailing_zeros, haystack_isL ? 7 : 15); + __ mv(ch2, wordSize / haystack_chr_size); + __ ble(needle_len, ch2, L_SMALL_CMP_LOOP_LAST_CMP2); + __ compute_index(haystack, trailing_zeros, match_mask, result, ch2, tmp, haystack_isL); + __ mv(trailing_zeros, wordSize / haystack_chr_size); + __ bne(ch1, ch2, L_SMALL_CMP_LOOP_NOMATCH); + + __ bind(L_SMALL_CMP_LOOP); + __ shadd(first, trailing_zeros, needle, first, needle_chr_shift); + __ shadd(ch2, trailing_zeros, haystack, ch2, haystack_chr_shift); + needle_isL ? __ lbu(first, Address(first)) : __ lhu(first, Address(first)); + haystack_isL ? __ lbu(ch2, Address(ch2)) : __ lhu(ch2, Address(ch2)); + __ add(trailing_zeros, trailing_zeros, 1); + __ bge(trailing_zeros, needle_len, L_SMALL_CMP_LOOP_LAST_CMP); + __ beq(first, ch2, L_SMALL_CMP_LOOP); + + __ bind(L_SMALL_CMP_LOOP_NOMATCH); + __ beqz(match_mask, NOMATCH); + __ ctzc_bit(trailing_zeros, match_mask, haystack_isL, tmp, ch2); + __ addi(trailing_zeros, trailing_zeros, haystack_isL ? 7 : 15); + __ add(result, result, 1); + __ add(haystack, haystack, haystack_chr_size); + __ j(L_SMALL_HAS_ZERO_LOOP); + + __ align(OptoLoopAlignment); + __ bind(L_SMALL_CMP_LOOP_LAST_CMP); + __ bne(first, ch2, L_SMALL_CMP_LOOP_NOMATCH); + __ j(DONE); + + __ align(OptoLoopAlignment); + __ bind(L_SMALL_CMP_LOOP_LAST_CMP2); + __ compute_index(haystack, trailing_zeros, match_mask, result, ch2, tmp, haystack_isL); + __ bne(ch1, ch2, L_SMALL_CMP_LOOP_NOMATCH); + __ j(DONE); + + __ align(OptoLoopAlignment); + __ bind(L_HAS_ZERO); + __ ctzc_bit(trailing_zeros, match_mask, haystack_isL, tmp, ch2); + __ addi(trailing_zeros, trailing_zeros, haystack_isL ? 7 : 15); + __ slli(needle_len, needle_len, BitsPerByte * wordSize / 2); + __ orr(haystack_len, haystack_len, needle_len); // restore needle_len(32bits) + __ sub(result, result, 1); // array index from 0, so result -= 1 + + __ bind(L_HAS_ZERO_LOOP); + __ mv(needle_len, wordSize / haystack_chr_size); + __ srli(ch2, haystack_len, BitsPerByte * wordSize / 2); + __ bge(needle_len, ch2, L_CMP_LOOP_LAST_CMP2); + // load next 8 bytes from haystack, and increase result index + __ compute_index(haystack, trailing_zeros, match_mask, result, ch2, tmp, haystack_isL); + __ add(result, result, 1); + __ mv(trailing_zeros, wordSize / haystack_chr_size); + __ bne(ch1, ch2, L_CMP_LOOP_NOMATCH); + + // compare one char + __ bind(L_CMP_LOOP); + __ shadd(needle_len, trailing_zeros, needle, needle_len, needle_chr_shift); + needle_isL ? __ lbu(needle_len, Address(needle_len)) : __ lhu(needle_len, Address(needle_len)); + __ shadd(ch2, trailing_zeros, haystack, ch2, haystack_chr_shift); + haystack_isL ? __ lbu(ch2, Address(ch2)) : __ lhu(ch2, Address(ch2)); + __ add(trailing_zeros, trailing_zeros, 1); // next char index + __ srli(tmp, haystack_len, BitsPerByte * wordSize / 2); + __ bge(trailing_zeros, tmp, L_CMP_LOOP_LAST_CMP); + __ beq(needle_len, ch2, L_CMP_LOOP); + + __ bind(L_CMP_LOOP_NOMATCH); + __ beqz(match_mask, L_HAS_ZERO_LOOP_NOMATCH); + __ ctzc_bit(trailing_zeros, match_mask, haystack_isL, needle_len, ch2); // find next "first" char index + __ addi(trailing_zeros, trailing_zeros, haystack_isL ? 7 : 15); + __ add(haystack, haystack, haystack_chr_size); + __ j(L_HAS_ZERO_LOOP); + + __ align(OptoLoopAlignment); + __ bind(L_CMP_LOOP_LAST_CMP); + __ bne(needle_len, ch2, L_CMP_LOOP_NOMATCH); + __ j(DONE); + + __ align(OptoLoopAlignment); + __ bind(L_CMP_LOOP_LAST_CMP2); + __ compute_index(haystack, trailing_zeros, match_mask, result, ch2, tmp, haystack_isL); + __ add(result, result, 1); + __ bne(ch1, ch2, L_CMP_LOOP_NOMATCH); + __ j(DONE); + + __ align(OptoLoopAlignment); + __ bind(L_HAS_ZERO_LOOP_NOMATCH); + // 1) Restore "result" index. Index was wordSize/str2_chr_size * N until + // L_HAS_ZERO block. Byte octet was analyzed in L_HAS_ZERO_LOOP, + // so, result was increased at max by wordSize/str2_chr_size - 1, so, + // respective high bit wasn't changed. L_LOOP_PROCEED will increase + // result by analyzed characters value, so, we can just reset lower bits + // in result here. Clear 2 lower bits for UU/UL and 3 bits for LL + // 2) restore needle_len and haystack_len values from "compressed" haystack_len + // 3) advance haystack value to represent next haystack octet. result & 7/3 is + // index of last analyzed substring inside current octet. So, haystack in at + // respective start address. We need to advance it to next octet + __ andi(match_mask, result, wordSize / haystack_chr_size - 1); + __ srli(needle_len, haystack_len, BitsPerByte * wordSize / 2); + __ andi(result, result, haystack_isL ? -8 : -4); + __ slli(tmp, match_mask, haystack_chr_shift); + __ sub(haystack, haystack, tmp); + __ sign_extend(haystack_len, haystack_len, 32); + __ j(L_LOOP_PROCEED); + + __ align(OptoLoopAlignment); + __ bind(NOMATCH); + __ mv(result, -1); + + __ bind(DONE); + __ pop_reg(spilled_regs, sp); + __ ret(); + return entry; + } + + void generate_string_indexof_stubs() + { + StubRoutines::riscv::_string_indexof_linear_ll = generate_string_indexof_linear(true, true); + StubRoutines::riscv::_string_indexof_linear_uu = generate_string_indexof_linear(false, false); + StubRoutines::riscv::_string_indexof_linear_ul = generate_string_indexof_linear(true, false); + } + +#ifdef COMPILER2 + address generate_mulAdd() + { + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", "mulAdd"); + + address entry = __ pc(); + + const Register out = x10; + const Register in = x11; + const Register offset = x12; + const Register len = x13; + const Register k = x14; + const Register tmp = x28; + + BLOCK_COMMENT("Entry:"); + __ enter(); + __ mul_add(out, in, offset, len, k, tmp); + __ leave(); + __ ret(); + + return entry; + } + + /** + * Arguments: + * + * Input: + * c_rarg0 - x address + * c_rarg1 - x length + * c_rarg2 - y address + * c_rarg3 - y length + * c_rarg4 - z address + * c_rarg5 - z length + */ + address generate_multiplyToLen() + { + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", "multiplyToLen"); + address entry = __ pc(); + + const Register x = x10; + const Register xlen = x11; + const Register y = x12; + const Register ylen = x13; + const Register z = x14; + const Register zlen = x15; + + const Register tmp1 = x16; + const Register tmp2 = x17; + const Register tmp3 = x7; + const Register tmp4 = x28; + const Register tmp5 = x29; + const Register tmp6 = x30; + const Register tmp7 = x31; + + BLOCK_COMMENT("Entry:"); + __ enter(); // required for proper stackwalking of RuntimeStub frame + __ multiply_to_len(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + __ leave(); // required for proper stackwalking of RuntimeStub frame + __ ret(); + + return entry; + } + + address generate_squareToLen() + { + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", "squareToLen"); + address entry = __ pc(); + + const Register x = x10; + const Register xlen = x11; + const Register z = x12; + const Register zlen = x13; + const Register y = x14; // == x + const Register ylen = x15; // == xlen + + const Register tmp1 = x16; + const Register tmp2 = x17; + const Register tmp3 = x7; + const Register tmp4 = x28; + const Register tmp5 = x29; + const Register tmp6 = x30; + const Register tmp7 = x31; + + BLOCK_COMMENT("Entry:"); + __ enter(); + __ mv(y, x); + __ mv(ylen, xlen); + __ multiply_to_len(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + __ leave(); + __ ret(); + + return entry; + } + + // Arguments: + // + // Input: + // c_rarg0 - newArr address + // c_rarg1 - oldArr address + // c_rarg2 - newIdx + // c_rarg3 - shiftCount + // c_rarg4 - numIter + // + address generate_bigIntegerLeftShift() { + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", "bigIntegerLeftShiftWorker"); + address entry = __ pc(); + + Label loop, exit; + + Register newArr = c_rarg0; + Register oldArr = c_rarg1; + Register newIdx = c_rarg2; + Register shiftCount = c_rarg3; + Register numIter = c_rarg4; + + Register shiftRevCount = c_rarg5; + Register oldArrNext = t1; + + __ beqz(numIter, exit); + __ shadd(newArr, newIdx, newArr, t0, 2); + + __ mv(shiftRevCount, 32); + __ sub(shiftRevCount, shiftRevCount, shiftCount); + + __ bind(loop); + __ addi(oldArrNext, oldArr, 4); + __ vsetvli(t0, numIter, Assembler::e32, Assembler::m4); + __ vle32_v(v0, oldArr); + __ vle32_v(v4, oldArrNext); + __ vsll_vx(v0, v0, shiftCount); + __ vsrl_vx(v4, v4, shiftRevCount); + __ vor_vv(v0, v0, v4); + __ vse32_v(v0, newArr); + __ sub(numIter, numIter, t0); + __ shadd(oldArr, t0, oldArr, t1, 2); + __ shadd(newArr, t0, newArr, t1, 2); + __ bnez(numIter, loop); + + __ bind(exit); + __ ret(); + + return entry; + } + + // Arguments: + // + // Input: + // c_rarg0 - newArr address + // c_rarg1 - oldArr address + // c_rarg2 - newIdx + // c_rarg3 - shiftCount + // c_rarg4 - numIter + // + address generate_bigIntegerRightShift() { + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", "bigIntegerRightShiftWorker"); + address entry = __ pc(); + + Label loop, exit; + + Register newArr = c_rarg0; + Register oldArr = c_rarg1; + Register newIdx = c_rarg2; + Register shiftCount = c_rarg3; + Register numIter = c_rarg4; + Register idx = numIter; + + Register shiftRevCount = c_rarg5; + Register oldArrNext = c_rarg6; + Register newArrCur = t0; + Register oldArrCur = t1; + + __ beqz(idx, exit); + __ shadd(newArr, newIdx, newArr, t0, 2); + + __ mv(shiftRevCount, 32); + __ sub(shiftRevCount, shiftRevCount, shiftCount); + + __ bind(loop); + __ vsetvli(t0, idx, Assembler::e32, Assembler::m4); + __ sub(idx, idx, t0); + __ shadd(oldArrNext, idx, oldArr, t1, 2); + __ shadd(newArrCur, idx, newArr, t1, 2); + __ addi(oldArrCur, oldArrNext, 4); + __ vle32_v(v0, oldArrCur); + __ vle32_v(v4, oldArrNext); + __ vsrl_vx(v0, v0, shiftCount); + __ vsll_vx(v4, v4, shiftRevCount); + __ vor_vv(v0, v0, v4); + __ vse32_v(v0, newArrCur); + __ bnez(idx, loop); + + __ bind(exit); + __ ret(); + + return entry; + } +#endif + +#ifdef COMPILER2 + class MontgomeryMultiplyGenerator : public MacroAssembler { + + Register Pa_base, Pb_base, Pn_base, Pm_base, inv, Rlen, Ra, Rb, Rm, Rn, + Pa, Pb, Pn, Pm, Rhi_ab, Rlo_ab, Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2, Ri, Rj; + + RegSet _toSave; + bool _squaring; + + public: + MontgomeryMultiplyGenerator (Assembler *as, bool squaring) + : MacroAssembler(as->code()), _squaring(squaring) { + + // Register allocation + + Register reg = c_rarg0; + Pa_base = reg; // Argument registers + if (squaring) { + Pb_base = Pa_base; + } else { + Pb_base = ++reg; + } + Pn_base = ++reg; + Rlen= ++reg; + inv = ++reg; + Pm_base = ++reg; + + // Working registers: + Ra = ++reg; // The current digit of a, b, n, and m. + Rb = ++reg; + Rm = ++reg; + Rn = ++reg; + + Pa = ++reg; // Pointers to the current/next digit of a, b, n, and m. + Pb = ++reg; + Pm = ++reg; + Pn = ++reg; + + tmp0 = ++reg; // Three registers which form a + tmp1 = ++reg; // triple-precision accumuator. + tmp2 = ++reg; + + Ri = x6; // Inner and outer loop indexes. + Rj = x7; + + Rhi_ab = x28; // Product registers: low and high parts + Rlo_ab = x29; // of a*b and m*n. + Rhi_mn = x30; + Rlo_mn = x31; + + // x18 and up are callee-saved. + _toSave = RegSet::range(x18, reg) + Pm_base; + } + + private: + void save_regs() { + push_reg(_toSave, sp); + } + + void restore_regs() { + pop_reg(_toSave, sp); + } + + template + void unroll_2(Register count, T block) { + Label loop, end, odd; + beqz(count, end); + test_bit(t0, count, 0); + bnez(t0, odd); + align(16); + bind(loop); + (this->*block)(); + bind(odd); + (this->*block)(); + addi(count, count, -2); + bgtz(count, loop); + bind(end); + } + + template + void unroll_2(Register count, T block, Register d, Register s, Register tmp) { + Label loop, end, odd; + beqz(count, end); + test_bit(tmp, count, 0); + bnez(tmp, odd); + align(16); + bind(loop); + (this->*block)(d, s, tmp); + bind(odd); + (this->*block)(d, s, tmp); + addi(count, count, -2); + bgtz(count, loop); + bind(end); + } + + void pre1(RegisterOrConstant i) { + block_comment("pre1"); + // Pa = Pa_base; + // Pb = Pb_base + i; + // Pm = Pm_base; + // Pn = Pn_base + i; + // Ra = *Pa; + // Rb = *Pb; + // Rm = *Pm; + // Rn = *Pn; + if (i.is_register()) { + slli(t0, i.as_register(), LogBytesPerWord); + } else { + mv(t0, i.as_constant()); + slli(t0, t0, LogBytesPerWord); + } + + mv(Pa, Pa_base); + add(Pb, Pb_base, t0); + mv(Pm, Pm_base); + add(Pn, Pn_base, t0); + + ld(Ra, Address(Pa)); + ld(Rb, Address(Pb)); + ld(Rm, Address(Pm)); + ld(Rn, Address(Pn)); + + // Zero the m*n result. + mv(Rhi_mn, zr); + mv(Rlo_mn, zr); + } + + // The core multiply-accumulate step of a Montgomery + // multiplication. The idea is to schedule operations as a + // pipeline so that instructions with long latencies (loads and + // multiplies) have time to complete before their results are + // used. This most benefits in-order implementations of the + // architecture but out-of-order ones also benefit. + void step() { + block_comment("step"); + // MACC(Ra, Rb, tmp0, tmp1, tmp2); + // Ra = *++Pa; + // Rb = *--Pb; + mulhu(Rhi_ab, Ra, Rb); + mul(Rlo_ab, Ra, Rb); + addi(Pa, Pa, wordSize); + ld(Ra, Address(Pa)); + addi(Pb, Pb, -wordSize); + ld(Rb, Address(Pb)); + acc(Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2); // The pending m*n from the + // previous iteration. + // MACC(Rm, Rn, tmp0, tmp1, tmp2); + // Rm = *++Pm; + // Rn = *--Pn; + mulhu(Rhi_mn, Rm, Rn); + mul(Rlo_mn, Rm, Rn); + addi(Pm, Pm, wordSize); + ld(Rm, Address(Pm)); + addi(Pn, Pn, -wordSize); + ld(Rn, Address(Pn)); + acc(Rhi_ab, Rlo_ab, tmp0, tmp1, tmp2); + } + + void post1() { + block_comment("post1"); + + // MACC(Ra, Rb, tmp0, tmp1, tmp2); + // Ra = *++Pa; + // Rb = *--Pb; + mulhu(Rhi_ab, Ra, Rb); + mul(Rlo_ab, Ra, Rb); + acc(Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2); // The pending m*n + acc(Rhi_ab, Rlo_ab, tmp0, tmp1, tmp2); + + // *Pm = Rm = tmp0 * inv; + mul(Rm, tmp0, inv); + sd(Rm, Address(Pm)); + + // MACC(Rm, Rn, tmp0, tmp1, tmp2); + // tmp0 = tmp1; tmp1 = tmp2; tmp2 = 0; + mulhu(Rhi_mn, Rm, Rn); + +#ifndef PRODUCT + // assert(m[i] * n[0] + tmp0 == 0, "broken Montgomery multiply"); + { + mul(Rlo_mn, Rm, Rn); + add(Rlo_mn, tmp0, Rlo_mn); + Label ok; + beqz(Rlo_mn, ok); + stop("broken Montgomery multiply"); + bind(ok); + } +#endif + // We have very carefully set things up so that + // m[i]*n[0] + tmp0 == 0 (mod b), so we don't have to calculate + // the lower half of Rm * Rn because we know the result already: + // it must be -tmp0. tmp0 + (-tmp0) must generate a carry iff + // tmp0 != 0. So, rather than do a mul and an cad we just set + // the carry flag iff tmp0 is nonzero. + // + // mul(Rlo_mn, Rm, Rn); + // cad(zr, tmp0, Rlo_mn); + addi(t0, tmp0, -1); + sltu(t0, t0, tmp0); // Set carry iff tmp0 is nonzero + cadc(tmp0, tmp1, Rhi_mn, t0); + adc(tmp1, tmp2, zr, t0); + mv(tmp2, zr); + } + + void pre2(Register i, Register len) { + block_comment("pre2"); + // Pa = Pa_base + i-len; + // Pb = Pb_base + len; + // Pm = Pm_base + i-len; + // Pn = Pn_base + len; + + sub(Rj, i, len); + // Rj == i-len + + // Ra as temp register + slli(Ra, Rj, LogBytesPerWord); + add(Pa, Pa_base, Ra); + add(Pm, Pm_base, Ra); + slli(Ra, len, LogBytesPerWord); + add(Pb, Pb_base, Ra); + add(Pn, Pn_base, Ra); + + // Ra = *++Pa; + // Rb = *--Pb; + // Rm = *++Pm; + // Rn = *--Pn; + add(Pa, Pa, wordSize); + ld(Ra, Address(Pa)); + add(Pb, Pb, -wordSize); + ld(Rb, Address(Pb)); + add(Pm, Pm, wordSize); + ld(Rm, Address(Pm)); + add(Pn, Pn, -wordSize); + ld(Rn, Address(Pn)); + + mv(Rhi_mn, zr); + mv(Rlo_mn, zr); + } + + void post2(Register i, Register len) { + block_comment("post2"); + sub(Rj, i, len); + + cad(tmp0, tmp0, Rlo_mn, t0); // The pending m*n, low part + + // As soon as we know the least significant digit of our result, + // store it. + // Pm_base[i-len] = tmp0; + // Rj as temp register + slli(Rj, Rj, LogBytesPerWord); + add(Rj, Pm_base, Rj); + sd(tmp0, Address(Rj)); + + // tmp0 = tmp1; tmp1 = tmp2; tmp2 = 0; + cadc(tmp0, tmp1, Rhi_mn, t0); // The pending m*n, high part + adc(tmp1, tmp2, zr, t0); + mv(tmp2, zr); + } + + // A carry in tmp0 after Montgomery multiplication means that we + // should subtract multiples of n from our result in m. We'll + // keep doing that until there is no carry. + void normalize(Register len) { + block_comment("normalize"); + // while (tmp0) + // tmp0 = sub(Pm_base, Pn_base, tmp0, len); + Label loop, post, again; + Register cnt = tmp1, i = tmp2; // Re-use registers; we're done with them now + beqz(tmp0, post); { + bind(again); { + mv(i, zr); + mv(cnt, len); + slli(Rn, i, LogBytesPerWord); + add(Rm, Pm_base, Rn); + ld(Rm, Address(Rm)); + add(Rn, Pn_base, Rn); + ld(Rn, Address(Rn)); + mv(t0, 1); // set carry flag, i.e. no borrow + align(16); + bind(loop); { + notr(Rn, Rn); + add(Rm, Rm, t0); + add(Rm, Rm, Rn); + sltu(t0, Rm, Rn); + slli(Rn, i, LogBytesPerWord); // Rn as temp register + add(Rn, Pm_base, Rn); + sd(Rm, Address(Rn)); + add(i, i, 1); + slli(Rn, i, LogBytesPerWord); + add(Rm, Pm_base, Rn); + ld(Rm, Address(Rm)); + add(Rn, Pn_base, Rn); + ld(Rn, Address(Rn)); + sub(cnt, cnt, 1); + } bnez(cnt, loop); + addi(tmp0, tmp0, -1); + add(tmp0, tmp0, t0); + } bnez(tmp0, again); + } bind(post); + } + + // Move memory at s to d, reversing words. + // Increments d to end of copied memory + // Destroys tmp1, tmp2 + // Preserves len + // Leaves s pointing to the address which was in d at start + void reverse(Register d, Register s, Register len, Register tmp1, Register tmp2) { + assert(tmp1 < x28 && tmp2 < x28, "register corruption"); + + slli(tmp1, len, LogBytesPerWord); + add(s, s, tmp1); + mv(tmp1, len); + unroll_2(tmp1, &MontgomeryMultiplyGenerator::reverse1, d, s, tmp2); + slli(tmp1, len, LogBytesPerWord); + sub(s, d, tmp1); + } + // [63...0] -> [31...0][63...32] + void reverse1(Register d, Register s, Register tmp) { + addi(s, s, -wordSize); + ld(tmp, Address(s)); + ror_imm(tmp, tmp, 32, t0); + sd(tmp, Address(d)); + addi(d, d, wordSize); + } + + void step_squaring() { + // An extra ACC + step(); + acc(Rhi_ab, Rlo_ab, tmp0, tmp1, tmp2); + } + + void last_squaring(Register i) { + Label dont; + // if ((i & 1) == 0) { + test_bit(t0, i, 0); + bnez(t0, dont); { + // MACC(Ra, Rb, tmp0, tmp1, tmp2); + // Ra = *++Pa; + // Rb = *--Pb; + mulhu(Rhi_ab, Ra, Rb); + mul(Rlo_ab, Ra, Rb); + acc(Rhi_ab, Rlo_ab, tmp0, tmp1, tmp2); + } bind(dont); + } + + void extra_step_squaring() { + acc(Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2); // The pending m*n + + // MACC(Rm, Rn, tmp0, tmp1, tmp2); + // Rm = *++Pm; + // Rn = *--Pn; + mulhu(Rhi_mn, Rm, Rn); + mul(Rlo_mn, Rm, Rn); + addi(Pm, Pm, wordSize); + ld(Rm, Address(Pm)); + addi(Pn, Pn, -wordSize); + ld(Rn, Address(Pn)); + } + + void post1_squaring() { + acc(Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2); // The pending m*n + + // *Pm = Rm = tmp0 * inv; + mul(Rm, tmp0, inv); + sd(Rm, Address(Pm)); + + // MACC(Rm, Rn, tmp0, tmp1, tmp2); + // tmp0 = tmp1; tmp1 = tmp2; tmp2 = 0; + mulhu(Rhi_mn, Rm, Rn); + +#ifndef PRODUCT + // assert(m[i] * n[0] + tmp0 == 0, "broken Montgomery multiply"); + { + mul(Rlo_mn, Rm, Rn); + add(Rlo_mn, tmp0, Rlo_mn); + Label ok; + beqz(Rlo_mn, ok); { + stop("broken Montgomery multiply"); + } bind(ok); + } +#endif + // We have very carefully set things up so that + // m[i]*n[0] + tmp0 == 0 (mod b), so we don't have to calculate + // the lower half of Rm * Rn because we know the result already: + // it must be -tmp0. tmp0 + (-tmp0) must generate a carry iff + // tmp0 != 0. So, rather than do a mul and a cad we just set + // the carry flag iff tmp0 is nonzero. + // + // mul(Rlo_mn, Rm, Rn); + // cad(zr, tmp, Rlo_mn); + addi(t0, tmp0, -1); + sltu(t0, t0, tmp0); // Set carry iff tmp0 is nonzero + cadc(tmp0, tmp1, Rhi_mn, t0); + adc(tmp1, tmp2, zr, t0); + mv(tmp2, zr); + } + + // use t0 as carry + void acc(Register Rhi, Register Rlo, + Register tmp0, Register tmp1, Register tmp2) { + cad(tmp0, tmp0, Rlo, t0); + cadc(tmp1, tmp1, Rhi, t0); + adc(tmp2, tmp2, zr, t0); + } + + public: + /** + * Fast Montgomery multiplication. The derivation of the + * algorithm is in A Cryptographic Library for the Motorola + * DSP56000, Dusse and Kaliski, Proc. EUROCRYPT 90, pp. 230-237. + * + * Arguments: + * + * Inputs for multiplication: + * c_rarg0 - int array elements a + * c_rarg1 - int array elements b + * c_rarg2 - int array elements n (the modulus) + * c_rarg3 - int length + * c_rarg4 - int inv + * c_rarg5 - int array elements m (the result) + * + * Inputs for squaring: + * c_rarg0 - int array elements a + * c_rarg1 - int array elements n (the modulus) + * c_rarg2 - int length + * c_rarg3 - int inv + * c_rarg4 - int array elements m (the result) + * + */ + address generate_multiply() { + Label argh, nothing; + bind(argh); + stop("MontgomeryMultiply total_allocation must be <= 8192"); + + align(CodeEntryAlignment); + address entry = pc(); + + beqz(Rlen, nothing); + + enter(); + + // Make room. + mv(Ra, 512); + bgt(Rlen, Ra, argh); + slli(Ra, Rlen, exact_log2(4 * sizeof(jint))); + sub(Ra, sp, Ra); + andi(sp, Ra, -2 * wordSize); + + srliw(Rlen, Rlen, 1); // length in longwords = len/2 + + { + // Copy input args, reversing as we go. We use Ra as a + // temporary variable. + reverse(Ra, Pa_base, Rlen, Ri, Rj); + if (!_squaring) + reverse(Ra, Pb_base, Rlen, Ri, Rj); + reverse(Ra, Pn_base, Rlen, Ri, Rj); + } + + // Push all call-saved registers and also Pm_base which we'll need + // at the end. + save_regs(); + +#ifndef PRODUCT + // assert(inv * n[0] == -1UL, "broken inverse in Montgomery multiply"); + { + ld(Rn, Address(Pn_base)); + mul(Rlo_mn, Rn, inv); + mv(t0, -1); + Label ok; + beq(Rlo_mn, t0, ok); + stop("broken inverse in Montgomery multiply"); + bind(ok); + } +#endif + + mv(Pm_base, Ra); + + mv(tmp0, zr); + mv(tmp1, zr); + mv(tmp2, zr); + + block_comment("for (int i = 0; i < len; i++) {"); + mv(Ri, zr); { + Label loop, end; + bge(Ri, Rlen, end); + + bind(loop); + pre1(Ri); + + block_comment(" for (j = i; j; j--) {"); { + mv(Rj, Ri); + unroll_2(Rj, &MontgomeryMultiplyGenerator::step); + } block_comment(" } // j"); + + post1(); + addw(Ri, Ri, 1); + blt(Ri, Rlen, loop); + bind(end); + block_comment("} // i"); + } + + block_comment("for (int i = len; i < 2*len; i++) {"); + mv(Ri, Rlen); { + Label loop, end; + slli(t0, Rlen, 1); + bge(Ri, t0, end); + + bind(loop); + pre2(Ri, Rlen); + + block_comment(" for (j = len*2-i-1; j; j--) {"); { + slliw(Rj, Rlen, 1); + subw(Rj, Rj, Ri); + subw(Rj, Rj, 1); + unroll_2(Rj, &MontgomeryMultiplyGenerator::step); + } block_comment(" } // j"); + + post2(Ri, Rlen); + addw(Ri, Ri, 1); + slli(t0, Rlen, 1); + blt(Ri, t0, loop); + bind(end); + } + block_comment("} // i"); + + normalize(Rlen); + + mv(Ra, Pm_base); // Save Pm_base in Ra + restore_regs(); // Restore caller's Pm_base + + // Copy our result into caller's Pm_base + reverse(Pm_base, Ra, Rlen, Ri, Rj); + + leave(); + bind(nothing); + ret(); + + return entry; + } + + /** + * + * Arguments: + * + * Inputs: + * c_rarg0 - int array elements a + * c_rarg1 - int array elements n (the modulus) + * c_rarg2 - int length + * c_rarg3 - int inv + * c_rarg4 - int array elements m (the result) + * + */ + address generate_square() { + Label argh; + bind(argh); + stop("MontgomeryMultiply total_allocation must be <= 8192"); + + align(CodeEntryAlignment); + address entry = pc(); + + enter(); + + // Make room. + mv(Ra, 512); + bgt(Rlen, Ra, argh); + slli(Ra, Rlen, exact_log2(4 * sizeof(jint))); + sub(Ra, sp, Ra); + andi(sp, Ra, -2 * wordSize); + + srliw(Rlen, Rlen, 1); // length in longwords = len/2 + + { + // Copy input args, reversing as we go. We use Ra as a + // temporary variable. + reverse(Ra, Pa_base, Rlen, Ri, Rj); + reverse(Ra, Pn_base, Rlen, Ri, Rj); + } + + // Push all call-saved registers and also Pm_base which we'll need + // at the end. + save_regs(); + + mv(Pm_base, Ra); + + mv(tmp0, zr); + mv(tmp1, zr); + mv(tmp2, zr); + + block_comment("for (int i = 0; i < len; i++) {"); + mv(Ri, zr); { + Label loop, end; + bind(loop); + bge(Ri, Rlen, end); + + pre1(Ri); + + block_comment("for (j = (i+1)/2; j; j--) {"); { + addi(Rj, Ri, 1); + srliw(Rj, Rj, 1); + unroll_2(Rj, &MontgomeryMultiplyGenerator::step_squaring); + } block_comment(" } // j"); + + last_squaring(Ri); + + block_comment(" for (j = i/2; j; j--) {"); { + srliw(Rj, Ri, 1); + unroll_2(Rj, &MontgomeryMultiplyGenerator::extra_step_squaring); + } block_comment(" } // j"); + + post1_squaring(); + addi(Ri, Ri, 1); + blt(Ri, Rlen, loop); + + bind(end); + block_comment("} // i"); + } + + block_comment("for (int i = len; i < 2*len; i++) {"); + mv(Ri, Rlen); { + Label loop, end; + bind(loop); + slli(t0, Rlen, 1); + bge(Ri, t0, end); + + pre2(Ri, Rlen); + + block_comment(" for (j = (2*len-i-1)/2; j; j--) {"); { + slli(Rj, Rlen, 1); + sub(Rj, Rj, Ri); + sub(Rj, Rj, 1); + srliw(Rj, Rj, 1); + unroll_2(Rj, &MontgomeryMultiplyGenerator::step_squaring); + } block_comment(" } // j"); + + last_squaring(Ri); + + block_comment(" for (j = (2*len-i)/2; j; j--) {"); { + slli(Rj, Rlen, 1); + sub(Rj, Rj, Ri); + srliw(Rj, Rj, 1); + unroll_2(Rj, &MontgomeryMultiplyGenerator::extra_step_squaring); + } block_comment(" } // j"); + + post2(Ri, Rlen); + addi(Ri, Ri, 1); + slli(t0, Rlen, 1); + blt(Ri, t0, loop); + + bind(end); + block_comment("} // i"); + } + + normalize(Rlen); + + mv(Ra, Pm_base); // Save Pm_base in Ra + restore_regs(); // Restore caller's Pm_base + + // Copy our result into caller's Pm_base + reverse(Pm_base, Ra, Rlen, Ri, Rj); + + leave(); + ret(); + + return entry; + } + }; +#endif // COMPILER2 + + // Continuation point for throwing of implicit exceptions that are + // not handled in the current activation. Fabricates an exception + // oop and initiates normal exception dispatching in this + // frame. Since we need to preserve callee-saved values (currently + // only for C2, but done for C1 as well) we need a callee-saved oop + // map and therefore have to make these stubs into RuntimeStubs + // rather than BufferBlobs. If the compiler needs all registers to + // be preserved between the fault point and the exception handler + // then it must assume responsibility for that in + // AbstractCompiler::continuation_for_implicit_null_exception or + // continuation_for_implicit_division_by_zero_exception. All other + // implicit exceptions (e.g., NullPointerException or + // AbstractMethodError on entry) are either at call sites or + // otherwise assume that stack unwinding will be initiated, so + // caller saved registers were assumed volatile in the compiler. + +#undef __ +#define __ masm-> + + address generate_throw_exception(const char* name, + address runtime_entry, + Register arg1 = noreg, + Register arg2 = noreg) { + // Information about frame layout at time of blocking runtime call. + // Note that we only have to preserve callee-saved registers since + // the compilers are responsible for supplying a continuation point + // if they expect all registers to be preserved. + // n.b. riscv asserts that frame::arg_reg_save_area_bytes == 0 + assert_cond(runtime_entry != NULL); + enum layout { + fp_off = 0, + fp_off2, + return_off, + return_off2, + framesize // inclusive of return address + }; + + const int insts_size = 512; + const int locs_size = 64; + + CodeBuffer code(name, insts_size, locs_size); + OopMapSet* oop_maps = new OopMapSet(); + MacroAssembler* masm = new MacroAssembler(&code); + assert_cond(oop_maps != NULL && masm != NULL); + + address start = __ pc(); + + // This is an inlined and slightly modified version of call_VM + // which has the ability to fetch the return PC out of + // thread-local storage and also sets up last_Java_sp slightly + // differently than the real call_VM + + __ enter(); // Save FP and RA before call + + assert(is_even(framesize / 2), "sp not 16-byte aligned"); + + // ra and fp are already in place + __ addi(sp, fp, 0 - ((unsigned)framesize << LogBytesPerInt)); // prolog + + int frame_complete = __ pc() - start; + + // Set up last_Java_sp and last_Java_fp + address the_pc = __ pc(); + __ set_last_Java_frame(sp, fp, the_pc, t0); + + // Call runtime + if (arg1 != noreg) { + assert(arg2 != c_rarg1, "clobbered"); + __ mv(c_rarg1, arg1); + } + if (arg2 != noreg) { + __ mv(c_rarg2, arg2); + } + __ mv(c_rarg0, xthread); + BLOCK_COMMENT("call runtime_entry"); + __ call(runtime_entry); + + // Generate oop map + OopMap* map = new OopMap(framesize, 0); + assert_cond(map != NULL); + + oop_maps->add_gc_map(the_pc - start, map); + + __ reset_last_Java_frame(true); + + __ leave(); + + // check for pending exceptions +#ifdef ASSERT + Label L; + __ ld(t0, Address(xthread, Thread::pending_exception_offset())); + __ bnez(t0, L); + __ should_not_reach_here(); + __ bind(L); +#endif // ASSERT + __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry())); + + + // codeBlob framesize is in words (not VMRegImpl::slot_size) + RuntimeStub* stub = + RuntimeStub::new_runtime_stub(name, + &code, + frame_complete, + (framesize >> (LogBytesPerWord - LogBytesPerInt)), + oop_maps, false); + assert(stub != NULL, "create runtime stub fail!"); + return stub->entry_point(); + } + + // Initialization + void generate_initial() { + // Generate initial stubs and initializes the entry points + + // entry points that exist in all platforms Note: This is code + // that could be shared among different platforms - however the + // benefit seems to be smaller than the disadvantage of having a + // much more complicated generator structure. See also comment in + // stubRoutines.hpp. + + StubRoutines::_forward_exception_entry = generate_forward_exception(); + + StubRoutines::_call_stub_entry = + generate_call_stub(StubRoutines::_call_stub_return_address); + + // is referenced by megamorphic call + StubRoutines::_catch_exception_entry = generate_catch_exception(); + + // Build this early so it's available for the interpreter. + StubRoutines::_throw_StackOverflowError_entry = + generate_throw_exception("StackOverflowError throw_exception", + CAST_FROM_FN_PTR(address, + SharedRuntime::throw_StackOverflowError)); + StubRoutines::_throw_delayed_StackOverflowError_entry = + generate_throw_exception("delayed StackOverflowError throw_exception", + CAST_FROM_FN_PTR(address, + SharedRuntime::throw_delayed_StackOverflowError)); + } + + void generate_all() { + // support for verify_oop (must happen after universe_init) + StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop(); + StubRoutines::_throw_AbstractMethodError_entry = + generate_throw_exception("AbstractMethodError throw_exception", + CAST_FROM_FN_PTR(address, + SharedRuntime:: + throw_AbstractMethodError)); + + StubRoutines::_throw_IncompatibleClassChangeError_entry = + generate_throw_exception("IncompatibleClassChangeError throw_exception", + CAST_FROM_FN_PTR(address, + SharedRuntime:: + throw_IncompatibleClassChangeError)); + + StubRoutines::_throw_NullPointerException_at_call_entry = + generate_throw_exception("NullPointerException at call throw_exception", + CAST_FROM_FN_PTR(address, + SharedRuntime:: + throw_NullPointerException_at_call)); + // arraycopy stubs used by compilers + generate_arraycopy_stubs(); + +#ifdef COMPILER2 + if (UseMulAddIntrinsic) { + StubRoutines::_mulAdd = generate_mulAdd(); + } + + if (UseMultiplyToLenIntrinsic) { + StubRoutines::_multiplyToLen = generate_multiplyToLen(); + } + + if (UseSquareToLenIntrinsic) { + StubRoutines::_squareToLen = generate_squareToLen(); + } + + if (UseMontgomeryMultiplyIntrinsic) { + StubCodeMark mark(this, "StubRoutines", "montgomeryMultiply"); + MontgomeryMultiplyGenerator g(_masm, /*squaring*/false); + StubRoutines::_montgomeryMultiply = g.generate_multiply(); + } + + if (UseMontgomerySquareIntrinsic) { + StubCodeMark mark(this, "StubRoutines", "montgomerySquare"); + MontgomeryMultiplyGenerator g(_masm, /*squaring*/true); + StubRoutines::_montgomerySquare = g.generate_square(); + } + + if (UseRVVForBigIntegerShiftIntrinsics) { + StubRoutines::_bigIntegerLeftShiftWorker = generate_bigIntegerLeftShift(); + StubRoutines::_bigIntegerRightShiftWorker = generate_bigIntegerRightShift(); + } +#endif + + generate_compare_long_strings(); + + generate_string_indexof_stubs(); + + BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); + if (bs_nm != NULL) { + StubRoutines::riscv::_method_entry_barrier = generate_method_entry_barrier(); + } + + StubRoutines::riscv::set_completed(); + } + + public: + StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) { + if (all) { + generate_all(); + } else { + generate_initial(); + } + } + + ~StubGenerator() {} +}; // end class declaration + +#define UCM_TABLE_MAX_ENTRIES 8 +void StubGenerator_generate(CodeBuffer* code, bool all) { + if (UnsafeCopyMemory::_table == NULL) { + UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES); + } + + StubGenerator g(code, all); +} diff --git a/src/hotspot/cpu/riscv/stubRoutines_riscv.cpp b/src/hotspot/cpu/riscv/stubRoutines_riscv.cpp new file mode 100644 index 0000000000000..395a2d338e4c3 --- /dev/null +++ b/src/hotspot/cpu/riscv/stubRoutines_riscv.cpp @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "runtime/deoptimization.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/stubRoutines.hpp" +#include "runtime/thread.inline.hpp" +#include "utilities/globalDefinitions.hpp" + +// Implementation of the platform-specific part of StubRoutines - for +// a description of how to extend it, see the stubRoutines.hpp file. + +address StubRoutines::riscv::_get_previous_sp_entry = NULL; + +address StubRoutines::riscv::_f2i_fixup = NULL; +address StubRoutines::riscv::_f2l_fixup = NULL; +address StubRoutines::riscv::_d2i_fixup = NULL; +address StubRoutines::riscv::_d2l_fixup = NULL; +address StubRoutines::riscv::_float_sign_mask = NULL; +address StubRoutines::riscv::_float_sign_flip = NULL; +address StubRoutines::riscv::_double_sign_mask = NULL; +address StubRoutines::riscv::_double_sign_flip = NULL; +address StubRoutines::riscv::_zero_blocks = NULL; +address StubRoutines::riscv::_compare_long_string_LL = NULL; +address StubRoutines::riscv::_compare_long_string_UU = NULL; +address StubRoutines::riscv::_compare_long_string_LU = NULL; +address StubRoutines::riscv::_compare_long_string_UL = NULL; +address StubRoutines::riscv::_string_indexof_linear_ll = NULL; +address StubRoutines::riscv::_string_indexof_linear_uu = NULL; +address StubRoutines::riscv::_string_indexof_linear_ul = NULL; +address StubRoutines::riscv::_large_byte_array_inflate = NULL; +address StubRoutines::riscv::_method_entry_barrier = NULL; + +bool StubRoutines::riscv::_completed = false; diff --git a/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp b/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp new file mode 100644 index 0000000000000..51f07819c33a1 --- /dev/null +++ b/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_STUBROUTINES_RISCV_HPP +#define CPU_RISCV_STUBROUTINES_RISCV_HPP + +// This file holds the platform specific parts of the StubRoutines +// definition. See stubRoutines.hpp for a description on how to +// extend it. + +static bool returns_to_call_stub(address return_pc) { + return return_pc == _call_stub_return_address; +} + +enum platform_dependent_constants { + code_size1 = 19000, // simply increase if too small (assembler will crash if too small) + code_size2 = 28000 // simply increase if too small (assembler will crash if too small) +}; + +class riscv { + friend class StubGenerator; + + private: + static address _get_previous_sp_entry; + + static address _f2i_fixup; + static address _f2l_fixup; + static address _d2i_fixup; + static address _d2l_fixup; + + static address _float_sign_mask; + static address _float_sign_flip; + static address _double_sign_mask; + static address _double_sign_flip; + + static address _zero_blocks; + + static address _compare_long_string_LL; + static address _compare_long_string_LU; + static address _compare_long_string_UL; + static address _compare_long_string_UU; + static address _string_indexof_linear_ll; + static address _string_indexof_linear_uu; + static address _string_indexof_linear_ul; + static address _large_byte_array_inflate; + + static address _method_entry_barrier; + + static bool _completed; + + public: + + static address get_previous_sp_entry() { + return _get_previous_sp_entry; + } + + static address f2i_fixup() { + return _f2i_fixup; + } + + static address f2l_fixup() { + return _f2l_fixup; + } + + static address d2i_fixup() { + return _d2i_fixup; + } + + static address d2l_fixup() { + return _d2l_fixup; + } + + static address float_sign_mask() { + return _float_sign_mask; + } + + static address float_sign_flip() { + return _float_sign_flip; + } + + static address double_sign_mask() { + return _double_sign_mask; + } + + static address double_sign_flip() { + return _double_sign_flip; + } + + static address zero_blocks() { + return _zero_blocks; + } + + static address compare_long_string_LL() { + return _compare_long_string_LL; + } + + static address compare_long_string_LU() { + return _compare_long_string_LU; + } + + static address compare_long_string_UL() { + return _compare_long_string_UL; + } + + static address compare_long_string_UU() { + return _compare_long_string_UU; + } + + static address string_indexof_linear_ul() { + return _string_indexof_linear_ul; + } + + static address string_indexof_linear_ll() { + return _string_indexof_linear_ll; + } + + static address string_indexof_linear_uu() { + return _string_indexof_linear_uu; + } + + static address large_byte_array_inflate() { + return _large_byte_array_inflate; + } + + static address method_entry_barrier() { + return _method_entry_barrier; + } + + static bool complete() { + return _completed; + } + + static void set_completed() { + _completed = true; + } +}; + +#endif // CPU_RISCV_STUBROUTINES_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp b/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp new file mode 100644 index 0000000000000..95fc27f9e7105 --- /dev/null +++ b/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp @@ -0,0 +1,1760 @@ +/* + * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "classfile/javaClasses.hpp" +#include "gc/shared/barrierSetAssembler.hpp" +#include "interpreter/bytecodeHistogram.hpp" +#include "interpreter/bytecodeTracer.hpp" +#include "interpreter/interp_masm.hpp" +#include "interpreter/interpreter.hpp" +#include "interpreter/interpreterRuntime.hpp" +#include "interpreter/templateInterpreterGenerator.hpp" +#include "interpreter/templateTable.hpp" +#include "memory/resourceArea.hpp" +#include "oops/arrayOop.hpp" +#include "oops/method.hpp" +#include "oops/methodData.hpp" +#include "oops/oop.inline.hpp" +#include "prims/jvmtiExport.hpp" +#include "prims/jvmtiThreadState.hpp" +#include "runtime/arguments.hpp" +#include "runtime/deoptimization.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/jniHandles.hpp" +#include "runtime/sharedRuntime.hpp" +#include "runtime/stubRoutines.hpp" +#include "runtime/synchronizer.hpp" +#include "runtime/timer.hpp" +#include "runtime/vframeArray.hpp" +#include "utilities/debug.hpp" +#include "utilities/powerOfTwo.hpp" +#include + +#ifndef PRODUCT +#include "oops/method.hpp" +#endif // !PRODUCT + +// Size of interpreter code. Increase if too small. Interpreter will +// fail with a guarantee ("not enough space for interpreter generation"); +// if too small. +// Run with +PrintInterpreter to get the VM to print out the size. +// Max size with JVMTI +int TemplateInterpreter::InterpreterCodeSize = 256 * 1024; + +#define __ _masm-> + +//----------------------------------------------------------------------------- + +address TemplateInterpreterGenerator::generate_slow_signature_handler() { + address entry = __ pc(); + + __ andi(esp, esp, -16); + __ mv(c_rarg3, esp); + // xmethod + // xlocals + // c_rarg3: first stack arg - wordSize + // adjust sp + + __ addi(sp, c_rarg3, -18 * wordSize); + __ addi(sp, sp, -2 * wordSize); + __ sd(ra, Address(sp, 0)); + + __ call_VM(noreg, + CAST_FROM_FN_PTR(address, + InterpreterRuntime::slow_signature_handler), + xmethod, xlocals, c_rarg3); + + // x10: result handler + + // Stack layout: + // sp: return address <- sp + // 1 garbage + // 8 integer args (if static first is unused) + // 1 float/double identifiers + // 8 double args + // stack args <- esp + // garbage + // expression stack bottom + // bcp (NULL) + // ... + + // Restore ra + __ ld(ra, Address(sp, 0)); + __ addi(sp, sp , 2 * wordSize); + + // Do FP first so we can use c_rarg3 as temp + __ lwu(c_rarg3, Address(sp, 9 * wordSize)); // float/double identifiers + + for (int i = 0; i < Argument::n_float_register_parameters_c; i++) { + const FloatRegister r = g_FPArgReg[i]; + Label d, done; + + __ test_bit(t0, c_rarg3, i); + __ bnez(t0, d); + __ flw(r, Address(sp, (10 + i) * wordSize)); + __ j(done); + __ bind(d); + __ fld(r, Address(sp, (10 + i) * wordSize)); + __ bind(done); + } + + // c_rarg0 contains the result from the call of + // InterpreterRuntime::slow_signature_handler so we don't touch it + // here. It will be loaded with the JNIEnv* later. + for (int i = 1; i < Argument::n_int_register_parameters_c; i++) { + const Register rm = g_INTArgReg[i]; + __ ld(rm, Address(sp, i * wordSize)); + } + + __ addi(sp, sp, 18 * wordSize); + __ ret(); + + return entry; +} + +// Various method entries +address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::MethodKind kind) { + // xmethod: Method* + // x30: sender sp + // esp: args + + if (!InlineIntrinsics) { + return NULL; // Generate a vanilla entry + } + + // These don't need a safepoint check because they aren't virtually + // callable. We won't enter these intrinsics from compiled code. + // If in the future we added an intrinsic which was virtually callable + // we'd have to worry about how to safepoint so that this code is used. + + // mathematical functions inlined by compiler + // (interpreter must provide identical implementation + // in order to avoid monotonicity bugs when switching + // from interpreter to compiler in the middle of some + // computation) + // + // stack: + // [ arg ] <-- esp + // [ arg ] + // retaddr in ra + + address fn = NULL; + address entry_point = NULL; + Register continuation = ra; + switch (kind) { + case Interpreter::java_lang_math_abs: + entry_point = __ pc(); + __ fld(f10, Address(esp)); + __ fabs_d(f10, f10); + __ mv(sp, x30); // Restore caller's SP + break; + case Interpreter::java_lang_math_sqrt: + entry_point = __ pc(); + __ fld(f10, Address(esp)); + __ fsqrt_d(f10, f10); + __ mv(sp, x30); + break; + case Interpreter::java_lang_math_sin : + entry_point = __ pc(); + __ fld(f10, Address(esp)); + __ mv(sp, x30); + __ mv(x9, ra); + continuation = x9; // The first callee-saved register + if (StubRoutines::dsin() == NULL) { + fn = CAST_FROM_FN_PTR(address, SharedRuntime::dsin); + } else { + fn = CAST_FROM_FN_PTR(address, StubRoutines::dsin()); + } + __ call(fn); + break; + case Interpreter::java_lang_math_cos : + entry_point = __ pc(); + __ fld(f10, Address(esp)); + __ mv(sp, x30); + __ mv(x9, ra); + continuation = x9; // The first callee-saved register + if (StubRoutines::dcos() == NULL) { + fn = CAST_FROM_FN_PTR(address, SharedRuntime::dcos); + } else { + fn = CAST_FROM_FN_PTR(address, StubRoutines::dcos()); + } + __ call(fn); + break; + case Interpreter::java_lang_math_tan : + entry_point = __ pc(); + __ fld(f10, Address(esp)); + __ mv(sp, x30); + __ mv(x9, ra); + continuation = x9; // The first callee-saved register + if (StubRoutines::dtan() == NULL) { + fn = CAST_FROM_FN_PTR(address, SharedRuntime::dtan); + } else { + fn = CAST_FROM_FN_PTR(address, StubRoutines::dtan()); + } + __ call(fn); + break; + case Interpreter::java_lang_math_log : + entry_point = __ pc(); + __ fld(f10, Address(esp)); + __ mv(sp, x30); + __ mv(x9, ra); + continuation = x9; // The first callee-saved register + if (StubRoutines::dlog() == NULL) { + fn = CAST_FROM_FN_PTR(address, SharedRuntime::dlog); + } else { + fn = CAST_FROM_FN_PTR(address, StubRoutines::dlog()); + } + __ call(fn); + break; + case Interpreter::java_lang_math_log10 : + entry_point = __ pc(); + __ fld(f10, Address(esp)); + __ mv(sp, x30); + __ mv(x9, ra); + continuation = x9; // The first callee-saved register + if (StubRoutines::dlog10() == NULL) { + fn = CAST_FROM_FN_PTR(address, SharedRuntime::dlog10); + } else { + fn = CAST_FROM_FN_PTR(address, StubRoutines::dlog10()); + } + __ call(fn); + break; + case Interpreter::java_lang_math_exp : + entry_point = __ pc(); + __ fld(f10, Address(esp)); + __ mv(sp, x30); + __ mv(x9, ra); + continuation = x9; // The first callee-saved register + if (StubRoutines::dexp() == NULL) { + fn = CAST_FROM_FN_PTR(address, SharedRuntime::dexp); + } else { + fn = CAST_FROM_FN_PTR(address, StubRoutines::dexp()); + } + __ call(fn); + break; + case Interpreter::java_lang_math_pow : + entry_point = __ pc(); + __ mv(x9, ra); + continuation = x9; + __ fld(f10, Address(esp, 2 * Interpreter::stackElementSize)); + __ fld(f11, Address(esp)); + __ mv(sp, x30); + if (StubRoutines::dpow() == NULL) { + fn = CAST_FROM_FN_PTR(address, SharedRuntime::dpow); + } else { + fn = CAST_FROM_FN_PTR(address, StubRoutines::dpow()); + } + __ call(fn); + break; + case Interpreter::java_lang_math_fmaD : + if (UseFMA) { + entry_point = __ pc(); + __ fld(f10, Address(esp, 4 * Interpreter::stackElementSize)); + __ fld(f11, Address(esp, 2 * Interpreter::stackElementSize)); + __ fld(f12, Address(esp)); + __ fmadd_d(f10, f10, f11, f12); + __ mv(sp, x30); // Restore caller's SP + } + break; + case Interpreter::java_lang_math_fmaF : + if (UseFMA) { + entry_point = __ pc(); + __ flw(f10, Address(esp, 2 * Interpreter::stackElementSize)); + __ flw(f11, Address(esp, Interpreter::stackElementSize)); + __ flw(f12, Address(esp)); + __ fmadd_s(f10, f10, f11, f12); + __ mv(sp, x30); // Restore caller's SP + } + break; + default: + ; + } + if (entry_point != NULL) { + __ jr(continuation); + } + + return entry_point; +} + +// Abstract method entry +// Attempt to execute abstract method. Throw exception +address TemplateInterpreterGenerator::generate_abstract_entry(void) { + // xmethod: Method* + // x30: sender SP + + address entry_point = __ pc(); + + // abstract method entry + + // pop return address, reset last_sp to NULL + __ empty_expression_stack(); + __ restore_bcp(); // bcp must be correct for exception handler (was destroyed) + __ restore_locals(); // make sure locals pointer is correct as well (was destroyed) + + // throw exception + __ call_VM(noreg, CAST_FROM_FN_PTR(address, + InterpreterRuntime::throw_AbstractMethodErrorWithMethod), + xmethod); + // the call_VM checks for exception, so we should never return here. + __ should_not_reach_here(); + + return entry_point; +} + +address TemplateInterpreterGenerator::generate_StackOverflowError_handler() { + address entry = __ pc(); + +#ifdef ASSERT + { + Label L; + __ ld(t0, Address(fp, frame::interpreter_frame_monitor_block_top_offset * wordSize)); + __ mv(t1, sp); + // maximal sp for current fp (stack grows negative) + // check if frame is complete + __ bge(t0, t1, L); + __ stop ("interpreter frame not set up"); + __ bind(L); + } +#endif // ASSERT + // Restore bcp under the assumption that the current frame is still + // interpreted + __ restore_bcp(); + + // expression stack must be empty before entering the VM if an + // exception happened + __ empty_expression_stack(); + // throw exception + __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_StackOverflowError)); + return entry; +} + +address TemplateInterpreterGenerator::generate_ArrayIndexOutOfBounds_handler() { + address entry = __ pc(); + // expression stack must be empty before entering the VM if an + // exception happened + __ empty_expression_stack(); + // setup parameters + + // convention: expect aberrant index in register x11 + __ zero_extend(c_rarg2, x11, 32); + // convention: expect array in register x13 + __ mv(c_rarg1, x13); + __ call_VM(noreg, + CAST_FROM_FN_PTR(address, + InterpreterRuntime:: + throw_ArrayIndexOutOfBoundsException), + c_rarg1, c_rarg2); + return entry; +} + +address TemplateInterpreterGenerator::generate_ClassCastException_handler() { + address entry = __ pc(); + + // object is at TOS + __ pop_reg(c_rarg1); + + // expression stack must be empty before entering the VM if an + // exception happened + __ empty_expression_stack(); + + __ call_VM(noreg, + CAST_FROM_FN_PTR(address, + InterpreterRuntime:: + throw_ClassCastException), + c_rarg1); + return entry; +} + +address TemplateInterpreterGenerator::generate_exception_handler_common( + const char* name, const char* message, bool pass_oop) { + assert(!pass_oop || message == NULL, "either oop or message but not both"); + address entry = __ pc(); + if (pass_oop) { + // object is at TOS + __ pop_reg(c_rarg2); + } + // expression stack must be empty before entering the VM if an + // exception happened + __ empty_expression_stack(); + // setup parameters + __ la(c_rarg1, Address((address)name)); + if (pass_oop) { + __ call_VM(x10, CAST_FROM_FN_PTR(address, + InterpreterRuntime:: + create_klass_exception), + c_rarg1, c_rarg2); + } else { + // kind of lame ExternalAddress can't take NULL because + // external_word_Relocation will assert. + if (message != NULL) { + __ la(c_rarg2, Address((address)message)); + } else { + __ mv(c_rarg2, NULL_WORD); + } + __ call_VM(x10, + CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception), + c_rarg1, c_rarg2); + } + // throw exception + __ j(address(Interpreter::throw_exception_entry())); + return entry; +} + +address TemplateInterpreterGenerator::generate_return_entry_for(TosState state, int step, size_t index_size) { + address entry = __ pc(); + + // Restore stack bottom in case i2c adjusted stack + __ ld(esp, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize)); + // and NULL it as marker that esp is now tos until next java call + __ sd(zr, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize)); + __ restore_bcp(); + __ restore_locals(); + __ restore_constant_pool_cache(); + __ get_method(xmethod); + + if (state == atos) { + Register obj = x10; + Register mdp = x11; + Register tmp = x12; + __ ld(mdp, Address(xmethod, Method::method_data_offset())); + __ profile_return_type(mdp, obj, tmp); + } + + // Pop N words from the stack + __ get_cache_and_index_at_bcp(x11, x12, 1, index_size); + __ ld(x11, Address(x11, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset())); + __ andi(x11, x11, ConstantPoolCacheEntry::parameter_size_mask); + + __ shadd(esp, x11, esp, t0, 3); + + // Restore machine SP + __ ld(t0, Address(xmethod, Method::const_offset())); + __ lhu(t0, Address(t0, ConstMethod::max_stack_offset())); + __ addi(t0, t0, frame::interpreter_frame_monitor_size() + 2); + __ ld(t1, + Address(fp, frame::interpreter_frame_initial_sp_offset * wordSize)); + __ slli(t0, t0, 3); + __ sub(t0, t1, t0); + __ andi(sp, t0, -16); + + __ check_and_handle_popframe(xthread); + __ check_and_handle_earlyret(xthread); + + __ get_dispatch(); + __ dispatch_next(state, step); + + return entry; +} + +address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state, + int step, + address continuation) { + address entry = __ pc(); + __ restore_bcp(); + __ restore_locals(); + __ restore_constant_pool_cache(); + __ get_method(xmethod); + __ get_dispatch(); + + // Calculate stack limit + __ ld(t0, Address(xmethod, Method::const_offset())); + __ lhu(t0, Address(t0, ConstMethod::max_stack_offset())); + __ addi(t0, t0, frame::interpreter_frame_monitor_size() + 2); + __ ld(t1, Address(fp, frame::interpreter_frame_initial_sp_offset * wordSize)); + __ slli(t0, t0, 3); + __ sub(t0, t1, t0); + __ andi(sp, t0, -16); + + // Restore expression stack pointer + __ ld(esp, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize)); + // NULL last_sp until next java call + __ sd(zr, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize)); + + // handle exceptions + { + Label L; + __ ld(t0, Address(xthread, Thread::pending_exception_offset())); + __ beqz(t0, L); + __ call_VM(noreg, + CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_pending_exception)); + __ should_not_reach_here(); + __ bind(L); + } + + if (continuation == NULL) { + __ dispatch_next(state, step); + } else { + __ jump_to_entry(continuation); + } + return entry; +} + +address TemplateInterpreterGenerator::generate_result_handler_for(BasicType type) { + address entry = __ pc(); + if (type == T_OBJECT) { + // retrieve result from frame + __ ld(x10, Address(fp, frame::interpreter_frame_oop_temp_offset * wordSize)); + // and verify it + __ verify_oop(x10); + } else { + __ cast_primitive_type(type, x10); + } + + __ ret(); // return from result handler + return entry; +} + +address TemplateInterpreterGenerator::generate_safept_entry_for(TosState state, + address runtime_entry) { + assert_cond(runtime_entry != NULL); + address entry = __ pc(); + __ push(state); + __ call_VM(noreg, runtime_entry); + __ membar(MacroAssembler::AnyAny); + __ dispatch_via(vtos, Interpreter::_normal_table.table_for(vtos)); + return entry; +} + +// Helpers for commoning out cases in the various type of method entries. +// + + +// increment invocation count & check for overflow +// +// Note: checking for negative value instead of overflow +// so we have a 'sticky' overflow test +// +// xmethod: method +// +void TemplateInterpreterGenerator::generate_counter_incr(Label* overflow) { + Label done; + // Note: In tiered we increment either counters in Method* or in MDO depending if we're profiling or not. + int increment = InvocationCounter::count_increment; + Label no_mdo; + if (ProfileInterpreter) { + // Are we profiling? + __ ld(x10, Address(xmethod, Method::method_data_offset())); + __ beqz(x10, no_mdo); + // Increment counter in the MDO + const Address mdo_invocation_counter(x10, in_bytes(MethodData::invocation_counter_offset()) + + in_bytes(InvocationCounter::counter_offset())); + const Address mask(x10, in_bytes(MethodData::invoke_mask_offset())); + __ increment_mask_and_jump(mdo_invocation_counter, increment, mask, t0, t1, false, overflow); + __ j(done); + } + __ bind(no_mdo); + // Increment counter in MethodCounters + const Address invocation_counter(t1, + MethodCounters::invocation_counter_offset() + + InvocationCounter::counter_offset()); + __ get_method_counters(xmethod, t1, done); + const Address mask(t1, in_bytes(MethodCounters::invoke_mask_offset())); + __ increment_mask_and_jump(invocation_counter, increment, mask, t0, x11, false, overflow); + __ bind(done); +} + +void TemplateInterpreterGenerator::generate_counter_overflow(Label& do_continue) { + __ mv(c_rarg1, zr); + __ call_VM(noreg, + CAST_FROM_FN_PTR(address, InterpreterRuntime::frequency_counter_overflow), c_rarg1); + __ j(do_continue); +} + +// See if we've got enough room on the stack for locals plus overhead +// below JavaThread::stack_overflow_limit(). If not, throw a StackOverflowError +// without going through the signal handler, i.e., reserved and yellow zones +// will not be made usable. The shadow zone must suffice to handle the +// overflow. +// The expression stack grows down incrementally, so the normal guard +// page mechanism will work for that. +// +// NOTE: Since the additional locals are also always pushed (wasn't +// obvious in generate_method_entry) so the guard should work for them +// too. +// +// Args: +// x13: number of additional locals this frame needs (what we must check) +// xmethod: Method* +// +// Kills: +// x10 +void TemplateInterpreterGenerator::generate_stack_overflow_check(void) { + + // monitor entry size: see picture of stack set + // (generate_method_entry) and frame_amd64.hpp + const int entry_size = frame::interpreter_frame_monitor_size() * wordSize; + + // total overhead size: entry_size + (saved fp through expr stack + // bottom). be sure to change this if you add/subtract anything + // to/from the overhead area + const int overhead_size = + -(frame::interpreter_frame_initial_sp_offset * wordSize) + entry_size; + + const int page_size = os::vm_page_size(); + + Label after_frame_check; + + // see if the frame is greater than one page in size. If so, + // then we need to verify there is enough stack space remaining + // for the additional locals. + __ mv(t0, (page_size - overhead_size) / Interpreter::stackElementSize); + __ bleu(x13, t0, after_frame_check); + + // compute sp as if this were going to be the last frame on + // the stack before the red zone + + // locals + overhead, in bytes + __ mv(x10, overhead_size); + __ shadd(x10, x13, x10, t0, Interpreter::logStackElementSize); // 2 slots per parameter. + + const Address stack_limit(xthread, JavaThread::stack_overflow_limit_offset()); + __ ld(t0, stack_limit); + +#ifdef ASSERT + Label limit_okay; + // Verify that thread stack limit is non-zero. + __ bnez(t0, limit_okay); + __ stop("stack overflow limit is zero"); + __ bind(limit_okay); +#endif + + // Add stack limit to locals. + __ add(x10, x10, t0); + + // Check against the current stack bottom. + __ bgtu(sp, x10, after_frame_check); + + // Remove the incoming args, peeling the machine SP back to where it + // was in the caller. This is not strictly necessary, but unless we + // do so the stack frame may have a garbage FP; this ensures a + // correct call stack that we can always unwind. The ANDI should be + // unnecessary because the sender SP in x30 is always aligned, but + // it doesn't hurt. + __ andi(sp, x30, -16); + + // Note: the restored frame is not necessarily interpreted. + // Use the shared runtime version of the StackOverflowError. + assert(StubRoutines::throw_StackOverflowError_entry() != NULL, "stub not yet generated"); + __ far_jump(RuntimeAddress(StubRoutines::throw_StackOverflowError_entry())); + + // all done with frame size check + __ bind(after_frame_check); +} + +// Allocate monitor and lock method (asm interpreter) +// +// Args: +// xmethod: Method* +// xlocals: locals +// +// Kills: +// x10 +// c_rarg0, c_rarg1, c_rarg2, c_rarg3, ...(param regs) +// t0, t1 (temporary regs) +void TemplateInterpreterGenerator::lock_method() { + // synchronize method + const Address access_flags(xmethod, Method::access_flags_offset()); + const Address monitor_block_top(fp, frame::interpreter_frame_monitor_block_top_offset * wordSize); + const int entry_size = frame::interpreter_frame_monitor_size() * wordSize; + +#ifdef ASSERT + __ lwu(x10, access_flags); + __ verify_access_flags(x10, JVM_ACC_SYNCHRONIZED, "method doesn't need synchronization", false); +#endif // ASSERT + + // get synchronization object + { + Label done; + __ lwu(x10, access_flags); + __ andi(t0, x10, JVM_ACC_STATIC); + // get receiver (assume this is frequent case) + __ ld(x10, Address(xlocals, Interpreter::local_offset_in_bytes(0))); + __ beqz(t0, done); + __ load_mirror(x10, xmethod); + +#ifdef ASSERT + { + Label L; + __ bnez(x10, L); + __ stop("synchronization object is NULL"); + __ bind(L); + } +#endif // ASSERT + + __ bind(done); + } + + // add space for monitor & lock + __ add(sp, sp, - entry_size); // add space for a monitor entry + __ add(esp, esp, - entry_size); + __ mv(t0, esp); + __ sd(t0, monitor_block_top); // set new monitor block top + // store object + __ sd(x10, Address(esp, BasicObjectLock::obj_offset_in_bytes())); + __ mv(c_rarg1, esp); // object address + __ lock_object(c_rarg1); +} + +// Generate a fixed interpreter frame. This is identical setup for +// interpreted methods and for native methods hence the shared code. +// +// Args: +// ra: return address +// xmethod: Method* +// xlocals: pointer to locals +// xcpool: cp cache +// stack_pointer: previous sp +void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) { + // initialize fixed part of activation frame + if (native_call) { + __ add(esp, sp, - 14 * wordSize); + __ mv(xbcp, zr); + __ add(sp, sp, - 14 * wordSize); + // add 2 zero-initialized slots for native calls + __ sd(zr, Address(sp, 13 * wordSize)); + __ sd(zr, Address(sp, 12 * wordSize)); + } else { + __ add(esp, sp, - 12 * wordSize); + __ ld(t0, Address(xmethod, Method::const_offset())); // get ConstMethod + __ add(xbcp, t0, in_bytes(ConstMethod::codes_offset())); // get codebase + __ add(sp, sp, - 12 * wordSize); + } + __ sd(xbcp, Address(sp, wordSize)); + __ sd(esp, Address(sp, 0)); + + if (ProfileInterpreter) { + Label method_data_continue; + __ ld(t0, Address(xmethod, Method::method_data_offset())); + __ beqz(t0, method_data_continue); + __ la(t0, Address(t0, in_bytes(MethodData::data_offset()))); + __ bind(method_data_continue); + } + + __ sd(xmethod, Address(sp, 7 * wordSize)); + __ sd(ProfileInterpreter ? t0 : zr, Address(sp, 6 * wordSize)); + + // Get mirror and store it in the frame as GC root for this Method* + __ load_mirror(t2, xmethod); + __ sd(zr, Address(sp, 5 * wordSize)); + __ sd(t2, Address(sp, 4 * wordSize)); + + __ ld(xcpool, Address(xmethod, Method::const_offset())); + __ ld(xcpool, Address(xcpool, ConstMethod::constants_offset())); + __ ld(xcpool, Address(xcpool, ConstantPool::cache_offset_in_bytes())); + __ sd(xcpool, Address(sp, 3 * wordSize)); + __ sd(xlocals, Address(sp, 2 * wordSize)); + + __ sd(ra, Address(sp, 11 * wordSize)); + __ sd(fp, Address(sp, 10 * wordSize)); + __ la(fp, Address(sp, 12 * wordSize)); // include ra & fp + + // set sender sp + // leave last_sp as null + __ sd(x30, Address(sp, 9 * wordSize)); + __ sd(zr, Address(sp, 8 * wordSize)); + + // Move SP out of the way + if (!native_call) { + __ ld(t0, Address(xmethod, Method::const_offset())); + __ lhu(t0, Address(t0, ConstMethod::max_stack_offset())); + __ add(t0, t0, frame::interpreter_frame_monitor_size() + 2); + __ slli(t0, t0, 3); + __ sub(t0, sp, t0); + __ andi(sp, t0, -16); + } +} + +// End of helpers + +// Various method entries +//------------------------------------------------------------------------------------------------------------------------ +// +// + +// Method entry for java.lang.ref.Reference.get. +address TemplateInterpreterGenerator::generate_Reference_get_entry(void) { + // Code: _aload_0, _getfield, _areturn + // parameter size = 1 + // + // The code that gets generated by this routine is split into 2 parts: + // 1. The "intrinsified" code for G1 (or any SATB based GC), + // 2. The slow path - which is an expansion of the regular method entry. + // + // Notes:- + // * In the G1 code we do not check whether we need to block for + // a safepoint. If G1 is enabled then we must execute the specialized + // code for Reference.get (except when the Reference object is null) + // so that we can log the value in the referent field with an SATB + // update buffer. + // If the code for the getfield template is modified so that the + // G1 pre-barrier code is executed when the current method is + // Reference.get() then going through the normal method entry + // will be fine. + // * The G1 code can, however, check the receiver object (the instance + // of java.lang.Reference) and jump to the slow path if null. If the + // Reference object is null then we obviously cannot fetch the referent + // and so we don't need to call the G1 pre-barrier. Thus we can use the + // regular method entry code to generate the NPE. + // + // This code is based on generate_accessor_entry. + // + // xmethod: Method* + // x30: senderSP must preserve for slow path, set SP to it on fast path + + // ra is live. It must be saved around calls. + + address entry = __ pc(); + + const int referent_offset = java_lang_ref_Reference::referent_offset(); + guarantee(referent_offset > 0, "referent offset not initialized"); + + Label slow_path; + const Register local_0 = c_rarg0; + // Check if local 0 != NULL + // If the receiver is null then it is OK to jump to the slow path. + __ ld(local_0, Address(esp, 0)); + __ beqz(local_0, slow_path); + + __ mv(x9, x30); // Move senderSP to a callee-saved register + + // Load the value of the referent field. + const Address field_address(local_0, referent_offset); + BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->load_at(_masm, IN_HEAP | ON_WEAK_OOP_REF, T_OBJECT, local_0, field_address, /*tmp1*/ t1, /*tmp2*/ t0); + + // areturn + __ andi(sp, x9, -16); // done with stack + __ ret(); + + // generate a vanilla interpreter entry as the slow path + __ bind(slow_path); + __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::zerolocals)); + return entry; +} + +/** + * Method entry for static native methods: + * int java.util.zip.CRC32.update(int crc, int b) + */ +address TemplateInterpreterGenerator::generate_CRC32_update_entry() { + // TODO: Unimplemented generate_CRC32_update_entry + return 0; +} + +/** + * Method entry for static native methods: + * int java.util.zip.CRC32.updateBytes(int crc, byte[] b, int off, int len) + * int java.util.zip.CRC32.updateByteBuffer(int crc, long buf, int off, int len) + */ +address TemplateInterpreterGenerator::generate_CRC32_updateBytes_entry(AbstractInterpreter::MethodKind kind) { + // TODO: Unimplemented generate_CRC32_updateBytes_entry + return 0; +} + +/** + * Method entry for intrinsic-candidate (non-native) methods: + * int java.util.zip.CRC32C.updateBytes(int crc, byte[] b, int off, int end) + * int java.util.zip.CRC32C.updateDirectByteBuffer(int crc, long buf, int off, int end) + * Unlike CRC32, CRC32C does not have any methods marked as native + * CRC32C also uses an "end" variable instead of the length variable CRC32 uses + */ +address TemplateInterpreterGenerator::generate_CRC32C_updateBytes_entry(AbstractInterpreter::MethodKind kind) { + // TODO: Unimplemented generate_CRC32C_updateBytes_entry + return 0; +} + +void TemplateInterpreterGenerator::bang_stack_shadow_pages(bool native_call) { + // Bang each page in the shadow zone. We can't assume it's been done for + // an interpreter frame with greater than a page of locals, so each page + // needs to be checked. Only true for non-native. + const int n_shadow_pages = (int)(StackOverflow::stack_shadow_zone_size() / os::vm_page_size()); + const int start_page = native_call ? n_shadow_pages : 1; + const int page_size = os::vm_page_size(); + for (int pages = start_page; pages <= n_shadow_pages ; pages++) { + __ sub(t1, sp, pages * page_size); + __ sd(zr, Address(t1)); + } +} + +// Interpreter stub for calling a native method. (asm interpreter) +// This sets up a somewhat different looking stack for calling the +// native method than the typical interpreter frame setup. +address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { + // determine code generation flags + bool inc_counter = UseCompiler || CountCompiledCalls || LogTouchedMethods; + + // x11: Method* + // x30: sender sp + + address entry_point = __ pc(); + + const Address constMethod (xmethod, Method::const_offset()); + const Address access_flags (xmethod, Method::access_flags_offset()); + const Address size_of_parameters(x12, ConstMethod:: + size_of_parameters_offset()); + + // get parameter size (always needed) + __ ld(x12, constMethod); + __ load_unsigned_short(x12, size_of_parameters); + + // Native calls don't need the stack size check since they have no + // expression stack and the arguments are already on the stack and + // we only add a handful of words to the stack. + + // xmethod: Method* + // x12: size of parameters + // x30: sender sp + + // for natives the size of locals is zero + + // compute beginning of parameters (xlocals) + __ shadd(xlocals, x12, esp, xlocals, 3); + __ addi(xlocals, xlocals, -wordSize); + + // Pull SP back to minimum size: this avoids holes in the stack + __ andi(sp, esp, -16); + + // initialize fixed part of activation frame + generate_fixed_frame(true); + + // make sure method is native & not abstract +#ifdef ASSERT + __ lwu(x10, access_flags); + __ verify_access_flags(x10, JVM_ACC_NATIVE, "tried to execute non-native method as native", false); + __ verify_access_flags(x10, JVM_ACC_ABSTRACT, "tried to execute abstract method in interpreter"); +#endif + + // Since at this point in the method invocation the exception + // handler would try to exit the monitor of synchronized methods + // which hasn't been entered yet, we set the thread local variable + // _do_not_unlock_if_synchronized to true. The remove_activation + // will check this flag. + + const Address do_not_unlock_if_synchronized(xthread, + in_bytes(JavaThread::do_not_unlock_if_synchronized_offset())); + __ mv(t1, true); + __ sb(t1, do_not_unlock_if_synchronized); + + // increment invocation count & check for overflow + Label invocation_counter_overflow; + if (inc_counter) { + generate_counter_incr(&invocation_counter_overflow); + } + + Label continue_after_compile; + __ bind(continue_after_compile); + + bang_stack_shadow_pages(true); + + // reset the _do_not_unlock_if_synchronized flag + __ sb(zr, do_not_unlock_if_synchronized); + + // check for synchronized methods + // Must happen AFTER invocation_counter check and stack overflow check, + // so method is not locked if overflows. + if (synchronized) { + lock_method(); + } else { + // no synchronization necessary +#ifdef ASSERT + __ lwu(x10, access_flags); + __ verify_access_flags(x10, JVM_ACC_SYNCHRONIZED, "method needs synchronization"); +#endif + } + + // start execution +#ifdef ASSERT + __ verify_frame_setup(); +#endif + + // jvmti support + __ notify_method_entry(); + + // work registers + const Register t = x18; + const Register result_handler = x19; + + // allocate space for parameters + __ ld(t, Address(xmethod, Method::const_offset())); + __ load_unsigned_short(t, Address(t, ConstMethod::size_of_parameters_offset())); + + __ slli(t, t, Interpreter::logStackElementSize); + __ sub(x30, esp, t); + __ andi(sp, x30, -16); + __ mv(esp, x30); + + // get signature handler + { + Label L; + __ ld(t, Address(xmethod, Method::signature_handler_offset())); + __ bnez(t, L); + __ call_VM(noreg, + CAST_FROM_FN_PTR(address, + InterpreterRuntime::prepare_native_call), + xmethod); + __ ld(t, Address(xmethod, Method::signature_handler_offset())); + __ bind(L); + } + + // call signature handler + assert(InterpreterRuntime::SignatureHandlerGenerator::from() == xlocals, + "adjust this code"); + assert(InterpreterRuntime::SignatureHandlerGenerator::to() == sp, + "adjust this code"); + assert(InterpreterRuntime::SignatureHandlerGenerator::temp() == t0, + "adjust this code"); + + // The generated handlers do not touch xmethod (the method). + // However, large signatures cannot be cached and are generated + // each time here. The slow-path generator can do a GC on return, + // so we must reload it after the call. + __ jalr(t); + __ get_method(xmethod); // slow path can do a GC, reload xmethod + + + // result handler is in x10 + // set result handler + __ mv(result_handler, x10); + // pass mirror handle if static call + { + Label L; + __ lwu(t, Address(xmethod, Method::access_flags_offset())); + __ test_bit(t0, t, exact_log2(JVM_ACC_STATIC)); + __ beqz(t0, L); + // get mirror + __ load_mirror(t, xmethod); + // copy mirror into activation frame + __ sd(t, Address(fp, frame::interpreter_frame_oop_temp_offset * wordSize)); + // pass handle to mirror + __ addi(c_rarg1, fp, frame::interpreter_frame_oop_temp_offset * wordSize); + __ bind(L); + } + + // get native function entry point in x28 + { + Label L; + __ ld(x28, Address(xmethod, Method::native_function_offset())); + address unsatisfied = (SharedRuntime::native_method_throw_unsatisfied_link_error_entry()); + __ mv(t1, unsatisfied); + __ ld(t1, Address(t1, 0)); + __ bne(x28, t1, L); + __ call_VM(noreg, + CAST_FROM_FN_PTR(address, + InterpreterRuntime::prepare_native_call), + xmethod); + __ get_method(xmethod); + __ ld(x28, Address(xmethod, Method::native_function_offset())); + __ bind(L); + } + + // pass JNIEnv + __ add(c_rarg0, xthread, in_bytes(JavaThread::jni_environment_offset())); + + // It is enough that the pc() points into the right code + // segment. It does not have to be the correct return pc. + Label native_return; + __ set_last_Java_frame(esp, fp, native_return, x30); + + // change thread state +#ifdef ASSERT + { + Label L; + __ lwu(t, Address(xthread, JavaThread::thread_state_offset())); + __ addi(t0, zr, (u1)_thread_in_Java); + __ beq(t, t0, L); + __ stop("Wrong thread state in native stub"); + __ bind(L); + } +#endif + + // Change state to native + __ la(t1, Address(xthread, JavaThread::thread_state_offset())); + __ mv(t0, _thread_in_native); + __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore); + __ sw(t0, Address(t1)); + + // Call the native method. + __ jalr(x28); + __ bind(native_return); + __ get_method(xmethod); + // result potentially in x10 or f10 + + // make room for the pushes we're about to do + __ sub(t0, esp, 4 * wordSize); + __ andi(sp, t0, -16); + + // NOTE: The order of these pushes is known to frame::interpreter_frame_result + // in order to extract the result of a method call. If the order of these + // pushes change or anything else is added to the stack then the code in + // interpreter_frame_result must also change. + __ push(dtos); + __ push(ltos); + + // change thread state + // Force all preceding writes to be observed prior to thread state change + __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore); + + __ mv(t0, _thread_in_native_trans); + __ sw(t0, Address(xthread, JavaThread::thread_state_offset())); + + // Force this write out before the read below + __ membar(MacroAssembler::AnyAny); + + // check for safepoint operation in progress and/or pending suspend requests + { + Label L, Continue; + + // We need an acquire here to ensure that any subsequent load of the + // global SafepointSynchronize::_state flag is ordered after this load + // of the thread-local polling word. We don't want this poll to + // return false (i.e. not safepointing) and a later poll of the global + // SafepointSynchronize::_state spuriously to return true. + // + // This is to avoid a race when we're in a native->Java transition + // racing the code which wakes up from a safepoint. + __ safepoint_poll(L, true /* at_return */, true /* acquire */, false /* in_nmethod */); + __ lwu(t1, Address(xthread, JavaThread::suspend_flags_offset())); + __ beqz(t1, Continue); + __ bind(L); + + // Don't use call_VM as it will see a possible pending exception + // and forward it and never return here preventing us from + // clearing _last_native_pc down below. So we do a runtime call by + // hand. + // + __ mv(c_rarg0, xthread); + __ call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)); + __ get_method(xmethod); + __ reinit_heapbase(); + __ bind(Continue); + } + + // change thread state + // Force all preceding writes to be observed prior to thread state change + __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore); + + __ mv(t0, _thread_in_Java); + __ sw(t0, Address(xthread, JavaThread::thread_state_offset())); + + // reset_last_Java_frame + __ reset_last_Java_frame(true); + + if (CheckJNICalls) { + // clear_pending_jni_exception_check + __ sd(zr, Address(xthread, JavaThread::pending_jni_exception_check_fn_offset())); + } + + // reset handle block + __ ld(t, Address(xthread, JavaThread::active_handles_offset())); + __ sd(zr, Address(t, JNIHandleBlock::top_offset_in_bytes())); + + // If result is an oop unbox and store it in frame where gc will see it + // and result handler will pick it up + + { + Label no_oop; + __ la(t, ExternalAddress(AbstractInterpreter::result_handler(T_OBJECT))); + __ bne(t, result_handler, no_oop); + // Unbox oop result, e.g. JNIHandles::resolve result. + __ pop(ltos); + __ resolve_jobject(x10, xthread, t); + __ sd(x10, Address(fp, frame::interpreter_frame_oop_temp_offset * wordSize)); + // keep stack depth as expected by pushing oop which will eventually be discarded + __ push(ltos); + __ bind(no_oop); + } + + { + Label no_reguard; + __ lwu(t0, Address(xthread, in_bytes(JavaThread::stack_guard_state_offset()))); + __ addi(t1, zr, (u1)StackOverflow::stack_guard_yellow_reserved_disabled); + __ bne(t0, t1, no_reguard); + + __ push_call_clobbered_registers(); + __ mv(c_rarg0, xthread); + __ call(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages)); + __ pop_call_clobbered_registers(); + __ bind(no_reguard); + } + + // The method register is junk from after the thread_in_native transition + // until here. Also can't call_VM until the bcp has been + // restored. Need bcp for throwing exception below so get it now. + __ get_method(xmethod); + + // restore bcp to have legal interpreter frame, i.e., bci == 0 <=> + // xbcp == code_base() + __ ld(xbcp, Address(xmethod, Method::const_offset())); // get ConstMethod* + __ add(xbcp, xbcp, in_bytes(ConstMethod::codes_offset())); // get codebase + // handle exceptions (exception handling will handle unlocking!) + { + Label L; + __ ld(t0, Address(xthread, Thread::pending_exception_offset())); + __ beqz(t0, L); + // Note: At some point we may want to unify this with the code + // used in call_VM_base(); i.e., we should use the + // StubRoutines::forward_exception code. For now this doesn't work + // here because the sp is not correctly set at this point. + __ MacroAssembler::call_VM(noreg, + CAST_FROM_FN_PTR(address, + InterpreterRuntime::throw_pending_exception)); + __ should_not_reach_here(); + __ bind(L); + } + + // do unlocking if necessary + { + Label L; + __ lwu(t, Address(xmethod, Method::access_flags_offset())); + __ test_bit(t0, t, exact_log2(JVM_ACC_SYNCHRONIZED)); + __ beqz(t0, L); + // the code below should be shared with interpreter macro + // assembler implementation + { + Label unlock; + // BasicObjectLock will be first in list, since this is a + // synchronized method. However, need to check that the object + // has not been unlocked by an explicit monitorexit bytecode. + + // monitor expect in c_rarg1 for slow unlock path + __ la(c_rarg1, Address(fp, // address of first monitor + (intptr_t)(frame::interpreter_frame_initial_sp_offset * + wordSize - sizeof(BasicObjectLock)))); + + __ ld(t, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes())); + __ bnez(t, unlock); + + // Entry already unlocked, need to throw exception + __ MacroAssembler::call_VM(noreg, + CAST_FROM_FN_PTR(address, + InterpreterRuntime::throw_illegal_monitor_state_exception)); + __ should_not_reach_here(); + + __ bind(unlock); + __ unlock_object(c_rarg1); + } + __ bind(L); + } + + // jvmti support + // Note: This must happen _after_ handling/throwing any exceptions since + // the exception handler code notifies the runtime of method exits + // too. If this happens before, method entry/exit notifications are + // not properly paired (was bug - gri 11/22/99). + __ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI); + + __ pop(ltos); + __ pop(dtos); + + __ jalr(result_handler); + + // remove activation + __ ld(esp, Address(fp, frame::interpreter_frame_sender_sp_offset * wordSize)); // get sender sp + // remove frame anchor + __ leave(); + + // restore sender sp + __ mv(sp, esp); + + __ ret(); + + if (inc_counter) { + // Handle overflow of counter and compile method + __ bind(invocation_counter_overflow); + generate_counter_overflow(continue_after_compile); + } + + return entry_point; +} + +// +// Generic interpreted method entry to (asm) interpreter +// +address TemplateInterpreterGenerator::generate_normal_entry(bool synchronized) { + + // determine code generation flags + const bool inc_counter = UseCompiler || CountCompiledCalls || LogTouchedMethods; + + // t0: sender sp + address entry_point = __ pc(); + + const Address constMethod(xmethod, Method::const_offset()); + const Address access_flags(xmethod, Method::access_flags_offset()); + const Address size_of_parameters(x13, + ConstMethod::size_of_parameters_offset()); + const Address size_of_locals(x13, ConstMethod::size_of_locals_offset()); + + // get parameter size (always needed) + // need to load the const method first + __ ld(x13, constMethod); + __ load_unsigned_short(x12, size_of_parameters); + + // x12: size of parameters + + __ load_unsigned_short(x13, size_of_locals); // get size of locals in words + __ sub(x13, x13, x12); // x13 = no. of additional locals + + // see if we've got enough room on the stack for locals plus overhead. + generate_stack_overflow_check(); + + // compute beginning of parameters (xlocals) + __ shadd(xlocals, x12, esp, t1, 3); + __ add(xlocals, xlocals, -wordSize); + + // Make room for additional locals + __ slli(t1, x13, 3); + __ sub(t0, esp, t1); + + // Padding between locals and fixed part of activation frame to ensure + // SP is always 16-byte aligned. + __ andi(sp, t0, -16); + + // x13 - # of additional locals + // allocate space for locals + // explicitly initialize locals + { + Label exit, loop; + __ blez(x13, exit); // do nothing if x13 <= 0 + __ bind(loop); + __ sd(zr, Address(t0)); + __ add(t0, t0, wordSize); + __ add(x13, x13, -1); // until everything initialized + __ bnez(x13, loop); + __ bind(exit); + } + + // And the base dispatch table + __ get_dispatch(); + + // initialize fixed part of activation frame + generate_fixed_frame(false); + + // make sure method is not native & not abstract +#ifdef ASSERT + __ lwu(x10, access_flags); + __ verify_access_flags(x10, JVM_ACC_NATIVE, "tried to execute native method as non-native"); + __ verify_access_flags(x10, JVM_ACC_ABSTRACT, "tried to execute abstract method in interpreter"); +#endif + + // Since at this point in the method invocation the exception + // handler would try to exit the monitor of synchronized methods + // which hasn't been entered yet, we set the thread local variable + // _do_not_unlock_if_synchronized to true. The remove_activation + // will check this flag. + + const Address do_not_unlock_if_synchronized(xthread, + in_bytes(JavaThread::do_not_unlock_if_synchronized_offset())); + __ mv(t1, true); + __ sb(t1, do_not_unlock_if_synchronized); + + Label no_mdp; + const Register mdp = x13; + __ ld(mdp, Address(xmethod, Method::method_data_offset())); + __ beqz(mdp, no_mdp); + __ add(mdp, mdp, in_bytes(MethodData::data_offset())); + __ profile_parameters_type(mdp, x11, x12, x14); // use x11, x12, x14 as tmp registers + __ bind(no_mdp); + + // increment invocation count & check for overflow + Label invocation_counter_overflow; + if (inc_counter) { + generate_counter_incr(&invocation_counter_overflow); + } + + Label continue_after_compile; + __ bind(continue_after_compile); + + bang_stack_shadow_pages(false); + + // reset the _do_not_unlock_if_synchronized flag + __ sb(zr, do_not_unlock_if_synchronized); + + // check for synchronized methods + // Must happen AFTER invocation_counter check and stack overflow check, + // so method is not locked if overflows. + if (synchronized) { + // Allocate monitor and lock method + lock_method(); + } else { + // no synchronization necessary +#ifdef ASSERT + __ lwu(x10, access_flags); + __ verify_access_flags(x10, JVM_ACC_SYNCHRONIZED, "method needs synchronization"); +#endif + } + + // start execution +#ifdef ASSERT + __ verify_frame_setup(); +#endif + + // jvmti support + __ notify_method_entry(); + + __ dispatch_next(vtos); + + // invocation counter overflow + if (inc_counter) { + // Handle overflow of counter and compile method + __ bind(invocation_counter_overflow); + generate_counter_overflow(continue_after_compile); + } + + return entry_point; +} + +//----------------------------------------------------------------------------- +// Exceptions + +void TemplateInterpreterGenerator::generate_throw_exception() { + // Entry point in previous activation (i.e., if the caller was + // interpreted) + Interpreter::_rethrow_exception_entry = __ pc(); + // Restore sp to interpreter_frame_last_sp even though we are going + // to empty the expression stack for the exception processing. + __ sd(zr, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize)); + // x10: exception + // x13: return address/pc that threw exception + __ restore_bcp(); // xbcp points to call/send + __ restore_locals(); + __ restore_constant_pool_cache(); + __ reinit_heapbase(); // restore xheapbase as heapbase. + __ get_dispatch(); + + // Entry point for exceptions thrown within interpreter code + Interpreter::_throw_exception_entry = __ pc(); + // If we came here via a NullPointerException on the receiver of a + // method, xthread may be corrupt. + __ get_method(xmethod); + // expression stack is undefined here + // x10: exception + // xbcp: exception bcp + __ verify_oop(x10); + __ mv(c_rarg1, x10); + + // expression stack must be empty before entering the VM in case of + // an exception + __ empty_expression_stack(); + // find exception handler address and preserve exception oop + __ call_VM(x13, + CAST_FROM_FN_PTR(address, + InterpreterRuntime::exception_handler_for_exception), + c_rarg1); + + // Calculate stack limit + __ ld(t0, Address(xmethod, Method::const_offset())); + __ lhu(t0, Address(t0, ConstMethod::max_stack_offset())); + __ add(t0, t0, frame::interpreter_frame_monitor_size() + 4); + __ ld(t1, Address(fp, frame::interpreter_frame_initial_sp_offset * wordSize)); + __ slli(t0, t0, 3); + __ sub(t0, t1, t0); + __ andi(sp, t0, -16); + + // x10: exception handler entry point + // x13: preserved exception oop + // xbcp: bcp for exception handler + __ push_ptr(x13); // push exception which is now the only value on the stack + __ jr(x10); // jump to exception handler (may be _remove_activation_entry!) + + // If the exception is not handled in the current frame the frame is + // removed and the exception is rethrown (i.e. exception + // continuation is _rethrow_exception). + // + // Note: At this point the bci is still the bxi for the instruction + // which caused the exception and the expression stack is + // empty. Thus, for any VM calls at this point, GC will find a legal + // oop map (with empty expression stack). + + // + // JVMTI PopFrame support + // + + Interpreter::_remove_activation_preserving_args_entry = __ pc(); + __ empty_expression_stack(); + // Set the popframe_processing bit in pending_popframe_condition + // indicating that we are currently handling popframe, so that + // call_VMs that may happen later do not trigger new popframe + // handling cycles. + __ lwu(x13, Address(xthread, JavaThread::popframe_condition_offset())); + __ ori(x13, x13, JavaThread::popframe_processing_bit); + __ sw(x13, Address(xthread, JavaThread::popframe_condition_offset())); + + { + // Check to see whether we are returning to a deoptimized frame. + // (The PopFrame call ensures that the caller of the popped frame is + // either interpreted or compiled and deoptimizes it if compiled.) + // In this case, we can't call dispatch_next() after the frame is + // popped, but instead must save the incoming arguments and restore + // them after deoptimization has occurred. + // + // Note that we don't compare the return PC against the + // deoptimization blob's unpack entry because of the presence of + // adapter frames in C2. + Label caller_not_deoptimized; + __ ld(c_rarg1, Address(fp, frame::return_addr_offset * wordSize)); + __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::interpreter_contains), c_rarg1); + __ bnez(x10, caller_not_deoptimized); + + // Compute size of arguments for saving when returning to + // deoptimized caller + __ get_method(x10); + __ ld(x10, Address(x10, Method::const_offset())); + __ load_unsigned_short(x10, Address(x10, in_bytes(ConstMethod:: + size_of_parameters_offset()))); + __ slli(x10, x10, Interpreter::logStackElementSize); + __ restore_locals(); + __ sub(xlocals, xlocals, x10); + __ add(xlocals, xlocals, wordSize); + // Save these arguments + __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, + Deoptimization:: + popframe_preserve_args), + xthread, x10, xlocals); + + __ remove_activation(vtos, + /* throw_monitor_exception */ false, + /* install_monitor_exception */ false, + /* notify_jvmdi */ false); + + // Inform deoptimization that it is responsible for restoring + // these arguments + __ mv(t0, JavaThread::popframe_force_deopt_reexecution_bit); + __ sw(t0, Address(xthread, JavaThread::popframe_condition_offset())); + + // Continue in deoptimization handler + __ ret(); + + __ bind(caller_not_deoptimized); + } + + __ remove_activation(vtos, + /* throw_monitor_exception */ false, + /* install_monitor_exception */ false, + /* notify_jvmdi */ false); + + // Restore the last_sp and null it out + __ ld(esp, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize)); + __ sd(zr, Address(fp, frame::interpreter_frame_last_sp_offset * wordSize)); + + __ restore_bcp(); + __ restore_locals(); + __ restore_constant_pool_cache(); + __ get_method(xmethod); + __ get_dispatch(); + + // The method data pointer was incremented already during + // call profiling. We have to restore the mdp for the current bcp. + if (ProfileInterpreter) { + __ set_method_data_pointer_for_bcp(); + } + + // Clear the popframe condition flag + __ sw(zr, Address(xthread, JavaThread::popframe_condition_offset())); + assert(JavaThread::popframe_inactive == 0, "fix popframe_inactive"); + +#if INCLUDE_JVMTI + { + Label L_done; + + __ lbu(t0, Address(xbcp, 0)); + __ mv(t1, Bytecodes::_invokestatic); + __ bne(t1, t0, L_done); + + // The member name argument must be restored if _invokestatic is re-executed after a PopFrame call. + // Detect such a case in the InterpreterRuntime function and return the member name argument,or NULL. + + __ ld(c_rarg0, Address(xlocals, 0)); + __ call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::member_name_arg_or_null),c_rarg0, xmethod, xbcp); + + __ beqz(x10, L_done); + + __ sd(x10, Address(esp, 0)); + __ bind(L_done); + } +#endif // INCLUDE_JVMTI + + // Restore machine SP + __ ld(t0, Address(xmethod, Method::const_offset())); + __ lhu(t0, Address(t0, ConstMethod::max_stack_offset())); + __ add(t0, t0, frame::interpreter_frame_monitor_size() + 4); + __ ld(t1, Address(fp, frame::interpreter_frame_initial_sp_offset * wordSize)); + __ slliw(t0, t0, 3); + __ sub(t0, t1, t0); + __ andi(sp, t0, -16); + + __ dispatch_next(vtos); + // end of PopFrame support + + Interpreter::_remove_activation_entry = __ pc(); + + // preserve exception over this code sequence + __ pop_ptr(x10); + __ sd(x10, Address(xthread, JavaThread::vm_result_offset())); + // remove the activation (without doing throws on illegalMonitorExceptions) + __ remove_activation(vtos, false, true, false); + // restore exception + __ get_vm_result(x10, xthread); + + // In between activations - previous activation type unknown yet + // compute continuation point - the continuation point expects the + // following registers set up: + // + // x10: exception + // ra: return address/pc that threw exception + // sp: expression stack of caller + // fp: fp of caller + // FIXME: There's no point saving ra here because VM calls don't trash it + __ sub(sp, sp, 2 * wordSize); + __ sd(x10, Address(sp, 0)); // save exception + __ sd(ra, Address(sp, wordSize)); // save return address + __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, + SharedRuntime::exception_handler_for_return_address), + xthread, ra); + __ mv(x11, x10); // save exception handler + __ ld(x10, Address(sp, 0)); // restore exception + __ ld(ra, Address(sp, wordSize)); // restore return address + __ add(sp, sp, 2 * wordSize); + // We might be returning to a deopt handler that expects x13 to + // contain the exception pc + __ mv(x13, ra); + // Note that an "issuing PC" is actually the next PC after the call + __ jr(x11); // jump to exception + // handler of caller +} + +// +// JVMTI ForceEarlyReturn support +// +address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state) { + address entry = __ pc(); + + __ restore_bcp(); + __ restore_locals(); + __ empty_expression_stack(); + __ load_earlyret_value(state); + + __ ld(t0, Address(xthread, JavaThread::jvmti_thread_state_offset())); + Address cond_addr(t0, JvmtiThreadState::earlyret_state_offset()); + + // Clear the earlyret state + assert(JvmtiThreadState::earlyret_inactive == 0, "should be"); + __ sd(zr, cond_addr); + + __ remove_activation(state, + false, /* throw_monitor_exception */ + false, /* install_monitor_exception */ + true); /* notify_jvmdi */ + __ ret(); + + return entry; +} +// end of ForceEarlyReturn support + +//----------------------------------------------------------------------------- +// Helper for vtos entry point generation + +void TemplateInterpreterGenerator::set_vtos_entry_points(Template* t, + address& bep, + address& cep, + address& sep, + address& aep, + address& iep, + address& lep, + address& fep, + address& dep, + address& vep) { + assert(t != NULL && t->is_valid() && t->tos_in() == vtos, "illegal template"); + Label L; + aep = __ pc(); __ push_ptr(); __ j(L); + fep = __ pc(); __ push_f(); __ j(L); + dep = __ pc(); __ push_d(); __ j(L); + lep = __ pc(); __ push_l(); __ j(L); + bep = cep = sep = + iep = __ pc(); __ push_i(); + vep = __ pc(); + __ bind(L); + generate_and_dispatch(t); +} + +//----------------------------------------------------------------------------- + +// Non-product code +#ifndef PRODUCT +address TemplateInterpreterGenerator::generate_trace_code(TosState state) { + address entry = __ pc(); + + __ push_reg(ra); + __ push(state); + __ push_reg(RegSet::range(x10, x17) + RegSet::range(x5, x7) + RegSet::range(x28, x31), sp); + __ mv(c_rarg2, x10); // Pass itos + __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::trace_bytecode), c_rarg1, c_rarg2, c_rarg3); + __ pop_reg(RegSet::range(x10, x17) + RegSet::range(x5, x7) + RegSet::range(x28, x31), sp); + __ pop(state); + __ pop_reg(ra); + __ ret(); // return from result handler + + return entry; +} + +void TemplateInterpreterGenerator::count_bytecode() { + __ push_reg(t0); + __ push_reg(x10); + __ mv(x10, (address) &BytecodeCounter::_counter_value); + __ mv(t0, 1); + __ amoadd_d(zr, x10, t0, Assembler::aqrl); + __ pop_reg(x10); + __ pop_reg(t0); +} + +void TemplateInterpreterGenerator::histogram_bytecode(Template* t) { ; } + +void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) { ; } + +void TemplateInterpreterGenerator::trace_bytecode(Template* t) { + // Call a little run-time stub to avoid blow-up for each bytecode. + // The run-time runtime saves the right registers, depending on + // the tosca in-state for the given template. + + assert(Interpreter::trace_code(t->tos_in()) != NULL, "entry must have been generated"); + __ jal(Interpreter::trace_code(t->tos_in())); + __ reinit_heapbase(); +} + +void TemplateInterpreterGenerator::stop_interpreter_at() { + Label L; + __ push_reg(t0); + __ mv(t0, (address) &BytecodeCounter::_counter_value); + __ ld(t0, Address(t0)); + __ mv(t1, StopInterpreterAt); + __ bne(t0, t1, L); + __ ebreak(); + __ bind(L); + __ pop_reg(t0); +} + +#endif // !PRODUCT diff --git a/src/hotspot/cpu/riscv/templateTable_riscv.cpp b/src/hotspot/cpu/riscv/templateTable_riscv.cpp new file mode 100644 index 0000000000000..19b31c969e314 --- /dev/null +++ b/src/hotspot/cpu/riscv/templateTable_riscv.cpp @@ -0,0 +1,3949 @@ +/* + * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "gc/shared/barrierSetAssembler.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/tlab_globals.hpp" +#include "interpreter/interp_masm.hpp" +#include "interpreter/interpreter.hpp" +#include "interpreter/interpreterRuntime.hpp" +#include "interpreter/templateTable.hpp" +#include "memory/universe.hpp" +#include "oops/method.hpp" +#include "oops/methodData.hpp" +#include "oops/objArrayKlass.hpp" +#include "oops/oop.inline.hpp" +#include "prims/jvmtiExport.hpp" +#include "prims/methodHandles.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/sharedRuntime.hpp" +#include "runtime/stubRoutines.hpp" +#include "runtime/synchronizer.hpp" +#include "utilities/powerOfTwo.hpp" + +#define __ _masm-> + +// Address computation: local variables + +static inline Address iaddress(int n) { + return Address(xlocals, Interpreter::local_offset_in_bytes(n)); +} + +static inline Address laddress(int n) { + return iaddress(n + 1); +} + +static inline Address faddress(int n) { + return iaddress(n); +} + +static inline Address daddress(int n) { + return laddress(n); +} + +static inline Address aaddress(int n) { + return iaddress(n); +} + +static inline Address iaddress(Register r, Register temp, InterpreterMacroAssembler* _masm) { + _masm->shadd(temp, r, xlocals, temp, 3); + return Address(temp, 0); +} + +static inline Address laddress(Register r, Register temp, InterpreterMacroAssembler* _masm) { + _masm->shadd(temp, r, xlocals, temp, 3); + return Address(temp, Interpreter::local_offset_in_bytes(1));; +} + +static inline Address faddress(Register r, Register temp, InterpreterMacroAssembler* _masm) { + return iaddress(r, temp, _masm); +} + +static inline Address daddress(Register r, Register temp, InterpreterMacroAssembler* _masm) { + return laddress(r, temp, _masm); +} + +static inline Address aaddress(Register r, Register temp, InterpreterMacroAssembler* _masm) { + return iaddress(r, temp, _masm); +} + +static inline Address at_rsp() { + return Address(esp, 0); +} + +// At top of Java expression stack which may be different than esp(). It +// isn't for category 1 objects. +static inline Address at_tos () { + return Address(esp, Interpreter::expr_offset_in_bytes(0)); +} + +static inline Address at_tos_p1() { + return Address(esp, Interpreter::expr_offset_in_bytes(1)); +} + +static inline Address at_tos_p2() { + return Address(esp, Interpreter::expr_offset_in_bytes(2)); +} + +static inline Address at_tos_p3() { + return Address(esp, Interpreter::expr_offset_in_bytes(3)); +} + +static inline Address at_tos_p4() { + return Address(esp, Interpreter::expr_offset_in_bytes(4)); +} + +static inline Address at_tos_p5() { + return Address(esp, Interpreter::expr_offset_in_bytes(5)); +} + +// Miscelaneous helper routines +// Store an oop (or NULL) at the Address described by obj. +// If val == noreg this means store a NULL +static void do_oop_store(InterpreterMacroAssembler* _masm, + Address dst, + Register val, + DecoratorSet decorators) { + assert(val == noreg || val == x10, "parameter is just for looks"); + __ store_heap_oop(dst, val, x29, x11, decorators); +} + +static void do_oop_load(InterpreterMacroAssembler* _masm, + Address src, + Register dst, + DecoratorSet decorators) { + __ load_heap_oop(dst, src, x7, x11, decorators); +} + +Address TemplateTable::at_bcp(int offset) { + assert(_desc->uses_bcp(), "inconsistent uses_bcp information"); + return Address(xbcp, offset); +} + +void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg, + Register temp_reg, bool load_bc_into_bc_reg/*=true*/, + int byte_no) +{ + if (!RewriteBytecodes) { return; } + Label L_patch_done; + + switch (bc) { + case Bytecodes::_fast_aputfield: // fall through + case Bytecodes::_fast_bputfield: // fall through + case Bytecodes::_fast_zputfield: // fall through + case Bytecodes::_fast_cputfield: // fall through + case Bytecodes::_fast_dputfield: // fall through + case Bytecodes::_fast_fputfield: // fall through + case Bytecodes::_fast_iputfield: // fall through + case Bytecodes::_fast_lputfield: // fall through + case Bytecodes::_fast_sputfield: { + // We skip bytecode quickening for putfield instructions when + // the put_code written to the constant pool cache is zero. + // This is required so that every execution of this instruction + // calls out to InterpreterRuntime::resolve_get_put to do + // additional, required work. + assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range"); + assert(load_bc_into_bc_reg, "we use bc_reg as temp"); + __ get_cache_and_index_and_bytecode_at_bcp(temp_reg, bc_reg, temp_reg, byte_no, 1); + __ mv(bc_reg, bc); + __ beqz(temp_reg, L_patch_done); + break; + } + default: + assert(byte_no == -1, "sanity"); + // the pair bytecodes have already done the load. + if (load_bc_into_bc_reg) { + __ mv(bc_reg, bc); + } + } + + if (JvmtiExport::can_post_breakpoint()) { + Label L_fast_patch; + // if a breakpoint is present we can't rewrite the stream directly + __ load_unsigned_byte(temp_reg, at_bcp(0)); + __ addi(temp_reg, temp_reg, -Bytecodes::_breakpoint); // temp_reg is temporary register. + __ bnez(temp_reg, L_fast_patch); + // Let breakpoint table handling rewrite to quicker bytecode + __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), xmethod, xbcp, bc_reg); + __ j(L_patch_done); + __ bind(L_fast_patch); + } + +#ifdef ASSERT + Label L_okay; + __ load_unsigned_byte(temp_reg, at_bcp(0)); + __ beq(temp_reg, bc_reg, L_okay); + __ addi(temp_reg, temp_reg, -(int) Bytecodes::java_code(bc)); + __ beqz(temp_reg, L_okay); + __ stop("patching the wrong bytecode"); + __ bind(L_okay); +#endif + + // patch bytecode + __ sb(bc_reg, at_bcp(0)); + __ bind(L_patch_done); +} + +// Individual instructions + +void TemplateTable::nop() { + transition(vtos, vtos); + // nothing to do +} + +void TemplateTable::shouldnotreachhere() { + transition(vtos, vtos); + __ stop("should not reach here bytecode"); +} + +void TemplateTable::aconst_null() +{ + transition(vtos, atos); + __ mv(x10, zr); +} + +void TemplateTable::iconst(int value) +{ + transition(vtos, itos); + __ mv(x10, value); +} + +void TemplateTable::lconst(int value) +{ + transition(vtos, ltos); + __ mv(x10, value); +} + +void TemplateTable::fconst(int value) +{ + transition(vtos, ftos); + static float fBuf[2] = {1.0, 2.0}; + __ mv(t0, (intptr_t)fBuf); + switch (value) { + case 0: + __ fmv_w_x(f10, zr); + break; + case 1: + __ flw(f10, Address(t0, 0)); + break; + case 2: + __ flw(f10, Address(t0, sizeof(float))); + break; + default: + ShouldNotReachHere(); + } +} + +void TemplateTable::dconst(int value) +{ + transition(vtos, dtos); + static double dBuf[2] = {1.0, 2.0}; + __ mv(t0, (intptr_t)dBuf); + switch (value) { + case 0: + __ fmv_d_x(f10, zr); + break; + case 1: + __ fld(f10, Address(t0, 0)); + break; + case 2: + __ fld(f10, Address(t0, sizeof(double))); + break; + default: + ShouldNotReachHere(); + } +} + +void TemplateTable::bipush() +{ + transition(vtos, itos); + __ load_signed_byte(x10, at_bcp(1)); +} + +void TemplateTable::sipush() +{ + transition(vtos, itos); + __ load_unsigned_short(x10, at_bcp(1)); + __ revb_w_w(x10, x10); + __ sraiw(x10, x10, 16); +} + +void TemplateTable::ldc(bool wide) +{ + transition(vtos, vtos); + Label call_ldc, notFloat, notClass, notInt, Done; + + if (wide) { + __ get_unsigned_2_byte_index_at_bcp(x11, 1); + } else { + __ load_unsigned_byte(x11, at_bcp(1)); + } + __ get_cpool_and_tags(x12, x10); + + const int base_offset = ConstantPool::header_size() * wordSize; + const int tags_offset = Array::base_offset_in_bytes(); + + // get type + __ addi(x13, x11, tags_offset); + __ add(x13, x10, x13); + __ membar(MacroAssembler::AnyAny); + __ lbu(x13, Address(x13, 0)); + __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); + + // unresolved class - get the resolved class + __ mv(t1, (u1)JVM_CONSTANT_UnresolvedClass); + __ beq(x13, t1, call_ldc); + + // unresolved class in error state - call into runtime to throw the error + // from the first resolution attempt + __ mv(t1, (u1)JVM_CONSTANT_UnresolvedClassInError); + __ beq(x13, t1, call_ldc); + + // resolved class - need to call vm to get java mirror of the class + __ mv(t1, (u1)JVM_CONSTANT_Class); + __ bne(x13, t1, notClass); + + __ bind(call_ldc); + __ mv(c_rarg1, wide); + call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), c_rarg1); + __ push_ptr(x10); + __ verify_oop(x10); + __ j(Done); + + __ bind(notClass); + __ mv(t1, (u1)JVM_CONSTANT_Float); + __ bne(x13, t1, notFloat); + + // ftos + __ shadd(x11, x11, x12, x11, 3); + __ flw(f10, Address(x11, base_offset)); + __ push_f(f10); + __ j(Done); + + __ bind(notFloat); + + __ mv(t1, (u1)JVM_CONSTANT_Integer); + __ bne(x13, t1, notInt); + + // itos + __ shadd(x11, x11, x12, x11, 3); + __ lw(x10, Address(x11, base_offset)); + __ push_i(x10); + __ j(Done); + + __ bind(notInt); + condy_helper(Done); + + __ bind(Done); +} + +// Fast path for caching oop constants. +void TemplateTable::fast_aldc(bool wide) +{ + transition(vtos, atos); + + const Register result = x10; + const Register tmp = x11; + const Register rarg = x12; + + const int index_size = wide ? sizeof(u2) : sizeof(u1); + + Label resolved; + + // We are resolved if the resolved reference cache entry contains a + // non-null object (String, MethodType, etc.) + assert_different_registers(result, tmp); + __ get_cache_index_at_bcp(tmp, 1, index_size); + __ load_resolved_reference_at_index(result, tmp); + __ bnez(result, resolved); + + const address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc); + + // first time invocation - must resolve first + __ mv(rarg, (int)bytecode()); + __ call_VM(result, entry, rarg); + + __ bind(resolved); + + { // Check for the null sentinel. + // If we just called the VM, it already did the mapping for us, + // but it's harmless to retry. + Label notNull; + + // Stash null_sentinel address to get its value later + int32_t offset = 0; + __ mv(rarg, Universe::the_null_sentinel_addr(), offset); + __ ld(tmp, Address(rarg, offset)); + __ resolve_oop_handle(tmp); + __ bne(result, tmp, notNull); + __ mv(result, zr); // NULL object reference + __ bind(notNull); + } + + if (VerifyOops) { + // Safe to call with 0 result + __ verify_oop(result); + } +} + +void TemplateTable::ldc2_w() +{ + transition(vtos, vtos); + Label notDouble, notLong, Done; + __ get_unsigned_2_byte_index_at_bcp(x10, 1); + + __ get_cpool_and_tags(x11, x12); + const int base_offset = ConstantPool::header_size() * wordSize; + const int tags_offset = Array::base_offset_in_bytes(); + + // get type + __ add(x12, x12, x10); + __ load_unsigned_byte(x12, Address(x12, tags_offset)); + __ mv(t1, JVM_CONSTANT_Double); + __ bne(x12, t1, notDouble); + + // dtos + __ shadd(x12, x10, x11, x12, 3); + __ fld(f10, Address(x12, base_offset)); + __ push_d(f10); + __ j(Done); + + __ bind(notDouble); + __ mv(t1, (int)JVM_CONSTANT_Long); + __ bne(x12, t1, notLong); + + // ltos + __ shadd(x10, x10, x11, x10, 3); + __ ld(x10, Address(x10, base_offset)); + __ push_l(x10); + __ j(Done); + + __ bind(notLong); + condy_helper(Done); + __ bind(Done); +} + +void TemplateTable::condy_helper(Label& Done) +{ + const Register obj = x10; + const Register rarg = x11; + const Register flags = x12; + const Register off = x13; + + const address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc); + + __ mv(rarg, (int) bytecode()); + __ call_VM(obj, entry, rarg); + + __ get_vm_result_2(flags, xthread); + + // VMr = obj = base address to find primitive value to push + // VMr2 = flags = (tos, off) using format of CPCE::_flags + __ mv(off, flags); + __ mv(t0, ConstantPoolCacheEntry::field_index_mask); + __ andrw(off, off, t0); + + __ add(off, obj, off); + const Address field(off, 0); // base + R---->base + offset + + __ slli(flags, flags, XLEN - (ConstantPoolCacheEntry::tos_state_shift + ConstantPoolCacheEntry::tos_state_bits)); + __ srli(flags, flags, XLEN - ConstantPoolCacheEntry::tos_state_bits); // (1 << 5) - 4 --> 28~31==> flags:0~3 + + switch (bytecode()) { + case Bytecodes::_ldc: // fall through + case Bytecodes::_ldc_w: { + // tos in (itos, ftos, stos, btos, ctos, ztos) + Label notInt, notFloat, notShort, notByte, notChar, notBool; + __ mv(t1, itos); + __ bne(flags, t1, notInt); + // itos + __ lw(x10, field); + __ push(itos); + __ j(Done); + + __ bind(notInt); + __ mv(t1, ftos); + __ bne(flags, t1, notFloat); + // ftos + __ load_float(field); + __ push(ftos); + __ j(Done); + + __ bind(notFloat); + __ mv(t1, stos); + __ bne(flags, t1, notShort); + // stos + __ load_signed_short(x10, field); + __ push(stos); + __ j(Done); + + __ bind(notShort); + __ mv(t1, btos); + __ bne(flags, t1, notByte); + // btos + __ load_signed_byte(x10, field); + __ push(btos); + __ j(Done); + + __ bind(notByte); + __ mv(t1, ctos); + __ bne(flags, t1, notChar); + // ctos + __ load_unsigned_short(x10, field); + __ push(ctos); + __ j(Done); + + __ bind(notChar); + __ mv(t1, ztos); + __ bne(flags, t1, notBool); + // ztos + __ load_signed_byte(x10, field); + __ push(ztos); + __ j(Done); + + __ bind(notBool); + break; + } + + case Bytecodes::_ldc2_w: { + Label notLong, notDouble; + __ mv(t1, ltos); + __ bne(flags, t1, notLong); + // ltos + __ ld(x10, field); + __ push(ltos); + __ j(Done); + + __ bind(notLong); + __ mv(t1, dtos); + __ bne(flags, t1, notDouble); + // dtos + __ load_double(field); + __ push(dtos); + __ j(Done); + + __ bind(notDouble); + break; + } + + default: + ShouldNotReachHere(); + } + + __ stop("bad ldc/condy"); +} + +void TemplateTable::locals_index(Register reg, int offset) +{ + __ lbu(reg, at_bcp(offset)); + __ neg(reg, reg); +} + +void TemplateTable::iload() { + iload_internal(); +} + +void TemplateTable::nofast_iload() { + iload_internal(may_not_rewrite); +} + +void TemplateTable::iload_internal(RewriteControl rc) { + transition(vtos, itos); + if (RewriteFrequentPairs && rc == may_rewrite) { + Label rewrite, done; + const Register bc = x14; + + // get next bytecode + __ load_unsigned_byte(x11, at_bcp(Bytecodes::length_for(Bytecodes::_iload))); + + // if _iload, wait to rewrite to iload2. We only want to rewrite the + // last two iloads in a pair. Comparing against fast_iload means that + // the next bytecode is neither an iload or a caload, and therefore + // an iload pair. + __ mv(t1, Bytecodes::_iload); + __ beq(x11, t1, done); + + // if _fast_iload rewrite to _fast_iload2 + __ mv(t1, Bytecodes::_fast_iload); + __ mv(bc, Bytecodes::_fast_iload2); + __ beq(x11, t1, rewrite); + + // if _caload rewrite to _fast_icaload + __ mv(t1, Bytecodes::_caload); + __ mv(bc, Bytecodes::_fast_icaload); + __ beq(x11, t1, rewrite); + + // else rewrite to _fast_iload + __ mv(bc, Bytecodes::_fast_iload); + + // rewrite + // bc: new bytecode + __ bind(rewrite); + patch_bytecode(Bytecodes::_iload, bc, x11, false); + __ bind(done); + + } + + // do iload, get the local value into tos + locals_index(x11); + __ lw(x10, iaddress(x11, x10, _masm)); +} + +void TemplateTable::fast_iload2() +{ + transition(vtos, itos); + locals_index(x11); + __ lw(x10, iaddress(x11, x10, _masm)); + __ push(itos); + locals_index(x11, 3); + __ lw(x10, iaddress(x11, x10, _masm)); +} + +void TemplateTable::fast_iload() +{ + transition(vtos, itos); + locals_index(x11); + __ lw(x10, iaddress(x11, x10, _masm)); +} + +void TemplateTable::lload() +{ + transition(vtos, ltos); + __ lbu(x11, at_bcp(1)); + __ slli(x11, x11, LogBytesPerWord); + __ sub(x11, xlocals, x11); + __ ld(x10, Address(x11, Interpreter::local_offset_in_bytes(1))); +} + +void TemplateTable::fload() +{ + transition(vtos, ftos); + locals_index(x11); + __ flw(f10, faddress(x11, t0, _masm)); +} + +void TemplateTable::dload() +{ + transition(vtos, dtos); + __ lbu(x11, at_bcp(1)); + __ slli(x11, x11, LogBytesPerWord); + __ sub(x11, xlocals, x11); + __ fld(f10, Address(x11, Interpreter::local_offset_in_bytes(1))); +} + +void TemplateTable::aload() +{ + transition(vtos, atos); + locals_index(x11); + __ ld(x10, iaddress(x11, x10, _masm)); + +} + +void TemplateTable::locals_index_wide(Register reg) { + __ lhu(reg, at_bcp(2)); + __ revb_h_h_u(reg, reg); // reverse bytes in half-word and zero-extend + __ neg(reg, reg); +} + +void TemplateTable::wide_iload() { + transition(vtos, itos); + locals_index_wide(x11); + __ lw(x10, iaddress(x11, t0, _masm)); +} + +void TemplateTable::wide_lload() +{ + transition(vtos, ltos); + __ lhu(x11, at_bcp(2)); + __ revb_h_h_u(x11, x11); // reverse bytes in half-word and zero-extend + __ slli(x11, x11, LogBytesPerWord); + __ sub(x11, xlocals, x11); + __ ld(x10, Address(x11, Interpreter::local_offset_in_bytes(1))); +} + +void TemplateTable::wide_fload() +{ + transition(vtos, ftos); + locals_index_wide(x11); + __ flw(f10, faddress(x11, t0, _masm)); +} + +void TemplateTable::wide_dload() +{ + transition(vtos, dtos); + __ lhu(x11, at_bcp(2)); + __ revb_h_h_u(x11, x11); // reverse bytes in half-word and zero-extend + __ slli(x11, x11, LogBytesPerWord); + __ sub(x11, xlocals, x11); + __ fld(f10, Address(x11, Interpreter::local_offset_in_bytes(1))); +} + +void TemplateTable::wide_aload() +{ + transition(vtos, atos); + locals_index_wide(x11); + __ ld(x10, aaddress(x11, t0, _masm)); +} + +void TemplateTable::index_check(Register array, Register index) +{ + // destroys x11, t0 + // check array + __ null_check(array, arrayOopDesc::length_offset_in_bytes()); + // sign extend index for use by indexed load + // check index + const Register length = t0; + __ lwu(length, Address(array, arrayOopDesc::length_offset_in_bytes())); + if (index != x11) { + assert(x11 != array, "different registers"); + __ mv(x11, index); + } + Label ok; + __ sign_extend(index, index, 32); + __ bltu(index, length, ok); + __ mv(x13, array); + __ mv(t0, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry); + __ jr(t0); + __ bind(ok); +} + +void TemplateTable::iaload() +{ + transition(itos, itos); + __ mv(x11, x10); + __ pop_ptr(x10); + // x10: array + // x11: index + index_check(x10, x11); // leaves index in x11 + __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_INT) >> 2); + __ shadd(x10, x11, x10, t0, 2); + __ access_load_at(T_INT, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg); + __ sign_extend(x10, x10, 32); +} + +void TemplateTable::laload() +{ + transition(itos, ltos); + __ mv(x11, x10); + __ pop_ptr(x10); + // x10: array + // x11: index + index_check(x10, x11); // leaves index in x11 + __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_LONG) >> 3); + __ shadd(x10, x11, x10, t0, 3); + __ access_load_at(T_LONG, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg); +} + +void TemplateTable::faload() +{ + transition(itos, ftos); + __ mv(x11, x10); + __ pop_ptr(x10); + // x10: array + // x11: index + index_check(x10, x11); // leaves index in x11 + __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_FLOAT) >> 2); + __ shadd(x10, x11, x10, t0, 2); + __ access_load_at(T_FLOAT, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg); +} + +void TemplateTable::daload() +{ + transition(itos, dtos); + __ mv(x11, x10); + __ pop_ptr(x10); + // x10: array + // x11: index + index_check(x10, x11); // leaves index in x11 + __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_DOUBLE) >> 3); + __ shadd(x10, x11, x10, t0, 3); + __ access_load_at(T_DOUBLE, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg); +} + +void TemplateTable::aaload() +{ + transition(itos, atos); + __ mv(x11, x10); + __ pop_ptr(x10); + // x10: array + // x11: index + index_check(x10, x11); // leaves index in x11 + __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop); + __ shadd(x10, x11, x10, t0, LogBytesPerHeapOop); + do_oop_load(_masm, + Address(x10), + x10, + IS_ARRAY); +} + +void TemplateTable::baload() +{ + transition(itos, itos); + __ mv(x11, x10); + __ pop_ptr(x10); + // x10: array + // x11: index + index_check(x10, x11); // leaves index in x11 + __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_BYTE) >> 0); + __ shadd(x10, x11, x10, t0, 0); + __ access_load_at(T_BYTE, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg); +} + +void TemplateTable::caload() +{ + transition(itos, itos); + __ mv(x11, x10); + __ pop_ptr(x10); + // x10: array + // x11: index + index_check(x10, x11); // leaves index in x11 + __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1); + __ shadd(x10, x11, x10, t0, 1); + __ access_load_at(T_CHAR, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg); +} + +// iload followed by caload frequent pair +void TemplateTable::fast_icaload() +{ + transition(vtos, itos); + // load index out of locals + locals_index(x12); + __ lw(x11, iaddress(x12, x11, _masm)); + __ pop_ptr(x10); + + // x10: array + // x11: index + index_check(x10, x11); // leaves index in x11, kills t0 + __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1); // addi, max imm is 2^11 + __ shadd(x10, x11, x10, t0, 1); + __ access_load_at(T_CHAR, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg); +} + +void TemplateTable::saload() +{ + transition(itos, itos); + __ mv(x11, x10); + __ pop_ptr(x10); + // x10: array + // x11: index + index_check(x10, x11); // leaves index in x11, kills t0 + __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_SHORT) >> 1); + __ shadd(x10, x11, x10, t0, 1); + __ access_load_at(T_SHORT, IN_HEAP | IS_ARRAY, x10, Address(x10), noreg, noreg); +} + +void TemplateTable::iload(int n) +{ + transition(vtos, itos); + __ lw(x10, iaddress(n)); +} + +void TemplateTable::lload(int n) +{ + transition(vtos, ltos); + __ ld(x10, laddress(n)); +} + +void TemplateTable::fload(int n) +{ + transition(vtos, ftos); + __ flw(f10, faddress(n)); +} + +void TemplateTable::dload(int n) +{ + transition(vtos, dtos); + __ fld(f10, daddress(n)); +} + +void TemplateTable::aload(int n) +{ + transition(vtos, atos); + __ ld(x10, iaddress(n)); +} + +void TemplateTable::aload_0() { + aload_0_internal(); +} + +void TemplateTable::nofast_aload_0() { + aload_0_internal(may_not_rewrite); +} + +void TemplateTable::aload_0_internal(RewriteControl rc) { + // According to bytecode histograms, the pairs: + // + // _aload_0, _fast_igetfield + // _aload_0, _fast_agetfield + // _aload_0, _fast_fgetfield + // + // occur frequently. If RewriteFrequentPairs is set, the (slow) + // _aload_0 bytecode checks if the next bytecode is either + // _fast_igetfield, _fast_agetfield or _fast_fgetfield and then + // rewrites the current bytecode into a pair bytecode; otherwise it + // rewrites the current bytecode into _fast_aload_0 that doesn't do + // the pair check anymore. + // + // Note: If the next bytecode is _getfield, the rewrite must be + // delayed, otherwise we may miss an opportunity for a pair. + // + // Also rewrite frequent pairs + // aload_0, aload_1 + // aload_0, iload_1 + // These bytecodes with a small amount of code are most profitable + // to rewrite + if (RewriteFrequentPairs && rc == may_rewrite) { + Label rewrite, done; + const Register bc = x14; + + // get next bytecode + __ load_unsigned_byte(x11, at_bcp(Bytecodes::length_for(Bytecodes::_aload_0))); + + // if _getfield then wait with rewrite + __ mv(t1, Bytecodes::Bytecodes::_getfield); + __ beq(x11, t1, done); + + // if _igetfield then rewrite to _fast_iaccess_0 + assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "fix bytecode definition"); + __ mv(t1, Bytecodes::_fast_igetfield); + __ mv(bc, Bytecodes::_fast_iaccess_0); + __ beq(x11, t1, rewrite); + + // if _agetfield then rewrite to _fast_aaccess_0 + assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "fix bytecode definition"); + __ mv(t1, Bytecodes::_fast_agetfield); + __ mv(bc, Bytecodes::_fast_aaccess_0); + __ beq(x11, t1, rewrite); + + // if _fgetfield then rewrite to _fast_faccess_0 + assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "fix bytecode definition"); + __ mv(t1, Bytecodes::_fast_fgetfield); + __ mv(bc, Bytecodes::_fast_faccess_0); + __ beq(x11, t1, rewrite); + + // else rewrite to _fast_aload0 + assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "fix bytecode definition"); + __ mv(bc, Bytecodes::Bytecodes::_fast_aload_0); + + // rewrite + // bc: new bytecode + __ bind(rewrite); + patch_bytecode(Bytecodes::_aload_0, bc, x11, false); + + __ bind(done); + } + + // Do actual aload_0 (must do this after patch_bytecode which might call VM and GC might change oop). + aload(0); +} + +void TemplateTable::istore() +{ + transition(itos, vtos); + locals_index(x11); + __ sw(x10, iaddress(x11, t0, _masm)); +} + +void TemplateTable::lstore() +{ + transition(ltos, vtos); + locals_index(x11); + __ sd(x10, laddress(x11, t0, _masm)); +} + +void TemplateTable::fstore() { + transition(ftos, vtos); + locals_index(x11); + __ fsw(f10, iaddress(x11, t0, _masm)); +} + +void TemplateTable::dstore() { + transition(dtos, vtos); + locals_index(x11); + __ fsd(f10, daddress(x11, t0, _masm)); +} + +void TemplateTable::astore() +{ + transition(vtos, vtos); + __ pop_ptr(x10); + locals_index(x11); + __ sd(x10, aaddress(x11, t0, _masm)); +} + +void TemplateTable::wide_istore() { + transition(vtos, vtos); + __ pop_i(); + locals_index_wide(x11); + __ sw(x10, iaddress(x11, t0, _masm)); +} + +void TemplateTable::wide_lstore() { + transition(vtos, vtos); + __ pop_l(); + locals_index_wide(x11); + __ sd(x10, laddress(x11, t0, _masm)); +} + +void TemplateTable::wide_fstore() { + transition(vtos, vtos); + __ pop_f(); + locals_index_wide(x11); + __ fsw(f10, faddress(x11, t0, _masm)); +} + +void TemplateTable::wide_dstore() { + transition(vtos, vtos); + __ pop_d(); + locals_index_wide(x11); + __ fsd(f10, daddress(x11, t0, _masm)); +} + +void TemplateTable::wide_astore() { + transition(vtos, vtos); + __ pop_ptr(x10); + locals_index_wide(x11); + __ sd(x10, aaddress(x11, t0, _masm)); +} + +void TemplateTable::iastore() { + transition(itos, vtos); + __ pop_i(x11); + __ pop_ptr(x13); + // x10: value + // x11: index + // x13: array + index_check(x13, x11); // prefer index in x11 + __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_INT) >> 2); + __ shadd(t0, x11, x13, t0, 2); + __ access_store_at(T_INT, IN_HEAP | IS_ARRAY, Address(t0, 0), x10, noreg, noreg); +} + +void TemplateTable::lastore() { + transition(ltos, vtos); + __ pop_i(x11); + __ pop_ptr(x13); + // x10: value + // x11: index + // x13: array + index_check(x13, x11); // prefer index in x11 + __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_LONG) >> 3); + __ shadd(t0, x11, x13, t0, 3); + __ access_store_at(T_LONG, IN_HEAP | IS_ARRAY, Address(t0, 0), x10, noreg, noreg); +} + +void TemplateTable::fastore() { + transition(ftos, vtos); + __ pop_i(x11); + __ pop_ptr(x13); + // f10: value + // x11: index + // x13: array + index_check(x13, x11); // prefer index in x11 + __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_FLOAT) >> 2); + __ shadd(t0, x11, x13, t0, 2); + __ access_store_at(T_FLOAT, IN_HEAP | IS_ARRAY, Address(t0, 0), noreg /* ftos */, noreg, noreg); +} + +void TemplateTable::dastore() { + transition(dtos, vtos); + __ pop_i(x11); + __ pop_ptr(x13); + // f10: value + // x11: index + // x13: array + index_check(x13, x11); // prefer index in x11 + __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_DOUBLE) >> 3); + __ shadd(t0, x11, x13, t0, 3); + __ access_store_at(T_DOUBLE, IN_HEAP | IS_ARRAY, Address(t0, 0), noreg /* dtos */, noreg, noreg); +} + +void TemplateTable::aastore() { + Label is_null, ok_is_subtype, done; + transition(vtos, vtos); + // stack: ..., array, index, value + __ ld(x10, at_tos()); // value + __ ld(x12, at_tos_p1()); // index + __ ld(x13, at_tos_p2()); // array + + index_check(x13, x12); // kills x11 + __ add(x14, x12, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop); + __ shadd(x14, x14, x13, x14, LogBytesPerHeapOop); + + Address element_address(x14, 0); + + // do array store check - check for NULL value first + __ beqz(x10, is_null); + + // Move subklass into x11 + __ load_klass(x11, x10); + // Move superklass into x10 + __ load_klass(x10, x13); + __ ld(x10, Address(x10, + ObjArrayKlass::element_klass_offset())); + // Compress array + index * oopSize + 12 into a single register. Frees x12. + + // Generate subtype check. Blows x12, x15 + // Superklass in x10. Subklass in x11. + __ gen_subtype_check(x11, ok_is_subtype); //todo + + // Come here on failure + // object is at TOS + __ j(Interpreter::_throw_ArrayStoreException_entry); + + // Come here on success + __ bind(ok_is_subtype); + + // Get the value we will store + __ ld(x10, at_tos()); + // Now store using the appropriate barrier + do_oop_store(_masm, element_address, x10, IS_ARRAY); + __ j(done); + + // Have a NULL in x10, x13=array, x12=index. Store NULL at ary[idx] + __ bind(is_null); + __ profile_null_seen(x12); + + // Store a NULL + do_oop_store(_masm, element_address, noreg, IS_ARRAY); + + // Pop stack arguments + __ bind(done); + __ add(esp, esp, 3 * Interpreter::stackElementSize); + +} + +void TemplateTable::bastore() +{ + transition(itos, vtos); + __ pop_i(x11); + __ pop_ptr(x13); + // x10: value + // x11: index + // x13: array + index_check(x13, x11); // prefer index in x11 + + // Need to check whether array is boolean or byte + // since both types share the bastore bytecode. + __ load_klass(x12, x13); + __ lwu(x12, Address(x12, Klass::layout_helper_offset())); + Label L_skip; + __ test_bit(t0, x12, exact_log2(Klass::layout_helper_boolean_diffbit())); + __ beqz(t0, L_skip); + __ andi(x10, x10, 1); // if it is a T_BOOLEAN array, mask the stored value to 0/1 + __ bind(L_skip); + + __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_BYTE) >> 0); + + __ add(x11, x13, x11); + __ access_store_at(T_BYTE, IN_HEAP | IS_ARRAY, Address(x11, 0), x10, noreg, noreg); +} + +void TemplateTable::castore() +{ + transition(itos, vtos); + __ pop_i(x11); + __ pop_ptr(x13); + // x10: value + // x11: index + // x13: array + index_check(x13, x11); // prefer index in x11 + __ add(x11, x11, arrayOopDesc::base_offset_in_bytes(T_CHAR) >> 1); + __ shadd(t0, x11, x13, t0, 1); + __ access_store_at(T_CHAR, IN_HEAP | IS_ARRAY, Address(t0, 0), x10, noreg, noreg); +} + +void TemplateTable::sastore() +{ + castore(); +} + +void TemplateTable::istore(int n) +{ + transition(itos, vtos); + __ sd(x10, iaddress(n)); +} + +void TemplateTable::lstore(int n) +{ + transition(ltos, vtos); + __ sd(x10, laddress(n)); +} + +void TemplateTable::fstore(int n) +{ + transition(ftos, vtos); + __ fsw(f10, faddress(n)); +} + +void TemplateTable::dstore(int n) +{ + transition(dtos, vtos); + __ fsd(f10, daddress(n)); +} + +void TemplateTable::astore(int n) +{ + transition(vtos, vtos); + __ pop_ptr(x10); + __ sd(x10, iaddress(n)); +} + +void TemplateTable::pop() +{ + transition(vtos, vtos); + __ addi(esp, esp, Interpreter::stackElementSize); +} + +void TemplateTable::pop2() +{ + transition(vtos, vtos); + __ addi(esp, esp, 2 * Interpreter::stackElementSize); +} + +void TemplateTable::dup() +{ + transition(vtos, vtos); + __ ld(x10, Address(esp, 0)); + __ push_reg(x10); + // stack: ..., a, a +} + +void TemplateTable::dup_x1() +{ + transition(vtos, vtos); + // stack: ..., a, b + __ ld(x10, at_tos()); // load b + __ ld(x12, at_tos_p1()); // load a + __ sd(x10, at_tos_p1()); // store b + __ sd(x12, at_tos()); // store a + __ push_reg(x10); // push b + // stack: ..., b, a, b +} + +void TemplateTable::dup_x2() +{ + transition(vtos, vtos); + // stack: ..., a, b, c + __ ld(x10, at_tos()); // load c + __ ld(x12, at_tos_p2()); // load a + __ sd(x10, at_tos_p2()); // store c in a + __ push_reg(x10); // push c + // stack: ..., c, b, c, c + __ ld(x10, at_tos_p2()); // load b + __ sd(x12, at_tos_p2()); // store a in b + // stack: ..., c, a, c, c + __ sd(x10, at_tos_p1()); // store b in c + // stack: ..., c, a, b, c +} + +void TemplateTable::dup2() +{ + transition(vtos, vtos); + // stack: ..., a, b + __ ld(x10, at_tos_p1()); // load a + __ push_reg(x10); // push a + __ ld(x10, at_tos_p1()); // load b + __ push_reg(x10); // push b + // stack: ..., a, b, a, b +} + +void TemplateTable::dup2_x1() +{ + transition(vtos, vtos); + // stack: ..., a, b, c + __ ld(x12, at_tos()); // load c + __ ld(x10, at_tos_p1()); // load b + __ push_reg(x10); // push b + __ push_reg(x12); // push c + // stack: ..., a, b, c, b, c + __ sd(x12, at_tos_p3()); // store c in b + // stack: ..., a, c, c, b, c + __ ld(x12, at_tos_p4()); // load a + __ sd(x12, at_tos_p2()); // store a in 2nd c + // stack: ..., a, c, a, b, c + __ sd(x10, at_tos_p4()); // store b in a + // stack: ..., b, c, a, b, c +} + +void TemplateTable::dup2_x2() +{ + transition(vtos, vtos); + // stack: ..., a, b, c, d + __ ld(x12, at_tos()); // load d + __ ld(x10, at_tos_p1()); // load c + __ push_reg(x10); // push c + __ push_reg(x12); // push d + // stack: ..., a, b, c, d, c, d + __ ld(x10, at_tos_p4()); // load b + __ sd(x10, at_tos_p2()); // store b in d + __ sd(x12, at_tos_p4()); // store d in b + // stack: ..., a, d, c, b, c, d + __ ld(x12, at_tos_p5()); // load a + __ ld(x10, at_tos_p3()); // load c + __ sd(x12, at_tos_p3()); // store a in c + __ sd(x10, at_tos_p5()); // store c in a + // stack: ..., c, d, a, b, c, d +} + +void TemplateTable::swap() +{ + transition(vtos, vtos); + // stack: ..., a, b + __ ld(x12, at_tos_p1()); // load a + __ ld(x10, at_tos()); // load b + __ sd(x12, at_tos()); // store a in b + __ sd(x10, at_tos_p1()); // store b in a + // stack: ..., b, a +} + +void TemplateTable::iop2(Operation op) +{ + transition(itos, itos); + // x10 <== x11 op x10 + __ pop_i(x11); + switch (op) { + case add : __ addw(x10, x11, x10); break; + case sub : __ subw(x10, x11, x10); break; + case mul : __ mulw(x10, x11, x10); break; + case _and : __ andrw(x10, x11, x10); break; + case _or : __ orrw(x10, x11, x10); break; + case _xor : __ xorrw(x10, x11, x10); break; + case shl : __ sllw(x10, x11, x10); break; + case shr : __ sraw(x10, x11, x10); break; + case ushr : __ srlw(x10, x11, x10); break; + default : ShouldNotReachHere(); + } +} + +void TemplateTable::lop2(Operation op) +{ + transition(ltos, ltos); + // x10 <== x11 op x10 + __ pop_l(x11); + switch (op) { + case add : __ add(x10, x11, x10); break; + case sub : __ sub(x10, x11, x10); break; + case mul : __ mul(x10, x11, x10); break; + case _and : __ andr(x10, x11, x10); break; + case _or : __ orr(x10, x11, x10); break; + case _xor : __ xorr(x10, x11, x10); break; + default : ShouldNotReachHere(); + } +} + +void TemplateTable::idiv() +{ + transition(itos, itos); + // explicitly check for div0 + Label no_div0; + __ bnez(x10, no_div0); + __ mv(t0, Interpreter::_throw_ArithmeticException_entry); + __ jr(t0); + __ bind(no_div0); + __ pop_i(x11); + // x10 <== x11 idiv x10 + __ corrected_idivl(x10, x11, x10, /* want_remainder */ false); +} + +void TemplateTable::irem() +{ + transition(itos, itos); + // explicitly check for div0 + Label no_div0; + __ bnez(x10, no_div0); + __ mv(t0, Interpreter::_throw_ArithmeticException_entry); + __ jr(t0); + __ bind(no_div0); + __ pop_i(x11); + // x10 <== x11 irem x10 + __ corrected_idivl(x10, x11, x10, /* want_remainder */ true); +} + +void TemplateTable::lmul() +{ + transition(ltos, ltos); + __ pop_l(x11); + __ mul(x10, x10, x11); +} + +void TemplateTable::ldiv() +{ + transition(ltos, ltos); + // explicitly check for div0 + Label no_div0; + __ bnez(x10, no_div0); + __ mv(t0, Interpreter::_throw_ArithmeticException_entry); + __ jr(t0); + __ bind(no_div0); + __ pop_l(x11); + // x10 <== x11 ldiv x10 + __ corrected_idivq(x10, x11, x10, /* want_remainder */ false); +} + +void TemplateTable::lrem() +{ + transition(ltos, ltos); + // explicitly check for div0 + Label no_div0; + __ bnez(x10, no_div0); + __ mv(t0, Interpreter::_throw_ArithmeticException_entry); + __ jr(t0); + __ bind(no_div0); + __ pop_l(x11); + // x10 <== x11 lrem x10 + __ corrected_idivq(x10, x11, x10, /* want_remainder */ true); +} + +void TemplateTable::lshl() +{ + transition(itos, ltos); + // shift count is in x10 + __ pop_l(x11); + __ sll(x10, x11, x10); +} + +void TemplateTable::lshr() +{ + transition(itos, ltos); + // shift count is in x10 + __ pop_l(x11); + __ sra(x10, x11, x10); +} + +void TemplateTable::lushr() +{ + transition(itos, ltos); + // shift count is in x10 + __ pop_l(x11); + __ srl(x10, x11, x10); +} + +void TemplateTable::fop2(Operation op) +{ + transition(ftos, ftos); + switch (op) { + case add: + __ pop_f(f11); + __ fadd_s(f10, f11, f10); + break; + case sub: + __ pop_f(f11); + __ fsub_s(f10, f11, f10); + break; + case mul: + __ pop_f(f11); + __ fmul_s(f10, f11, f10); + break; + case div: + __ pop_f(f11); + __ fdiv_s(f10, f11, f10); + break; + case rem: + __ fmv_s(f11, f10); + __ pop_f(f10); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::frem)); + break; + default: + ShouldNotReachHere(); + } +} + +void TemplateTable::dop2(Operation op) +{ + transition(dtos, dtos); + switch (op) { + case add: + __ pop_d(f11); + __ fadd_d(f10, f11, f10); + break; + case sub: + __ pop_d(f11); + __ fsub_d(f10, f11, f10); + break; + case mul: + __ pop_d(f11); + __ fmul_d(f10, f11, f10); + break; + case div: + __ pop_d(f11); + __ fdiv_d(f10, f11, f10); + break; + case rem: + __ fmv_d(f11, f10); + __ pop_d(f10); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::drem)); + break; + default: + ShouldNotReachHere(); + } +} + +void TemplateTable::ineg() +{ + transition(itos, itos); + __ negw(x10, x10); +} + +void TemplateTable::lneg() +{ + transition(ltos, ltos); + __ neg(x10, x10); +} + +void TemplateTable::fneg() +{ + transition(ftos, ftos); + __ fneg_s(f10, f10); +} + +void TemplateTable::dneg() +{ + transition(dtos, dtos); + __ fneg_d(f10, f10); +} + +void TemplateTable::iinc() +{ + transition(vtos, vtos); + __ load_signed_byte(x11, at_bcp(2)); // get constant + locals_index(x12); + __ ld(x10, iaddress(x12, x10, _masm)); + __ addw(x10, x10, x11); + __ sd(x10, iaddress(x12, t0, _masm)); +} + +void TemplateTable::wide_iinc() +{ + transition(vtos, vtos); + __ lwu(x11, at_bcp(2)); // get constant and index + __ revb_h_w_u(x11, x11); // reverse bytes in half-word (32bit) and zero-extend + __ zero_extend(x12, x11, 16); + __ neg(x12, x12); + __ slli(x11, x11, 32); + __ srai(x11, x11, 48); + __ ld(x10, iaddress(x12, t0, _masm)); + __ addw(x10, x10, x11); + __ sd(x10, iaddress(x12, t0, _masm)); +} + +void TemplateTable::convert() +{ + // Checking +#ifdef ASSERT + { + TosState tos_in = ilgl; + TosState tos_out = ilgl; + switch (bytecode()) { + case Bytecodes::_i2l: // fall through + case Bytecodes::_i2f: // fall through + case Bytecodes::_i2d: // fall through + case Bytecodes::_i2b: // fall through + case Bytecodes::_i2c: // fall through + case Bytecodes::_i2s: tos_in = itos; break; + case Bytecodes::_l2i: // fall through + case Bytecodes::_l2f: // fall through + case Bytecodes::_l2d: tos_in = ltos; break; + case Bytecodes::_f2i: // fall through + case Bytecodes::_f2l: // fall through + case Bytecodes::_f2d: tos_in = ftos; break; + case Bytecodes::_d2i: // fall through + case Bytecodes::_d2l: // fall through + case Bytecodes::_d2f: tos_in = dtos; break; + default : ShouldNotReachHere(); + } + switch (bytecode()) { + case Bytecodes::_l2i: // fall through + case Bytecodes::_f2i: // fall through + case Bytecodes::_d2i: // fall through + case Bytecodes::_i2b: // fall through + case Bytecodes::_i2c: // fall through + case Bytecodes::_i2s: tos_out = itos; break; + case Bytecodes::_i2l: // fall through + case Bytecodes::_f2l: // fall through + case Bytecodes::_d2l: tos_out = ltos; break; + case Bytecodes::_i2f: // fall through + case Bytecodes::_l2f: // fall through + case Bytecodes::_d2f: tos_out = ftos; break; + case Bytecodes::_i2d: // fall through + case Bytecodes::_l2d: // fall through + case Bytecodes::_f2d: tos_out = dtos; break; + default : ShouldNotReachHere(); + } + transition(tos_in, tos_out); + } +#endif // ASSERT + + // Conversion + switch (bytecode()) { + case Bytecodes::_i2l: + __ sign_extend(x10, x10, 32); + break; + case Bytecodes::_i2f: + __ fcvt_s_w(f10, x10); + break; + case Bytecodes::_i2d: + __ fcvt_d_w(f10, x10); + break; + case Bytecodes::_i2b: + __ sign_extend(x10, x10, 8); + break; + case Bytecodes::_i2c: + __ zero_extend(x10, x10, 16); + break; + case Bytecodes::_i2s: + __ sign_extend(x10, x10, 16); + break; + case Bytecodes::_l2i: + __ sign_extend(x10, x10, 32); + break; + case Bytecodes::_l2f: + __ fcvt_s_l(f10, x10); + break; + case Bytecodes::_l2d: + __ fcvt_d_l(f10, x10); + break; + case Bytecodes::_f2i: + __ fcvt_w_s_safe(x10, f10); + break; + case Bytecodes::_f2l: + __ fcvt_l_s_safe(x10, f10); + break; + case Bytecodes::_f2d: + __ fcvt_d_s(f10, f10); + break; + case Bytecodes::_d2i: + __ fcvt_w_d_safe(x10, f10); + break; + case Bytecodes::_d2l: + __ fcvt_l_d_safe(x10, f10); + break; + case Bytecodes::_d2f: + __ fcvt_s_d(f10, f10); + break; + default: + ShouldNotReachHere(); + } +} + +void TemplateTable::lcmp() +{ + transition(ltos, itos); + __ pop_l(x11); + __ cmp_l2i(t0, x11, x10); + __ mv(x10, t0); +} + +void TemplateTable::float_cmp(bool is_float, int unordered_result) +{ + // For instruction feq, flt and fle, the result is 0 if either operand is NaN + if (is_float) { + __ pop_f(f11); + // if unordered_result < 0: + // we want -1 for unordered or less than, 0 for equal and 1 for + // greater than. + // else: + // we want -1 for less than, 0 for equal and 1 for unordered or + // greater than. + // f11 primary, f10 secondary + __ float_compare(x10, f11, f10, unordered_result); + } else { + __ pop_d(f11); + // if unordered_result < 0: + // we want -1 for unordered or less than, 0 for equal and 1 for + // greater than. + // else: + // we want -1 for less than, 0 for equal and 1 for unordered or + // greater than. + // f11 primary, f10 secondary + __ double_compare(x10, f11, f10, unordered_result); + } +} + +void TemplateTable::branch(bool is_jsr, bool is_wide) +{ + __ profile_taken_branch(x10, x11); + const ByteSize be_offset = MethodCounters::backedge_counter_offset() + + InvocationCounter::counter_offset(); + const ByteSize inv_offset = MethodCounters::invocation_counter_offset() + + InvocationCounter::counter_offset(); + + // load branch displacement + if (!is_wide) { + __ lhu(x12, at_bcp(1)); + __ revb_h_h(x12, x12); // reverse bytes in half-word and sign-extend + } else { + __ lwu(x12, at_bcp(1)); + __ revb_w_w(x12, x12); // reverse bytes in word and sign-extend + } + + // Handle all the JSR stuff here, then exit. + // It's much shorter and cleaner than intermingling with the non-JSR + // normal-branch stuff occurring below. + + if (is_jsr) { + // compute return address as bci + __ ld(t1, Address(xmethod, Method::const_offset())); + __ add(t1, t1, + in_bytes(ConstMethod::codes_offset()) - (is_wide ? 5 : 3)); + __ sub(x11, xbcp, t1); + __ push_i(x11); + // Adjust the bcp by the 16-bit displacement in x12 + __ add(xbcp, xbcp, x12); + __ load_unsigned_byte(t0, Address(xbcp, 0)); + // load the next target bytecode into t0, it is the argument of dispatch_only + __ dispatch_only(vtos, /*generate_poll*/true); + return; + } + + // Normal (non-jsr) branch handling + + // Adjust the bcp by the displacement in x12 + __ add(xbcp, xbcp, x12); + + assert(UseLoopCounter || !UseOnStackReplacement, + "on-stack-replacement requires loop counters"); + Label backedge_counter_overflow; + Label dispatch; + if (UseLoopCounter) { + // increment backedge counter for backward branches + // x10: MDO + // x11: MDO bumped taken-count + // x12: target offset + __ bgtz(x12, dispatch); // count only if backward branch + + // check if MethodCounters exists + Label has_counters; + __ ld(t0, Address(xmethod, Method::method_counters_offset())); + __ bnez(t0, has_counters); + __ push_reg(x10); + __ push_reg(x11); + __ push_reg(x12); + __ call_VM(noreg, CAST_FROM_FN_PTR(address, + InterpreterRuntime::build_method_counters), xmethod); + __ pop_reg(x12); + __ pop_reg(x11); + __ pop_reg(x10); + __ ld(t0, Address(xmethod, Method::method_counters_offset())); + __ beqz(t0, dispatch); // No MethodCounters allocated, OutOfMemory + __ bind(has_counters); + + Label no_mdo; + int increment = InvocationCounter::count_increment; + if (ProfileInterpreter) { + // Are we profiling? + __ ld(x11, Address(xmethod, in_bytes(Method::method_data_offset()))); + __ beqz(x11, no_mdo); + // Increment the MDO backedge counter + const Address mdo_backedge_counter(x11, in_bytes(MethodData::backedge_counter_offset()) + + in_bytes(InvocationCounter::counter_offset())); + const Address mask(x11, in_bytes(MethodData::backedge_mask_offset())); + __ increment_mask_and_jump(mdo_backedge_counter, increment, mask, + x10, t0, false, + UseOnStackReplacement ? &backedge_counter_overflow : &dispatch); + __ j(dispatch); + } + __ bind(no_mdo); + // Increment backedge counter in MethodCounters* + __ ld(t0, Address(xmethod, Method::method_counters_offset())); + const Address mask(t0, in_bytes(MethodCounters::backedge_mask_offset())); + __ increment_mask_and_jump(Address(t0, be_offset), increment, mask, + x10, t1, false, + UseOnStackReplacement ? &backedge_counter_overflow : &dispatch); + __ bind(dispatch); + } + + // Pre-load the next target bytecode into t0 + __ load_unsigned_byte(t0, Address(xbcp, 0)); + + // continue with the bytecode @ target + // t0: target bytecode + // xbcp: target bcp + __ dispatch_only(vtos, /*generate_poll*/true); + + if (UseLoopCounter && UseOnStackReplacement) { + // invocation counter overflow + __ bind(backedge_counter_overflow); + __ neg(x12, x12); + __ add(x12, x12, xbcp); // branch xbcp + // IcoResult frequency_counter_overflow([JavaThread*], address branch_bcp) + __ call_VM(noreg, + CAST_FROM_FN_PTR(address, + InterpreterRuntime::frequency_counter_overflow), + x12); + __ load_unsigned_byte(x11, Address(xbcp, 0)); // restore target bytecode + + // x10: osr nmethod (osr ok) or NULL (osr not possible) + // w11: target bytecode + // x12: temporary + __ beqz(x10, dispatch); // test result -- no osr if null + // nmethod may have been invalidated (VM may block upon call_VM return) + __ lbu(x12, Address(x10, nmethod::state_offset())); + if (nmethod::in_use != 0) { + __ sub(x12, x12, nmethod::in_use); + } + __ bnez(x12, dispatch); + + // We have the address of an on stack replacement routine in x10 + // We need to prepare to execute the OSR method. First we must + // migrate the locals and monitors off of the stack. + + __ mv(x9, x10); // save the nmethod + + call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin)); + + // x10 is OSR buffer, move it to expected parameter location + __ mv(j_rarg0, x10); + + // remove activation + // get sender esp + __ ld(esp, + Address(fp, frame::interpreter_frame_sender_sp_offset * wordSize)); + // remove frame anchor + __ leave(); + // Ensure compiled code always sees stack at proper alignment + __ andi(sp, esp, -16); + + // and begin the OSR nmethod + __ ld(t0, Address(x9, nmethod::osr_entry_point_offset())); + __ jr(t0); + } +} + +void TemplateTable::if_0cmp(Condition cc) +{ + transition(itos, vtos); + // assume branch is more often taken than not (loops use backward branches) + Label not_taken; + + __ sign_extend(x10, x10, 32); + switch (cc) { + case equal: + __ bnez(x10, not_taken); + break; + case not_equal: + __ beqz(x10, not_taken); + break; + case less: + __ bgez(x10, not_taken); + break; + case less_equal: + __ bgtz(x10, not_taken); + break; + case greater: + __ blez(x10, not_taken); + break; + case greater_equal: + __ bltz(x10, not_taken); + break; + default: + break; + } + + branch(false, false); + __ bind(not_taken); + __ profile_not_taken_branch(x10); +} + +void TemplateTable::if_icmp(Condition cc) +{ + transition(itos, vtos); + // assume branch is more often taken than not (loops use backward branches) + Label not_taken; + __ pop_i(x11); + __ sign_extend(x10, x10, 32); + switch (cc) { + case equal: + __ bne(x11, x10, not_taken); + break; + case not_equal: + __ beq(x11, x10, not_taken); + break; + case less: + __ bge(x11, x10, not_taken); + break; + case less_equal: + __ bgt(x11, x10, not_taken); + break; + case greater: + __ ble(x11, x10, not_taken); + break; + case greater_equal: + __ blt(x11, x10, not_taken); + break; + default: + break; + } + + branch(false, false); + __ bind(not_taken); + __ profile_not_taken_branch(x10); +} + +void TemplateTable::if_nullcmp(Condition cc) +{ + transition(atos, vtos); + // assume branch is more often taken than not (loops use backward branches) + Label not_taken; + if (cc == equal) { + __ bnez(x10, not_taken); + } else { + __ beqz(x10, not_taken); + } + branch(false, false); + __ bind(not_taken); + __ profile_not_taken_branch(x10); +} + +void TemplateTable::if_acmp(Condition cc) +{ + transition(atos, vtos); + // assume branch is more often taken than not (loops use backward branches) + Label not_taken; + __ pop_ptr(x11); + + if (cc == equal) { + __ bne(x11, x10, not_taken); + } else if (cc == not_equal) { + __ beq(x11, x10, not_taken); + } + branch(false, false); + __ bind(not_taken); + __ profile_not_taken_branch(x10); +} + +void TemplateTable::ret() { + transition(vtos, vtos); + locals_index(x11); + __ ld(x11, aaddress(x11, t1, _masm)); // get return bci, compute return bcp + __ profile_ret(x11, x12); + __ ld(xbcp, Address(xmethod, Method::const_offset())); + __ add(xbcp, xbcp, x11); + __ addi(xbcp, xbcp, in_bytes(ConstMethod::codes_offset())); + __ dispatch_next(vtos, 0, /*generate_poll*/true); +} + +void TemplateTable::wide_ret() { + transition(vtos, vtos); + locals_index_wide(x11); + __ ld(x11, aaddress(x11, t0, _masm)); // get return bci, compute return bcp + __ profile_ret(x11, x12); + __ ld(xbcp, Address(xmethod, Method::const_offset())); + __ add(xbcp, xbcp, x11); + __ add(xbcp, xbcp, in_bytes(ConstMethod::codes_offset())); + __ dispatch_next(vtos, 0, /*generate_poll*/true); +} + +void TemplateTable::tableswitch() { + Label default_case, continue_execution; + transition(itos, vtos); + // align xbcp + __ la(x11, at_bcp(BytesPerInt)); + __ andi(x11, x11, -BytesPerInt); + // load lo & hi + __ lwu(x12, Address(x11, BytesPerInt)); + __ lwu(x13, Address(x11, 2 * BytesPerInt)); + __ revb_w_w(x12, x12); // reverse bytes in word (32bit) and sign-extend + __ revb_w_w(x13, x13); // reverse bytes in word (32bit) and sign-extend + // check against lo & hi + __ blt(x10, x12, default_case); + __ bgt(x10, x13, default_case); + // lookup dispatch offset + __ subw(x10, x10, x12); + __ shadd(x13, x10, x11, t0, 2); + __ lwu(x13, Address(x13, 3 * BytesPerInt)); + __ profile_switch_case(x10, x11, x12); + // continue execution + __ bind(continue_execution); + __ revb_w_w(x13, x13); // reverse bytes in word (32bit) and sign-extend + __ add(xbcp, xbcp, x13); + __ load_unsigned_byte(t0, Address(xbcp)); + __ dispatch_only(vtos, /*generate_poll*/true); + // handle default + __ bind(default_case); + __ profile_switch_default(x10); + __ lwu(x13, Address(x11, 0)); + __ j(continue_execution); +} + +void TemplateTable::lookupswitch() { + transition(itos, itos); + __ stop("lookupswitch bytecode should have been rewritten"); +} + +void TemplateTable::fast_linearswitch() { + transition(itos, vtos); + Label loop_entry, loop, found, continue_execution; + // bswap x10 so we can avoid bswapping the table entries + __ revb_w_w(x10, x10); // reverse bytes in word (32bit) and sign-extend + // align xbcp + __ la(x9, at_bcp(BytesPerInt)); // btw: should be able to get rid of + // this instruction (change offsets + // below) + __ andi(x9, x9, -BytesPerInt); + // set counter + __ lwu(x11, Address(x9, BytesPerInt)); + __ revb_w(x11, x11); + __ j(loop_entry); + // table search + __ bind(loop); + __ shadd(t0, x11, x9, t0, 3); + __ lw(t0, Address(t0, 2 * BytesPerInt)); + __ beq(x10, t0, found); + __ bind(loop_entry); + __ addi(x11, x11, -1); + __ bgez(x11, loop); + // default case + __ profile_switch_default(x10); + __ lwu(x13, Address(x9, 0)); + __ j(continue_execution); + // entry found -> get offset + __ bind(found); + __ shadd(t0, x11, x9, t0, 3); + __ lwu(x13, Address(t0, 3 * BytesPerInt)); + __ profile_switch_case(x11, x10, x9); + // continue execution + __ bind(continue_execution); + __ revb_w_w(x13, x13); // reverse bytes in word (32bit) and sign-extend + __ add(xbcp, xbcp, x13); + __ lbu(t0, Address(xbcp, 0)); + __ dispatch_only(vtos, /*generate_poll*/true); +} + +void TemplateTable::fast_binaryswitch() { + transition(itos, vtos); + // Implementation using the following core algorithm: + // + // int binary_search(int key, LookupswitchPair* array, int n) + // binary_search start: + // #Binary search according to "Methodik des Programmierens" by + // # Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985. + // int i = 0; + // int j = n; + // while (i + 1 < j) do + // # invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q) + // # with Q: for all i: 0 <= i < n: key < a[i] + // # where a stands for the array and assuming that the (inexisting) + // # element a[n] is infinitely big. + // int h = (i + j) >> 1 + // # i < h < j + // if (key < array[h].fast_match()) + // then [j = h] + // else [i = h] + // end + // # R: a[i] <= key < a[i+1] or Q + // # (i.e., if key is within array, i is the correct index) + // return i + // binary_search end + + + // Register allocation + const Register key = x10; // already set (tosca) + const Register array = x11; + const Register i = x12; + const Register j = x13; + const Register h = x14; + const Register temp = x15; + + // Find array start + __ la(array, at_bcp(3 * BytesPerInt)); // btw: should be able to + // get rid of this + // instruction (change + // offsets below) + __ andi(array, array, -BytesPerInt); + + // Initialize i & j + __ mv(i, zr); // i = 0 + __ lwu(j, Address(array, -BytesPerInt)); // j = length(array) + + // Convert j into native byteordering + __ revb_w(j, j); + + // And start + Label entry; + __ j(entry); + + // binary search loop + { + Label loop; + __ bind(loop); + __ addw(h, i, j); // h = i + j + __ srliw(h, h, 1); // h = (i + j) >> 1 + // if [key < array[h].fast_match()] + // then [j = h] + // else [i = h] + // Convert array[h].match to native byte-ordering before compare + __ shadd(temp, h, array, temp, 3); + __ ld(temp, Address(temp, 0)); + __ revb_w_w(temp, temp); // reverse bytes in word (32bit) and sign-extend + + Label L_done, L_greater; + __ bge(key, temp, L_greater); + // if [key < array[h].fast_match()] then j = h + __ mv(j, h); + __ j(L_done); + __ bind(L_greater); + // if [key >= array[h].fast_match()] then i = h + __ mv(i, h); + __ bind(L_done); + + // while [i + 1 < j] + __ bind(entry); + __ addiw(h, i, 1); // i + 1 + __ blt(h, j, loop); // i + 1 < j + } + + // end of binary search, result index is i (must check again!) + Label default_case; + // Convert array[i].match to native byte-ordering before compare + __ shadd(temp, i, array, temp, 3); + __ ld(temp, Address(temp, 0)); + __ revb_w_w(temp, temp); // reverse bytes in word (32bit) and sign-extend + __ bne(key, temp, default_case); + + // entry found -> j = offset + __ shadd(temp, i, array, temp, 3); + __ lwu(j, Address(temp, BytesPerInt)); + __ profile_switch_case(i, key, array); + __ revb_w_w(j, j); // reverse bytes in word (32bit) and sign-extend + + __ add(temp, xbcp, j); + __ load_unsigned_byte(t0, Address(temp, 0)); + + __ add(xbcp, xbcp, j); + __ la(xbcp, Address(xbcp, 0)); + __ dispatch_only(vtos, /*generate_poll*/true); + + // default case -> j = default offset + __ bind(default_case); + __ profile_switch_default(i); + __ lwu(j, Address(array, -2 * BytesPerInt)); + __ revb_w_w(j, j); // reverse bytes in word (32bit) and sign-extend + + __ add(temp, xbcp, j); + __ load_unsigned_byte(t0, Address(temp, 0)); + + __ add(xbcp, xbcp, j); + __ la(xbcp, Address(xbcp, 0)); + __ dispatch_only(vtos, /*generate_poll*/true); +} + +void TemplateTable::_return(TosState state) +{ + transition(state, state); + assert(_desc->calls_vm(), + "inconsistent calls_vm information"); // call in remove_activation + + if (_desc->bytecode() == Bytecodes::_return_register_finalizer) { + assert(state == vtos, "only valid state"); + + __ ld(c_rarg1, aaddress(0)); + __ load_klass(x13, c_rarg1); + __ lwu(x13, Address(x13, Klass::access_flags_offset())); + Label skip_register_finalizer; + __ test_bit(t0, x13, exact_log2(JVM_ACC_HAS_FINALIZER)); + __ beqz(t0, skip_register_finalizer); + + __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), c_rarg1); + + __ bind(skip_register_finalizer); + } + + // Issue a StoreStore barrier after all stores but before return + // from any constructor for any class with a final field. We don't + // know if this is a finalizer, so we always do so. + if (_desc->bytecode() == Bytecodes::_return) { + __ membar(MacroAssembler::StoreStore); + } + + // Narrow result if state is itos but result type is smaller. + // Need to narrow in the return bytecode rather than in generate_return_entry + // since compiled code callers expect the result to already be narrowed. + if (state == itos) { + __ narrow(x10); + } + + __ remove_activation(state); + __ ret(); +} + + +// ---------------------------------------------------------------------------- +// Volatile variables demand their effects be made known to all CPU's +// in order. Store buffers on most chips allow reads & writes to +// reorder; the JMM's ReadAfterWrite.java test fails in -Xint mode +// without some kind of memory barrier (i.e., it's not sufficient that +// the interpreter does not reorder volatile references, the hardware +// also must not reorder them). +// +// According to the new Java Memory Model (JMM): +// (1) All volatiles are serialized wrt to each other. ALSO reads & +// writes act as aquire & release, so: +// (2) A read cannot let unrelated NON-volatile memory refs that +// happen after the read float up to before the read. It's OK for +// non-volatile memory refs that happen before the volatile read to +// float down below it. +// (3) Similar a volatile write cannot let unrelated NON-volatile +// memory refs that happen BEFORE the write float down to after the +// write. It's OK for non-volatile memory refs that happen after the +// volatile write to float up before it. +// +// We only put in barriers around volatile refs (they are expensive), +// not _between_ memory refs (that would require us to track the +// flavor of the previous memory refs). Requirements (2) and (3) +// require some barriers before volatile stores and after volatile +// loads. These nearly cover requirement (1) but miss the +// volatile-store-volatile-load case. This final case is placed after +// volatile-stores although it could just as well go before +// volatile-loads. + +void TemplateTable::resolve_cache_and_index(int byte_no, + Register Rcache, + Register index, + size_t index_size) { + const Register temp = x9; + assert_different_registers(Rcache, index, temp); + + Label resolved, clinit_barrier_slow; + + Bytecodes::Code code = bytecode(); + switch (code) { + case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break; + case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break; + default: break; + } + + assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range"); + __ get_cache_and_index_and_bytecode_at_bcp(Rcache, index, temp, byte_no, 1, index_size); + __ mv(t0, (int) code); + __ beq(temp, t0, resolved); + + // resolve first time through + // Class initialization barrier slow path lands here as well. + __ bind(clinit_barrier_slow); + + address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache); + __ mv(temp, (int) code); + __ call_VM(noreg, entry, temp); + + // Update registers with resolved info + __ get_cache_and_index_at_bcp(Rcache, index, 1, index_size); + // n.b. unlike x86 Rcache is now rcpool plus the indexed offset + // so all clients ofthis method must be modified accordingly + __ bind(resolved); + + // Class initialization barrier for static methods + if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) { + __ load_resolved_method_at_index(byte_no, temp, Rcache); + __ load_method_holder(temp, temp); + __ clinit_barrier(temp, t0, NULL, &clinit_barrier_slow); + } +} + +// The Rcache and index registers must be set before call +// n.b unlike x86 cache already includes the index offset +void TemplateTable::load_field_cp_cache_entry(Register obj, + Register cache, + Register index, + Register off, + Register flags, + bool is_static = false) { + assert_different_registers(cache, index, flags, off); + + ByteSize cp_base_offset = ConstantPoolCache::base_offset(); + // Field offset + __ ld(off, Address(cache, in_bytes(cp_base_offset + + ConstantPoolCacheEntry::f2_offset()))); + // Flags + __ lwu(flags, Address(cache, in_bytes(cp_base_offset + + ConstantPoolCacheEntry::flags_offset()))); + + // klass overwrite register + if (is_static) { + __ ld(obj, Address(cache, in_bytes(cp_base_offset + + ConstantPoolCacheEntry::f1_offset()))); + const int mirror_offset = in_bytes(Klass::java_mirror_offset()); + __ ld(obj, Address(obj, mirror_offset)); + __ resolve_oop_handle(obj); + } +} + +void TemplateTable::load_invoke_cp_cache_entry(int byte_no, + Register method, + Register itable_index, + Register flags, + bool is_invokevirtual, + bool is_invokevfinal, /*unused*/ + bool is_invokedynamic) { + // setup registers + const Register cache = t1; + const Register index = x14; + assert_different_registers(method, flags); + assert_different_registers(method, cache, index); + assert_different_registers(itable_index, flags); + assert_different_registers(itable_index, cache, index); + // determine constant pool cache field offsets + assert(is_invokevirtual == (byte_no == f2_byte), "is_invokevirtual flag redundant"); + const int method_offset = in_bytes(ConstantPoolCache::base_offset() + + (is_invokevirtual ? + ConstantPoolCacheEntry::f2_offset() : + ConstantPoolCacheEntry::f1_offset())); + const int flags_offset = in_bytes(ConstantPoolCache::base_offset() + + ConstantPoolCacheEntry::flags_offset()); + // access constant pool cache fields + const int index_offset = in_bytes(ConstantPoolCache::base_offset() + + ConstantPoolCacheEntry::f2_offset()); + + const size_t index_size = (is_invokedynamic ? sizeof(u4) : sizeof(u2)); + resolve_cache_and_index(byte_no, cache, index, index_size); + __ ld(method, Address(cache, method_offset)); + + if (itable_index != noreg) { + __ ld(itable_index, Address(cache, index_offset)); + } + __ lwu(flags, Address(cache, flags_offset)); +} + +// The registers cache and index expected to be set before call. +// Correct values of the cache and index registers are preserved. +void TemplateTable::jvmti_post_field_access(Register cache, Register index, + bool is_static, bool has_tos) { + // do the JVMTI work here to avoid disturbing the register state below + // We use c_rarg registers here beacause we want to use the register used in + // the call to the VM + if (JvmtiExport::can_post_field_access()) { + // Check to see if a field access watch has been set before we + // take the time to call into the VM. + Label L1; + assert_different_registers(cache, index, x10); + ExternalAddress target((address) JvmtiExport::get_field_access_count_addr()); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ lwu(x10, Address(t0, offset)); + }); + + __ beqz(x10, L1); + + __ get_cache_and_index_at_bcp(c_rarg2, c_rarg3, 1); + __ la(c_rarg2, Address(c_rarg2, in_bytes(ConstantPoolCache::base_offset()))); + + if (is_static) { + __ mv(c_rarg1, zr); // NULL object reference + } else { + __ ld(c_rarg1, at_tos()); // get object pointer without popping it + __ verify_oop(c_rarg1); + } + // c_rarg1: object pointer or NULL + // c_rarg2: cache entry pointer + // c_rarg3: jvalue object on the stack + __ call_VM(noreg, CAST_FROM_FN_PTR(address, + InterpreterRuntime::post_field_access), + c_rarg1, c_rarg2, c_rarg3); + __ get_cache_and_index_at_bcp(cache, index, 1); + __ bind(L1); + } +} + +void TemplateTable::pop_and_check_object(Register r) +{ + __ pop_ptr(r); + __ null_check(r); // for field access must check obj. + __ verify_oop(r); +} + +void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc) +{ + const Register cache = x12; + const Register index = x13; + const Register obj = x14; + const Register off = x9; + const Register flags = x10; + const Register raw_flags = x16; + const Register bc = x14; // uses same reg as obj, so don't mix them + + resolve_cache_and_index(byte_no, cache, index, sizeof(u2)); + jvmti_post_field_access(cache, index, is_static, false); + load_field_cp_cache_entry(obj, cache, index, off, raw_flags, is_static); + + if (!is_static) { + // obj is on the stack + pop_and_check_object(obj); + } + + __ add(off, obj, off); + const Address field(off); + + Label Done, notByte, notBool, notInt, notShort, notChar, + notLong, notFloat, notObj, notDouble; + + __ slli(flags, raw_flags, XLEN - (ConstantPoolCacheEntry::tos_state_shift + + ConstantPoolCacheEntry::tos_state_bits)); + __ srli(flags, flags, XLEN - ConstantPoolCacheEntry::tos_state_bits); + + assert(btos == 0, "change code, btos != 0"); + __ bnez(flags, notByte); + + // Dont't rewrite getstatic, only getfield + if (is_static) { + rc = may_not_rewrite; + } + + // btos + __ access_load_at(T_BYTE, IN_HEAP, x10, field, noreg, noreg); + __ push(btos); + // Rewrite bytecode to be faster + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_bgetfield, bc, x11); + } + __ j(Done); + + __ bind(notByte); + __ sub(t0, flags, (u1)ztos); + __ bnez(t0, notBool); + + // ztos (same code as btos) + __ access_load_at(T_BOOLEAN, IN_HEAP, x10, field, noreg, noreg); + __ push(ztos); + // Rewirte bytecode to be faster + if (rc == may_rewrite) { + // uses btos rewriting, no truncating to t/f bit is needed for getfield + patch_bytecode(Bytecodes::_fast_bgetfield, bc, x11); + } + __ j(Done); + + __ bind(notBool); + __ sub(t0, flags, (u1)atos); + __ bnez(t0, notObj); + // atos + do_oop_load(_masm, field, x10, IN_HEAP); + __ push(atos); + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_agetfield, bc, x11); + } + __ j(Done); + + __ bind(notObj); + __ sub(t0, flags, (u1)itos); + __ bnez(t0, notInt); + // itos + __ access_load_at(T_INT, IN_HEAP, x10, field, noreg, noreg); + __ sign_extend(x10, x10, 32); + __ push(itos); + // Rewrite bytecode to be faster + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_igetfield, bc, x11); + } + __ j(Done); + + __ bind(notInt); + __ sub(t0, flags, (u1)ctos); + __ bnez(t0, notChar); + // ctos + __ access_load_at(T_CHAR, IN_HEAP, x10, field, noreg, noreg); + __ push(ctos); + // Rewrite bytecode to be faster + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_cgetfield, bc, x11); + } + __ j(Done); + + __ bind(notChar); + __ sub(t0, flags, (u1)stos); + __ bnez(t0, notShort); + // stos + __ access_load_at(T_SHORT, IN_HEAP, x10, field, noreg, noreg); + __ push(stos); + // Rewrite bytecode to be faster + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_sgetfield, bc, x11); + } + __ j(Done); + + __ bind(notShort); + __ sub(t0, flags, (u1)ltos); + __ bnez(t0, notLong); + // ltos + __ access_load_at(T_LONG, IN_HEAP, x10, field, noreg, noreg); + __ push(ltos); + // Rewrite bytecode to be faster + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_lgetfield, bc, x11); + } + __ j(Done); + + __ bind(notLong); + __ sub(t0, flags, (u1)ftos); + __ bnez(t0, notFloat); + // ftos + __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg); + __ push(ftos); + // Rewrite bytecode to be faster + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_fgetfield, bc, x11); + } + __ j(Done); + + __ bind(notFloat); +#ifdef ASSERT + __ sub(t0, flags, (u1)dtos); + __ bnez(t0, notDouble); +#endif + // dtos + __ access_load_at(T_DOUBLE, IN_HEAP, noreg /* ftos */, field, noreg, noreg); + __ push(dtos); + // Rewrite bytecode to be faster + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_dgetfield, bc, x11); + } +#ifdef ASSERT + __ j(Done); + + __ bind(notDouble); + __ stop("Bad state"); +#endif + + __ bind(Done); + + Label notVolatile; + __ test_bit(t0, raw_flags, ConstantPoolCacheEntry::is_volatile_shift); + __ beqz(t0, notVolatile); + __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); + __ bind(notVolatile); +} + +void TemplateTable::getfield(int byte_no) +{ + getfield_or_static(byte_no, false); +} + +void TemplateTable::nofast_getfield(int byte_no) { + getfield_or_static(byte_no, false, may_not_rewrite); +} + +void TemplateTable::getstatic(int byte_no) +{ + getfield_or_static(byte_no, true); +} + +// The registers cache and index expected to be set before call. +// The function may destroy various registers, just not the cache and index registers. +void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) { + transition(vtos, vtos); + + ByteSize cp_base_offset = ConstantPoolCache::base_offset(); + + if (JvmtiExport::can_post_field_modification()) { + // Check to see if a field modification watch has been set before + // we take the time to call into the VM. + Label L1; + assert_different_registers(cache, index, x10); + ExternalAddress target((address)JvmtiExport::get_field_modification_count_addr()); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ lwu(x10, Address(t0, offset)); + }); + __ beqz(x10, L1); + + __ get_cache_and_index_at_bcp(c_rarg2, t0, 1); + + if (is_static) { + // Life is simple. Null out the object pointer. + __ mv(c_rarg1, zr); + } else { + // Life is harder. The stack holds the value on top, followed by + // the object. We don't know the size of the value, though; it + // could be one or two words depending on its type. As a result, + // we must find the type to determine where the object is. + __ lwu(c_rarg3, Address(c_rarg2, + in_bytes(cp_base_offset + + ConstantPoolCacheEntry::flags_offset()))); + __ srli(c_rarg3, c_rarg3, ConstantPoolCacheEntry::tos_state_shift); + ConstantPoolCacheEntry::verify_tos_state_shift(); + Label nope2, done, ok; + __ ld(c_rarg1, at_tos_p1()); // initially assume a one word jvalue + __ sub(t0, c_rarg3, ltos); + __ beqz(t0, ok); + __ sub(t0, c_rarg3, dtos); + __ bnez(t0, nope2); + __ bind(ok); + __ ld(c_rarg1, at_tos_p2()); // ltos (two word jvalue); + __ bind(nope2); + } + // cache entry pointer + __ add(c_rarg2, c_rarg2, in_bytes(cp_base_offset)); + // object (tos) + __ mv(c_rarg3, esp); + // c_rarg1: object pointer set up above (NULL if static) + // c_rarg2: cache entry pointer + // c_rarg3: jvalue object on the stack + __ call_VM(noreg, + CAST_FROM_FN_PTR(address, + InterpreterRuntime::post_field_modification), + c_rarg1, c_rarg2, c_rarg3); + __ get_cache_and_index_at_bcp(cache, index, 1); + __ bind(L1); + } +} + +void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteControl rc) { + transition(vtos, vtos); + + const Register cache = x12; + const Register index = x13; + const Register obj = x12; + const Register off = x9; + const Register flags = x10; + const Register bc = x14; + + resolve_cache_and_index(byte_no, cache, index, sizeof(u2)); + jvmti_post_field_mod(cache, index, is_static); + load_field_cp_cache_entry(obj, cache, index, off, flags, is_static); + + Label Done; + __ mv(x15, flags); + + { + Label notVolatile; + __ test_bit(t0, x15, ConstantPoolCacheEntry::is_volatile_shift); + __ beqz(t0, notVolatile); + __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore); + __ bind(notVolatile); + } + + Label notByte, notBool, notInt, notShort, notChar, + notLong, notFloat, notObj, notDouble; + + __ slli(flags, flags, XLEN - (ConstantPoolCacheEntry::tos_state_shift + + ConstantPoolCacheEntry::tos_state_bits)); + __ srli(flags, flags, XLEN - ConstantPoolCacheEntry::tos_state_bits); + + assert(btos == 0, "change code, btos != 0"); + __ bnez(flags, notByte); + + // Don't rewrite putstatic, only putfield + if (is_static) { + rc = may_not_rewrite; + } + + // btos + { + __ pop(btos); + // field address + if (!is_static) { + pop_and_check_object(obj); + } + __ add(off, obj, off); // if static, obj from cache, else obj from stack. + const Address field(off, 0); // off register as temparator register. + __ access_store_at(T_BYTE, IN_HEAP, field, x10, noreg, noreg); + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_bputfield, bc, x11, true, byte_no); + } + __ j(Done); + } + + __ bind(notByte); + __ sub(t0, flags, (u1)ztos); + __ bnez(t0, notBool); + + // ztos + { + __ pop(ztos); + // field address + if (!is_static) { + pop_and_check_object(obj); + } + __ add(off, obj, off); // if static, obj from cache, else obj from stack. + const Address field(off, 0); + __ access_store_at(T_BOOLEAN, IN_HEAP, field, x10, noreg, noreg); + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_zputfield, bc, x11, true, byte_no); + } + __ j(Done); + } + + __ bind(notBool); + __ sub(t0, flags, (u1)atos); + __ bnez(t0, notObj); + + // atos + { + __ pop(atos); + // field address + if (!is_static) { + pop_and_check_object(obj); + } + __ add(off, obj, off); // if static, obj from cache, else obj from stack. + const Address field(off, 0); + // Store into the field + do_oop_store(_masm, field, x10, IN_HEAP); + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_aputfield, bc, x11, true, byte_no); + } + __ j(Done); + } + + __ bind(notObj); + __ sub(t0, flags, (u1)itos); + __ bnez(t0, notInt); + + // itos + { + __ pop(itos); + // field address + if (!is_static) { + pop_and_check_object(obj); + } + __ add(off, obj, off); // if static, obj from cache, else obj from stack. + const Address field(off, 0); + __ access_store_at(T_INT, IN_HEAP, field, x10, noreg, noreg); + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_iputfield, bc, x11, true, byte_no); + } + __ j(Done); + } + + __ bind(notInt); + __ sub(t0, flags, (u1)ctos); + __ bnez(t0, notChar); + + // ctos + { + __ pop(ctos); + // field address + if (!is_static) { + pop_and_check_object(obj); + } + __ add(off, obj, off); // if static, obj from cache, else obj from stack. + const Address field(off, 0); + __ access_store_at(T_CHAR, IN_HEAP, field, x10, noreg, noreg); + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_cputfield, bc, x11, true, byte_no); + } + __ j(Done); + } + + __ bind(notChar); + __ sub(t0, flags, (u1)stos); + __ bnez(t0, notShort); + + // stos + { + __ pop(stos); + // field address + if (!is_static) { + pop_and_check_object(obj); + } + __ add(off, obj, off); // if static, obj from cache, else obj from stack. + const Address field(off, 0); + __ access_store_at(T_SHORT, IN_HEAP, field, x10, noreg, noreg); + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_sputfield, bc, x11, true, byte_no); + } + __ j(Done); + } + + __ bind(notShort); + __ sub(t0, flags, (u1)ltos); + __ bnez(t0, notLong); + + // ltos + { + __ pop(ltos); + // field address + if (!is_static) { + pop_and_check_object(obj); + } + __ add(off, obj, off); // if static, obj from cache, else obj from stack. + const Address field(off, 0); + __ access_store_at(T_LONG, IN_HEAP, field, x10, noreg, noreg); + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_lputfield, bc, x11, true, byte_no); + } + __ j(Done); + } + + __ bind(notLong); + __ sub(t0, flags, (u1)ftos); + __ bnez(t0, notFloat); + + // ftos + { + __ pop(ftos); + // field address + if (!is_static) { + pop_and_check_object(obj); + } + __ add(off, obj, off); // if static, obj from cache, else obj from stack. + const Address field(off, 0); + __ access_store_at(T_FLOAT, IN_HEAP, field, noreg /* ftos */, noreg, noreg); + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_fputfield, bc, x11, true, byte_no); + } + __ j(Done); + } + + __ bind(notFloat); +#ifdef ASSERT + __ sub(t0, flags, (u1)dtos); + __ bnez(t0, notDouble); +#endif + + // dtos + { + __ pop(dtos); + // field address + if (!is_static) { + pop_and_check_object(obj); + } + __ add(off, obj, off); // if static, obj from cache, else obj from stack. + const Address field(off, 0); + __ access_store_at(T_DOUBLE, IN_HEAP, field, noreg /* dtos */, noreg, noreg); + if (rc == may_rewrite) { + patch_bytecode(Bytecodes::_fast_dputfield, bc, x11, true, byte_no); + } + } + +#ifdef ASSERT + __ j(Done); + + __ bind(notDouble); + __ stop("Bad state"); +#endif + + __ bind(Done); + + { + Label notVolatile; + __ test_bit(t0, x15, ConstantPoolCacheEntry::is_volatile_shift); + __ beqz(t0, notVolatile); + __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore); + __ bind(notVolatile); + } +} + +void TemplateTable::putfield(int byte_no) +{ + putfield_or_static(byte_no, false); +} + +void TemplateTable::nofast_putfield(int byte_no) { + putfield_or_static(byte_no, false, may_not_rewrite); +} + +void TemplateTable::putstatic(int byte_no) { + putfield_or_static(byte_no, true); +} + +void TemplateTable::jvmti_post_fast_field_mod() +{ + if (JvmtiExport::can_post_field_modification()) { + // Check to see if a field modification watch has been set before + // we take the time to call into the VM. + Label L2; + ExternalAddress target((address)JvmtiExport::get_field_modification_count_addr()); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ lwu(c_rarg3, Address(t0, offset)); + }); + __ beqz(c_rarg3, L2); + __ pop_ptr(x9); // copy the object pointer from tos + __ verify_oop(x9); + __ push_ptr(x9); // put the object pointer back on tos + // Save tos values before call_VM() clobbers them. Since we have + // to do it for every data type, we use the saved values as the + // jvalue object. + switch (bytecode()) { // load values into the jvalue object + case Bytecodes::_fast_aputfield: __ push_ptr(x10); break; + case Bytecodes::_fast_bputfield: // fall through + case Bytecodes::_fast_zputfield: // fall through + case Bytecodes::_fast_sputfield: // fall through + case Bytecodes::_fast_cputfield: // fall through + case Bytecodes::_fast_iputfield: __ push_i(x10); break; + case Bytecodes::_fast_dputfield: __ push_d(); break; + case Bytecodes::_fast_fputfield: __ push_f(); break; + case Bytecodes::_fast_lputfield: __ push_l(x10); break; + + default: + ShouldNotReachHere(); + } + __ mv(c_rarg3, esp); // points to jvalue on the stack + // access constant pool cache entry + __ get_cache_entry_pointer_at_bcp(c_rarg2, x10, 1); + __ verify_oop(x9); + // x9: object pointer copied above + // c_rarg2: cache entry pointer + // c_rarg3: jvalue object on the stack + __ call_VM(noreg, + CAST_FROM_FN_PTR(address, + InterpreterRuntime::post_field_modification), + x9, c_rarg2, c_rarg3); + + switch (bytecode()) { // restore tos values + case Bytecodes::_fast_aputfield: __ pop_ptr(x10); break; + case Bytecodes::_fast_bputfield: // fall through + case Bytecodes::_fast_zputfield: // fall through + case Bytecodes::_fast_sputfield: // fall through + case Bytecodes::_fast_cputfield: // fall through + case Bytecodes::_fast_iputfield: __ pop_i(x10); break; + case Bytecodes::_fast_dputfield: __ pop_d(); break; + case Bytecodes::_fast_fputfield: __ pop_f(); break; + case Bytecodes::_fast_lputfield: __ pop_l(x10); break; + default: break; + } + __ bind(L2); + } +} + +void TemplateTable::fast_storefield(TosState state) +{ + transition(state, vtos); + + ByteSize base = ConstantPoolCache::base_offset(); + + jvmti_post_fast_field_mod(); + + // access constant pool cache + __ get_cache_and_index_at_bcp(x12, x11, 1); + + // Must prevent reordering of the following cp cache loads with bytecode load + __ membar(MacroAssembler::LoadLoad); + + // test for volatile with x13 + __ lwu(x13, Address(x12, in_bytes(base + + ConstantPoolCacheEntry::flags_offset()))); + + // replace index with field offset from cache entry + __ ld(x11, Address(x12, in_bytes(base + ConstantPoolCacheEntry::f2_offset()))); + + { + Label notVolatile; + __ test_bit(t0, x13, ConstantPoolCacheEntry::is_volatile_shift); + __ beqz(t0, notVolatile); + __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore); + __ bind(notVolatile); + } + + // Get object from stack + pop_and_check_object(x12); + + // field address + __ add(x11, x12, x11); + const Address field(x11, 0); + + // access field + switch (bytecode()) { + case Bytecodes::_fast_aputfield: + do_oop_store(_masm, field, x10, IN_HEAP); + break; + case Bytecodes::_fast_lputfield: + __ access_store_at(T_LONG, IN_HEAP, field, x10, noreg, noreg); + break; + case Bytecodes::_fast_iputfield: + __ access_store_at(T_INT, IN_HEAP, field, x10, noreg, noreg); + break; + case Bytecodes::_fast_zputfield: + __ access_store_at(T_BOOLEAN, IN_HEAP, field, x10, noreg, noreg); + break; + case Bytecodes::_fast_bputfield: + __ access_store_at(T_BYTE, IN_HEAP, field, x10, noreg, noreg); + break; + case Bytecodes::_fast_sputfield: + __ access_store_at(T_SHORT, IN_HEAP, field, x10, noreg, noreg); + break; + case Bytecodes::_fast_cputfield: + __ access_store_at(T_CHAR, IN_HEAP, field, x10, noreg, noreg); + break; + case Bytecodes::_fast_fputfield: + __ access_store_at(T_FLOAT, IN_HEAP, field, noreg /* ftos */, noreg, noreg); + break; + case Bytecodes::_fast_dputfield: + __ access_store_at(T_DOUBLE, IN_HEAP, field, noreg /* dtos */, noreg, noreg); + break; + default: + ShouldNotReachHere(); + } + + { + Label notVolatile; + __ test_bit(t0, x13, ConstantPoolCacheEntry::is_volatile_shift); + __ beqz(t0, notVolatile); + __ membar(MacroAssembler::StoreLoad | MacroAssembler::StoreStore); + __ bind(notVolatile); + } +} + +void TemplateTable::fast_accessfield(TosState state) +{ + transition(atos, state); + // Do the JVMTI work here to avoid disturbing the register state below + if (JvmtiExport::can_post_field_access()) { + // Check to see if a field access watch has been set before we + // take the time to call into the VM. + Label L1; + ExternalAddress target((address)JvmtiExport::get_field_access_count_addr()); + __ relocate(target.rspec(), [&] { + int32_t offset; + __ la_patchable(t0, target, offset); + __ lwu(x12, Address(t0, offset)); + }); + __ beqz(x12, L1); + // access constant pool cache entry + __ get_cache_entry_pointer_at_bcp(c_rarg2, t1, 1); + __ verify_oop(x10); + __ push_ptr(x10); // save object pointer before call_VM() clobbers it + __ mv(c_rarg1, x10); + // c_rarg1: object pointer copied above + // c_rarg2: cache entry pointer + __ call_VM(noreg, + CAST_FROM_FN_PTR(address, + InterpreterRuntime::post_field_access), + c_rarg1, c_rarg2); + __ pop_ptr(x10); // restore object pointer + __ bind(L1); + } + + // access constant pool cache + __ get_cache_and_index_at_bcp(x12, x11, 1); + + // Must prevent reordering of the following cp cache loads with bytecode load + __ membar(MacroAssembler::LoadLoad); + + __ ld(x11, Address(x12, in_bytes(ConstantPoolCache::base_offset() + + ConstantPoolCacheEntry::f2_offset()))); + __ lwu(x13, Address(x12, in_bytes(ConstantPoolCache::base_offset() + + ConstantPoolCacheEntry::flags_offset()))); + + // x10: object + __ verify_oop(x10); + __ null_check(x10); + __ add(x11, x10, x11); + const Address field(x11, 0); + + // access field + switch (bytecode()) { + case Bytecodes::_fast_agetfield: + do_oop_load(_masm, field, x10, IN_HEAP); + __ verify_oop(x10); + break; + case Bytecodes::_fast_lgetfield: + __ access_load_at(T_LONG, IN_HEAP, x10, field, noreg, noreg); + break; + case Bytecodes::_fast_igetfield: + __ access_load_at(T_INT, IN_HEAP, x10, field, noreg, noreg); + __ sign_extend(x10, x10, 32); + break; + case Bytecodes::_fast_bgetfield: + __ access_load_at(T_BYTE, IN_HEAP, x10, field, noreg, noreg); + break; + case Bytecodes::_fast_sgetfield: + __ access_load_at(T_SHORT, IN_HEAP, x10, field, noreg, noreg); + break; + case Bytecodes::_fast_cgetfield: + __ access_load_at(T_CHAR, IN_HEAP, x10, field, noreg, noreg); + break; + case Bytecodes::_fast_fgetfield: + __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, field, noreg, noreg); + break; + case Bytecodes::_fast_dgetfield: + __ access_load_at(T_DOUBLE, IN_HEAP, noreg /* dtos */, field, noreg, noreg); + break; + default: + ShouldNotReachHere(); + } + { + Label notVolatile; + __ test_bit(t0, x13, ConstantPoolCacheEntry::is_volatile_shift); + __ beqz(t0, notVolatile); + __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); + __ bind(notVolatile); + } +} + +void TemplateTable::fast_xaccess(TosState state) +{ + transition(vtos, state); + + // get receiver + __ ld(x10, aaddress(0)); + // access constant pool cache + __ get_cache_and_index_at_bcp(x12, x13, 2); + __ ld(x11, Address(x12, in_bytes(ConstantPoolCache::base_offset() + + ConstantPoolCacheEntry::f2_offset()))); + + // make sure exception is reported in correct bcp range (getfield is + // next instruction) + __ addi(xbcp, xbcp, 1); + __ null_check(x10); + switch (state) { + case itos: + __ add(x10, x10, x11); + __ access_load_at(T_INT, IN_HEAP, x10, Address(x10, 0), noreg, noreg); + __ sign_extend(x10, x10, 32); + break; + case atos: + __ add(x10, x10, x11); + do_oop_load(_masm, Address(x10, 0), x10, IN_HEAP); + __ verify_oop(x10); + break; + case ftos: + __ add(x10, x10, x11); + __ access_load_at(T_FLOAT, IN_HEAP, noreg /* ftos */, Address(x10), noreg, noreg); + break; + default: + ShouldNotReachHere(); + } + + { + Label notVolatile; + __ lwu(x13, Address(x12, in_bytes(ConstantPoolCache::base_offset() + + ConstantPoolCacheEntry::flags_offset()))); + __ test_bit(t0, x13, ConstantPoolCacheEntry::is_volatile_shift); + __ beqz(t0, notVolatile); + __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); + __ bind(notVolatile); + } + + __ sub(xbcp, xbcp, 1); +} + +//----------------------------------------------------------------------------- +// Calls + +void TemplateTable::prepare_invoke(int byte_no, + Register method, // linked method (or i-klass) + Register index, // itable index, MethodType, etc. + Register recv, // if caller wants to see it + Register flags // if caller wants to test it + ) { + // determine flags + const Bytecodes::Code code = bytecode(); + const bool is_invokeinterface = code == Bytecodes::_invokeinterface; + const bool is_invokedynamic = code == Bytecodes::_invokedynamic; + const bool is_invokehandle = code == Bytecodes::_invokehandle; + const bool is_invokevirtual = code == Bytecodes::_invokevirtual; + const bool is_invokespecial = code == Bytecodes::_invokespecial; + const bool load_receiver = (recv != noreg); + const bool save_flags = (flags != noreg); + assert(load_receiver == (code != Bytecodes::_invokestatic && code != Bytecodes::_invokedynamic), ""); + assert(save_flags == (is_invokeinterface || is_invokevirtual), "need flags for vfinal"); + assert(flags == noreg || flags == x13, ""); + assert(recv == noreg || recv == x12, ""); + + // setup registers & access constant pool cache + if (recv == noreg) { + recv = x12; + } + if (flags == noreg) { + flags = x13; + } + assert_different_registers(method, index, recv, flags); + + // save 'interpreter return address' + __ save_bcp(); + + load_invoke_cp_cache_entry(byte_no, method, index, flags, is_invokevirtual, false, is_invokedynamic); + + // maybe push appendix to arguments (just before return address) + if (is_invokedynamic || is_invokehandle) { + Label L_no_push; + __ test_bit(t0, flags, ConstantPoolCacheEntry::has_appendix_shift); + __ beqz(t0, L_no_push); + // Push the appendix as a trailing parameter. + // This must be done before we get the receiver, + // since the parameter_size includes it. + __ push_reg(x9); + __ mv(x9, index); + __ load_resolved_reference_at_index(index, x9); + __ pop_reg(x9); + __ push_reg(index); // push appendix (MethodType, CallSite, etc.) + __ bind(L_no_push); + } + + // load receiver if needed (note: no return address pushed yet) + if (load_receiver) { + __ andi(recv, flags, ConstantPoolCacheEntry::parameter_size_mask); // parameter_size_mask = 1 << 8 + __ shadd(t0, recv, esp, t0, 3); + __ ld(recv, Address(t0, -Interpreter::expr_offset_in_bytes(1))); + __ verify_oop(recv); + } + + // compute return type + __ slli(t1, flags, XLEN - (ConstantPoolCacheEntry::tos_state_shift + ConstantPoolCacheEntry::tos_state_bits)); + __ srli(t1, t1, XLEN - ConstantPoolCacheEntry::tos_state_bits); // (1 << 5) - 4 --> 28~31==> t1:0~3 + + // load return address + { + const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code); + __ mv(t0, table_addr); + __ shadd(t0, t1, t0, t1, 3); + __ ld(ra, Address(t0, 0)); + } +} + +void TemplateTable::invokevirtual_helper(Register index, + Register recv, + Register flags) +{ + // Uses temporary registers x10, x13 + assert_different_registers(index, recv, x10, x13); + // Test for an invoke of a final method + Label notFinal; + __ test_bit(t0, flags, ConstantPoolCacheEntry::is_vfinal_shift); + __ beqz(t0, notFinal); + + const Register method = index; // method must be xmethod + assert(method == xmethod, "Method must be xmethod for interpreter calling convention"); + + // do the call - the index is actually the method to call + // that is, f2 is a vtable index if !is_vfinal, else f2 is a Method* + + // It's final, need a null check here! + __ null_check(recv); + + // profile this call + __ profile_final_call(x10); + __ profile_arguments_type(x10, method, x14, true); + + __ jump_from_interpreted(method); + + __ bind(notFinal); + + // get receiver klass + __ null_check(recv, oopDesc::klass_offset_in_bytes()); + __ load_klass(x10, recv); + + // profile this call + __ profile_virtual_call(x10, xlocals, x13); + + // get target Method & entry point + __ lookup_virtual_method(x10, index, method); + __ profile_arguments_type(x13, method, x14, true); + __ jump_from_interpreted(method); +} + +void TemplateTable::invokevirtual(int byte_no) +{ + transition(vtos, vtos); + assert(byte_no == f2_byte, "use this argument"); + + prepare_invoke(byte_no, xmethod, noreg, x12, x13); + + // xmethod: index (actually a Method*) + // x12: receiver + // x13: flags + + invokevirtual_helper(xmethod, x12, x13); +} + +void TemplateTable::invokespecial(int byte_no) +{ + transition(vtos, vtos); + assert(byte_no == f1_byte, "use this argument"); + + prepare_invoke(byte_no, xmethod, noreg, // get f1 Method* + x12); // get receiver also for null check + __ verify_oop(x12); + __ null_check(x12); + // do the call + __ profile_call(x10); + __ profile_arguments_type(x10, xmethod, xbcp, false); + __ jump_from_interpreted(xmethod); +} + +void TemplateTable::invokestatic(int byte_no) +{ + transition(vtos, vtos); + assert(byte_no == f1_byte, "use this arugment"); + + prepare_invoke(byte_no, xmethod); // get f1 Method* + // do the call + __ profile_call(x10); + __ profile_arguments_type(x10, xmethod, x14, false); + __ jump_from_interpreted(xmethod); +} + +void TemplateTable::fast_invokevfinal(int byte_no) +{ + __ call_Unimplemented(); +} + +void TemplateTable::invokeinterface(int byte_no) { + transition(vtos, vtos); + assert(byte_no == f1_byte, "use this argument"); + + prepare_invoke(byte_no, x10, xmethod, // get f1 Klass*, f2 Method* + x12, x13); // recv, flags + + // x10: interface klass (from f1) + // xmethod: method (from f2) + // x12: receiver + // x13: flags + + // First check for Object case, then private interface method, + // then regular interface method. + + // Special case of invokeinterface called for virtual method of + // java.lang.Object. See cpCache.cpp for details + Label notObjectMethod; + __ test_bit(t0, x13, ConstantPoolCacheEntry::is_forced_virtual_shift); + __ beqz(t0, notObjectMethod); + + invokevirtual_helper(xmethod, x12, x13); + __ bind(notObjectMethod); + + Label no_such_interface; + + // Check for private method invocation - indicated by vfinal + Label notVFinal; + __ test_bit(t0, x13, ConstantPoolCacheEntry::is_vfinal_shift); + __ beqz(t0, notVFinal); + + // Check receiver klass into x13 - also a null check + __ null_check(x12, oopDesc::klass_offset_in_bytes()); + __ load_klass(x13, x12); + + Label subtype; + __ check_klass_subtype(x13, x10, x14, subtype); + // If we get here the typecheck failed + __ j(no_such_interface); + __ bind(subtype); + + __ profile_final_call(x10); + __ profile_arguments_type(x10, xmethod, x14, true); + __ jump_from_interpreted(xmethod); + + __ bind(notVFinal); + + // Get receiver klass into x13 - also a null check + __ restore_locals(); + __ null_check(x12, oopDesc::klass_offset_in_bytes()); + __ load_klass(x13, x12); + + Label no_such_method; + + // Preserve method for the throw_AbstractMethodErrorVerbose. + __ mv(x28, xmethod); + // Receiver subtype check against REFC. + // Superklass in x10. Subklass in x13. Blows t1, x30 + __ lookup_interface_method(// inputs: rec. class, interface, itable index + x13, x10, noreg, + // outputs: scan temp. reg, scan temp. reg + t1, x30, + no_such_interface, + /*return_method=*/false); + + // profile this call + __ profile_virtual_call(x13, x30, x9); + + // Get declaring interface class from method, and itable index + __ load_method_holder(x10, xmethod); + __ lwu(xmethod, Address(xmethod, Method::itable_index_offset())); + __ subw(xmethod, xmethod, Method::itable_index_max); + __ negw(xmethod, xmethod); + + // Preserve recvKlass for throw_AbstractMethodErrorVerbose + __ mv(xlocals, x13); + __ lookup_interface_method(// inputs: rec. class, interface, itable index + xlocals, x10, xmethod, + // outputs: method, scan temp. reg + xmethod, x30, + no_such_interface); + + // xmethod: Method to call + // x12: receiver + // Check for abstract method error + // Note: This should be done more efficiently via a throw_abstract_method_error + // interpreter entry point and a conditional jump to it in case of a null + // method. + __ beqz(xmethod, no_such_method); + + __ profile_arguments_type(x13, xmethod, x30, true); + + // do the call + // x12: receiver + // xmethod: Method + __ jump_from_interpreted(xmethod); + __ should_not_reach_here(); + + // exception handling code follows ... + // note: must restore interpreter registers to canonical + // state for exception handling to work correctly! + + __ bind(no_such_method); + // throw exception + __ restore_bcp(); // bcp must be correct for exception handler (was destroyed) + __ restore_locals(); // make sure locals pointer is correct as well (was destroyed) + // Pass arguments for generating a verbose error message. + __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodErrorVerbose), x13, x28); + // the call_VM checks for exception, so we should never return here. + __ should_not_reach_here(); + + __ bind(no_such_interface); + // throw exceptiong + __ restore_bcp(); // bcp must be correct for exception handler (was destroyed) + __ restore_locals(); // make sure locals pointer is correct as well (was destroyed) + // Pass arguments for generating a verbose error message. + __ call_VM(noreg, CAST_FROM_FN_PTR(address, + InterpreterRuntime::throw_IncompatibleClassChangeErrorVerbose), x13, x10); + // the call_VM checks for exception, so we should never return here. + __ should_not_reach_here(); + return; +} + +void TemplateTable::invokehandle(int byte_no) { + transition(vtos, vtos); + assert(byte_no == f1_byte, "use this argument"); + + prepare_invoke(byte_no, xmethod, x10, x12); + __ verify_method_ptr(x12); + __ verify_oop(x12); + __ null_check(x12); + + // FIXME: profile the LambdaForm also + + // x30 is safe to use here as a temp reg because it is about to + // be clobbered by jump_from_interpreted(). + __ profile_final_call(x30); + __ profile_arguments_type(x30, xmethod, x14, true); + + __ jump_from_interpreted(xmethod); +} + +void TemplateTable::invokedynamic(int byte_no) { + transition(vtos, vtos); + assert(byte_no == f1_byte, "use this argument"); + + prepare_invoke(byte_no, xmethod, x10); + + // x10: CallSite object (from cpool->resolved_references[]) + // xmethod: MH.linkToCallSite method (from f2) + + // Note: x10_callsite is already pushed by prepare_invoke + + // %%% should make a type profile for any invokedynamic that takes a ref argument + // profile this call + __ profile_call(xbcp); + __ profile_arguments_type(x13, xmethod, x30, false); + + __ verify_oop(x10); + + __ jump_from_interpreted(xmethod); +} + +//----------------------------------------------------------------------------- +// Allocation + +void TemplateTable::_new() { + transition(vtos, atos); + + __ get_unsigned_2_byte_index_at_bcp(x13, 1); + Label slow_case; + Label done; + Label initialize_header; + Label initialize_object; // including clearing the fields + + __ get_cpool_and_tags(x14, x10); + // Make sure the class we're about to instantiate has been resolved. + // This is done before loading InstanceKlass to be consistent with the order + // how Constant Pool is update (see ConstantPool::klass_at_put) + const int tags_offset = Array::base_offset_in_bytes(); + __ add(t0, x10, x13); + __ la(t0, Address(t0, tags_offset)); + __ membar(MacroAssembler::AnyAny); + __ lbu(t0, t0); + __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); + __ sub(t1, t0, (u1)JVM_CONSTANT_Class); + __ bnez(t1, slow_case); + + // get InstanceKlass + __ load_resolved_klass_at_offset(x14, x13, x14, t0); + + // make sure klass is initialized & doesn't have finalizer + // make sure klass is fully initialized + __ lbu(t0, Address(x14, InstanceKlass::init_state_offset())); + __ sub(t1, t0, (u1)InstanceKlass::fully_initialized); + __ bnez(t1, slow_case); + + // get instance_size in InstanceKlass (scaled to a count of bytes) + __ lwu(x13, Address(x14, Klass::layout_helper_offset())); + // test to see if it has a finalizer or is malformed in some way + __ test_bit(t0, x13, exact_log2(Klass::_lh_instance_slow_path_bit)); + __ bnez(t0, slow_case); + + // Allocate the instance: + // If TLAB is enabled: + // Try to allocate in the TLAB. + // If fails, go to the slow path. + // Else If inline contiguous allocations are enabled: + // Try to allocate in eden. + // If fails due to heap end, go to slow path + // + // If TLAB is enabled OR inline contiguous is enabled: + // Initialize the allocation. + // Exit. + // Go to slow path. + const bool allow_shared_alloc = Universe::heap()->supports_inline_contig_alloc(); + + if (UseTLAB) { + __ tlab_allocate(x10, x13, 0, noreg, x11, slow_case); + + if (ZeroTLAB) { + // the fields have been already cleared + __ j(initialize_header); + } else { + // initialize both the header and fields + __ j(initialize_object); + } + } else { + // Allocation in the shared Eden, if allowed. + // + // x13: instance size in bytes + if (allow_shared_alloc) { + __ eden_allocate(x10, x13, 0, x28, slow_case); + } + } + + // If USETLAB or allow_shared_alloc are true, the object is created above and + // there is an initialized need. Otherwise, skip and go to the slow path. + if (UseTLAB || allow_shared_alloc) { + // The object is initialized before the header. If the object size is + // zero, go directly to the header initialization. + __ bind(initialize_object); + __ sub(x13, x13, sizeof(oopDesc)); + __ beqz(x13, initialize_header); + + // Initialize obejct fields + { + __ add(x12, x10, sizeof(oopDesc)); + Label loop; + __ bind(loop); + __ sd(zr, Address(x12)); + __ add(x12, x12, BytesPerLong); + __ sub(x13, x13, BytesPerLong); + __ bnez(x13, loop); + } + + // initialize object hader only. + __ bind(initialize_header); + if (UseBiasedLocking) { + __ ld(t0, Address(x14, Klass::prototype_header_offset())); + } else { + __ mv(t0, (intptr_t)markWord::prototype().value()); + } + __ sd(t0, Address(x10, oopDesc::mark_offset_in_bytes())); + __ store_klass_gap(x10, zr); // zero klass gap for compressed oops + __ store_klass(x10, x14); // store klass last + + { + SkipIfEqual skip(_masm, &DTraceAllocProbes, false); + // Trigger dtrace event for fastpath + __ push(atos); // save the return value + __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), x10); + __ pop(atos); // restore the return value + } + __ j(done); + } + + // slow case + __ bind(slow_case); + __ get_constant_pool(c_rarg1); + __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1); + call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2); + __ verify_oop(x10); + + // continue + __ bind(done); + // Must prevent reordering of stores for object initialization with stores that publish the new object. + __ membar(MacroAssembler::StoreStore); +} + +void TemplateTable::newarray() { + transition(itos, atos); + __ load_unsigned_byte(c_rarg1, at_bcp(1)); + __ mv(c_rarg2, x10); + call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray), + c_rarg1, c_rarg2); + // Must prevent reordering of stores for object initialization with stores that publish the new object. + __ membar(MacroAssembler::StoreStore); +} + +void TemplateTable::anewarray() { + transition(itos, atos); + __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1); + __ get_constant_pool(c_rarg1); + __ mv(c_rarg3, x10); + call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray), + c_rarg1, c_rarg2, c_rarg3); + // Must prevent reordering of stores for object initialization with stores that publish the new object. + __ membar(MacroAssembler::StoreStore); +} + +void TemplateTable::arraylength() { + transition(atos, itos); + __ null_check(x10, arrayOopDesc::length_offset_in_bytes()); + __ lwu(x10, Address(x10, arrayOopDesc::length_offset_in_bytes())); +} + +void TemplateTable::checkcast() +{ + transition(atos, atos); + Label done, is_null, ok_is_subtype, quicked, resolved; + __ beqz(x10, is_null); + + // Get cpool & tags index + __ get_cpool_and_tags(x12, x13); // x12=cpool, x13=tags array + __ get_unsigned_2_byte_index_at_bcp(x9, 1); // x9=index + // See if bytecode has already been quicked + __ add(t0, x13, Array::base_offset_in_bytes()); + __ add(x11, t0, x9); + __ membar(MacroAssembler::AnyAny); + __ lbu(x11, x11); + __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); + __ sub(t0, x11, (u1)JVM_CONSTANT_Class); + __ beqz(t0, quicked); + + __ push(atos); // save receiver for result, and for GC + call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc)); + // vm_result_2 has metadata result + __ get_vm_result_2(x10, xthread); + __ pop_reg(x13); // restore receiver + __ j(resolved); + + // Get superklass in x10 and subklass in x13 + __ bind(quicked); + __ mv(x13, x10); // Save object in x13; x10 needed for subtype check + __ load_resolved_klass_at_offset(x12, x9, x10, t0); // x10 = klass + + __ bind(resolved); + __ load_klass(x9, x13); + + // Generate subtype check. Blows x12, x15. Object in x13. + // Superklass in x10. Subklass in x9. + __ gen_subtype_check(x9, ok_is_subtype); + + // Come here on failure + __ push_reg(x13); + // object is at TOS + __ j(Interpreter::_throw_ClassCastException_entry); + + // Come here on success + __ bind(ok_is_subtype); + __ mv(x10, x13); // Restore object in x13 + + // Collect counts on whether this test sees NULLs a lot or not. + if (ProfileInterpreter) { + __ j(done); + __ bind(is_null); + __ profile_null_seen(x12); + } else { + __ bind(is_null); // same as 'done' + } + __ bind(done); +} + +void TemplateTable::instanceof() { + transition(atos, itos); + Label done, is_null, ok_is_subtype, quicked, resolved; + __ beqz(x10, is_null); + + // Get cpool & tags index + __ get_cpool_and_tags(x12, x13); // x12=cpool, x13=tags array + __ get_unsigned_2_byte_index_at_bcp(x9, 1); // x9=index + // See if bytecode has already been quicked + __ add(t0, x13, Array::base_offset_in_bytes()); + __ add(x11, t0, x9); + __ membar(MacroAssembler::AnyAny); + __ lbu(x11, x11); + __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); + __ sub(t0, x11, (u1)JVM_CONSTANT_Class); + __ beqz(t0, quicked); + + __ push(atos); // save receiver for result, and for GC + call_VM(x10, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc)); + // vm_result_2 has metadata result + __ get_vm_result_2(x10, xthread); + __ pop_reg(x13); // restore receiver + __ verify_oop(x13); + __ load_klass(x13, x13); + __ j(resolved); + + // Get superklass in x10 and subklass in x13 + __ bind(quicked); + __ load_klass(x13, x10); + __ load_resolved_klass_at_offset(x12, x9, x10, t0); + + __ bind(resolved); + + // Generate subtype check. Blows x12, x15 + // Superklass in x10. Subklass in x13. + __ gen_subtype_check(x13, ok_is_subtype); + + // Come here on failure + __ mv(x10, zr); + __ j(done); + // Come here on success + __ bind(ok_is_subtype); + __ mv(x10, 1); + + // Collect counts on whether this test sees NULLs a lot or not. + if (ProfileInterpreter) { + __ j(done); + __ bind(is_null); + __ profile_null_seen(x12); + } else { + __ bind(is_null); // same as 'done' + } + __ bind(done); + // x10 = 0: obj == NULL or obj is not an instanceof the specified klass + // x10 = 1: obj != NULL and obj is an instanceof the specified klass +} + +//----------------------------------------------------------------------------- +// Breakpoints +void TemplateTable::_breakpoint() { + // Note: We get here even if we are single stepping.. + // jbug inists on setting breakpoints at every bytecode + // even if we are in single step mode. + + transition(vtos, vtos); + + // get the unpatched byte code + __ get_method(c_rarg1); + __ call_VM(noreg, + CAST_FROM_FN_PTR(address, + InterpreterRuntime::get_original_bytecode_at), + c_rarg1, xbcp); + __ mv(x9, x10); + + // post the breakpoint event + __ call_VM(noreg, + CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint), + xmethod, xbcp); + + // complete the execution of original bytecode + __ mv(t0, x9); + __ dispatch_only_normal(vtos); +} + +//----------------------------------------------------------------------------- +// Exceptions + +void TemplateTable::athrow() { + transition(atos, vtos); + __ null_check(x10); + __ j(Interpreter::throw_exception_entry()); +} + +//----------------------------------------------------------------------------- +// Synchronization +// +// Note: monitorenter & exit are symmetric routines; which is reflected +// in the assembly code structure as well +// +// Stack layout: +// +// [expressions ] <--- esp = expression stack top +// .. +// [expressions ] +// [monitor entry] <--- monitor block top = expression stack bot +// .. +// [monitor entry] +// [frame data ] <--- monitor block bot +// ... +// [saved fp ] <--- fp +void TemplateTable::monitorenter() +{ + transition(atos, vtos); + + // check for NULL object + __ null_check(x10); + + const Address monitor_block_top( + fp, frame::interpreter_frame_monitor_block_top_offset * wordSize); + const Address monitor_block_bot( + fp, frame::interpreter_frame_initial_sp_offset * wordSize); + const int entry_size = frame::interpreter_frame_monitor_size() * wordSize; + + Label allocated; + + // initialize entry pointer + __ mv(c_rarg1, zr); // points to free slot or NULL + + // find a free slot in the monitor block (result in c_rarg1) + { + Label entry, loop, exit, notUsed; + __ ld(c_rarg3, monitor_block_top); // points to current entry, + // starting with top-most entry + __ la(c_rarg2, monitor_block_bot); // points to word before bottom + + __ j(entry); + + __ bind(loop); + // check if current entry is used + // if not used then remember entry in c_rarg1 + __ ld(t0, Address(c_rarg3, BasicObjectLock::obj_offset_in_bytes())); + __ bnez(t0, notUsed); + __ mv(c_rarg1, c_rarg3); + __ bind(notUsed); + // check if current entry is for same object + // if same object then stop searching + __ beq(x10, t0, exit); + // otherwise advance to next entry + __ add(c_rarg3, c_rarg3, entry_size); + __ bind(entry); + // check if bottom reached + // if not at bottom then check this entry + __ bne(c_rarg3, c_rarg2, loop); + __ bind(exit); + } + + __ bnez(c_rarg1, allocated); // check if a slot has been found and + // if found, continue with that on + + // allocate one if there's no free slot + { + Label entry, loop; + // 1. compute new pointers // esp: old expression stack top + __ ld(c_rarg1, monitor_block_bot); // c_rarg1: old expression stack bottom + __ sub(esp, esp, entry_size); // move expression stack top + __ sub(c_rarg1, c_rarg1, entry_size); // move expression stack bottom + __ mv(c_rarg3, esp); // set start value for copy loop + __ sd(c_rarg1, monitor_block_bot); // set new monitor block bottom + __ sub(sp, sp, entry_size); // make room for the monitor + + __ j(entry); + // 2. move expression stack contents + __ bind(loop); + __ ld(c_rarg2, Address(c_rarg3, entry_size)); // load expression stack + // word from old location + __ sd(c_rarg2, Address(c_rarg3, 0)); // and store it at new location + __ add(c_rarg3, c_rarg3, wordSize); // advance to next word + __ bind(entry); + __ bne(c_rarg3, c_rarg1, loop); // check if bottom reached.if not at bottom + // then copy next word + } + + // call run-time routine + // c_rarg1: points to monitor entry + __ bind(allocated); + + // Increment bcp to point to the next bytecode, so exception + // handling for async. exceptions work correctly. + // The object has already been poped from the stack, so the + // expression stack looks correct. + __ addi(xbcp, xbcp, 1); + + // store object + __ sd(x10, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes())); + __ lock_object(c_rarg1); + + // check to make sure this monitor doesn't cause stack overflow after locking + __ save_bcp(); // in case of exception + __ generate_stack_overflow_check(0); + + // The bcp has already been incremented. Just need to dispatch to + // next instruction. + __ dispatch_next(vtos); +} + +void TemplateTable::monitorexit() +{ + transition(atos, vtos); + + // check for NULL object + __ null_check(x10); + + const Address monitor_block_top( + fp, frame::interpreter_frame_monitor_block_top_offset * wordSize); + const Address monitor_block_bot( + fp, frame::interpreter_frame_initial_sp_offset * wordSize); + const int entry_size = frame::interpreter_frame_monitor_size() * wordSize; + + Label found; + + // find matching slot + { + Label entry, loop; + __ ld(c_rarg1, monitor_block_top); // points to current entry, + // starting with top-most entry + __ la(c_rarg2, monitor_block_bot); // points to word before bottom + // of monitor block + __ j(entry); + + __ bind(loop); + // check if current entry is for same object + __ ld(t0, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes())); + // if same object then stop searching + __ beq(x10, t0, found); + // otherwise advance to next entry + __ add(c_rarg1, c_rarg1, entry_size); + __ bind(entry); + // check if bottom reached + // if not at bottom then check this entry + __ bne(c_rarg1, c_rarg2, loop); + } + + // error handling. Unlocking was not block-structured + __ call_VM(noreg, CAST_FROM_FN_PTR(address, + InterpreterRuntime::throw_illegal_monitor_state_exception)); + __ should_not_reach_here(); + + // call run-time routine + __ bind(found); + __ push_ptr(x10); // make sure object is on stack (contract with oopMaps) + __ unlock_object(c_rarg1); + __ pop_ptr(x10); // discard object +} + +// Wide instructions +void TemplateTable::wide() +{ + __ load_unsigned_byte(x9, at_bcp(1)); + __ mv(t0, (address)Interpreter::_wentry_point); + __ shadd(t0, x9, t0, t1, 3); + __ ld(t0, Address(t0)); + __ jr(t0); +} + +// Multi arrays +void TemplateTable::multianewarray() { + transition(vtos, atos); + __ load_unsigned_byte(x10, at_bcp(3)); // get number of dimensions + // last dim is on top of stack; we want address of first one: + // first_addr = last_addr + (ndims - 1) * wordSize + __ shadd(c_rarg1, x10, esp, c_rarg1, 3); + __ sub(c_rarg1, c_rarg1, wordSize); + call_VM(x10, + CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray), + c_rarg1); + __ load_unsigned_byte(x11, at_bcp(3)); + __ shadd(esp, x11, esp, t0, 3); +} diff --git a/src/hotspot/cpu/riscv/templateTable_riscv.hpp b/src/hotspot/cpu/riscv/templateTable_riscv.hpp new file mode 100644 index 0000000000000..fcc86108d2839 --- /dev/null +++ b/src/hotspot/cpu/riscv/templateTable_riscv.hpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_TEMPLATETABLE_RISCV_HPP +#define CPU_RISCV_TEMPLATETABLE_RISCV_HPP + +static void prepare_invoke(int byte_no, + Register method, // linked method (or i-klass) + Register index = noreg, // itable index, MethodType, etc. + Register recv = noreg, // if caller wants to see it + Register flags = noreg // if caller wants to test it + ); +static void invokevirtual_helper(Register index, Register recv, + Register flags); + +// Helpers +static void index_check(Register array, Register index); + +#endif // CPU_RISCV_TEMPLATETABLE_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/universalNativeInvoker_riscv.cpp b/src/hotspot/cpu/riscv/universalNativeInvoker_riscv.cpp new file mode 100644 index 0000000000000..4f50adb05c315 --- /dev/null +++ b/src/hotspot/cpu/riscv/universalNativeInvoker_riscv.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "prims/universalNativeInvoker.hpp" +#include "utilities/debug.hpp" + +address ProgrammableInvoker::generate_adapter(jobject jabi, jobject jlayout) { + Unimplemented(); + return nullptr; +} diff --git a/src/hotspot/cpu/riscv/universalUpcallHandle_riscv.cpp b/src/hotspot/cpu/riscv/universalUpcallHandle_riscv.cpp new file mode 100644 index 0000000000000..ce70da72f2e46 --- /dev/null +++ b/src/hotspot/cpu/riscv/universalUpcallHandle_riscv.cpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "prims/universalUpcallHandler.hpp" +#include "utilities/debug.hpp" + +address ProgrammableUpcallHandler::generate_upcall_stub(jobject jrec, jobject jabi, jobject jlayout) { + Unimplemented(); + return nullptr; +} + +address ProgrammableUpcallHandler::generate_optimized_upcall_stub(jobject mh, Method* entry, jobject jabi, jobject jconv) { + ShouldNotCallThis(); + return nullptr; +} + +bool ProgrammableUpcallHandler::supports_optimized_upcalls() { + return false; +} diff --git a/src/hotspot/cpu/riscv/vmStructs_riscv.hpp b/src/hotspot/cpu/riscv/vmStructs_riscv.hpp new file mode 100644 index 0000000000000..6c89133de0280 --- /dev/null +++ b/src/hotspot/cpu/riscv/vmStructs_riscv.hpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_VMSTRUCTS_RISCV_HPP +#define CPU_RISCV_VMSTRUCTS_RISCV_HPP + +// These are the CPU-specific fields, types and integer +// constants required by the Serviceability Agent. This file is +// referenced by vmStructs.cpp. + +#define VM_STRUCTS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \ + volatile_nonstatic_field(JavaFrameAnchor, _last_Java_fp, intptr_t*) + +#define VM_TYPES_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) + +#define VM_INT_CONSTANTS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant) + +#define VM_LONG_CONSTANTS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant) + +#endif // CPU_RISCV_VMSTRUCTS_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/vm_version_ext_riscv.cpp b/src/hotspot/cpu/riscv/vm_version_ext_riscv.cpp new file mode 100644 index 0000000000000..f6f2bbe228976 --- /dev/null +++ b/src/hotspot/cpu/riscv/vm_version_ext_riscv.cpp @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "memory/allocation.hpp" +#include "memory/allocation.inline.hpp" +#include "runtime/os.inline.hpp" +#include "vm_version_ext_riscv.hpp" + +// VM_Version_Ext statics +int VM_Version_Ext::_no_of_threads = 0; +int VM_Version_Ext::_no_of_cores = 0; +int VM_Version_Ext::_no_of_sockets = 0; +bool VM_Version_Ext::_initialized = false; +char VM_Version_Ext::_cpu_name[CPU_TYPE_DESC_BUF_SIZE] = {0}; +char VM_Version_Ext::_cpu_desc[CPU_DETAILED_DESC_BUF_SIZE] = {0}; + +void VM_Version_Ext::initialize_cpu_information(void) { + // do nothing if cpu info has been initialized + if (_initialized) { + return; + } + + _no_of_cores = os::processor_count(); + _no_of_threads = _no_of_cores; + _no_of_sockets = _no_of_cores; + snprintf(_cpu_name, CPU_TYPE_DESC_BUF_SIZE - 1, "RISCV64"); + snprintf(_cpu_desc, CPU_DETAILED_DESC_BUF_SIZE, "RISCV64 %s", _features_string); + _initialized = true; +} + +int VM_Version_Ext::number_of_threads(void) { + initialize_cpu_information(); + return _no_of_threads; +} + +int VM_Version_Ext::number_of_cores(void) { + initialize_cpu_information(); + return _no_of_cores; +} + +int VM_Version_Ext::number_of_sockets(void) { + initialize_cpu_information(); + return _no_of_sockets; +} + +const char* VM_Version_Ext::cpu_name(void) { + initialize_cpu_information(); + char* tmp = NEW_C_HEAP_ARRAY_RETURN_NULL(char, CPU_TYPE_DESC_BUF_SIZE, mtTracing); + if (NULL == tmp) { + return NULL; + } + strncpy(tmp, _cpu_name, CPU_TYPE_DESC_BUF_SIZE); + return tmp; +} + +const char* VM_Version_Ext::cpu_description(void) { + initialize_cpu_information(); + char* tmp = NEW_C_HEAP_ARRAY_RETURN_NULL(char, CPU_DETAILED_DESC_BUF_SIZE, mtTracing); + if (NULL == tmp) { + return NULL; + } + strncpy(tmp, _cpu_desc, CPU_DETAILED_DESC_BUF_SIZE); + return tmp; +} diff --git a/src/hotspot/cpu/riscv/vm_version_ext_riscv.hpp b/src/hotspot/cpu/riscv/vm_version_ext_riscv.hpp new file mode 100644 index 0000000000000..ad9785b7c6776 --- /dev/null +++ b/src/hotspot/cpu/riscv/vm_version_ext_riscv.hpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_VM_VERSION_EXT_RISCV_HPP +#define CPU_RISCV_VM_VERSION_EXT_RISCV_HPP + +#include "runtime/vm_version.hpp" +#include "utilities/macros.hpp" + +class VM_Version_Ext : public VM_Version { + private: + static const size_t CPU_TYPE_DESC_BUF_SIZE = 256; + static const size_t CPU_DETAILED_DESC_BUF_SIZE = 4096; + + static int _no_of_threads; + static int _no_of_cores; + static int _no_of_sockets; + static bool _initialized; + static char _cpu_name[CPU_TYPE_DESC_BUF_SIZE]; + static char _cpu_desc[CPU_DETAILED_DESC_BUF_SIZE]; + + public: + static int number_of_threads(void); + static int number_of_cores(void); + static int number_of_sockets(void); + + static const char* cpu_name(void); + static const char* cpu_description(void); + static void initialize_cpu_information(void); + +}; + +#endif // CPU_RISCV_VM_VERSION_EXT_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.cpp b/src/hotspot/cpu/riscv/vm_version_riscv.cpp new file mode 100644 index 0000000000000..63eb94d018bc7 --- /dev/null +++ b/src/hotspot/cpu/riscv/vm_version_riscv.cpp @@ -0,0 +1,225 @@ +/* + * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "runtime/java.hpp" +#include "runtime/os.hpp" +#include "runtime/vm_version.hpp" +#include "utilities/formatBuffer.hpp" +#include "utilities/macros.hpp" + +#include OS_HEADER_INLINE(os) + +const char* VM_Version::_uarch = ""; +const char* VM_Version::_vm_mode = ""; +uint32_t VM_Version::_initial_vector_length = 0; + +void VM_Version::initialize() { + _supports_cx8 = true; + _supports_atomic_getset4 = true; + _supports_atomic_getadd4 = true; + _supports_atomic_getset8 = true; + _supports_atomic_getadd8 = true; + + get_os_cpu_info(); + + // check if satp.mode is supported, currently supports up to SV48(RV64) + if (get_satp_mode() > VM_SV48) { + vm_exit_during_initialization( + err_msg("Unsupported satp mode: %s. Only satp modes up to sv48 are supported for now.", + _vm_mode)); + } + + if (FLAG_IS_DEFAULT(UseFMA)) { + FLAG_SET_DEFAULT(UseFMA, true); + } + + if (FLAG_IS_DEFAULT(AllocatePrefetchDistance)) { + FLAG_SET_DEFAULT(AllocatePrefetchDistance, 0); + } + + if (UseAES || UseAESIntrinsics) { + if (UseAES && !FLAG_IS_DEFAULT(UseAES)) { + warning("AES instructions are not available on this CPU"); + FLAG_SET_DEFAULT(UseAES, false); + } + if (UseAESIntrinsics && !FLAG_IS_DEFAULT(UseAESIntrinsics)) { + warning("AES intrinsics are not available on this CPU"); + FLAG_SET_DEFAULT(UseAESIntrinsics, false); + } + } + + if (UseAESCTRIntrinsics) { + warning("AES/CTR intrinsics are not available on this CPU"); + FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false); + } + + if (UseSHA) { + warning("SHA instructions are not available on this CPU"); + FLAG_SET_DEFAULT(UseSHA, false); + } + + if (UseSHA1Intrinsics) { + warning("Intrinsics for SHA-1 crypto hash functions not available on this CPU."); + FLAG_SET_DEFAULT(UseSHA1Intrinsics, false); + } + + if (UseSHA256Intrinsics) { + warning("Intrinsics for SHA-224 and SHA-256 crypto hash functions not available on this CPU."); + FLAG_SET_DEFAULT(UseSHA256Intrinsics, false); + } + + if (UseSHA512Intrinsics) { + warning("Intrinsics for SHA-384 and SHA-512 crypto hash functions not available on this CPU."); + FLAG_SET_DEFAULT(UseSHA512Intrinsics, false); + } + + if (UseSHA3Intrinsics) { + warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU."); + FLAG_SET_DEFAULT(UseSHA3Intrinsics, false); + } + + if (UseCRC32Intrinsics) { + warning("CRC32 intrinsics are not available on this CPU."); + FLAG_SET_DEFAULT(UseCRC32Intrinsics, false); + } + + if (UseCRC32CIntrinsics) { + warning("CRC32C intrinsics are not available on this CPU."); + FLAG_SET_DEFAULT(UseCRC32CIntrinsics, false); + } + + if (UseMD5Intrinsics) { + warning("MD5 intrinsics are not available on this CPU."); + FLAG_SET_DEFAULT(UseMD5Intrinsics, false); + } + + if (UseRVV) { + if (!(_features & CPU_V)) { + warning("RVV is not supported on this CPU"); + FLAG_SET_DEFAULT(UseRVV, false); + } else { + // read vector length from vector CSR vlenb + _initial_vector_length = get_current_vector_length(); + } + } + + if (UseRVC && !(_features & CPU_C)) { + warning("RVC is not supported on this CPU"); + FLAG_SET_DEFAULT(UseRVC, false); + } + + if (FLAG_IS_DEFAULT(AvoidUnalignedAccesses)) { + FLAG_SET_DEFAULT(AvoidUnalignedAccesses, true); + } + + if (UseZbb) { + if (FLAG_IS_DEFAULT(UsePopCountInstruction)) { + FLAG_SET_DEFAULT(UsePopCountInstruction, true); + } + } else { + FLAG_SET_DEFAULT(UsePopCountInstruction, false); + } + + char buf[512]; + buf[0] = '\0'; + if (_uarch != NULL && strcmp(_uarch, "") != 0) snprintf(buf, sizeof(buf), "%s,", _uarch); + strcat(buf, "rv64"); +#define ADD_FEATURE_IF_SUPPORTED(id, name, bit) if (_features & CPU_##id) strcat(buf, name); + CPU_FEATURE_FLAGS(ADD_FEATURE_IF_SUPPORTED) +#undef ADD_FEATURE_IF_SUPPORTED + + _features_string = os::strdup(buf); + +#ifdef COMPILER2 + c2_initialize(); +#endif // COMPILER2 +} + +#ifdef COMPILER2 +void VM_Version::c2_initialize() { + if (UseCMoveUnconditionally) { + FLAG_SET_DEFAULT(UseCMoveUnconditionally, false); + } + + if (ConditionalMoveLimit > 0) { + FLAG_SET_DEFAULT(ConditionalMoveLimit, 0); + } + + if (!UseRVV) { + FLAG_SET_DEFAULT(SpecialEncodeISOArray, false); + } + + if (!UseRVV && MaxVectorSize) { + FLAG_SET_DEFAULT(MaxVectorSize, 0); + } + + if (!UseRVV) { + FLAG_SET_DEFAULT(UseRVVForBigIntegerShiftIntrinsics, false); + } + + if (UseRVV) { + if (FLAG_IS_DEFAULT(MaxVectorSize)) { + MaxVectorSize = _initial_vector_length; + } else if (MaxVectorSize < 16) { + warning("RVV does not support vector length less than 16 bytes. Disabling RVV."); + UseRVV = false; + } else if (is_power_of_2(MaxVectorSize)) { + if (MaxVectorSize > _initial_vector_length) { + warning("Current system only supports max RVV vector length %d. Set MaxVectorSize to %d", + _initial_vector_length, _initial_vector_length); + } + MaxVectorSize = _initial_vector_length; + } else { + vm_exit_during_initialization(err_msg("Unsupported MaxVectorSize: %d", (int)MaxVectorSize)); + } + } + + // disable prefetch + if (FLAG_IS_DEFAULT(AllocatePrefetchStyle)) { + FLAG_SET_DEFAULT(AllocatePrefetchStyle, 0); + } + + if (FLAG_IS_DEFAULT(UseMulAddIntrinsic)) { + FLAG_SET_DEFAULT(UseMulAddIntrinsic, true); + } + + if (FLAG_IS_DEFAULT(UseMultiplyToLenIntrinsic)) { + FLAG_SET_DEFAULT(UseMultiplyToLenIntrinsic, true); + } + + if (FLAG_IS_DEFAULT(UseSquareToLenIntrinsic)) { + FLAG_SET_DEFAULT(UseSquareToLenIntrinsic, true); + } + + if (FLAG_IS_DEFAULT(UseMontgomeryMultiplyIntrinsic)) { + FLAG_SET_DEFAULT(UseMontgomeryMultiplyIntrinsic, true); + } + + if (FLAG_IS_DEFAULT(UseMontgomerySquareIntrinsic)) { + FLAG_SET_DEFAULT(UseMontgomerySquareIntrinsic, true); + } +} +#endif // COMPILER2 diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.hpp b/src/hotspot/cpu/riscv/vm_version_riscv.hpp new file mode 100644 index 0000000000000..39c127e9285b7 --- /dev/null +++ b/src/hotspot/cpu/riscv/vm_version_riscv.hpp @@ -0,0 +1,82 @@ +/* + * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_VM_VERSION_RISCV_HPP +#define CPU_RISCV_VM_VERSION_RISCV_HPP + +#include "runtime/abstract_vm_version.hpp" +#include "runtime/arguments.hpp" +#include "runtime/globals_extension.hpp" +#include "utilities/sizes.hpp" + +class VM_Version : public Abstract_VM_Version { +#ifdef COMPILER2 +private: + static void c2_initialize(); +#endif // COMPILER2 + +// VM modes (satp.mode) privileged ISA 1.10 +enum VM_MODE { + VM_MBARE = 0, + VM_SV39 = 8, + VM_SV48 = 9, + VM_SV57 = 10, + VM_SV64 = 11 +}; + +protected: + static const char* _uarch; + static const char* _vm_mode; + static uint32_t _initial_vector_length; + static void get_os_cpu_info(); + static uint32_t get_current_vector_length(); + static VM_MODE get_satp_mode(); + +public: + // Initialization + static void initialize(); + + constexpr static bool supports_stack_watermark_barrier() { return true; } + + enum Feature_Flag { +#define CPU_FEATURE_FLAGS(decl) \ + decl(I, "i", 8) \ + decl(M, "m", 12) \ + decl(A, "a", 0) \ + decl(F, "f", 5) \ + decl(D, "d", 3) \ + decl(C, "c", 2) \ + decl(V, "v", 21) + +#define DECLARE_CPU_FEATURE_FLAG(id, name, bit) CPU_##id = (1 << bit), + CPU_FEATURE_FLAGS(DECLARE_CPU_FEATURE_FLAG) +#undef DECLARE_CPU_FEATURE_FLAG + }; + + static void initialize_cpu_information(void); +}; + +#endif // CPU_RISCV_VM_VERSION_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/vmreg_riscv.cpp b/src/hotspot/cpu/riscv/vmreg_riscv.cpp new file mode 100644 index 0000000000000..36a1869c57313 --- /dev/null +++ b/src/hotspot/cpu/riscv/vmreg_riscv.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/assembler.hpp" +#include "code/vmreg.hpp" + +void VMRegImpl::set_regName() { + int i = 0; + Register reg = ::as_Register(0); + for ( ; i < ConcreteRegisterImpl::max_gpr ; ) { + for (int j = 0 ; j < RegisterImpl::max_slots_per_register ; j++) { + regName[i++] = reg->name(); + } + reg = reg->successor(); + } + + FloatRegister freg = ::as_FloatRegister(0); + for ( ; i < ConcreteRegisterImpl::max_fpr ; ) { + for (int j = 0 ; j < FloatRegisterImpl::max_slots_per_register ; j++) { + regName[i++] = freg->name(); + } + freg = freg->successor(); + } + + VectorRegister vreg = ::as_VectorRegister(0); + for ( ; i < ConcreteRegisterImpl::max_vpr ; ) { + for (int j = 0 ; j < VectorRegisterImpl::max_slots_per_register ; j++) { + regName[i++] = vreg->name(); + } + vreg = vreg->successor(); + } + + for ( ; i < ConcreteRegisterImpl::number_of_registers ; i++) { + regName[i] = "NON-GPR-FPR-VPR"; + } +} + +VMReg VMRegImpl::vmStorageToVMReg(int type, int index) { + Unimplemented(); + return VMRegImpl::Bad(); +} diff --git a/src/hotspot/cpu/riscv/vmreg_riscv.hpp b/src/hotspot/cpu/riscv/vmreg_riscv.hpp new file mode 100644 index 0000000000000..9e611b1f67110 --- /dev/null +++ b/src/hotspot/cpu/riscv/vmreg_riscv.hpp @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2006, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_VMREG_RISCV_HPP +#define CPU_RISCV_VMREG_RISCV_HPP + +inline bool is_Register() { + return (unsigned int) value() < (unsigned int) ConcreteRegisterImpl::max_gpr; +} + +inline bool is_FloatRegister() { + return value() >= ConcreteRegisterImpl::max_gpr && value() < ConcreteRegisterImpl::max_fpr; +} + +inline bool is_VectorRegister() { + return value() >= ConcreteRegisterImpl::max_fpr && value() < ConcreteRegisterImpl::max_vpr; +} + +inline Register as_Register() { + assert(is_Register(), "must be"); + return ::as_Register(value() / RegisterImpl::max_slots_per_register); +} + +inline FloatRegister as_FloatRegister() { + assert(is_FloatRegister() && is_even(value()), "must be"); + return ::as_FloatRegister((value() - ConcreteRegisterImpl::max_gpr) / + FloatRegisterImpl::max_slots_per_register); +} + +inline VectorRegister as_VectorRegister() { + assert(is_VectorRegister() && ((value() & (VectorRegisterImpl::max_slots_per_register - 1)) == 0), "must be"); + return ::as_VectorRegister((value() - ConcreteRegisterImpl::max_fpr) / + VectorRegisterImpl::max_slots_per_register); +} + +inline bool is_concrete() { + assert(is_reg(), "must be"); + if (is_VectorRegister()) { + int base = value() - ConcreteRegisterImpl::max_fpr; + return (base % VectorRegisterImpl::max_slots_per_register) == 0; + } else { + return is_even(value()); + } +} + +#endif // CPU_RISCV_VMREG_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/vmreg_riscv.inline.hpp b/src/hotspot/cpu/riscv/vmreg_riscv.inline.hpp new file mode 100644 index 0000000000000..09e07f5896404 --- /dev/null +++ b/src/hotspot/cpu/riscv/vmreg_riscv.inline.hpp @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2006, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_VM_VMREG_RISCV_INLINE_HPP +#define CPU_RISCV_VM_VMREG_RISCV_INLINE_HPP + +inline VMReg RegisterImpl::as_VMReg() { + if (this == noreg) { + return VMRegImpl::Bad(); + } + return VMRegImpl::as_VMReg(encoding() * RegisterImpl::max_slots_per_register); +} + +inline VMReg FloatRegisterImpl::as_VMReg() { + return VMRegImpl::as_VMReg((encoding() * FloatRegisterImpl::max_slots_per_register) + + ConcreteRegisterImpl::max_gpr); +} + +inline VMReg VectorRegisterImpl::as_VMReg() { + return VMRegImpl::as_VMReg((encoding() * VectorRegisterImpl::max_slots_per_register) + + ConcreteRegisterImpl::max_fpr); +} + +#endif // CPU_RISCV_VM_VMREG_RISCV_INLINE_HPP diff --git a/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp b/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp new file mode 100644 index 0000000000000..af62f7d77eb43 --- /dev/null +++ b/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp @@ -0,0 +1,260 @@ +/* + * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, Red Hat Inc. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/macroAssembler.inline.hpp" +#include "assembler_riscv.inline.hpp" +#include "code/vtableStubs.hpp" +#include "interp_masm_riscv.hpp" +#include "memory/resourceArea.hpp" +#include "oops/compiledICHolder.hpp" +#include "oops/instanceKlass.hpp" +#include "oops/klassVtable.hpp" +#include "runtime/sharedRuntime.hpp" +#include "vmreg_riscv.inline.hpp" +#ifdef COMPILER2 +#include "opto/runtime.hpp" +#endif + +// machine-dependent part of VtableStubs: create VtableStub of correct size and +// initialize its code + +#define __ masm-> + +#ifndef PRODUCT +extern "C" void bad_compiled_vtable_index(JavaThread* thread, oop receiver, int index); +#endif + +VtableStub* VtableStubs::create_vtable_stub(int vtable_index) { + // Read "A word on VtableStub sizing" in share/code/vtableStubs.hpp for details on stub sizing. + const int stub_code_length = code_size_limit(true); + VtableStub* s = new(stub_code_length) VtableStub(true, vtable_index); + // Can be NULL if there is no free space in the code cache. + if (s == NULL) { + return NULL; + } + + // Count unused bytes in instruction sequences of variable size. + // We add them to the computed buffer size in order to avoid + // overflow in subsequently generated stubs. + address start_pc = NULL; + int slop_bytes = 0; + int slop_delta = 0; + + ResourceMark rm; + CodeBuffer cb(s->entry_point(), stub_code_length); + MacroAssembler* masm = new MacroAssembler(&cb); + assert_cond(masm != NULL); + +#if (!defined(PRODUCT) && defined(COMPILER2)) + if (CountCompiledCalls) { + __ la(t2, ExternalAddress((address) SharedRuntime::nof_megamorphic_calls_addr())); + __ increment(Address(t2)); + } +#endif + + // get receiver (need to skip return address on top of stack) + assert(VtableStub::receiver_location() == j_rarg0->as_VMReg(), "receiver expected in j_rarg0"); + + // get receiver klass + address npe_addr = __ pc(); + __ load_klass(t2, j_rarg0); + +#ifndef PRODUCT + if (DebugVtables) { + Label L; + start_pc = __ pc(); + + // check offset vs vtable length + __ lwu(t0, Address(t2, Klass::vtable_length_offset())); + __ mv(t1, vtable_index * vtableEntry::size()); + __ bgt(t0, t1, L); + __ enter(); + __ mv(x12, vtable_index); + + __ call_VM(noreg, CAST_FROM_FN_PTR(address, bad_compiled_vtable_index), j_rarg0, x12); + const ptrdiff_t estimate = 256; + const ptrdiff_t codesize = __ pc() - start_pc; + slop_delta = estimate - codesize; // call_VM varies in length, depending on data + slop_bytes += slop_delta; + assert(slop_delta >= 0, "vtable #%d: Code size estimate (%d) for DebugVtables too small, required: %d", vtable_index, (int)estimate, (int)codesize); + + __ leave(); + __ bind(L); + } +#endif // PRODUCT + + start_pc = __ pc(); + __ lookup_virtual_method(t2, vtable_index, xmethod); + // lookup_virtual_method generates + // 4 instructions (maximum value encountered in normal case):li(lui + addiw) + add + ld + // 1 instruction (best case):ld * 1 + slop_delta = 16 - (int)(__ pc() - start_pc); + slop_bytes += slop_delta; + assert(slop_delta >= 0, "negative slop(%d) encountered, adjust code size estimate!", slop_delta); + +#ifndef PRODUCT + if (DebugVtables) { + Label L; + __ beqz(xmethod, L); + __ ld(t0, Address(xmethod, Method::from_compiled_offset())); + __ bnez(t0, L); + __ stop("Vtable entry is NULL"); + __ bind(L); + } +#endif // PRODUCT + + // x10: receiver klass + // xmethod: Method* + // x12: receiver + address ame_addr = __ pc(); + __ ld(t0, Address(xmethod, Method::from_compiled_offset())); + __ jr(t0); + + masm->flush(); + bookkeeping(masm, tty, s, npe_addr, ame_addr, true, vtable_index, slop_bytes, 0); + + return s; +} + +VtableStub* VtableStubs::create_itable_stub(int itable_index) { + // Read "A word on VtableStub sizing" in share/code/vtableStubs.hpp for details on stub sizing. + const int stub_code_length = code_size_limit(false); + VtableStub* s = new(stub_code_length) VtableStub(false, itable_index); + // Can be NULL if there is no free space in the code cache. + if (s == NULL) { + return NULL; + } + // Count unused bytes in instruction sequences of variable size. + // We add them to the computed buffer size in order to avoid + // overflow in subsequently generated stubs. + address start_pc = NULL; + int slop_bytes = 0; + int slop_delta = 0; + + ResourceMark rm; + CodeBuffer cb(s->entry_point(), stub_code_length); + MacroAssembler* masm = new MacroAssembler(&cb); + assert_cond(masm != NULL); + +#if (!defined(PRODUCT) && defined(COMPILER2)) + if (CountCompiledCalls) { + __ la(x18, ExternalAddress((address) SharedRuntime::nof_megamorphic_calls_addr())); + __ increment(Address(x18)); + } +#endif + + // get receiver (need to skip return address on top of stack) + assert(VtableStub::receiver_location() == j_rarg0->as_VMReg(), "receiver expected in j_rarg0"); + + // Entry arguments: + // t1: CompiledICHolder + // j_rarg0: Receiver + + // This stub is called from compiled code which has no callee-saved registers, + // so all registers except arguments are free at this point. + const Register recv_klass_reg = x18; + const Register holder_klass_reg = x19; // declaring interface klass (DECC) + const Register resolved_klass_reg = xmethod; // resolved interface klass (REFC) + const Register temp_reg = x28; + const Register temp_reg2 = x29; + const Register icholder_reg = t1; + + Label L_no_such_interface; + + __ ld(resolved_klass_reg, Address(icholder_reg, CompiledICHolder::holder_klass_offset())); + __ ld(holder_klass_reg, Address(icholder_reg, CompiledICHolder::holder_metadata_offset())); + + start_pc = __ pc(); + + // get receiver klass (also an implicit null-check) + address npe_addr = __ pc(); + __ load_klass(recv_klass_reg, j_rarg0); + + // Receiver subtype check against REFC. + __ lookup_interface_method(// inputs: rec. class, interface + recv_klass_reg, resolved_klass_reg, noreg, + // outputs: scan temp. reg1, scan temp. reg2 + temp_reg2, temp_reg, + L_no_such_interface, + /*return_method=*/false); + + const ptrdiff_t typecheckSize = __ pc() - start_pc; + start_pc = __ pc(); + + // Get selected method from declaring class and itable index + __ lookup_interface_method(// inputs: rec. class, interface, itable index + recv_klass_reg, holder_klass_reg, itable_index, + // outputs: method, scan temp. reg + xmethod, temp_reg, + L_no_such_interface); + + const ptrdiff_t lookupSize = __ pc() - start_pc; + + // Reduce "estimate" such that "padding" does not drop below 8. + const ptrdiff_t estimate = 256; + const ptrdiff_t codesize = typecheckSize + lookupSize; + slop_delta = (int)(estimate - codesize); + slop_bytes += slop_delta; + assert(slop_delta >= 0, "itable #%d: Code size estimate (%d) for lookup_interface_method too small, required: %d", itable_index, (int)estimate, (int)codesize); + +#ifdef ASSERT + if (DebugVtables) { + Label L2; + __ beqz(xmethod, L2); + __ ld(t0, Address(xmethod, Method::from_compiled_offset())); + __ bnez(t0, L2); + __ stop("compiler entrypoint is null"); + __ bind(L2); + } +#endif // ASSERT + + // xmethod: Method* + // j_rarg0: receiver + address ame_addr = __ pc(); + __ ld(t0, Address(xmethod, Method::from_compiled_offset())); + __ jr(t0); + + __ bind(L_no_such_interface); + // Handle IncompatibleClassChangeError in itable stubs. + // More detailed error message. + // We force resolving of the call site by jumping to the "handle + // wrong method" stub, and so let the interpreter runtime do all the + // dirty work. + assert(SharedRuntime::get_handle_wrong_method_stub() != NULL, "check initialization order"); + __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); + + masm->flush(); + bookkeeping(masm, tty, s, npe_addr, ame_addr, false, itable_index, slop_bytes, 0); + + return s; +} + +int VtableStub::pd_code_alignment() { + // RISCV cache line size is not an architected constant. We just align on word size. + const unsigned int icache_line_size = wordSize; + return icache_line_size; +} diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp index a7d5a4a1c4284..eaf83d8b311a6 100644 --- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp @@ -950,12 +950,7 @@ void LIR_Assembler::mem2reg(LIR_Opr src_opr, LIR_Opr dest, BasicType type, LIR_P } break; case T_ADDRESS: - if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) { - __ z_llgf(dest->as_register(), disp_value, disp_reg, src); - __ decode_klass_not_null(dest->as_register()); - } else { - __ z_lg(dest->as_register(), disp_value, disp_reg, src); - } + __ z_lg(dest->as_register(), disp_value, disp_reg, src); break; case T_ARRAY : // fall through case T_OBJECT: @@ -2754,6 +2749,22 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) { __ bind(*op->stub()->continuation()); } +void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { + Register obj = op->obj()->as_pointer_register(); + Register result = op->result_opr()->as_pointer_register(); + + CodeEmitInfo* info = op->info(); + if (info != NULL) { + add_debug_info_for_null_check_here(info); + } + + if (UseCompressedClassPointers) { + __ z_llgf(result, Address(obj, oopDesc::klass_offset_in_bytes())); + __ decode_klass_not_null(result); + } else { + __ z_lg(result, Address(obj, oopDesc::klass_offset_in_bytes())); + } +} void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { ciMethod* method = op->profiled_method(); int bci = op->profiled_bci(); diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad index 6610e394e9552..0abe0ce4875e3 100644 --- a/src/hotspot/cpu/s390/s390.ad +++ b/src/hotspot/cpu/s390/s390.ad @@ -5048,6 +5048,7 @@ instruct membar_CPUOrder() %{ instruct membar_storestore() %{ match(MemBarStoreStore); + match(StoreStoreFence); ins_cost(0); size(0); format %{ "MEMBAR-storestore (empty)" %} diff --git a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp index 95facb3a2efa7..1c6c1713c77a8 100644 --- a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp +++ b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016, 2019 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -3409,8 +3409,9 @@ void SharedRuntime::montgomery_multiply(jint *a_ints, jint *b_ints, jint *n_ints // Make very sure we don't use so much space that the stack might // overflow. 512 jints corresponds to an 16384-bit integer and // will use here a total of 8k bytes of stack space. + int divisor = sizeof(unsigned long) * 4; + guarantee(longwords <= 8192 / divisor, "must be"); int total_allocation = longwords * sizeof (unsigned long) * 4; - guarantee(total_allocation <= 8192, "must be"); unsigned long *scratch = (unsigned long *)alloca(total_allocation); // Local scratch arrays @@ -3439,8 +3440,9 @@ void SharedRuntime::montgomery_square(jint *a_ints, jint *n_ints, // Make very sure we don't use so much space that the stack might // overflow. 512 jints corresponds to an 16384-bit integer and // will use here a total of 6k bytes of stack space. + int divisor = sizeof(unsigned long) * 3; + guarantee(longwords <= (8192 / divisor), "must be"); int total_allocation = longwords * sizeof (unsigned long) * 3; - guarantee(total_allocation <= 8192, "must be"); unsigned long *scratch = (unsigned long *)alloca(total_allocation); // Local scratch arrays diff --git a/src/hotspot/cpu/s390/stubGenerator_s390.cpp b/src/hotspot/cpu/s390/stubGenerator_s390.cpp index 102200fc08f35..3aa5ead87b190 100644 --- a/src/hotspot/cpu/s390/stubGenerator_s390.cpp +++ b/src/hotspot/cpu/s390/stubGenerator_s390.cpp @@ -1457,44 +1457,6 @@ class StubGenerator: public StubCodeGenerator { StubRoutines::_arrayof_oop_arraycopy_uninit = generate_conjoint_oop_copy (true, "arrayof_oop_arraycopy_uninit", true); } - void generate_safefetch(const char* name, int size, address* entry, address* fault_pc, address* continuation_pc) { - - // safefetch signatures: - // int SafeFetch32(int* adr, int errValue); - // intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue); - // - // arguments: - // Z_ARG1 = adr - // Z_ARG2 = errValue - // - // result: - // Z_RET = *adr or errValue - - StubCodeMark mark(this, "StubRoutines", name); - - // entry point - // Load *adr into Z_ARG2, may fault. - *entry = *fault_pc = __ pc(); - switch (size) { - case 4: - // Sign extended int32_t. - __ z_lgf(Z_ARG2, 0, Z_ARG1); - break; - case 8: - // int64_t - __ z_lg(Z_ARG2, 0, Z_ARG1); - break; - default: - ShouldNotReachHere(); - } - - // Return errValue or *adr. - *continuation_pc = __ pc(); - __ z_lgr(Z_RET, Z_ARG2); - __ z_br(Z_R14); - - } - // Call interface for AES_encryptBlock, AES_decryptBlock stubs. // // Z_ARG1 - source data block. Ptr to leftmost byte to be processed. @@ -2337,10 +2299,6 @@ class StubGenerator: public StubCodeGenerator { // Comapct string intrinsics: Translate table for string inflate intrinsic. Used by trot instruction. StubRoutines::zarch::_trot_table_addr = (address)StubRoutines::zarch::_trot_table; - - // safefetch stubs - generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry, &StubRoutines::_safefetch32_fault_pc, &StubRoutines::_safefetch32_continuation_pc); - generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry, &StubRoutines::_safefetchN_fault_pc, &StubRoutines::_safefetchN_continuation_pc); } diff --git a/src/hotspot/cpu/x86/assembler_x86.cpp b/src/hotspot/cpu/x86/assembler_x86.cpp index ba07e84c4c649..f7a46091ac26f 100644 --- a/src/hotspot/cpu/x86/assembler_x86.cpp +++ b/src/hotspot/cpu/x86/assembler_x86.cpp @@ -218,6 +218,17 @@ void Assembler::init_attributes(void) { _attributes = NULL; } +void Assembler::set_attributes(InstructionAttr* attributes) { + // Record the assembler in the attributes, so the attributes destructor can + // clear the assembler's attributes, cleaning up the otherwise dangling + // pointer. gcc13 has a false positive warning, because it doesn't tie that + // cleanup to the assignment of _attributes here. + attributes->set_current_assembler(this); + PRAGMA_DIAG_PUSH + PRAGMA_DANGLING_POINTER_IGNORED + _attributes = attributes; + PRAGMA_DIAG_POP +} void Assembler::membar(Membar_mask_bits order_constraint) { // We only have to handle StoreLoad @@ -2633,6 +2644,16 @@ void Assembler::ktestd(KRegister src1, KRegister src2) { emit_int16((unsigned char)0x99, (0xC0 | encode)); } + +void Assembler::kshiftlbl(KRegister dst, KRegister src, int imm8) { + assert(VM_Version::supports_avx512dq(), ""); + InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false); + int encode = vex_prefix_and_encode(dst->encoding(), 0 , src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); + emit_int16(0x32, (0xC0 | encode)); + emit_int8(imm8); +} + + void Assembler::movb(Address dst, int imm8) { InstructionMark im(this); prefix(dst); @@ -3948,6 +3969,14 @@ void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, Compa emit_int24(0x3E, (0xC0 | encode), vcc); } +void Assembler::evpcmpuq(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) { + assert(VM_Version::supports_avx512vlbw(), ""); + InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); + attributes.set_is_evex_instruction(); + int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes); + emit_int24(0x1E, (0xC0 | encode), vcc); +} + void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) { assert(VM_Version::supports_avx512vlbw(), ""); InstructionMark im(this); @@ -6574,6 +6603,19 @@ void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector emit_operand(dst, src); } +void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { + assert(VM_Version::supports_evex(), ""); + assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), ""); + InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true); + attributes.set_is_evex_instruction(); + attributes.set_embedded_opmask_register_specifier(mask); + if (merge) { + attributes.reset_is_clear_context(); + } + int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes); + emit_int16((unsigned char)0xD4, (0xC0 | encode)); +} + void Assembler::psubb(XMMRegister dst, XMMRegister src) { NOT_LP64(assert(VM_Version::supports_sse2(), "")); InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true); @@ -8958,7 +9000,6 @@ void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix vex_x = adr.index_needs_rex(); } set_attributes(attributes); - attributes->set_current_assembler(this); // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction // is allowed in legacy mode and has resources which will fit in it. @@ -9005,7 +9046,6 @@ int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexS bool vex_b = (src_enc & 8) == 8; bool vex_x = false; set_attributes(attributes); - attributes->set_current_assembler(this); // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction // is allowed in legacy mode and has resources which will fit in it. @@ -9683,7 +9723,7 @@ void Assembler::prefix(Register dst, Address adr, Prefix p) { if (adr.index_needs_rex()) { assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X"); } else { - prefix(REX_B); + p = (Prefix)(p | REX_B); } } else { if (adr.index_needs_rex()) { diff --git a/src/hotspot/cpu/x86/assembler_x86.hpp b/src/hotspot/cpu/x86/assembler_x86.hpp index 3c72c93ad4014..c3d25ad5974c8 100644 --- a/src/hotspot/cpu/x86/assembler_x86.hpp +++ b/src/hotspot/cpu/x86/assembler_x86.hpp @@ -678,7 +678,8 @@ class Assembler : public AbstractAssembler { bool _legacy_mode_vlbw; NOT_LP64(bool _is_managed;) - class InstructionAttr *_attributes; + InstructionAttr *_attributes; + void set_attributes(InstructionAttr* attributes); // 64bit prefixes void prefix(Register reg); @@ -896,8 +897,6 @@ class Assembler : public AbstractAssembler { // belong in macro assembler but there is no need for both varieties to exist void init_attributes(void); - - void set_attributes(InstructionAttr *attributes) { _attributes = attributes; } void clear_attributes(void) { _attributes = NULL; } void set_managed(void) { NOT_LP64(_is_managed = true;) } @@ -1492,6 +1491,8 @@ class Assembler : public AbstractAssembler { void ktestql(KRegister dst, KRegister src); + void kshiftlbl(KRegister dst, KRegister src, int imm8); + void movdl(XMMRegister dst, Register src); void movdl(Register dst, XMMRegister src); void movdl(XMMRegister dst, Address src); @@ -1727,6 +1728,8 @@ class Assembler : public AbstractAssembler { void evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len); void evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len); + void evpcmpuq(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len); + void pcmpeqw(XMMRegister dst, XMMRegister src); void vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); void evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len); @@ -2249,6 +2252,10 @@ class Assembler : public AbstractAssembler { void vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len); void vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len); + // Leaf level assembler routines for masked operations. + void evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); +// void evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); + // Sub packed integers void psubb(XMMRegister dst, XMMRegister src); void psubw(XMMRegister dst, XMMRegister src); @@ -2587,7 +2594,6 @@ class InstructionAttr { if (_current_assembler != NULL) { _current_assembler->clear_attributes(); } - _current_assembler = NULL; } private: diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index e2454f32481c9..d73e730f3e3c8 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -1184,7 +1184,6 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch LIR_Address* addr = src->as_address_ptr(); Address from_addr = as_Address(addr); - Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg); if (addr->base()->type() == T_OBJECT) { __ verify_oop(addr->base()->as_pointer_register()); @@ -1257,11 +1256,7 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch break; case T_ADDRESS: - if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) { - __ movl(dest->as_register(), from_addr); - } else { - __ movptr(dest->as_register(), from_addr); - } + __ movptr(dest->as_register(), from_addr); break; case T_INT: __ movl(dest->as_register(), from_addr); @@ -1367,12 +1362,6 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch if (!UseZGC) { __ verify_oop(dest->as_register()); } - } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) { -#ifdef _LP64 - if (UseCompressedClassPointers) { - __ decode_klass_not_null(dest->as_register(), tmp_load_klass); - } -#endif } } @@ -3532,6 +3521,23 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) { __ bind(*op->stub()->continuation()); } +void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { + Register obj = op->obj()->as_pointer_register(); + Register result = op->result_opr()->as_pointer_register(); + + CodeEmitInfo* info = op->info(); + if (info != NULL) { + add_debug_info_for_null_check_here(info); + } + +#ifdef _LP64 + if (UseCompressedClassPointers) { + __ movl(result, Address(obj, oopDesc::klass_offset_in_bytes())); + __ decode_klass_not_null(result, rscratch1); + } else +#endif + __ movptr(result, Address(obj, oopDesc::klass_offset_in_bytes())); +} void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { ciMethod* method = op->profiled_method(); diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp index 8f16747b6e166..e1c9f0221504c 100644 --- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp @@ -2094,6 +2094,14 @@ XMMRegister C2_MacroAssembler::get_lane(BasicType typ, XMMRegister dst, XMMRegis } } +void C2_MacroAssembler::movsxl(BasicType typ, Register dst) { + if (typ == T_BYTE) { + movsbl(dst, dst); + } else if (typ == T_SHORT) { + movswl(dst, dst); + } +} + void C2_MacroAssembler::get_elem(BasicType typ, Register dst, XMMRegister src, int elemindex) { int esize = type2aelembytes(typ); int elem_per_lane = 16/esize; @@ -2105,13 +2113,11 @@ void C2_MacroAssembler::get_elem(BasicType typ, Register dst, XMMRegister src, i movq(dst, src); } else { movdl(dst, src); - if (typ == T_BYTE) - movsbl(dst, dst); - else if (typ == T_SHORT) - movswl(dst, dst); + movsxl(typ, dst); } } else { extract(typ, dst, src, eindex); + movsxl(typ, dst); } } diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp index 5527cff637d08..01830e5edff54 100644 --- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp @@ -132,6 +132,7 @@ XMMRegister get_lane(BasicType typ, XMMRegister dst, XMMRegister src, int elemindex); void get_elem(BasicType typ, Register dst, XMMRegister src, int elemindex); void get_elem(BasicType typ, XMMRegister dst, XMMRegister src, int elemindex, Register tmp = noreg, XMMRegister vtmp = xnoreg); + void movsxl(BasicType typ, Register dst); // vector test void vectortest(int bt, int vlen, XMMRegister src1, XMMRegister src2, diff --git a/src/hotspot/cpu/x86/crc32c.h b/src/hotspot/cpu/x86/crc32c.h index dfdbd525e7765..b2d4a90619dc7 100644 --- a/src/hotspot/cpu/x86/crc32c.h +++ b/src/hotspot/cpu/x86/crc32c.h @@ -1,5 +1,5 @@ /* -* Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. +* Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -39,15 +39,15 @@ enum { // based on ubench study using methodology described in // V. Gopal et al. / Fast CRC Computation for iSCSI Polynomial Using CRC32 Instruction April 2011 8 // - // arbitrary value between 27 and 256 - CRC32C_MIDDLE = 8 * 86, + // arbitrary value between 9 and 256 + CRC32C_MIDDLE = 8 * 74, // V. Gopal et al. / Fast CRC Computation for iSCSI Polynomial Using CRC32 Instruction April 2011 9 - // shows that 240 and 1024 are equally good choices as the 216==8*27 + // shows that 240 and 1024 are equally good choices as the 216==8*9*3 // // Selecting the smallest value which resulted in a significant performance improvement over // sequential version - CRC32C_LOW = 8 * 27, + CRC32C_LOW = 8 * 9, CRC32C_NUM_ChunkSizeInBytes = 3, diff --git a/src/hotspot/cpu/x86/frame_x86.cpp b/src/hotspot/cpu/x86/frame_x86.cpp index 3faf6f99db8ac..aa5628a7f7e60 100644 --- a/src/hotspot/cpu/x86/frame_x86.cpp +++ b/src/hotspot/cpu/x86/frame_x86.cpp @@ -64,8 +64,11 @@ bool frame::safe_for_sender(JavaThread *thread) { return false; } - // unextended sp must be within the stack and above or equal sp - if (!thread->is_in_stack_range_incl(unextended_sp, sp)) { + // unextended sp must be within the stack + // Note: sp can be greater than unextended_sp in the case of + // interpreted -> interpreted calls that go through a method handle linker, + // since those pop the last argument (the appendix) from the stack. + if (!thread->is_in_stack_range_incl(unextended_sp, sp - Interpreter::stackElementSize)) { return false; } diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index 740189c144fb0..da4a40ba3f905 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -7937,24 +7937,27 @@ void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Registe addl(tmp1, in2); addq(tmp1, in1); - BIND(L_wordByWord); cmpq(in1, tmp1); - jcc(Assembler::greaterEqual, L_byteByByteProlog); - crc32(in_out, Address(in1, 0), 4); - addq(in1, 4); - jmp(L_wordByWord); + jccb(Assembler::greaterEqual, L_byteByByteProlog); + align(16); + BIND(L_wordByWord); + crc32(in_out, Address(in1, 0), 8); + addq(in1, 8); + cmpq(in1, tmp1); + jcc(Assembler::less, L_wordByWord); BIND(L_byteByByteProlog); andl(in2, 0x00000007); movl(tmp2, 1); - BIND(L_byteByByte); cmpl(tmp2, in2); jccb(Assembler::greater, L_exit); + BIND(L_byteByByte); crc32(in_out, Address(in1, 0), 1); incq(in1); incl(tmp2); - jmp(L_byteByByte); + cmpl(tmp2, in2); + jcc(Assembler::lessEqual, L_byteByByte); BIND(L_exit); } diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp index fcde34b275174..dc5c9023c0dcd 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp @@ -964,6 +964,8 @@ class MacroAssembler: public Assembler { void roundDec(XMMRegister key, int rnum); void lastroundDec(XMMRegister key, int rnum); void ev_load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask); + void ev_add128(XMMRegister xmmdst, XMMRegister xmmsrc1, XMMRegister xmmsrc2, + int vector_len, KRegister ktmp, Register rscratch = noreg); public: void aesecb_encrypt(Register source_addr, Register dest_addr, Register key, Register len); diff --git a/src/hotspot/cpu/x86/macroAssembler_x86_aes.cpp b/src/hotspot/cpu/x86/macroAssembler_x86_aes.cpp index 4f86bc5b6954e..dcc48a6c1f03b 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86_aes.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86_aes.cpp @@ -779,6 +779,19 @@ void MacroAssembler::avx_ghash(Register input_state, Register htbl, vpxor(xmm15, xmm15, xmm15, Assembler::AVX_128bit); } +// Add 128-bit integers in xmmsrc1 to xmmsrc2, then place the result in xmmdst. +// Clobber ktmp and rscratch. +// Used by aesctr_encrypt. +void MacroAssembler::ev_add128(XMMRegister xmmdst, XMMRegister xmmsrc1, XMMRegister xmmsrc2, + int vector_len, KRegister ktmp, Register rscratch) { + vpaddq(xmmdst, xmmsrc1, xmmsrc2, vector_len); + evpcmpuq(ktmp, xmmdst, xmmsrc2, lt, vector_len); // set mask[0/1] bit if addq to dst[0/1] wraps + kshiftlbl(ktmp, ktmp, 1); // mask[1] <- mask[0], mask[0] <- 0, etc + + evpaddq(xmmdst, ktmp, xmmdst, xmm17, /*merge*/true, + vector_len); // dst[1]++ if mask[1] set +} + // AES Counter Mode using VAES instructions void MacroAssembler::aesctr_encrypt(Register src_addr, Register dest_addr, Register key, Register counter, Register len_reg, Register used, Register used_addr, Register saved_encCounter_start) { @@ -831,19 +844,23 @@ void MacroAssembler::aesctr_encrypt(Register src_addr, Register dest_addr, Regis //shuffle counter using lbswap_mask vpshufb(xmm8, xmm8, xmm16, Assembler::AVX_512bit); + // Vector value to propagate carries + evmovdquq(xmm17, ExternalAddress(StubRoutines::x86::counter_mask_ones_addr()), Assembler::AVX_512bit, r15); // pre-increment and propagate counter values to zmm9-zmm15 registers. // Linc0 increments the zmm8 by 1 (initial value being 0), Linc4 increments the counters zmm9-zmm15 by 4 // The counter is incremented after each block i.e. 16 bytes is processed; // each zmm register has 4 counter values as its MSB // the counters are incremented in parallel - vpaddd(xmm8, xmm8, ExternalAddress(StubRoutines::x86::counter_mask_addr() + 64), Assembler::AVX_512bit, r15);//linc0 - vpaddd(xmm9, xmm8, ExternalAddress(StubRoutines::x86::counter_mask_addr() + 128), Assembler::AVX_512bit, r15);//linc4(rip) - vpaddd(xmm10, xmm9, ExternalAddress(StubRoutines::x86::counter_mask_addr() + 128), Assembler::AVX_512bit, r15);//Linc4(rip) - vpaddd(xmm11, xmm10, ExternalAddress(StubRoutines::x86::counter_mask_addr() + 128), Assembler::AVX_512bit, r15);//Linc4(rip) - vpaddd(xmm12, xmm11, ExternalAddress(StubRoutines::x86::counter_mask_addr() + 128), Assembler::AVX_512bit, r15);//Linc4(rip) - vpaddd(xmm13, xmm12, ExternalAddress(StubRoutines::x86::counter_mask_addr() + 128), Assembler::AVX_512bit, r15);//Linc4(rip) - vpaddd(xmm14, xmm13, ExternalAddress(StubRoutines::x86::counter_mask_addr() + 128), Assembler::AVX_512bit, r15);//Linc4(rip) - vpaddd(xmm15, xmm14, ExternalAddress(StubRoutines::x86::counter_mask_addr() + 128), Assembler::AVX_512bit, r15);//Linc4(rip) + evmovdquq(xmm19, ExternalAddress(StubRoutines::x86::counter_mask_addr() + 64), Assembler::AVX_512bit, r15 /*rscratch*/);//linc0 + ev_add128(xmm8, xmm8, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15); + evmovdquq(xmm19, ExternalAddress(StubRoutines::x86::counter_mask_addr() + 128), Assembler::AVX_512bit, r15 /*rscratch*/);//linc4 + ev_add128(xmm9, xmm8, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15); + ev_add128(xmm10, xmm9, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15); + ev_add128(xmm11, xmm10, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15); + ev_add128(xmm12, xmm11, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15); + ev_add128(xmm13, xmm12, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15); + ev_add128(xmm14, xmm13, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15); + ev_add128(xmm15, xmm14, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15); // load linc32 mask in zmm register.linc32 increments counter by 32 evmovdquq(xmm19, ExternalAddress(StubRoutines::x86::counter_mask_addr() + 256), Assembler::AVX_512bit, r15);//Linc32 @@ -891,21 +908,21 @@ void MacroAssembler::aesctr_encrypt(Register src_addr, Register dest_addr, Regis // This is followed by incrementing counter values in zmm8-zmm15. // Since we will be processing 32 blocks at a time, the counter is incremented by 32. roundEnc(xmm21, 7); - vpaddq(xmm8, xmm8, xmm19, Assembler::AVX_512bit); + ev_add128/*!!!*/(xmm8, xmm8, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15 /*rscratch*/); roundEnc(xmm22, 7); - vpaddq(xmm9, xmm9, xmm19, Assembler::AVX_512bit); + ev_add128/*!!!*/(xmm9, xmm9, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15 /*rscratch*/); roundEnc(xmm23, 7); - vpaddq(xmm10, xmm10, xmm19, Assembler::AVX_512bit); + ev_add128/*!!!*/(xmm10, xmm10, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15 /*rscratch*/); roundEnc(xmm24, 7); - vpaddq(xmm11, xmm11, xmm19, Assembler::AVX_512bit); + ev_add128/*!!!*/(xmm11, xmm11, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15 /*rscratch*/); roundEnc(xmm25, 7); - vpaddq(xmm12, xmm12, xmm19, Assembler::AVX_512bit); + ev_add128/*!!!*/(xmm12, xmm12, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15 /*rscratch*/); roundEnc(xmm26, 7); - vpaddq(xmm13, xmm13, xmm19, Assembler::AVX_512bit); + ev_add128/*!!!*/(xmm13, xmm13, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15 /*rscratch*/); roundEnc(xmm27, 7); - vpaddq(xmm14, xmm14, xmm19, Assembler::AVX_512bit); + ev_add128/*!!!*/(xmm14, xmm14, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15 /*rscratch*/); roundEnc(xmm28, 7); - vpaddq(xmm15, xmm15, xmm19, Assembler::AVX_512bit); + ev_add128/*!!!*/(xmm15, xmm15, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15 /*rscratch*/); roundEnc(xmm29, 7); cmpl(rounds, 52); @@ -983,8 +1000,8 @@ void MacroAssembler::aesctr_encrypt(Register src_addr, Register dest_addr, Regis vpshufb(xmm3, xmm11, xmm16, Assembler::AVX_512bit); evpxorq(xmm3, xmm3, xmm20, Assembler::AVX_512bit); // Increment counter values by 16 - vpaddq(xmm8, xmm8, xmm19, Assembler::AVX_512bit); - vpaddq(xmm9, xmm9, xmm19, Assembler::AVX_512bit); + ev_add128/*!!!*/(xmm8, xmm8, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15 /*rscratch*/); + ev_add128/*!!!*/(xmm9, xmm9, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15 /*rscratch*/); // AES encode rounds roundEnc(xmm21, 3); roundEnc(xmm22, 3); @@ -1051,7 +1068,7 @@ void MacroAssembler::aesctr_encrypt(Register src_addr, Register dest_addr, Regis vpshufb(xmm1, xmm9, xmm16, Assembler::AVX_512bit); evpxorq(xmm1, xmm1, xmm20, Assembler::AVX_512bit); // increment counter by 8 - vpaddq(xmm8, xmm8, xmm19, Assembler::AVX_512bit); + ev_add128/*!!!*/(xmm8, xmm8, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15 /*rscratch*/); // AES encode roundEnc(xmm21, 1); roundEnc(xmm22, 1); @@ -1109,7 +1126,7 @@ void MacroAssembler::aesctr_encrypt(Register src_addr, Register dest_addr, Regis vpshufb(xmm0, xmm8, xmm16, Assembler::AVX_512bit); evpxorq(xmm0, xmm0, xmm20, Assembler::AVX_512bit); // Increment counter - vpaddq(xmm8, xmm8, xmm19, Assembler::AVX_512bit); + ev_add128/*!!!*/(xmm8, xmm8, xmm19, Assembler::AVX_512bit, /*ktmp*/k1, r15 /*rscratch*/); vaesenc(xmm0, xmm0, xmm21, Assembler::AVX_512bit); vaesenc(xmm0, xmm0, xmm22, Assembler::AVX_512bit); vaesenc(xmm0, xmm0, xmm23, Assembler::AVX_512bit); @@ -1159,7 +1176,7 @@ void MacroAssembler::aesctr_encrypt(Register src_addr, Register dest_addr, Regis evpxorq(xmm0, xmm0, xmm20, Assembler::AVX_128bit); vaesenc(xmm0, xmm0, xmm21, Assembler::AVX_128bit); // Increment counter by 1 - vpaddq(xmm8, xmm8, xmm19, Assembler::AVX_128bit); + ev_add128/*!!!*/(xmm8, xmm8, xmm19, Assembler::AVX_128bit, /*ktmp*/k1, r15 /*rscratch*/); vaesenc(xmm0, xmm0, xmm22, Assembler::AVX_128bit); vaesenc(xmm0, xmm0, xmm23, Assembler::AVX_128bit); vaesenc(xmm0, xmm0, xmm24, Assembler::AVX_128bit); diff --git a/src/hotspot/cpu/x86/macroAssembler_x86_md5.cpp b/src/hotspot/cpu/x86/macroAssembler_x86_md5.cpp index 9f6ed1b9d657d..439c17b10d37a 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86_md5.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86_md5.cpp @@ -52,19 +52,16 @@ // int com.sun.security.provider.MD5.implCompress0(byte[] b, int ofs) void MacroAssembler::fast_md5(Register buf, Address state, Address ofs, Address limit, bool multi_block) { - Label start, done_hash, loop0; + Label done_hash, loop0; - bind(start); - - bind(loop0); - - // Save hash values for addition after rounds movptr(rdi, state); movl(rax, Address(rdi, 0)); movl(rbx, Address(rdi, 4)); movl(rcx, Address(rdi, 8)); movl(rdx, Address(rdi, 12)); + bind(loop0); + #define FF(r1, r2, r3, r4, k, s, t) \ addl(r1, t); \ movl(rsi, r3); \ @@ -189,10 +186,14 @@ void MacroAssembler::fast_md5(Register buf, Address state, Address ofs, Address // write hash values back in the correct order movptr(rdi, state); - addl(Address(rdi, 0), rax); - addl(Address(rdi, 4), rbx); - addl(Address(rdi, 8), rcx); - addl(Address(rdi, 12), rdx); + addl(rax, Address(rdi, 0)); + movl(Address(rdi, 0), rax); + addl(rbx, Address(rdi, 4)); + movl(Address(rdi, 4), rbx); + addl(rcx, Address(rdi, 8)); + movl(Address(rdi, 8), rcx); + addl(rdx, Address(rdi, 12)); + movl(Address(rdi, 12), rdx); if (multi_block) { // increment data pointer and loop if more to process diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp index 492ec5962d097..0c178ac21ef37 100644 --- a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp @@ -998,7 +998,6 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - __ flush(); return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry); } diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp index 768f2dabdd3bb..f912b7650a170 100644 --- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1032,7 +1032,6 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup); - __ flush(); return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry); } @@ -3714,8 +3713,9 @@ void SharedRuntime::montgomery_multiply(jint *a_ints, jint *b_ints, jint *n_ints // Make very sure we don't use so much space that the stack might // overflow. 512 jints corresponds to an 16384-bit integer and // will use here a total of 8k bytes of stack space. + int divisor = sizeof(julong) * 4; + guarantee(longwords <= 8192 / divisor, "must be"); int total_allocation = longwords * sizeof (julong) * 4; - guarantee(total_allocation <= 8192, "must be"); julong *scratch = (julong *)alloca(total_allocation); // Local scratch arrays @@ -3743,8 +3743,9 @@ void SharedRuntime::montgomery_square(jint *a_ints, jint *n_ints, // Make very sure we don't use so much space that the stack might // overflow. 512 jints corresponds to an 16384-bit integer and // will use here a total of 6k bytes of stack space. + int divisor = sizeof(julong) * 3; + guarantee(longwords <= (8192 / divisor), "must be"); int total_allocation = longwords * sizeof (julong) * 3; - guarantee(total_allocation <= 8192, "must be"); julong *scratch = (julong *)alloca(total_allocation); // Local scratch arrays diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_32.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_32.cpp index 654066ac87262..f0ed72c290056 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_32.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_32.cpp @@ -3629,40 +3629,6 @@ class StubGenerator: public StubCodeGenerator { } - // Safefetch stubs. - void generate_safefetch(const char* name, int size, address* entry, - address* fault_pc, address* continuation_pc) { - // safefetch signatures: - // int SafeFetch32(int* adr, int errValue); - // intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue); - - StubCodeMark mark(this, "StubRoutines", name); - - // Entry point, pc or function descriptor. - *entry = __ pc(); - - __ movl(rax, Address(rsp, 0x8)); - __ movl(rcx, Address(rsp, 0x4)); - // Load *adr into eax, may fault. - *fault_pc = __ pc(); - switch (size) { - case 4: - // int32_t - __ movl(rax, Address(rcx, 0)); - break; - case 8: - // int64_t - Unimplemented(); - break; - default: - ShouldNotReachHere(); - } - - // Return errValue or *adr. - *continuation_pc = __ pc(); - __ ret(0); - } - address generate_method_entry_barrier() { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", "nmethod_entry_barrier"); @@ -3961,14 +3927,6 @@ class StubGenerator: public StubCodeGenerator { StubRoutines::_dtan = generate_libmTan(); } } - - // Safefetch stubs. - generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry, - &StubRoutines::_safefetch32_fault_pc, - &StubRoutines::_safefetch32_continuation_pc); - StubRoutines::_safefetchN_entry = StubRoutines::_safefetch32_entry; - StubRoutines::_safefetchN_fault_pc = StubRoutines::_safefetch32_fault_pc; - StubRoutines::_safefetchN_continuation_pc = StubRoutines::_safefetch32_continuation_pc; } void generate_all() { diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp index c7634a2e59627..163455cb16a2b 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp @@ -3829,46 +3829,6 @@ class StubGenerator: public StubCodeGenerator { return start; } - // Safefetch stubs. - void generate_safefetch(const char* name, int size, address* entry, - address* fault_pc, address* continuation_pc) { - // safefetch signatures: - // int SafeFetch32(int* adr, int errValue); - // intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue); - // - // arguments: - // c_rarg0 = adr - // c_rarg1 = errValue - // - // result: - // PPC_RET = *adr or errValue - - StubCodeMark mark(this, "StubRoutines", name); - - // Entry point, pc or function descriptor. - *entry = __ pc(); - - // Load *adr into c_rarg1, may fault. - *fault_pc = __ pc(); - switch (size) { - case 4: - // int32_t - __ movl(c_rarg1, Address(c_rarg0, 0)); - break; - case 8: - // int64_t - __ movq(c_rarg1, Address(c_rarg0, 0)); - break; - default: - ShouldNotReachHere(); - } - - // return errValue or *adr - *continuation_pc = __ pc(); - __ movq(rax, c_rarg1); - __ ret(0); - } - // This is a version of CBC/AES Decrypt which does 4 blocks in a loop at a time // to hide instruction latency // @@ -4424,7 +4384,19 @@ class StubGenerator: public StubCodeGenerator { return start; } - // Vector AES Counter implementation + // Vector AES Counter implementation + + address counter_mask_ones_addr() { + __ align64(); + StubCodeMark mark(this, "StubRoutines", "counter_mask_addr"); + address start = __ pc(); + for (int i = 0; i < 4; i ++) { + __ emit_data64(0x0000000000000000, relocInfo::none); + __ emit_data64(0x0000000000000001, relocInfo::none); + } + return start; + } + address generate_counterMode_VectorAESCrypt() { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", "counterMode_AESCrypt"); @@ -6494,26 +6466,38 @@ address generate_avx_ghash_processBlocks() { BLOCK_COMMENT("Entry:"); __ enter(); // required for proper stackwalking of RuntimeStub frame + Label L_continue; + if (VM_Version::supports_sse4_1() && VM_Version::supports_avx512_vpclmulqdq() && VM_Version::supports_avx512bw() && VM_Version::supports_avx512vl()) { + Label L_doSmall; + + __ cmpl(len, 384); + __ jcc(Assembler::lessEqual, L_doSmall); + __ lea(j, ExternalAddress(StubRoutines::x86::crc32c_table_avx512_addr())); __ kernel_crc32_avx512(crc, buf, len, j, l, k); - } else { + + __ jmp(L_continue); + + __ bind(L_doSmall); + } #ifdef _WIN64 - __ push(y); - __ push(z); + __ push(y); + __ push(z); #endif - __ crc32c_ipl_alg2_alt2(crc, buf, len, - a, j, k, - l, y, z, - c_farg0, c_farg1, c_farg2, - is_pclmulqdq_supported); + __ crc32c_ipl_alg2_alt2(crc, buf, len, + a, j, k, + l, y, z, + c_farg0, c_farg1, c_farg2, + is_pclmulqdq_supported); #ifdef _WIN64 - __ pop(z); - __ pop(y); + __ pop(z); + __ pop(y); #endif - } + + __ bind(L_continue); __ movl(rax, crc); __ vzeroupper(); __ leave(); // required for proper stackwalking of RuntimeStub frame @@ -7557,14 +7541,6 @@ address generate_avx_ghash_processBlocks() { StubRoutines::_dtan = generate_libmTan(); } } - - // Safefetch stubs. - generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry, - &StubRoutines::_safefetch32_fault_pc, - &StubRoutines::_safefetch32_continuation_pc); - generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry, - &StubRoutines::_safefetchN_fault_pc, - &StubRoutines::_safefetchN_continuation_pc); } void generate_all() { @@ -7641,6 +7617,7 @@ address generate_avx_ghash_processBlocks() { if (UseAESCTRIntrinsics) { if (VM_Version::supports_avx512_vaes() && VM_Version::supports_avx512bw() && VM_Version::supports_avx512vl()) { StubRoutines::x86::_counter_mask_addr = counter_mask_addr(); + StubRoutines::x86::_counter_mask_ones_addr = counter_mask_ones_addr(); StubRoutines::_counterMode_AESCrypt = generate_counterMode_VectorAESCrypt(); } else { StubRoutines::x86::_counter_shuffle_mask_addr = generate_counter_shuffle_mask(); diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.cpp b/src/hotspot/cpu/x86/stubRoutines_x86.cpp index 4471a5498bb46..904a5485ff6a6 100644 --- a/src/hotspot/cpu/x86/stubRoutines_x86.cpp +++ b/src/hotspot/cpu/x86/stubRoutines_x86.cpp @@ -71,6 +71,7 @@ address StubRoutines::x86::_avx2_shuffle_base64 = NULL; address StubRoutines::x86::_avx2_input_mask_base64 = NULL; address StubRoutines::x86::_avx2_lut_base64 = NULL; address StubRoutines::x86::_counter_mask_addr = NULL; +address StubRoutines::x86::_counter_mask_ones_addr = NULL; address StubRoutines::x86::_lookup_lo_base64 = NULL; address StubRoutines::x86::_lookup_hi_base64 = NULL; address StubRoutines::x86::_lookup_lo_base64url = NULL; diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.hpp b/src/hotspot/cpu/x86/stubRoutines_x86.hpp index 45f4870c37280..84eb964d98ff2 100644 --- a/src/hotspot/cpu/x86/stubRoutines_x86.hpp +++ b/src/hotspot/cpu/x86/stubRoutines_x86.hpp @@ -184,6 +184,7 @@ class x86 { // byte flip mask for sha512 static address _pshuffle_byte_flip_mask_addr_sha512; static address _counter_mask_addr; + static address _counter_mask_ones_addr; // Masks for base64 static address _encoding_table_base64; static address _shuffle_base64; @@ -343,6 +344,7 @@ class x86 { static address base64_avx2_input_mask_addr() { return _avx2_input_mask_base64; } static address base64_avx2_lut_addr() { return _avx2_lut_base64; } static address counter_mask_addr() { return _counter_mask_addr; } + static address counter_mask_ones_addr() { return _counter_mask_ones_addr; } static address base64_vbmi_lookup_lo_addr() { return _lookup_lo_base64; } static address base64_vbmi_lookup_hi_addr() { return _lookup_hi_base64; } static address base64_vbmi_lookup_lo_url_addr() { return _lookup_lo_base64url; } diff --git a/src/hotspot/cpu/x86/x86_32.ad b/src/hotspot/cpu/x86/x86_32.ad index 92cd95e744328..01005c73047cb 100644 --- a/src/hotspot/cpu/x86/x86_32.ad +++ b/src/hotspot/cpu/x86/x86_32.ad @@ -6646,6 +6646,7 @@ instruct unnecessary_membar_volatile() %{ instruct membar_storestore() %{ match(MemBarStoreStore); + match(StoreStoreFence); ins_cost(0); size(0); diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad index 3a95006819afc..654b92db5bfb2 100644 --- a/src/hotspot/cpu/x86/x86_64.ad +++ b/src/hotspot/cpu/x86/x86_64.ad @@ -6765,6 +6765,7 @@ instruct unnecessary_membar_volatile() instruct membar_storestore() %{ match(MemBarStoreStore); + match(StoreStoreFence); ins_cost(0); size(0); diff --git a/src/hotspot/cpu/zero/stubGenerator_zero.cpp b/src/hotspot/cpu/zero/stubGenerator_zero.cpp index d4f52d576d8ef..dbd31dbb5704e 100644 --- a/src/hotspot/cpu/zero/stubGenerator_zero.cpp +++ b/src/hotspot/cpu/zero/stubGenerator_zero.cpp @@ -43,18 +43,6 @@ #include "opto/runtime.hpp" #endif -// For SafeFetch we need POSIX tls and setjmp -#include -#include -static pthread_key_t g_jmpbuf_key; - -// return the currently active jump buffer for this thread -// - if there is any, NULL otherwise. Called from -// zero signal handlers. -extern sigjmp_buf* get_jmp_buf_for_continuation() { - return (sigjmp_buf*) pthread_getspecific(g_jmpbuf_key); -} - // Declaration and definition of StubGenerator (no .hpp file). // For a more detailed description of the stub routine structure // see the comment in stubRoutines.hpp @@ -188,57 +176,6 @@ class StubGenerator: public StubCodeGenerator { StubRoutines::_oop_arraycopy; } - static int SafeFetch32(int *adr, int errValue) { - - // set up a jump buffer; anchor the pointer to the jump buffer in tls; then - // do the pointer access. If pointer is invalid, we crash; in signal - // handler, we retrieve pointer to jmp buffer from tls, and jump back. - // - // Note: the jump buffer itself - which can get pretty large depending on - // the architecture - lives on the stack and that is fine, because we will - // not rewind the stack: either we crash, in which case signal handler - // frame is below us, or we don't crash, in which case it does not matter. - sigjmp_buf jb; - if (sigsetjmp(jb, 1)) { - // we crashed. clean up tls and return default value. - pthread_setspecific(g_jmpbuf_key, NULL); - return errValue; - } else { - // preparation phase - pthread_setspecific(g_jmpbuf_key, &jb); - } - - int value = errValue; - value = *adr; - - // all went well. clean tls. - pthread_setspecific(g_jmpbuf_key, NULL); - - return value; - } - - static intptr_t SafeFetchN(intptr_t *adr, intptr_t errValue) { - - sigjmp_buf jb; - if (sigsetjmp(jb, 1)) { - // we crashed. clean up tls and return default value. - pthread_setspecific(g_jmpbuf_key, NULL); - return errValue; - } else { - // preparation phase - pthread_setspecific(g_jmpbuf_key, &jb); - } - - intptr_t value = errValue; - value = *adr; - - // all went well. clean tls. - pthread_setspecific(g_jmpbuf_key, NULL); - - return value; - - } - void generate_initial() { // Generates all stubs and initializes the entry points @@ -285,15 +222,6 @@ class StubGenerator: public StubCodeGenerator { // arraycopy stubs used by compilers generate_arraycopy_stubs(); - // Safefetch stubs. - pthread_key_create(&g_jmpbuf_key, NULL); - StubRoutines::_safefetch32_entry = CAST_FROM_FN_PTR(address, StubGenerator::SafeFetch32); - StubRoutines::_safefetch32_fault_pc = NULL; - StubRoutines::_safefetch32_continuation_pc = NULL; - - StubRoutines::_safefetchN_entry = CAST_FROM_FN_PTR(address, StubGenerator::SafeFetchN); - StubRoutines::_safefetchN_fault_pc = NULL; - StubRoutines::_safefetchN_continuation_pc = NULL; } public: diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp index c2d2ecb204bc2..104bf127c278f 100644 --- a/src/hotspot/os/aix/os_aix.cpp +++ b/src/hotspot/os/aix/os_aix.cpp @@ -61,7 +61,7 @@ #include "runtime/os.hpp" #include "runtime/osThread.hpp" #include "runtime/perfMemory.hpp" -#include "runtime/safefetch.inline.hpp" +#include "runtime/safefetch.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/statSampler.hpp" #include "runtime/thread.inline.hpp" @@ -904,9 +904,9 @@ bool os::create_attached_thread(JavaThread* thread) { PosixSignals::hotspot_sigmask(thread); log_info(os, thread)("Thread attached (tid: " UINTX_FORMAT ", kernel thread id: " UINTX_FORMAT - ", stack: " PTR_FORMAT " - " PTR_FORMAT " (" SIZE_FORMAT "k) ).", + ", stack: " PTR_FORMAT " - " PTR_FORMAT " (" SIZE_FORMAT "K) ).", os::current_thread_id(), (uintx) kernel_thread_id, - p2i(thread->stack_base()), p2i(thread->stack_end()), thread->stack_size()); + p2i(thread->stack_base()), p2i(thread->stack_end()), thread->stack_size() / K); return true; } @@ -2067,36 +2067,33 @@ static bool checked_mprotect(char* addr, size_t size, int prot) { // if (!os::Aix::xpg_sus_mode()) { - if (CanUseSafeFetch32()) { + const bool read_protected = + (SafeFetch32((int*)addr, 0x12345678) == 0x12345678 && + SafeFetch32((int*)addr, 0x76543210) == 0x76543210) ? true : false; - const bool read_protected = - (SafeFetch32((int*)addr, 0x12345678) == 0x12345678 && - SafeFetch32((int*)addr, 0x76543210) == 0x76543210) ? true : false; - - if (prot & PROT_READ) { - rc = !read_protected; - } else { - rc = read_protected; - } + if (prot & PROT_READ) { + rc = !read_protected; + } else { + rc = read_protected; + } - if (!rc) { - if (os::Aix::on_pase()) { - // There is an issue on older PASE systems where mprotect() will return success but the - // memory will not be protected. - // This has nothing to do with the problem of using mproect() on SPEC1170 incompatible - // machines; we only see it rarely, when using mprotect() to protect the guard page of - // a stack. It is an OS error. - // - // A valid strategy is just to try again. This usually works. :-/ - - ::usleep(1000); - Events::log(NULL, "Protecting memory [" INTPTR_FORMAT "," INTPTR_FORMAT "] with protection modes %x", p2i(addr), p2i(addr+size), prot); - if (::mprotect(addr, size, prot) == 0) { - const bool read_protected_2 = - (SafeFetch32((int*)addr, 0x12345678) == 0x12345678 && - SafeFetch32((int*)addr, 0x76543210) == 0x76543210) ? true : false; - rc = true; - } + if (!rc) { + if (os::Aix::on_pase()) { + // There is an issue on older PASE systems where mprotect() will return success but the + // memory will not be protected. + // This has nothing to do with the problem of using mproect() on SPEC1170 incompatible + // machines; we only see it rarely, when using mprotect() to protect the guard page of + // a stack. It is an OS error. + // + // A valid strategy is just to try again. This usually works. :-/ + + ::usleep(1000); + Events::log(NULL, "Protecting memory [" INTPTR_FORMAT "," INTPTR_FORMAT "] with protection modes %x", p2i(addr), p2i(addr+size), prot); + if (::mprotect(addr, size, prot) == 0) { + const bool read_protected_2 = + (SafeFetch32((int*)addr, 0x12345678) == 0x12345678 && + SafeFetch32((int*)addr, 0x76543210) == 0x76543210) ? true : false; + rc = true; } } } diff --git a/src/hotspot/os/aix/os_aix.inline.hpp b/src/hotspot/os/aix/os_aix.inline.hpp index c0ec5faa56422..88a39b4b3700c 100644 --- a/src/hotspot/os/aix/os_aix.inline.hpp +++ b/src/hotspot/os/aix/os_aix.inline.hpp @@ -47,4 +47,8 @@ inline bool os::must_commit_stack_guard_pages() { inline void os::map_stack_shadow_pages(address sp) { } +// stubbed-out trim-native support +inline bool os::can_trim_native_heap() { return false; } +inline bool os::trim_native_heap(os::size_change_t* rss_change) { return false; } + #endif // OS_AIX_OS_AIX_INLINE_HPP diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp index d07ebcd7f7b58..071045c6a6e90 100644 --- a/src/hotspot/os/bsd/os_bsd.cpp +++ b/src/hotspot/os/bsd/os_bsd.cpp @@ -729,9 +729,9 @@ bool os::create_attached_thread(JavaThread* thread) { PosixSignals::hotspot_sigmask(thread); log_info(os, thread)("Thread attached (tid: " UINTX_FORMAT ", pthread id: " UINTX_FORMAT - ", stack: " PTR_FORMAT " - " PTR_FORMAT " (" SIZE_FORMAT "k) ).", + ", stack: " PTR_FORMAT " - " PTR_FORMAT " (" SIZE_FORMAT "K) ).", os::current_thread_id(), (uintx) pthread_self(), - p2i(thread->stack_base()), p2i(thread->stack_end()), thread->stack_size()); + p2i(thread->stack_base()), p2i(thread->stack_end()), thread->stack_size() / K); return true; } @@ -1377,8 +1377,34 @@ void os::print_os_info(outputStream* st) { VM_Version::print_platform_virtualization_info(st); } +#ifdef __APPLE__ +static void print_sysctl_info_string(const char* sysctlkey, outputStream* st, char* buf, size_t size) { + if (sysctlbyname(sysctlkey, buf, &size, nullptr, 0) >= 0) { + st->print_cr("%s:%s", sysctlkey, buf); + } +} + +static void print_sysctl_info_uint64(const char* sysctlkey, outputStream* st) { + uint64_t val; + size_t size=sizeof(uint64_t); + if (sysctlbyname(sysctlkey, &val, &size, nullptr, 0) >= 0) { + st->print_cr("%s:%llu", sysctlkey, val); + } +} +#endif + void os::pd_print_cpu_info(outputStream* st, char* buf, size_t buflen) { - // Nothing to do for now. +#ifdef __APPLE__ + print_sysctl_info_string("machdep.cpu.brand_string", st, buf, buflen); + print_sysctl_info_uint64("hw.cpufrequency", st); + print_sysctl_info_uint64("hw.cpufrequency_min", st); + print_sysctl_info_uint64("hw.cpufrequency_max", st); + print_sysctl_info_uint64("hw.cachelinesize", st); + print_sysctl_info_uint64("hw.l1icachesize", st); + print_sysctl_info_uint64("hw.l1dcachesize", st); + print_sysctl_info_uint64("hw.l2cachesize", st); + print_sysctl_info_uint64("hw.l3cachesize", st); +#endif } void os::get_summary_cpu_info(char* buf, size_t buflen) { diff --git a/src/hotspot/os/bsd/os_bsd.inline.hpp b/src/hotspot/os/bsd/os_bsd.inline.hpp index 0b9fc65b0d836..9fc70da1583a8 100644 --- a/src/hotspot/os/bsd/os_bsd.inline.hpp +++ b/src/hotspot/os/bsd/os_bsd.inline.hpp @@ -51,4 +51,8 @@ inline bool os::must_commit_stack_guard_pages() { inline void os::map_stack_shadow_pages(address sp) { } +// stubbed-out trim-native support +inline bool os::can_trim_native_heap() { return false; } +inline bool os::trim_native_heap(os::size_change_t* rss_change) { return false; } + #endif // OS_BSD_OS_BSD_INLINE_HPP diff --git a/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp b/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp index 79dd256ac61ef..e62dcf4f75975 100644 --- a/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp +++ b/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp @@ -38,40 +38,26 @@ * on the contents of the mountinfo and cgroup files. */ void CgroupV1Controller::set_subsystem_path(char *cgroup_path) { - char buf[MAXPATHLEN+1]; + stringStream ss; if (_root != NULL && cgroup_path != NULL) { if (strcmp(_root, "/") == 0) { - int buflen; - strncpy(buf, _mount_point, MAXPATHLEN); - buf[MAXPATHLEN-1] = '\0'; + ss.print_raw(_mount_point); if (strcmp(cgroup_path,"/") != 0) { - buflen = strlen(buf); - if ((buflen + strlen(cgroup_path)) > (MAXPATHLEN-1)) { - return; - } - strncat(buf, cgroup_path, MAXPATHLEN-buflen); - buf[MAXPATHLEN-1] = '\0'; + ss.print_raw(cgroup_path); } - _path = os::strdup(buf); + _path = os::strdup(ss.base()); } else { if (strcmp(_root, cgroup_path) == 0) { - strncpy(buf, _mount_point, MAXPATHLEN); - buf[MAXPATHLEN-1] = '\0'; - _path = os::strdup(buf); + ss.print_raw(_mount_point); + _path = os::strdup(ss.base()); } else { char *p = strstr(cgroup_path, _root); if (p != NULL && p == _root) { if (strlen(cgroup_path) > strlen(_root)) { - int buflen; - strncpy(buf, _mount_point, MAXPATHLEN); - buf[MAXPATHLEN-1] = '\0'; - buflen = strlen(buf); - if ((buflen + strlen(cgroup_path) - strlen(_root)) > (MAXPATHLEN-1)) { - return; - } - strncat(buf, cgroup_path + strlen(_root), MAXPATHLEN-buflen); - buf[MAXPATHLEN-1] = '\0'; - _path = os::strdup(buf); + ss.print_raw(_mount_point); + const char* cg_path_sub = cgroup_path + strlen(_root); + ss.print_raw(cg_path_sub); + _path = os::strdup(ss.base()); } } } diff --git a/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp b/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp index b1035f89bf056..2456385af7abd 100644 --- a/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp +++ b/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp @@ -234,17 +234,12 @@ void CgroupV2Subsystem::print_version_specific_info(outputStream* st) { } char* CgroupV2Controller::construct_path(char* mount_path, char *cgroup_path) { - char buf[MAXPATHLEN+1]; - int buflen; - strncpy(buf, mount_path, MAXPATHLEN); - buf[MAXPATHLEN] = '\0'; - buflen = strlen(buf); - if ((buflen + strlen(cgroup_path)) > MAXPATHLEN) { - return NULL; + stringStream ss; + ss.print_raw(mount_path); + if (strcmp(cgroup_path, "/") != 0) { + ss.print_raw(cgroup_path); } - strncat(buf, cgroup_path, MAXPATHLEN-buflen); - buf[MAXPATHLEN] = '\0'; - return os::strdup(buf); + return os::strdup(ss.base()); } char* CgroupV2Subsystem::pids_max_val() { diff --git a/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp b/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp index 7a113055423aa..1101ec9fe0b90 100644 --- a/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp +++ b/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp @@ -34,10 +34,11 @@ #include "logging/log.hpp" #include "runtime/init.hpp" #include "runtime/os.hpp" -#include "runtime/safefetch.inline.hpp" +#include "runtime/safefetch.hpp" #include "utilities/align.hpp" #include "utilities/debug.hpp" #include "utilities/growableArray.hpp" +#include "runtime/globals.hpp" #include #include diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp index ba139a3f32006..1bfdacbb2afef 100644 --- a/src/hotspot/os/linux/os_linux.cpp +++ b/src/hotspot/os/linux/os_linux.cpp @@ -744,20 +744,14 @@ static void *thread_native_entry(Thread *thread) { // As a workaround, we call a private but assumed-stable glibc function, // __pthread_get_minstack() to obtain the minstack size and derive the // static TLS size from it. We then increase the user requested stack -// size by this TLS size. +// size by this TLS size. The same function is used to determine whether +// adjustStackSizeForGuardPages() needs to be true. // // Due to compatibility concerns, this size adjustment is opt-in and // controlled via AdjustStackSizeForTLS. typedef size_t (*GetMinStack)(const pthread_attr_t *attr); -GetMinStack _get_minstack_func = NULL; - -static void get_minstack_init() { - _get_minstack_func = - (GetMinStack)dlsym(RTLD_DEFAULT, "__pthread_get_minstack"); - log_info(os, thread)("Lookup of __pthread_get_minstack %s", - _get_minstack_func == NULL ? "failed" : "succeeded"); -} +GetMinStack _get_minstack_func = nullptr; // Initialized via os::init_2() // Returns the size of the static TLS area glibc puts on thread stacks. // The value is cached on first use, which occurs when the first thread @@ -770,8 +764,8 @@ static size_t get_static_tls_area_size(const pthread_attr_t *attr) { // Remove non-TLS area size included in minstack size returned // by __pthread_get_minstack() to get the static TLS size. - // In glibc before 2.27, minstack size includes guard_size. - // In glibc 2.27 and later, guard_size is automatically added + // If adjustStackSizeForGuardPages() is true, minstack size includes + // guard_size. Otherwise guard_size is automatically added // to the stack size by pthread_create and is no longer included // in minstack size. In both cases, the guard_size is taken into // account, so there is no need to adjust the result for that. @@ -800,6 +794,42 @@ static size_t get_static_tls_area_size(const pthread_attr_t *attr) { return tls_size; } +// In glibc versions prior to 2.27 the guard size mechanism +// was not implemented properly. The POSIX standard requires adding +// the size of the guard pages to the stack size, instead glibc +// took the space out of 'stacksize'. Thus we need to adapt the requested +// stack_size by the size of the guard pages to mimic proper behaviour. +// The fix in glibc 2.27 has now been backported to numerous earlier +// glibc versions so we need to do a dynamic runtime check. +static bool _adjustStackSizeForGuardPages = true; +bool os::Linux::adjustStackSizeForGuardPages() { + return _adjustStackSizeForGuardPages; +} + +#ifdef __GLIBC__ +static void init_adjust_stacksize_for_guard_pages() { + assert(_get_minstack_func == nullptr, "initialization error"); + _get_minstack_func =(GetMinStack)dlsym(RTLD_DEFAULT, "__pthread_get_minstack"); + log_info(os, thread)("Lookup of __pthread_get_minstack %s", + _get_minstack_func == nullptr ? "failed" : "succeeded"); + + if (_get_minstack_func != nullptr) { + pthread_attr_t attr; + pthread_attr_init(&attr); + size_t min_stack = _get_minstack_func(&attr); + size_t guard = 16 * K; // Actual value doesn't matter as it is not examined + pthread_attr_setguardsize(&attr, guard); + size_t min_stack2 = _get_minstack_func(&attr); + pthread_attr_destroy(&attr); + // If the minimum stack size changed when we added the guard page space + // then we need to perform the adjustment. + _adjustStackSizeForGuardPages = (min_stack2 != min_stack); + log_info(os)("Glibc stack size guard page adjustment is %sneeded", + _adjustStackSizeForGuardPages ? "" : "not "); + } +} +#endif // GLIBC + bool os::create_thread(Thread* thread, ThreadType thr_type, size_t req_stack_size) { assert(thread->osthread() == NULL, "caller responsible"); @@ -825,23 +855,18 @@ bool os::create_thread(Thread* thread, ThreadType thr_type, // Calculate stack size if it's not specified by caller. size_t stack_size = os::Posix::get_initial_stack_size(thr_type, req_stack_size); - // In glibc versions prior to 2.7 the guard size mechanism - // is not implemented properly. The posix standard requires adding - // the size of the guard pages to the stack size, instead Linux - // takes the space out of 'stacksize'. Thus we adapt the requested - // stack_size by the size of the guard pages to mimick proper - // behaviour. However, be careful not to end up with a size - // of zero due to overflow. Don't add the guard page in that case. size_t guard_size = os::Linux::default_guard_size(thr_type); // Configure glibc guard page. Must happen before calling // get_static_tls_area_size(), which uses the guard_size. pthread_attr_setguardsize(&attr, guard_size); + // Apply stack size adjustments if needed. However, be careful not to end up + // with a size of zero due to overflow. Don't add the adjustment in that case. size_t stack_adjust_size = 0; if (AdjustStackSizeForTLS) { // Adjust the stack_size for on-stack TLS - see get_static_tls_area_size(). stack_adjust_size += get_static_tls_area_size(&attr); - } else { + } else if (os::Linux::adjustStackSizeForGuardPages()) { stack_adjust_size += guard_size; } @@ -851,6 +876,15 @@ bool os::create_thread(Thread* thread, ThreadType thr_type, } assert(is_aligned(stack_size, os::vm_page_size()), "stack_size not aligned"); + // Add an additional page to the stack size to reduce its chances of getting large page aligned + // so that the stack does not get backed by a transparent huge page. + size_t default_large_page_size = os::Linux::default_large_page_size(); + if (default_large_page_size != 0 && + stack_size >= default_large_page_size && + is_aligned(stack_size, default_large_page_size)) { + stack_size += os::vm_page_size(); + } + int status = pthread_attr_setstacksize(&attr, stack_size); if (status != 0) { // pthread_attr_setstacksize() function can fail @@ -984,9 +1018,9 @@ bool os::create_attached_thread(JavaThread* thread) { PosixSignals::hotspot_sigmask(thread); log_info(os, thread)("Thread attached (tid: " UINTX_FORMAT ", pthread id: " UINTX_FORMAT - ", stack: " PTR_FORMAT " - " PTR_FORMAT " (" SIZE_FORMAT "k) ).", + ", stack: " PTR_FORMAT " - " PTR_FORMAT " (" SIZE_FORMAT "K) ).", os::current_thread_id(), (uintx) pthread_self(), - p2i(thread->stack_base()), p2i(thread->stack_end()), thread->stack_size()); + p2i(thread->stack_base()), p2i(thread->stack_end()), thread->stack_size() / K); return true; } @@ -1326,7 +1360,7 @@ void os::Linux::fast_thread_clock_init() { // Note, that some kernels may support the current thread // clock (CLOCK_THREAD_CPUTIME_ID) but not the clocks // returned by the pthread_getcpuclockid(). - // If the fast Posix clocks are supported then the clock_getres() + // If the fast POSIX clocks are supported then the clock_getres() // must return at least tp.tv_sec == 0 which means a resolution // better than 1 sec. This is extra check for reliability. @@ -2559,6 +2593,8 @@ void os::get_summary_cpu_info(char* cpuinfo, size_t length) { strncpy(cpuinfo, "IA64", length); #elif defined(PPC) strncpy(cpuinfo, "PPC64", length); +#elif defined(RISCV) + strncpy(cpuinfo, "RISCV64", length); #elif defined(S390) strncpy(cpuinfo, "S390", length); #elif defined(SPARC) @@ -3801,8 +3837,11 @@ bool os::Linux::setup_large_page_type(size_t page_size) { } void os::large_page_init() { - // 1) Handle the case where we do not want to use huge pages and hence - // there is no need to scan the OS for related info + // Always initialize the default large page size even if large pages are not being used. + size_t default_large_page_size = scan_default_large_page_size(); + os::Linux::_default_large_page_size = default_large_page_size; + + // 1) Handle the case where we do not want to use huge pages if (!UseLargePages && !UseTransparentHugePages && !UseHugeTLBFS && @@ -3820,9 +3859,7 @@ void os::large_page_init() { return; } - // 2) Scan OS info - size_t default_large_page_size = scan_default_large_page_size(); - os::Linux::_default_large_page_size = default_large_page_size; + // 2) check if large pages are configured if (default_large_page_size == 0) { // No large pages configured, return. warn_no_large_pages_configured(); @@ -4560,10 +4597,6 @@ jint os::init_2(void) { return JNI_ERR; } - if (AdjustStackSizeForTLS) { - get_minstack_init(); - } - // Check and sets minimum stack sizes against command line options if (Posix::set_minimum_stack_sizes() == JNI_ERR) { return JNI_ERR; @@ -4586,6 +4619,11 @@ jint os::init_2(void) { log_info(os)("HotSpot is running with %s, %s", Linux::libc_version(), Linux::libpthread_version()); +#ifdef __GLIBC__ + // Check if we need to adjust the stack size for glibc guard pages. + init_adjust_stacksize_for_guard_pages(); +#endif + if (UseNUMA || UseNUMAInterleaving) { Linux::numa_init(); } @@ -5360,9 +5398,9 @@ bool os::start_debugging(char *buf, int buflen) { // // ** P1 (aka bottom) and size (P2 = P1 - size) are the address and stack size // returned from pthread_attr_getstack(). -// ** Due to NPTL implementation error, linux takes the glibc guard page out -// of the stack size given in pthread_attr. We work around this for -// threads created by the VM. (We adapt bottom to be P1 and size accordingly.) +// ** If adjustStackSizeForGuardPages() is true the guard pages have been taken +// out of the stack size given in pthread_attr. We work around this for +// threads created by the VM. We adjust bottom to be P1 and size accordingly. // #ifndef ZERO static void current_stack_region(address * bottom, size_t * size) { @@ -5389,14 +5427,15 @@ static void current_stack_region(address * bottom, size_t * size) { fatal("Cannot locate current stack attributes!"); } - // Work around NPTL stack guard error. - size_t guard_size = 0; - rslt = pthread_attr_getguardsize(&attr, &guard_size); - if (rslt != 0) { - fatal("pthread_attr_getguardsize failed with error = %d", rslt); + if (os::Linux::adjustStackSizeForGuardPages()) { + size_t guard_size = 0; + rslt = pthread_attr_getguardsize(&attr, &guard_size); + if (rslt != 0) { + fatal("pthread_attr_getguardsize failed with error = %d", rslt); + } + *bottom += guard_size; + *size -= guard_size; } - *bottom += guard_size; - *size -= guard_size; pthread_attr_destroy(&attr); @@ -5470,3 +5509,32 @@ void os::print_memory_mappings(char* addr, size_t bytes, outputStream* st) { st->cr(); } } + +bool os::trim_native_heap(os::size_change_t* rss_change) { +#ifdef __GLIBC__ + os::Linux::meminfo_t info1; + os::Linux::meminfo_t info2; + + bool have_info1 = rss_change != nullptr && + os::Linux::query_process_memory_info(&info1); + ::malloc_trim(0); + bool have_info2 = rss_change != nullptr && have_info1 && + os::Linux::query_process_memory_info(&info2); + ssize_t delta = (ssize_t) -1; + if (rss_change != nullptr) { + if (have_info1 && have_info2 && + info1.vmrss != -1 && info2.vmrss != -1 && + info1.vmswap != -1 && info2.vmswap != -1) { + // Note: query_process_memory_info returns values in K + rss_change->before = (info1.vmrss + info1.vmswap) * K; + rss_change->after = (info2.vmrss + info2.vmswap) * K; + } else { + rss_change->after = rss_change->before = SIZE_MAX; + } + } + + return true; +#else + return false; // musl +#endif +} diff --git a/src/hotspot/os/linux/os_linux.hpp b/src/hotspot/os/linux/os_linux.hpp index f71a4b805250f..d618e72bf64af 100644 --- a/src/hotspot/os/linux/os_linux.hpp +++ b/src/hotspot/os/linux/os_linux.hpp @@ -154,6 +154,8 @@ class Linux { // Return default guard size for the specified thread type static size_t default_guard_size(os::ThreadType thr_type); + static bool adjustStackSizeForGuardPages(); // See comments in os_linux.cpp + static void capture_initial_stack(size_t max_size); // Stack overflow handling diff --git a/src/hotspot/os/linux/os_linux.inline.hpp b/src/hotspot/os/linux/os_linux.inline.hpp index 7cbb72eed6525..4701e9aadf6fb 100644 --- a/src/hotspot/os/linux/os_linux.inline.hpp +++ b/src/hotspot/os/linux/os_linux.inline.hpp @@ -43,4 +43,13 @@ inline bool os::must_commit_stack_guard_pages() { inline void os::map_stack_shadow_pages(address sp) { } +// Trim-native support +inline bool os::can_trim_native_heap() { +#ifdef __GLIBC__ + return true; +#else + return false; // musl +#endif +} + #endif // OS_LINUX_OS_LINUX_INLINE_HPP diff --git a/src/hotspot/os/linux/trimCHeapDCmd.cpp b/src/hotspot/os/linux/trimCHeapDCmd.cpp index ee93ac5e8c8d7..33dd6f3a5bdc7 100644 --- a/src/hotspot/os/linux/trimCHeapDCmd.cpp +++ b/src/hotspot/os/linux/trimCHeapDCmd.cpp @@ -1,6 +1,6 @@ /* - * Copyright (c) 2021 SAP SE. All rights reserved. - * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022 SAP SE. All rights reserved. + * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,56 +24,29 @@ */ #include "precompiled.hpp" -#include "logging/log.hpp" -#include "runtime/os.hpp" +#include "runtime/os.inline.hpp" +#include "trimCHeapDCmd.hpp" #include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" #include "utilities/ostream.hpp" -#include "trimCHeapDCmd.hpp" #include void TrimCLibcHeapDCmd::execute(DCmdSource source, TRAPS) { -#ifdef __GLIBC__ - stringStream ss_report(1024); // Note: before calling trim - - os::Linux::meminfo_t info1; - os::Linux::meminfo_t info2; - // Query memory before... - bool have_info1 = os::Linux::query_process_memory_info(&info1); - - _output->print_cr("Attempting trim..."); - ::malloc_trim(0); - _output->print_cr("Done."); - - // ...and after trim. - bool have_info2 = os::Linux::query_process_memory_info(&info2); - - // Print report both to output stream as well to UL - bool wrote_something = false; - if (have_info1 && have_info2) { - if (info1.vmsize != -1 && info2.vmsize != -1) { - ss_report.print_cr("Virtual size before: " SSIZE_FORMAT "k, after: " SSIZE_FORMAT "k, (" SSIZE_FORMAT "k)", - info1.vmsize, info2.vmsize, (info2.vmsize - info1.vmsize)); - wrote_something = true; - } - if (info1.vmrss != -1 && info2.vmrss != -1) { - ss_report.print_cr("RSS before: " SSIZE_FORMAT "k, after: " SSIZE_FORMAT "k, (" SSIZE_FORMAT "k)", - info1.vmrss, info2.vmrss, (info2.vmrss - info1.vmrss)); - wrote_something = true; - } - if (info1.vmswap != -1 && info2.vmswap != -1) { - ss_report.print_cr("Swap before: " SSIZE_FORMAT "k, after: " SSIZE_FORMAT "k, (" SSIZE_FORMAT "k)", - info1.vmswap, info2.vmswap, (info2.vmswap - info1.vmswap)); - wrote_something = true; + if (os::can_trim_native_heap()) { + os::size_change_t sc; + if (os::trim_native_heap(&sc)) { + _output->print("Trim native heap: "); + if (sc.after != SIZE_MAX) { + const size_t delta = sc.after < sc.before ? (sc.before - sc.after) : (sc.after - sc.before); + const char sign = sc.after < sc.before ? '-' : '+'; + _output->print_cr("RSS+Swap: " PROPERFMT "->" PROPERFMT " (%c" PROPERFMT ")", + PROPERFMTARGS(sc.before), PROPERFMTARGS(sc.after), sign, PROPERFMTARGS(delta)); + } else { + _output->print_cr("(no details available)."); + } } + } else { + _output->print_cr("Not available."); } - if (!wrote_something) { - ss_report.print_raw("No details available."); - } - - _output->print_raw(ss_report.base()); - log_info(os)("malloc_trim:\n%s", ss_report.base()); -#else - _output->print_cr("Not available."); -#endif } diff --git a/src/hotspot/os/posix/os_posix.cpp b/src/hotspot/os/posix/os_posix.cpp index 5c34a30a07d6c..fd94f40282cac 100644 --- a/src/hotspot/os/posix/os_posix.cpp +++ b/src/hotspot/os/posix/os_posix.cpp @@ -881,8 +881,8 @@ char* os::Posix::describe_pthread_attr(char* buf, size_t buflen, const pthread_a int detachstate = 0; pthread_attr_getstacksize(attr, &stack_size); pthread_attr_getguardsize(attr, &guard_size); - // Work around linux NPTL implementation error, see also os::create_thread() in os_linux.cpp. - LINUX_ONLY(stack_size -= guard_size); + // Work around glibc stack guard issue, see os::create_thread() in os_linux.cpp. + LINUX_ONLY(if (os::Linux::adjustStackSizeForGuardPages()) stack_size -= guard_size;) pthread_attr_getdetachstate(attr, &detachstate); jio_snprintf(buf, buflen, "stacksize: " SIZE_FORMAT "k, guardsize: " SIZE_FORMAT "k, %s", stack_size / 1024, guard_size / 1024, diff --git a/src/hotspot/os/posix/safefetch_sigjmp.cpp b/src/hotspot/os/posix/safefetch_sigjmp.cpp new file mode 100644 index 0000000000000..71b6d4445e8d8 --- /dev/null +++ b/src/hotspot/os/posix/safefetch_sigjmp.cpp @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2022 SAP SE. All rights reserved. + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "runtime/safefetch.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +#ifdef SAFEFETCH_METHOD_SIGSETJMP + +// For SafeFetch we need POSIX TLS and sigsetjmp/longjmp. +#include +#include +static pthread_key_t g_jmpbuf_key; + +struct InitTLSKey { InitTLSKey() { pthread_key_create(&g_jmpbuf_key, NULL); } }; +static InitTLSKey g_init_tly_key; + +// Handle safefetch, sigsetjmp style: +// +// If a safefetch jump had been established and the sig qualifies, we +// jump back to the established jump point (and hence out of signal handling). +// +// Note that this function will never return for safefetch faults. We just +// keep the prototype the same as other handle_safefetch() versions to keep +// caller sites simple. +bool handle_safefetch(int sig, address ignored1, void* ignored2) { + if (sig == SIGSEGV || sig == SIGBUS) { + // Retrieve jump buffer pointer from TLS. If not NULL, it means we set the + // jump buffer and this is indeed a SafeFetch fault. + // Note signal safety: pthread_getspecific is not safe for signal handler + // usage, but in practice it works and we have done this in the JVM for many + // years (via Thread::current_or_null_safe()). + sigjmp_buf* const jb = (sigjmp_buf*) pthread_getspecific(g_jmpbuf_key); + if (jb) { + siglongjmp(*jb, 1); + } + } + return false; +} + +template +static bool _SafeFetchXX_internal(const T *adr, T* result) { + + T n = 0; + + // Set up a jump buffer. Anchor its pointer in TLS. Then read from the unsafe address. + // If that address was invalid, we fault, and in the signal handler we will jump back + // to the jump point. + sigjmp_buf jb; + if (sigsetjmp(jb, 1) != 0) { + // We faulted. Reset TLS slot, then return. + pthread_setspecific(g_jmpbuf_key, NULL); + *result = 0; + return false; + } + + // Anchor jump buffer in TLS + pthread_setspecific(g_jmpbuf_key, &jb); + + // unsafe access + n = *adr; + + // Still here... All went well, adr was valid. + // Reset TLS slot, then return result. + pthread_setspecific(g_jmpbuf_key, NULL); + *result = n; + + return true; + +} + +int SafeFetch32_impl(int *adr, int errValue) { + int result; + return _SafeFetchXX_internal(adr, &result) ? result : errValue; +} + +intptr_t SafeFetchN_impl(intptr_t *adr, intptr_t errValue) { + intptr_t result; + return _SafeFetchXX_internal(adr, &result) ? result : errValue; +} + +#endif // SAFEFETCH_METHOD_SIGSETJMP diff --git a/src/hotspot/os/posix/safefetch_sigjmp.hpp b/src/hotspot/os/posix/safefetch_sigjmp.hpp new file mode 100644 index 0000000000000..949522cec27b1 --- /dev/null +++ b/src/hotspot/os/posix/safefetch_sigjmp.hpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2022 SAP SE. All rights reserved. + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_POSIX_SAFEFETCH_SIGJMP_HPP +#define OS_POSIX_SAFEFETCH_SIGJMP_HPP + +#include "utilities/globalDefinitions.hpp" + +// On Posix platforms that don't do anything better - or cannot, like Zero - +// SafeFetch is implemented using setjmp/longjmp. That is reliable and portable, +// but slower than other methods, and needs more thread stack (the sigjmp buffer +// lives on the thread stack). + +int SafeFetch32_impl(int* adr, int errValue); +intptr_t SafeFetchN_impl(intptr_t* adr, intptr_t errValue); + +// Handle safefetch, sigsetjmp style. Only call from signal handler. +// If a safefetch jump had been established and the sig qualifies, we +// jump back to the established jump point (and hence out of signal handling). +bool handle_safefetch(int sig, address pc, void* context); + +#endif // OS_POSIX_SAFEFETCH_SIGJMP_HPP diff --git a/src/hotspot/os/posix/safefetch_static_posix.cpp b/src/hotspot/os/posix/safefetch_static_posix.cpp new file mode 100644 index 0000000000000..186f25e089d30 --- /dev/null +++ b/src/hotspot/os/posix/safefetch_static_posix.cpp @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2022 SAP SE. All rights reserved. + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "runtime/os.hpp" +#include "runtime/safefetch.hpp" +#include "utilities/globalDefinitions.hpp" + +#ifdef SAFEFETCH_METHOD_STATIC_ASSEMBLY + +// SafeFetch handling, static assembly style: +// +// SafeFetch32 and SafeFetchN are implemented via static assembly +// and live in os_cpu/xx_xx/safefetch_xx_xx.S + +extern "C" char _SafeFetch32_continuation[] __attribute__ ((visibility ("hidden"))); +extern "C" char _SafeFetch32_fault[] __attribute__ ((visibility ("hidden"))); + +#ifdef _LP64 +extern "C" char _SafeFetchN_continuation[] __attribute__ ((visibility ("hidden"))); +extern "C" char _SafeFetchN_fault[] __attribute__ ((visibility ("hidden"))); +#endif // _LP64 + +bool handle_safefetch(int sig, address pc, void* context) { + ucontext_t* uc = (ucontext_t*)context; + if ((sig == SIGSEGV || sig == SIGBUS) && uc != NULL) { + address pc = os::Posix::ucontext_get_pc(uc); + if (pc == (address)_SafeFetch32_fault) { + os::Posix::ucontext_set_pc(uc, (address)_SafeFetch32_continuation); + return true; + } +#ifdef _LP64 + if (pc == (address)_SafeFetchN_fault) { + os::Posix::ucontext_set_pc(uc, (address)_SafeFetchN_continuation); + return true; + } +#endif + } + return false; +} + +#endif // SAFEFETCH_METHOD_STATIC_ASSEMBLY diff --git a/src/hotspot/os/posix/signals_posix.cpp b/src/hotspot/os/posix/signals_posix.cpp index f95025202c9a8..d800fa3381a86 100644 --- a/src/hotspot/os/posix/signals_posix.cpp +++ b/src/hotspot/os/posix/signals_posix.cpp @@ -32,20 +32,14 @@ #include "runtime/java.hpp" #include "runtime/os.hpp" #include "runtime/osThread.hpp" +#include "runtime/safefetch.hpp" #include "runtime/semaphore.inline.hpp" -#include "runtime/stubRoutines.hpp" #include "runtime/thread.hpp" #include "signals_posix.hpp" #include "utilities/events.hpp" #include "utilities/ostream.hpp" #include "utilities/vmError.hpp" -#ifdef ZERO -// See stubGenerator_zero.cpp -#include -extern sigjmp_buf* get_jmp_buf_for_continuation(); -#endif - #include @@ -594,25 +588,26 @@ int JVM_HANDLE_XXX_SIGNAL(int sig, siginfo_t* info, } #endif - if (!signal_was_handled) { - // Handle SafeFetch access. -#ifndef ZERO - if ((sig == SIGSEGV || sig == SIGBUS) && uc != NULL) { - address pc = os::Posix::ucontext_get_pc(uc); - if (StubRoutines::is_safefetch_fault(pc)) { - os::Posix::ucontext_set_pc(uc, StubRoutines::continuation_for_safefetch_fault(pc)); - signal_was_handled = true; - } - } -#else - // See JDK-8076185 - if (sig == SIGSEGV || sig == SIGBUS) { - sigjmp_buf* const pjb = get_jmp_buf_for_continuation(); - if (pjb) { - siglongjmp(*pjb, 1); - } + // Extract pc from context. Note that for certain signals and certain + // architectures the pc in ucontext_t will point *after* the offending + // instruction. In those cases, use siginfo si_addr instead. + address pc = NULL; + if (uc != NULL) { + if (S390_ONLY(sig == SIGILL || sig == SIGFPE) NOT_S390(false)) { + pc = (address)info->si_addr; + } else if (ZERO_ONLY(true) NOT_ZERO(false)) { + // Non-arch-specific Zero code does not really know the pc. + // This can be alleviated by making arch-specific os::Posix::ucontext_get_pc + // available for Zero for known architectures. But for generic Zero + // code, it would still remain unknown. + pc = NULL; + } else { + pc = os::Posix::ucontext_get_pc(uc); } -#endif // ZERO + } + + if (!signal_was_handled) { + signal_was_handled = handle_safefetch(sig, pc, uc); } // Ignore SIGPIPE and SIGXFSZ (4229104, 6499219). @@ -637,22 +632,6 @@ int JVM_HANDLE_XXX_SIGNAL(int sig, siginfo_t* info, // Invoke fatal error handling. if (!signal_was_handled && abort_if_unrecognized) { - // Extract pc from context for the error handler to display. - address pc = NULL; - if (uc != NULL) { - // prepare fault pc address for error reporting. - if (S390_ONLY(sig == SIGILL || sig == SIGFPE) NOT_S390(false)) { - pc = (address)info->si_addr; - } else if (ZERO_ONLY(true) NOT_ZERO(false)) { - // Non-arch-specific Zero code does not really know the pc. - // This can be alleviated by making arch-specific os::Posix::ucontext_get_pc - // available for Zero for known architectures. But for generic Zero - // code, it would still remain unknown. - pc = NULL; - } else { - pc = os::Posix::ucontext_get_pc(uc); - } - } // For Zero, we ignore the crash context, because: // a) The crash would be in C++ interpreter code, so context is not really relevant; // b) Generic Zero code would not be able to parse it, so when generic error diff --git a/src/hotspot/os/posix/vmError_posix.cpp b/src/hotspot/os/posix/vmError_posix.cpp index d3c78e43cb7c9..2e49c5360a29e 100644 --- a/src/hotspot/os/posix/vmError_posix.cpp +++ b/src/hotspot/os/posix/vmError_posix.cpp @@ -26,7 +26,7 @@ #include "precompiled.hpp" #include "cds/metaspaceShared.hpp" #include "runtime/os.hpp" -#include "runtime/stubRoutines.hpp" +#include "runtime/safefetch.hpp" #include "runtime/thread.hpp" #include "signals_posix.hpp" #include "utilities/debug.hpp" @@ -67,7 +67,6 @@ static void crash_handler(int sig, siginfo_t* info, void* ucVoid) { PosixSignals::unblock_error_signals(); - // support safefetch faults in error handling ucontext_t* const uc = (ucontext_t*) ucVoid; address pc = (uc != NULL) ? os::Posix::ucontext_get_pc(uc) : NULL; @@ -76,22 +75,19 @@ static void crash_handler(int sig, siginfo_t* info, void* ucVoid) { pc = (address) info->si_addr; } - // Needed to make it possible to call SafeFetch.. APIs in error handling. - if (sig == SIGSEGV || sig == SIGBUS) { - if (uc && pc && StubRoutines::is_safefetch_fault(pc)) { - os::Posix::ucontext_set_pc(uc, StubRoutines::continuation_for_safefetch_fault(pc)); - return; - } + // Handle safefetch here too, to be able to use SafeFetch() inside the error handler + if (handle_safefetch(sig, pc, uc)) { + return; + } - // Needed because asserts may happen in error handling too. + // Needed because asserts may happen in error handling too. #ifdef CAN_SHOW_REGISTERS_ON_ASSERT - if (info != NULL && info->si_addr == g_assert_poison) { - if (handle_assert_poison_fault(ucVoid, info->si_addr)) { - return; - } + if ((sig == SIGSEGV || sig == SIGBUS) && info != NULL && info->si_addr == g_assert_poison) { + if (handle_assert_poison_fault(ucVoid, info->si_addr)) { + return; } -#endif // CAN_SHOW_REGISTERS_ON_ASSERT } +#endif // CAN_SHOW_REGISTERS_ON_ASSERT VMError::report_and_die(NULL, sig, pc, info, ucVoid); } diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp index 9b7541792fbe0..d3461977fa6cb 100644 --- a/src/hotspot/os/windows/os_windows.cpp +++ b/src/hotspot/os/windows/os_windows.cpp @@ -55,11 +55,12 @@ #include "runtime/orderAccess.hpp" #include "runtime/osThread.hpp" #include "runtime/perfMemory.hpp" -#include "runtime/safefetch.inline.hpp" +#include "runtime/safefetch.hpp" #include "runtime/safepointMechanism.hpp" #include "runtime/semaphore.inline.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/statSampler.hpp" +#include "runtime/stubRoutines.hpp" #include "runtime/thread.inline.hpp" #include "runtime/threadCritical.hpp" #include "runtime/timer.hpp" @@ -88,6 +89,7 @@ #include #include +#include #include #include #include @@ -625,9 +627,9 @@ bool os::create_attached_thread(JavaThread* thread) { thread->set_osthread(osthread); log_info(os, thread)("Thread attached (tid: " UINTX_FORMAT ", stack: " - PTR_FORMAT " - " PTR_FORMAT " (" SIZE_FORMAT "k) ).", + PTR_FORMAT " - " PTR_FORMAT " (" SIZE_FORMAT "K) ).", os::current_thread_id(), p2i(thread->stack_base()), - p2i(thread->stack_end()), thread->stack_size()); + p2i(thread->stack_end()), thread->stack_size() / K); return true; } @@ -1925,8 +1927,65 @@ void os::win32::print_windows_version(outputStream* st) { st->cr(); } +// Processor Power Information; missing from Windows headers +typedef struct _PROCESSOR_POWER_INFORMATION { + ULONG Number; + ULONG MaxMhz; // max specified clock frequency of the system processor + ULONG CurrentMhz; // max specified processor clock frequency mult. by current processor throttle + ULONG MhzLimit; // max specified processor clock frequency mult. by current processor thermal throttle limit + ULONG MaxIdleState; + ULONG CurrentIdleState; +} PROCESSOR_POWER_INFORMATION; + void os::pd_print_cpu_info(outputStream* st, char* buf, size_t buflen) { - // Nothing to do for now. + int proc_count = os::processor_count(); + // handle potential early cases where processor count is not yet set + if (proc_count < 1) { + SYSTEM_INFO si; + GetSystemInfo(&si); + proc_count = si.dwNumberOfProcessors; + } + + size_t sz_check = sizeof(PROCESSOR_POWER_INFORMATION) * (size_t)proc_count; + NTSTATUS status = ::CallNtPowerInformation(ProcessorInformation, nullptr, 0, buf, (ULONG) buflen); + int max_mhz = -1, current_mhz = -1, mhz_limit = -1; + bool same_vals_for_all_cpus = true; + + if (status == ERROR_SUCCESS) { + PROCESSOR_POWER_INFORMATION* pppi = (PROCESSOR_POWER_INFORMATION*) buf; + for (int i = 0; i < proc_count; i++) { + if (i == 0) { + max_mhz = (int) pppi->MaxMhz; + current_mhz = (int) pppi->CurrentMhz; + mhz_limit = (int) pppi->MhzLimit; + } else { + if (max_mhz != (int) pppi->MaxMhz || + current_mhz != (int) pppi->CurrentMhz || + mhz_limit != (int) pppi->MhzLimit) { + same_vals_for_all_cpus = false; + break; + } + } + // avoid iteration in case buf is too small to hold all proc infos + if (sz_check > buflen) break; + pppi++; + } + + if (same_vals_for_all_cpus && max_mhz != -1) { + st->print_cr("Processor Information for all %d processors :", proc_count); + st->print_cr(" Max Mhz: %d, Current Mhz: %d, Mhz Limit: %d", max_mhz, current_mhz, mhz_limit); + return; + } + // differing values, iterate again + pppi = (PROCESSOR_POWER_INFORMATION*) buf; + for (int i = 0; i < proc_count; i++) { + st->print_cr("Processor Information for processor %d", (int) pppi->Number); + st->print_cr(" Max Mhz: %d, Current Mhz: %d, Mhz Limit: %d", + (int) pppi->MaxMhz, (int) pppi->CurrentMhz, (int) pppi->MhzLimit); + if (sz_check > buflen) break; + pppi++; + } + } } void os::get_summary_cpu_info(char* buf, size_t buflen) { @@ -2526,11 +2585,6 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) { #endif Thread* t = Thread::current_or_null_safe(); - // Handle SafeFetch32 and SafeFetchN exceptions. - if (StubRoutines::is_safefetch_fault(pc)) { - return Handle_Exception(exceptionInfo, StubRoutines::continuation_for_safefetch_fault(pc)); - } - #ifndef _WIN64 // Execution protection violation - win32 running on AMD64 only // Handled first to avoid misdiagnosis as a "normal" access violation; diff --git a/src/hotspot/os/windows/os_windows.inline.hpp b/src/hotspot/os/windows/os_windows.inline.hpp index 8dab6f44180f1..1fb2f393f8a83 100644 --- a/src/hotspot/os/windows/os_windows.inline.hpp +++ b/src/hotspot/os/windows/os_windows.inline.hpp @@ -93,4 +93,8 @@ inline void os::PlatformMonitor::notify_all() { WakeAllConditionVariable(&_cond); } +// stubbed-out trim-native support +inline bool os::can_trim_native_heap() { return false; } +inline bool os::trim_native_heap(os::size_change_t* rss_change) { return false; } + #endif // OS_WINDOWS_OS_WINDOWS_INLINE_HPP diff --git a/src/hotspot/os/windows/safefetch_windows.hpp b/src/hotspot/os/windows/safefetch_windows.hpp new file mode 100644 index 0000000000000..01904ffa9b218 --- /dev/null +++ b/src/hotspot/os/windows/safefetch_windows.hpp @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2022 SAP SE. All rights reserved. + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_WINDOWS_SAFEFETCH_WINDOWS_HPP +#define OS_WINDOWS_SAFEFETCH_WINDOWS_HPP + +#include "utilities/globalDefinitions.hpp" + +// On windows, we use structured exception handling to implement SafeFetch + +template +inline T SafeFetchXX(const T* adr, T errValue) { + T v = 0; + __try { + v = *adr; + } + __except(EXCEPTION_EXECUTE_HANDLER) { + v = errValue; + } + return v; +} + +inline int SafeFetch32_impl(const int* adr, int errValue) { + return SafeFetchXX(adr, errValue); +} + +inline intptr_t SafeFetchN_impl(const intptr_t* adr, intptr_t errValue) { + return SafeFetchXX(adr, errValue); +} + +#endif diff --git a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp index b32c853347b49..3e1399f63c4e7 100644 --- a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp +++ b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp @@ -503,7 +503,7 @@ int os::extra_bang_size_in_bytes() { return 0; } -bool os::platform_print_native_stack(outputStream* st, void* context, char *buf, int buf_size) { +bool os::platform_print_native_stack(outputStream* st, void* context, char *buf, int buf_size, address& lastpc) { AixNativeCallstack::print_callstack_for_context(st, (const ucontext_t*)context, true, buf, (size_t) buf_size); return true; } diff --git a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.hpp b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.hpp index 130c1f4d48b53..414db7acb11a0 100644 --- a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.hpp +++ b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.hpp @@ -34,7 +34,7 @@ #define PLATFORM_PRINT_NATIVE_STACK 1 static bool platform_print_native_stack(outputStream* st, void* context, - char *buf, int buf_size); + char *buf, int buf_size, address& lastpc); #define HAVE_FUNCTION_DESCRIPTORS 1 static void* resolve_function_descriptor(void* p); diff --git a/src/hotspot/os_cpu/bsd_aarch64/safefetch_bsd_aarch64.S b/src/hotspot/os_cpu/bsd_aarch64/safefetch_bsd_aarch64.S new file mode 100644 index 0000000000000..34d7b8e34a739 --- /dev/null +++ b/src/hotspot/os_cpu/bsd_aarch64/safefetch_bsd_aarch64.S @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2022 SAP SE. All rights reserved. + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifdef __APPLE__ +# Darwin uses _ prefixed global symbols +#define SYMBOL(s) _ ## s +#define ELF_TYPE(name, description) +#else +#define SYMBOL(s) s +#define ELF_TYPE(name, description) .type name,description +#endif + + .global SYMBOL(SafeFetchN_impl) + .global SYMBOL(_SafeFetchN_fault) + .global SYMBOL(_SafeFetchN_continuation) + .global SYMBOL(SafeFetch32_impl) + .global SYMBOL(_SafeFetch32_fault) + .global SYMBOL(_SafeFetch32_continuation) + + # Support for int SafeFetch32(int* address, int defaultval); + # + # x0 : address + # w1 : defaultval + + # needed to align function start to 4 byte + .align 6 + ELF_TYPE(SafeFetch32_impl,@function) +SYMBOL(SafeFetch32_impl): +SYMBOL(_SafeFetch32_fault): + ldr w0, [x0] + ret +SYMBOL(_SafeFetch32_continuation): + mov x0, x1 + ret + + # Support for intptr_t SafeFetchN(intptr_t* address, intptr_t defaultval); + # + # x1 : address + # x0 : defaultval + + .align 6 + ELF_TYPE(SafeFetchN_impl,@function) +SYMBOL(SafeFetchN_impl): +SYMBOL(_SafeFetchN_fault): + ldr x0, [x0] + ret +SYMBOL(_SafeFetchN_continuation): + mov x0, x1 + ret diff --git a/src/hotspot/os_cpu/bsd_x86/safefetch_bsd_x86_64.S b/src/hotspot/os_cpu/bsd_x86/safefetch_bsd_x86_64.S new file mode 100644 index 0000000000000..2a75f3dac94b3 --- /dev/null +++ b/src/hotspot/os_cpu/bsd_x86/safefetch_bsd_x86_64.S @@ -0,0 +1,67 @@ +# +# Copyright (c) 2022 SAP SE. All rights reserved. +# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# This code is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 only, as +# published by the Free Software Foundation. +# +# This code is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# version 2 for more details (a copy is included in the LICENSE file that +# accompanied this code). +# +# You should have received a copy of the GNU General Public License version +# 2 along with this work; if not, write to the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +# or visit www.oracle.com if you need additional information or have any +# questions. +# + +#ifdef __APPLE__ +# Darwin uses _ prefixed global symbols +#define SYMBOL(s) _ ## s +#define ELF_TYPE(name, description) +#else +#define SYMBOL(s) s +#define ELF_TYPE(name, description) .type name,description +#endif + + .globl SYMBOL(SafeFetch32_impl) + .globl SYMBOL(SafeFetchN_impl) + .globl SYMBOL(_SafeFetch32_fault) + .globl SYMBOL(_SafeFetchN_fault) + .globl SYMBOL(_SafeFetch32_continuation) + .globl SYMBOL(_SafeFetchN_continuation) + + .text + + # Support for int SafeFetch32(int* address, int defaultval); + # + # %rdi : address + # %esi : defaultval + ELF_TYPE(SafeFetch32_impl,@function) +SYMBOL(SafeFetch32_impl:) +SYMBOL(_SafeFetch32_fault:) + movl (%rdi), %eax + ret +SYMBOL(_SafeFetch32_continuation:) + movl %esi, %eax + ret + + # Support for intptr_t SafeFetchN(intptr_t* address, intptr_t defaultval); + # + # %rdi : address + # %rsi : defaultval + ELF_TYPE(SafeFetchN_impl,@function) +SYMBOL(SafeFetchN_impl:) +SYMBOL(_SafeFetchN_fault:) + movq (%rdi), %rax + ret +SYMBOL(_SafeFetchN_continuation:) + movq %rsi, %rax + ret diff --git a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S index 3007587d9c22c..9c91942cb335a 100644 --- a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S +++ b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S @@ -47,6 +47,28 @@ aarch64_atomic_fetch_add_4_default_impl: mov w0, w2 ret + .global aarch64_atomic_fetch_add_8_relaxed_default_impl + .align 5 +aarch64_atomic_fetch_add_8_relaxed_default_impl: + prfm pstl1strm, [x0] +0: ldxr x2, [x0] + add x8, x2, x1 + stxr w9, x8, [x0] + cbnz w9, 0b + mov x0, x2 + ret + + .global aarch64_atomic_fetch_add_4_relaxed_default_impl + .align 5 +aarch64_atomic_fetch_add_4_relaxed_default_impl: + prfm pstl1strm, [x0] +0: ldxr w2, [x0] + add w8, w2, w1 + stxr w9, w8, [x0] + cbnz w9, 0b + mov w0, w2 + ret + .globl aarch64_atomic_xchg_4_default_impl .align 5 aarch64_atomic_xchg_4_default_impl: diff --git a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp index 316e877ec1f64..3208db5b4a6b7 100644 --- a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp +++ b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -87,9 +87,14 @@ inline D Atomic::PlatformAdd<4>::fetch_and_add(D volatile* dest, I add_value, atomic_memory_order order) const { STATIC_ASSERT(4 == sizeof(I)); STATIC_ASSERT(4 == sizeof(D)); - D old_value - = atomic_fastcall(aarch64_atomic_fetch_add_4_impl, dest, add_value); - return old_value; + aarch64_atomic_stub_t stub; + switch (order) { + case memory_order_relaxed: + stub = aarch64_atomic_fetch_add_4_relaxed_impl; break; + default: + stub = aarch64_atomic_fetch_add_4_impl; break; + } + return atomic_fastcall(stub, dest, add_value); } template<> @@ -98,9 +103,14 @@ inline D Atomic::PlatformAdd<8>::fetch_and_add(D volatile* dest, I add_value, atomic_memory_order order) const { STATIC_ASSERT(8 == sizeof(I)); STATIC_ASSERT(8 == sizeof(D)); - D old_value - = atomic_fastcall(aarch64_atomic_fetch_add_8_impl, dest, add_value); - return old_value; + aarch64_atomic_stub_t stub; + switch (order) { + case memory_order_relaxed: + stub = aarch64_atomic_fetch_add_8_relaxed_impl; break; + default: + stub = aarch64_atomic_fetch_add_8_impl; break; + } + return atomic_fastcall(stub, dest, add_value); } template<> diff --git a/src/hotspot/os_cpu/linux_aarch64/globals_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/globals_linux_aarch64.hpp index 6f33872bc2345..c4219c2bac3d8 100644 --- a/src/hotspot/os_cpu/linux_aarch64/globals_linux_aarch64.hpp +++ b/src/hotspot/os_cpu/linux_aarch64/globals_linux_aarch64.hpp @@ -30,10 +30,18 @@ // (see globals.hpp) define_pd_global(bool, DontYieldALot, false); -define_pd_global(intx, ThreadStackSize, 2048); // 0 => use system default -define_pd_global(intx, VMThreadStackSize, 2048); -define_pd_global(intx, CompilerThreadStackSize, 2048); +// Set default stack sizes < 2MB so as to prevent stacks from getting +// large-page aligned and backed by THPs on systems where 2MB is the +// default huge page size. For non-JavaThreads, glibc may add an additional +// guard page to the total stack size, so to keep the default sizes same +// for all the following flags, we set them to 2 pages less than 2MB. On +// systems where 2MB is the default large page size, 4KB is most commonly +// the regular page size. +define_pd_global(intx, ThreadStackSize, 2040); // 0 => use system default +define_pd_global(intx, VMThreadStackSize, 2040); + +define_pd_global(intx, CompilerThreadStackSize, 2040); define_pd_global(uintx,JVMInvokeMethodSlack, 8192); diff --git a/src/hotspot/os_cpu/linux_aarch64/safefetch_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/safefetch_linux_aarch64.S new file mode 100644 index 0000000000000..fcb7e62e6d5e9 --- /dev/null +++ b/src/hotspot/os_cpu/linux_aarch64/safefetch_linux_aarch64.S @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2022 SAP SE. All rights reserved. + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + + .globl SafeFetchN_impl + .globl _SafeFetchN_fault + .globl _SafeFetchN_continuation + .globl SafeFetch32_impl + .globl _SafeFetch32_fault + .globl _SafeFetch32_continuation + + # Support for int SafeFetch32(int* address, int defaultval); + # + # x0 : address + # x1 : defaultval +SafeFetch32_impl: +_SafeFetch32_fault: + ldr w0, [x0] + ret +_SafeFetch32_continuation: + mov x0, x1 + ret + + # Support for intptr_t SafeFetchN(intptr_t* address, intptr_t defaultval); + # + # x1 : address + # x0 : defaultval +SafeFetchN_impl: +_SafeFetchN_fault: + ldr x0, [x0] + ret +_SafeFetchN_continuation: + mov x0, x1 + ret diff --git a/src/hotspot/os_cpu/linux_arm/safefetch_linux_arm.S b/src/hotspot/os_cpu/linux_arm/safefetch_linux_arm.S new file mode 100644 index 0000000000000..5196b199f05f6 --- /dev/null +++ b/src/hotspot/os_cpu/linux_arm/safefetch_linux_arm.S @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 SAP SE. All rights reserved. + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + + .globl SafeFetch32_impl + .globl _SafeFetch32_fault + .globl _SafeFetch32_continuation + .type SafeFetch32_impl, %function + + # Support for int SafeFetch32(int* address, int defaultval); + # + # r0 : address + # r1 : defaultval +SafeFetch32_impl: +_SafeFetch32_fault: + ldr r0, [r0] + bx lr +_SafeFetch32_continuation: + mov r0, r1 + bx lr diff --git a/src/hotspot/os_cpu/linux_ppc/safefetch_linux_ppc.S b/src/hotspot/os_cpu/linux_ppc/safefetch_linux_ppc.S new file mode 100644 index 0000000000000..c8d20cc1b4328 --- /dev/null +++ b/src/hotspot/os_cpu/linux_ppc/safefetch_linux_ppc.S @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2022 SAP SE. All rights reserved. + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + + .globl SafeFetchN_impl + .globl _SafeFetchN_fault + .globl _SafeFetchN_continuation + .globl SafeFetch32_impl + .globl _SafeFetch32_fault + .globl _SafeFetch32_continuation + + # Support for int SafeFetch32(int* address, int defaultval); + # + # r3 : address + # r4 : defaultval + # r3 : retval +SafeFetch32_impl: +_SafeFetch32_fault: + lwa %r3, 0(%r3) + blr +_SafeFetch32_continuation: + mr %r3, %r4 + blr + + # Support for intptr_t SafeFetchN(intptr_t* address, intptr_t defaultval); + # + # r3 : address + # r4 : defaultval + # r3 : retval +SafeFetchN_impl: +_SafeFetchN_fault: + ld %r3, 0(%r3) + blr +_SafeFetchN_continuation: + mr %r3, %r4 + blr diff --git a/test/jdk/tools/jpackage/junit/junit.java b/src/hotspot/os_cpu/linux_riscv/assembler_linux_riscv.cpp similarity index 80% rename from test/jdk/tools/jpackage/junit/junit.java rename to src/hotspot/os_cpu/linux_riscv/assembler_linux_riscv.cpp index 839b14ab09122..f2610af6cdd0f 100644 --- a/test/jdk/tools/jpackage/junit/junit.java +++ b/src/hotspot/os_cpu/linux_riscv/assembler_linux_riscv.cpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2015, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -19,12 +20,7 @@ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. + * */ -/* - * @test - * @summary jpackage unit tests - * @library ${jtreg.home}/lib/junit.jar ${jtreg.home}/lib/hamcrest.jar - * @modules jdk.jpackage - * @run shell run_junit.sh - */ +// nothing required here diff --git a/src/hotspot/os_cpu/linux_riscv/atomic_linux_riscv.hpp b/src/hotspot/os_cpu/linux_riscv/atomic_linux_riscv.hpp new file mode 100644 index 0000000000000..11cd9e1d72da8 --- /dev/null +++ b/src/hotspot/os_cpu/linux_riscv/atomic_linux_riscv.hpp @@ -0,0 +1,151 @@ +/* + * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_CPU_LINUX_RISCV_ATOMIC_LINUX_RISCV_HPP +#define OS_CPU_LINUX_RISCV_ATOMIC_LINUX_RISCV_HPP + +#include "runtime/vm_version.hpp" + +// Implementation of class atomic + +// Note that memory_order_conservative requires a full barrier after atomic stores. +// See https://patchwork.kernel.org/patch/3575821/ + +template +struct Atomic::PlatformAdd { + template + D add_and_fetch(D volatile* dest, I add_value, atomic_memory_order order) const { + if (order != memory_order_relaxed) { + FULL_MEM_BARRIER; + } + + D res = __atomic_add_fetch(dest, add_value, __ATOMIC_RELAXED); + + if (order != memory_order_relaxed) { + FULL_MEM_BARRIER; + } + return res; + } + + template + D fetch_and_add(D volatile* dest, I add_value, atomic_memory_order order) const { + return add_and_fetch(dest, add_value, order) - add_value; + } +}; + +template +template +inline T Atomic::PlatformXchg::operator()(T volatile* dest, + T exchange_value, + atomic_memory_order order) const { + STATIC_ASSERT(byte_size == sizeof(T)); + if (order != memory_order_relaxed) { + FULL_MEM_BARRIER; + } + + T res = __atomic_exchange_n(dest, exchange_value, __ATOMIC_RELAXED); + + if (order != memory_order_relaxed) { + FULL_MEM_BARRIER; + } + return res; +} + +// __attribute__((unused)) on dest is to get rid of spurious GCC warnings. +template +template +inline T Atomic::PlatformCmpxchg::operator()(T volatile* dest __attribute__((unused)), + T compare_value, + T exchange_value, + atomic_memory_order order) const { + STATIC_ASSERT(byte_size == sizeof(T)); + T value = compare_value; + if (order != memory_order_relaxed) { + FULL_MEM_BARRIER; + } + + __atomic_compare_exchange(dest, &value, &exchange_value, /* weak */ false, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); + + if (order != memory_order_relaxed) { + FULL_MEM_BARRIER; + } + return value; +} + +template<> +template +inline T Atomic::PlatformCmpxchg<4>::operator()(T volatile* dest __attribute__((unused)), + T compare_value, + T exchange_value, + atomic_memory_order order) const { + STATIC_ASSERT(4 == sizeof(T)); + + T old_value; + long rc; + + if (order != memory_order_relaxed) { + FULL_MEM_BARRIER; + } + + __asm__ __volatile__ ( + "1: sext.w %1, %3 \n\t" // sign-extend compare_value + " lr.w %0, %2 \n\t" + " bne %0, %1, 2f \n\t" + " sc.w %1, %4, %2 \n\t" + " bnez %1, 1b \n\t" + "2: \n\t" + : /*%0*/"=&r" (old_value), /*%1*/"=&r" (rc), /*%2*/"+A" (*dest) + : /*%3*/"r" (compare_value), /*%4*/"r" (exchange_value) + : "memory" ); + + if (order != memory_order_relaxed) { + FULL_MEM_BARRIER; + } + return old_value; +} + +template +struct Atomic::PlatformOrderedLoad +{ + template + T operator()(const volatile T* p) const { T data; __atomic_load(const_cast(p), &data, __ATOMIC_ACQUIRE); return data; } +}; + +template +struct Atomic::PlatformOrderedStore +{ + template + void operator()(volatile T* p, T v) const { __atomic_store(const_cast(p), &v, __ATOMIC_RELEASE); } +}; + +template +struct Atomic::PlatformOrderedStore +{ + template + void operator()(volatile T* p, T v) const { release_store(p, v); OrderAccess::fence(); } +}; + +#endif // OS_CPU_LINUX_RISCV_ATOMIC_LINUX_RISCV_HPP diff --git a/src/hotspot/os_cpu/linux_riscv/bytes_linux_riscv.hpp b/src/hotspot/os_cpu/linux_riscv/bytes_linux_riscv.hpp new file mode 100644 index 0000000000000..28868c7640640 --- /dev/null +++ b/src/hotspot/os_cpu/linux_riscv/bytes_linux_riscv.hpp @@ -0,0 +1,45 @@ +/* + * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_CPU_LINUX_RISCV_BYTES_LINUX_RISCV_HPP +#define OS_CPU_LINUX_RISCV_BYTES_LINUX_RISCV_HPP + +#include + +// Efficient swapping of data bytes from Java byte +// ordering to native byte ordering and vice versa. +inline u2 Bytes::swap_u2(u2 x) { + return bswap_16(x); +} + +inline u4 Bytes::swap_u4(u4 x) { + return bswap_32(x); +} + +inline u8 Bytes::swap_u8(u8 x) { + return bswap_64(x); +} + +#endif // OS_CPU_LINUX_RISCV_BYTES_LINUX_RISCV_HPP diff --git a/src/hotspot/os_cpu/linux_riscv/copy_linux_riscv.hpp b/src/hotspot/os_cpu/linux_riscv/copy_linux_riscv.hpp new file mode 100644 index 0000000000000..147cfdf3c100d --- /dev/null +++ b/src/hotspot/os_cpu/linux_riscv/copy_linux_riscv.hpp @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_CPU_LINUX_RISCV_VM_COPY_LINUX_RISCV_HPP +#define OS_CPU_LINUX_RISCV_VM_COPY_LINUX_RISCV_HPP + +// Empty for build system + +#endif // OS_CPU_LINUX_RISCV_VM_COPY_LINUX_RISCV_HPP diff --git a/src/hotspot/os_cpu/linux_riscv/gc/z/zSyscall_linux_riscv.hpp b/src/hotspot/os_cpu/linux_riscv/gc/z/zSyscall_linux_riscv.hpp new file mode 100644 index 0000000000000..1aa58f27871d2 --- /dev/null +++ b/src/hotspot/os_cpu/linux_riscv/gc/z/zSyscall_linux_riscv.hpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_CPU_LINUX_RISCV_GC_Z_ZSYSCALL_LINUX_RISCV_HPP +#define OS_CPU_LINUX_RISCV_GC_Z_ZSYSCALL_LINUX_RISCV_HPP + +#include + +// +// Support for building on older Linux systems +// + +#ifndef SYS_memfd_create +#define SYS_memfd_create 279 +#endif +#ifndef SYS_fallocate +#define SYS_fallocate 47 +#endif + +#endif // OS_CPU_LINUX_RISCV_GC_Z_ZSYSCALL_LINUX_RISCV_HPP diff --git a/src/hotspot/os_cpu/linux_riscv/globals_linux_riscv.hpp b/src/hotspot/os_cpu/linux_riscv/globals_linux_riscv.hpp new file mode 100644 index 0000000000000..297414bfcd510 --- /dev/null +++ b/src/hotspot/os_cpu/linux_riscv/globals_linux_riscv.hpp @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_CPU_LINUX_RISCV_VM_GLOBALS_LINUX_RISCV_HPP +#define OS_CPU_LINUX_RISCV_VM_GLOBALS_LINUX_RISCV_HPP + +// Sets the default values for platform dependent flags used by the runtime system. +// (see globals.hpp) + +define_pd_global(bool, DontYieldALot, false); +define_pd_global(intx, ThreadStackSize, 2048); // 0 => use system default +define_pd_global(intx, VMThreadStackSize, 2048); + +define_pd_global(intx, CompilerThreadStackSize, 2048); + +define_pd_global(uintx, JVMInvokeMethodSlack, 8192); + +// Used on 64 bit platforms for UseCompressedOops base address +define_pd_global(uintx, HeapBaseMinAddress, 2 * G); + +#endif // OS_CPU_LINUX_RISCV_VM_GLOBALS_LINUX_RISCV_HPP diff --git a/src/hotspot/os_cpu/linux_riscv/orderAccess_linux_riscv.hpp b/src/hotspot/os_cpu/linux_riscv/orderAccess_linux_riscv.hpp new file mode 100644 index 0000000000000..2b500376f9b48 --- /dev/null +++ b/src/hotspot/os_cpu/linux_riscv/orderAccess_linux_riscv.hpp @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_CPU_LINUX_RISCV_ORDERACCESS_LINUX_RISCV_HPP +#define OS_CPU_LINUX_RISCV_ORDERACCESS_LINUX_RISCV_HPP + +// Included in orderAccess.hpp header file. + +#include "runtime/vm_version.hpp" + +// Implementation of class OrderAccess. + +inline void OrderAccess::loadload() { acquire(); } +inline void OrderAccess::storestore() { release(); } +inline void OrderAccess::loadstore() { acquire(); } +inline void OrderAccess::storeload() { fence(); } + +#define FULL_MEM_BARRIER __sync_synchronize() +#define READ_MEM_BARRIER __atomic_thread_fence(__ATOMIC_ACQUIRE); +#define WRITE_MEM_BARRIER __atomic_thread_fence(__ATOMIC_RELEASE); + +inline void OrderAccess::acquire() { + READ_MEM_BARRIER; +} + +inline void OrderAccess::release() { + WRITE_MEM_BARRIER; +} + +inline void OrderAccess::fence() { + FULL_MEM_BARRIER; +} + +inline void OrderAccess::cross_modify_fence_impl() { +} + +#endif // OS_CPU_LINUX_RISCV_ORDERACCESS_LINUX_RISCV_HPP diff --git a/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp b/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp new file mode 100644 index 0000000000000..d672b57182f74 --- /dev/null +++ b/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp @@ -0,0 +1,475 @@ +/* + * Copyright (c) 1999, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +// no precompiled headers +#include "asm/macroAssembler.hpp" +#include "classfile/vmSymbols.hpp" +#include "code/codeCache.hpp" +#include "code/icBuffer.hpp" +#include "code/nativeInst.hpp" +#include "code/vtableStubs.hpp" +#include "interpreter/interpreter.hpp" +#include "jvm.h" +#include "memory/allocation.inline.hpp" +#include "os_share_linux.hpp" +#include "prims/jniFastGetField.hpp" +#include "prims/jvm_misc.hpp" +#include "runtime/arguments.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/interfaceSupport.inline.hpp" +#include "runtime/java.hpp" +#include "runtime/javaCalls.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/osThread.hpp" +#include "runtime/safepointMechanism.hpp" +#include "runtime/sharedRuntime.hpp" +#include "runtime/stubRoutines.hpp" +#include "runtime/thread.inline.hpp" +#include "runtime/timer.hpp" +#include "signals_posix.hpp" +#include "utilities/debug.hpp" +#include "utilities/events.hpp" +#include "utilities/vmError.hpp" + +// put OS-includes here +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +#define REG_LR 1 +#define REG_FP 8 + +NOINLINE address os::current_stack_pointer() { + return (address)__builtin_frame_address(0); +} + +char* os::non_memory_address_word() { + // Must never look like an address returned by reserve_memory, + // even in its subfields (as defined by the CPU immediate fields, + // if the CPU splits constants across multiple instructions). + + return (char*) 0xffffffffffff; +} + +address os::Posix::ucontext_get_pc(const ucontext_t * uc) { + return (address)uc->uc_mcontext.__gregs[REG_PC]; +} + +void os::Posix::ucontext_set_pc(ucontext_t * uc, address pc) { + uc->uc_mcontext.__gregs[REG_PC] = (intptr_t)pc; +} + +intptr_t* os::Linux::ucontext_get_sp(const ucontext_t * uc) { + return (intptr_t*)uc->uc_mcontext.__gregs[REG_SP]; +} + +intptr_t* os::Linux::ucontext_get_fp(const ucontext_t * uc) { + return (intptr_t*)uc->uc_mcontext.__gregs[REG_FP]; +} + +address os::fetch_frame_from_context(const void* ucVoid, + intptr_t** ret_sp, intptr_t** ret_fp) { + address epc; + const ucontext_t* uc = (const ucontext_t*)ucVoid; + + if (uc != NULL) { + epc = os::Posix::ucontext_get_pc(uc); + if (ret_sp != NULL) { + *ret_sp = os::Linux::ucontext_get_sp(uc); + } + if (ret_fp != NULL) { + *ret_fp = os::Linux::ucontext_get_fp(uc); + } + } else { + epc = NULL; + if (ret_sp != NULL) { + *ret_sp = (intptr_t *)NULL; + } + if (ret_fp != NULL) { + *ret_fp = (intptr_t *)NULL; + } + } + + return epc; +} + +frame os::fetch_compiled_frame_from_context(const void* ucVoid) { + const ucontext_t* uc = (const ucontext_t*)ucVoid; + // In compiled code, the stack banging is performed before RA + // has been saved in the frame. RA is live, and SP and FP + // belong to the caller. + intptr_t* frame_fp = os::Linux::ucontext_get_fp(uc); + intptr_t* frame_sp = os::Linux::ucontext_get_sp(uc); + address frame_pc = (address)(uc->uc_mcontext.__gregs[REG_LR] + - NativeInstruction::instruction_size); + return frame(frame_sp, frame_fp, frame_pc); +} + +frame os::fetch_frame_from_context(const void* ucVoid) { + intptr_t* frame_sp = NULL; + intptr_t* frame_fp = NULL; + address epc = fetch_frame_from_context(ucVoid, &frame_sp, &frame_fp); + if (!is_readable_pointer(epc)) { + // Try to recover from calling into bad memory + // Assume new frame has not been set up, the same as + // compiled frame stack bang + return fetch_compiled_frame_from_context(ucVoid); + } + return frame(frame_sp, frame_fp, epc); +} + +// By default, gcc always saves frame pointer rfp on this stack. This +// may get turned off by -fomit-frame-pointer. +frame os::get_sender_for_C_frame(frame* fr) { + return frame(fr->sender_sp(), fr->link(), fr->sender_pc()); +} + +NOINLINE frame os::current_frame() { + intptr_t **sender_sp = (intptr_t **)__builtin_frame_address(0); + if (sender_sp != NULL) { + frame myframe((intptr_t*)os::current_stack_pointer(), + sender_sp[frame::link_offset], + CAST_FROM_FN_PTR(address, os::current_frame)); + if (os::is_first_C_frame(&myframe)) { + // stack is not walkable + return frame(); + } else { + return os::get_sender_for_C_frame(&myframe); + } + } else { + ShouldNotReachHere(); + return frame(); + } +} + +// Utility functions +bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info, + ucontext_t* uc, JavaThread* thread) { + + // decide if this trap can be handled by a stub + address stub = NULL; + + address pc = NULL; + + //%note os_trap_1 + if (info != NULL && uc != NULL && thread != NULL) { + pc = (address) os::Posix::ucontext_get_pc(uc); + + address addr = (address) info->si_addr; + + // Make sure the high order byte is sign extended, as it may be masked away by the hardware. + if ((uintptr_t(addr) & (uintptr_t(1) << 55)) != 0) { + addr = address(uintptr_t(addr) | (uintptr_t(0xFF) << 56)); + } + + // Handle ALL stack overflow variations here + if (sig == SIGSEGV) { + // check if fault address is within thread stack + if (thread->is_in_full_stack(addr)) { + if (os::Posix::handle_stack_overflow(thread, addr, pc, uc, &stub)) { + return true; // continue + } + } + } + + if (thread->thread_state() == _thread_in_Java) { + // Java thread running in Java code => find exception handler if any + // a fault inside compiled code, the interpreter, or a stub + + // Handle signal from NativeJump::patch_verified_entry(). + if ((sig == SIGILL || sig == SIGTRAP) + && nativeInstruction_at(pc)->is_sigill_zombie_not_entrant()) { + if (TraceTraps) { + tty->print_cr("trap: zombie_not_entrant (%s)", (sig == SIGTRAP) ? "SIGTRAP" : "SIGILL"); + } + stub = SharedRuntime::get_handle_wrong_method_stub(); + } else if (sig == SIGSEGV && SafepointMechanism::is_poll_address((address)info->si_addr)) { + stub = SharedRuntime::get_poll_stub(pc); + } else if (sig == SIGBUS /* && info->si_code == BUS_OBJERR */) { + // BugId 4454115: A read from a MappedByteBuffer can fault + // here if the underlying file has been truncated. + // Do not crash the VM in such a case. + CodeBlob* cb = CodeCache::find_blob_unsafe(pc); + CompiledMethod* nm = (cb != NULL) ? cb->as_compiled_method_or_null() : NULL; + bool is_unsafe_arraycopy = (thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc)); + if ((nm != NULL && nm->has_unsafe_access()) || is_unsafe_arraycopy) { + address next_pc = pc + NativeCall::instruction_size; + if (is_unsafe_arraycopy) { + next_pc = UnsafeCopyMemory::page_error_continue_pc(pc); + } + stub = SharedRuntime::handle_unsafe_access(thread, next_pc); + } + } else if (sig == SIGILL && nativeInstruction_at(pc)->is_stop()) { + // Pull a pointer to the error message out of the instruction + // stream. + const uint64_t *detail_msg_ptr + = (uint64_t*)(pc + NativeInstruction::instruction_size); + const char *detail_msg = (const char *)*detail_msg_ptr; + const char *msg = "stop"; + if (TraceTraps) { + tty->print_cr("trap: %s: (SIGILL)", msg); + } + + // End life with a fatal error, message and detail message and the context. + // Note: no need to do any post-processing here (e.g. signal chaining) + va_list va_dummy; + VMError::report_and_die(thread, uc, NULL, 0, msg, detail_msg, va_dummy); + va_end(va_dummy); + + ShouldNotReachHere(); + } else if (sig == SIGFPE && + (info->si_code == FPE_INTDIV || info->si_code == FPE_FLTDIV)) { + stub = + SharedRuntime:: + continuation_for_implicit_exception(thread, + pc, + SharedRuntime:: + IMPLICIT_DIVIDE_BY_ZERO); + } else if (sig == SIGSEGV && + MacroAssembler::uses_implicit_null_check((void*)addr)) { + // Determination of interpreter/vtable stub/compiled code null exception + stub = SharedRuntime::continuation_for_implicit_exception(thread, pc, SharedRuntime::IMPLICIT_NULL); + } + } else if ((thread->thread_state() == _thread_in_vm || + thread->thread_state() == _thread_in_native) && + sig == SIGBUS && /* info->si_code == BUS_OBJERR && */ + thread->doing_unsafe_access()) { + address next_pc = pc + NativeCall::instruction_size; + if (UnsafeCopyMemory::contains_pc(pc)) { + next_pc = UnsafeCopyMemory::page_error_continue_pc(pc); + } + stub = SharedRuntime::handle_unsafe_access(thread, next_pc); + } + + // jni_fast_GetField can trap at certain pc's if a GC kicks in + // and the heap gets shrunk before the field access. + if ((sig == SIGSEGV) || (sig == SIGBUS)) { + address addr_slow = JNI_FastGetField::find_slowcase_pc(pc); + if (addr_slow != (address)-1) { + stub = addr_slow; + } + } + } + + if (stub != NULL) { + // save all thread context in case we need to restore it + if (thread != NULL) { + thread->set_saved_exception_pc(pc); + } + + os::Posix::ucontext_set_pc(uc, stub); + return true; + } + + return false; // Mute compiler +} + +void os::Linux::init_thread_fpu_state(void) { +} + +int os::Linux::get_fpu_control_word(void) { + return 0; +} + +void os::Linux::set_fpu_control_word(int fpu_control) { +} + +//////////////////////////////////////////////////////////////////////////////// +// thread stack + +// Minimum usable stack sizes required to get to user code. Space for +// HotSpot guard pages is added later. +size_t os::Posix::_compiler_thread_min_stack_allowed = 72 * K; +size_t os::Posix::_java_thread_min_stack_allowed = 72 * K; +size_t os::Posix::_vm_internal_thread_min_stack_allowed = 72 * K; + +// return default stack size for thr_type +size_t os::Posix::default_stack_size(os::ThreadType thr_type) { + // default stack size (compiler thread needs larger stack) + size_t s = (thr_type == os::compiler_thread ? 4 * M : 1 * M); + return s; +} + +///////////////////////////////////////////////////////////////////////////// +// helper functions for fatal error handler + +static const char* reg_abi_names[] = { + "pc", + "x1(ra)", "x2(sp)", "x3(gp)", "x4(tp)", + "x5(t0)", "x6(t1)", "x7(t2)", + "x8(s0)", "x9(s1)", + "x10(a0)", "x11(a1)", "x12(a2)", "x13(a3)", "x14(a4)", "x15(a5)", "x16(a6)", "x17(a7)", + "x18(s2)", "x19(s3)", "x20(s4)", "x21(s5)", "x22(s6)", "x23(s7)", "x24(s8)", "x25(s9)", "x26(s10)", "x27(s11)", + "x28(t3)", "x29(t4)","x30(t5)", "x31(t6)" +}; + +void os::print_context(outputStream *st, const void *context) { + if (context == NULL) { + return; + } + + const ucontext_t *uc = (const ucontext_t*)context; + st->print_cr("Registers:"); + for (int r = 0; r < 32; r++) { + st->print("%-*.*s=", 8, 8, reg_abi_names[r]); + print_location(st, uc->uc_mcontext.__gregs[r]); + } + st->cr(); + + intptr_t *frame_sp = (intptr_t *)os::Linux::ucontext_get_sp(uc); + st->print_cr("Top of Stack: (sp=" PTR_FORMAT ")", p2i(frame_sp)); + print_hex_dump(st, (address)frame_sp, (address)(frame_sp + 64), sizeof(intptr_t)); + st->cr(); + + // Note: it may be unsafe to inspect memory near pc. For example, pc may + // point to garbage if entry point in an nmethod is corrupted. Leave + // this at the end, and hope for the best. + address pc = os::fetch_frame_from_context(uc).pc(); + print_instructions(st, pc, UseRVC ? sizeof(char) : (int)NativeInstruction::instruction_size); + st->cr(); +} + +void os::print_register_info(outputStream *st, const void *context) { + if (context == NULL) { + return; + } + + const ucontext_t *uc = (const ucontext_t*)context; + + st->print_cr("Register to memory mapping:"); + st->cr(); + + // this is horrendously verbose but the layout of the registers in the + // context does not match how we defined our abstract Register set, so + // we can't just iterate through the gregs area + + // this is only for the "general purpose" registers + + for (int r = 0; r < 32; r++) + st->print_cr("%-*.*s=" INTPTR_FORMAT, 8, 8, reg_abi_names[r], (uintptr_t)uc->uc_mcontext.__gregs[r]); + st->cr(); +} + +void os::setup_fpu() { +} + +#ifndef PRODUCT +void os::verify_stack_alignment() { + assert(((intptr_t)os::current_stack_pointer() & (StackAlignmentInBytes-1)) == 0, "incorrect stack alignment"); +} +#endif + +int os::extra_bang_size_in_bytes() { + return 0; +} + +extern "C" { + int SpinPause() { + return 0; + } + + void _Copy_conjoint_jshorts_atomic(const jshort* from, jshort* to, size_t count) { + if (from > to) { + const jshort *end = from + count; + while (from < end) { + *(to++) = *(from++); + } + } else if (from < to) { + const jshort *end = from; + from += count - 1; + to += count - 1; + while (from >= end) { + *(to--) = *(from--); + } + } + } + void _Copy_conjoint_jints_atomic(const jint* from, jint* to, size_t count) { + if (from > to) { + const jint *end = from + count; + while (from < end) { + *(to++) = *(from++); + } + } else if (from < to) { + const jint *end = from; + from += count - 1; + to += count - 1; + while (from >= end) { + *(to--) = *(from--); + } + } + } + void _Copy_conjoint_jlongs_atomic(const jlong* from, jlong* to, size_t count) { + if (from > to) { + const jlong *end = from + count; + while (from < end) { + os::atomic_copy64(from++, to++); + } + } else if (from < to) { + const jlong *end = from; + from += count - 1; + to += count - 1; + while (from >= end) { + os::atomic_copy64(from--, to--); + } + } + } + + void _Copy_arrayof_conjoint_bytes(const HeapWord* from, + HeapWord* to, + size_t count) { + memmove(to, from, count); + } + void _Copy_arrayof_conjoint_jshorts(const HeapWord* from, + HeapWord* to, + size_t count) { + memmove(to, from, count * 2); + } + void _Copy_arrayof_conjoint_jints(const HeapWord* from, + HeapWord* to, + size_t count) { + memmove(to, from, count * 4); + } + void _Copy_arrayof_conjoint_jlongs(const HeapWord* from, + HeapWord* to, + size_t count) { + memmove(to, from, count * 8); + } +}; diff --git a/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.hpp b/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.hpp new file mode 100644 index 0000000000000..f3e3a73bc5e61 --- /dev/null +++ b/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.hpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 1999, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_CPU_LINUX_RISCV_VM_OS_LINUX_RISCV_HPP +#define OS_CPU_LINUX_RISCV_VM_OS_LINUX_RISCV_HPP + + static void setup_fpu(); + + // Used to register dynamic code cache area with the OS + // Note: Currently only used in 64 bit Windows implementations + static bool register_code_area(char *low, char *high) { return true; } + + // Atomically copy 64 bits of data + static void atomic_copy64(const volatile void *src, volatile void *dst) { + *(jlong *) dst = *(const jlong *) src; + } + +#endif // OS_CPU_LINUX_RISCV_VM_OS_LINUX_RISCV_HPP diff --git a/src/hotspot/os_cpu/linux_riscv/prefetch_linux_riscv.inline.hpp b/src/hotspot/os_cpu/linux_riscv/prefetch_linux_riscv.inline.hpp new file mode 100644 index 0000000000000..2bd48e09c3406 --- /dev/null +++ b/src/hotspot/os_cpu/linux_riscv/prefetch_linux_riscv.inline.hpp @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_CPU_LINUX_RISCV_VM_PREFETCH_LINUX_RISCV_INLINE_HPP +#define OS_CPU_LINUX_RISCV_VM_PREFETCH_LINUX_RISCV_INLINE_HPP + +#include "runtime/prefetch.hpp" + + +inline void Prefetch::read (void *loc, intx interval) { +} + +inline void Prefetch::write(void *loc, intx interval) { +} + +#endif // OS_CPU_LINUX_RISCV_VM_PREFETCH_LINUX_RISCV_INLINE_HPP diff --git a/src/hotspot/os_cpu/linux_riscv/safefetch_linux_riscv.S b/src/hotspot/os_cpu/linux_riscv/safefetch_linux_riscv.S new file mode 100644 index 0000000000000..ecf0bac6f9e78 --- /dev/null +++ b/src/hotspot/os_cpu/linux_riscv/safefetch_linux_riscv.S @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2022 SAP SE. All rights reserved. + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + + .globl SafeFetchN_impl + .globl _SafeFetchN_fault + .globl _SafeFetchN_continuation + .globl SafeFetch32_impl + .globl _SafeFetch32_fault + .globl _SafeFetch32_continuation + + # Support for int SafeFetch32(int* address, int defaultval); + # + # x10 (a0) : address + # x11 (a1) : defaultval + # x10 (a0) : retval +SafeFetch32_impl: +_SafeFetch32_fault: + lw a0, 0(a0) + ret +_SafeFetch32_continuation: + mv a0, a1 + ret + + # Support for intptr_t SafeFetchN(intptr_t* address, intptr_t defaultval); + # + # x10 (a0) : address + # x11 (a1) : defaultval + # x10 (a0) : retval +SafeFetchN_impl: +_SafeFetchN_fault: + ld a0, 0(a0) + ret +_SafeFetchN_continuation: + mv a0, a1 + ret diff --git a/src/hotspot/os_cpu/linux_riscv/thread_linux_riscv.cpp b/src/hotspot/os_cpu/linux_riscv/thread_linux_riscv.cpp new file mode 100644 index 0000000000000..3100572e9fdec --- /dev/null +++ b/src/hotspot/os_cpu/linux_riscv/thread_linux_riscv.cpp @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/thread.inline.hpp" + +frame JavaThread::pd_last_frame() { + assert(has_last_Java_frame(), "must have last_Java_sp() when suspended"); + return frame(_anchor.last_Java_sp(), _anchor.last_Java_fp(), _anchor.last_Java_pc()); +} + +// For Forte Analyzer AsyncGetCallTrace profiling support - thread is +// currently interrupted by SIGPROF +bool JavaThread::pd_get_top_frame_for_signal_handler(frame* fr_addr, + void* ucontext, bool isInJava) { + + assert(Thread::current() == this, "caller must be current thread"); + return pd_get_top_frame(fr_addr, ucontext, isInJava); +} + +bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext, bool isInJava) { + return pd_get_top_frame(fr_addr, ucontext, isInJava); +} + +bool JavaThread::pd_get_top_frame(frame* fr_addr, void* ucontext, bool isInJava) { + // If we have a last_Java_frame, then we should use it even if + // isInJava == true. It should be more reliable than ucontext info. + if (has_last_Java_frame() && frame_anchor()->walkable()) { + *fr_addr = pd_last_frame(); + return true; + } + + // At this point, we don't have a last_Java_frame, so + // we try to glean some information out of the ucontext + // if we were running Java code when SIGPROF came in. + if (isInJava) { + ucontext_t* uc = (ucontext_t*) ucontext; + + intptr_t* ret_fp = NULL; + intptr_t* ret_sp = NULL; + address addr = os::fetch_frame_from_context(uc, &ret_sp, &ret_fp); + if (addr == NULL || ret_sp == NULL ) { + // ucontext wasn't useful + return false; + } + + frame ret_frame(ret_sp, ret_fp, addr); + if (!ret_frame.safe_for_sender(this)) { +#ifdef COMPILER2 + frame ret_frame2(ret_sp, NULL, addr); + if (!ret_frame2.safe_for_sender(this)) { + // nothing else to try if the frame isn't good + return false; + } + ret_frame = ret_frame2; +#else + // nothing else to try if the frame isn't good + return false; +#endif /* COMPILER2 */ + } + *fr_addr = ret_frame; + return true; + } + + // nothing else to try + return false; +} + +void JavaThread::cache_global_variables() { } diff --git a/src/hotspot/os_cpu/linux_riscv/thread_linux_riscv.hpp b/src/hotspot/os_cpu/linux_riscv/thread_linux_riscv.hpp new file mode 100644 index 0000000000000..61e2cf85b6300 --- /dev/null +++ b/src/hotspot/os_cpu/linux_riscv/thread_linux_riscv.hpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_CPU_LINUX_RISCV_THREAD_LINUX_RISCV_HPP +#define OS_CPU_LINUX_RISCV_THREAD_LINUX_RISCV_HPP + + private: + void pd_initialize() { + _anchor.clear(); + } + + frame pd_last_frame(); + + public: + static ByteSize last_Java_fp_offset() { + return byte_offset_of(JavaThread, _anchor) + JavaFrameAnchor::last_Java_fp_offset(); + } + + bool pd_get_top_frame_for_signal_handler(frame* fr_addr, void* ucontext, + bool isInJava); + + bool pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext, bool isInJava); +private: + bool pd_get_top_frame(frame* fr_addr, void* ucontext, bool isInJava); + +#endif // OS_CPU_LINUX_RISCV_THREAD_LINUX_RISCV_HPP diff --git a/src/hotspot/os_cpu/linux_riscv/vmStructs_linux_riscv.hpp b/src/hotspot/os_cpu/linux_riscv/vmStructs_linux_riscv.hpp new file mode 100644 index 0000000000000..6cf7683a58602 --- /dev/null +++ b/src/hotspot/os_cpu/linux_riscv/vmStructs_linux_riscv.hpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_CPU_LINUX_RISCV_VM_VMSTRUCTS_LINUX_RISCV_HPP +#define OS_CPU_LINUX_RISCV_VM_VMSTRUCTS_LINUX_RISCV_HPP + +// These are the OS and CPU-specific fields, types and integer +// constants required by the Serviceability Agent. This file is +// referenced by vmStructs.cpp. + +#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \ + \ + /******************************/ \ + /* Threads (NOTE: incomplete) */ \ + /******************************/ \ + nonstatic_field(OSThread, _thread_id, OSThread::thread_id_t) \ + nonstatic_field(OSThread, _pthread_id, pthread_t) + + +#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \ + \ + /**********************/ \ + /* Posix Thread IDs */ \ + /**********************/ \ + \ + declare_integer_type(OSThread::thread_id_t) \ + declare_unsigned_integer_type(pthread_t) + +#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant) + +#define VM_LONG_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant) + +#endif // OS_CPU_LINUX_RISCV_VM_VMSTRUCTS_LINUX_RISCV_HPP diff --git a/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp b/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp new file mode 100644 index 0000000000000..8d13894031287 --- /dev/null +++ b/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2006, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, Huawei Technologies Co., Ltd. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/register.hpp" +#include "runtime/os.hpp" +#include "runtime/os.inline.hpp" +#include "runtime/vm_version.hpp" + +#include +#include + +#ifndef HWCAP_ISA_I +#define HWCAP_ISA_I (1 << ('I' - 'A')) +#endif + +#ifndef HWCAP_ISA_M +#define HWCAP_ISA_M (1 << ('M' - 'A')) +#endif + +#ifndef HWCAP_ISA_A +#define HWCAP_ISA_A (1 << ('A' - 'A')) +#endif + +#ifndef HWCAP_ISA_F +#define HWCAP_ISA_F (1 << ('F' - 'A')) +#endif + +#ifndef HWCAP_ISA_D +#define HWCAP_ISA_D (1 << ('D' - 'A')) +#endif + +#ifndef HWCAP_ISA_C +#define HWCAP_ISA_C (1 << ('C' - 'A')) +#endif + +#ifndef HWCAP_ISA_V +#define HWCAP_ISA_V (1 << ('V' - 'A')) +#endif + +#define read_csr(csr) \ +({ \ + register unsigned long __v; \ + __asm__ __volatile__ ("csrr %0, %1" \ + : "=r" (__v) \ + : "i" (csr) \ + : "memory"); \ + __v; \ +}) + +uint32_t VM_Version::get_current_vector_length() { + assert(_features & CPU_V, "should not call this"); + return (uint32_t)read_csr(CSR_VLENB); +} + +VM_Version::VM_MODE VM_Version::get_satp_mode() { + if (!strcmp(_vm_mode, "sv39")) { + return VM_SV39; + } else if (!strcmp(_vm_mode, "sv48")) { + return VM_SV48; + } else if (!strcmp(_vm_mode, "sv57")) { + return VM_SV57; + } else if (!strcmp(_vm_mode, "sv64")) { + return VM_SV64; + } else { + return VM_MBARE; + } +} + +void VM_Version::get_os_cpu_info() { + + uint64_t auxv = getauxval(AT_HWCAP); + + static_assert(CPU_I == HWCAP_ISA_I, "Flag CPU_I must follow Linux HWCAP"); + static_assert(CPU_M == HWCAP_ISA_M, "Flag CPU_M must follow Linux HWCAP"); + static_assert(CPU_A == HWCAP_ISA_A, "Flag CPU_A must follow Linux HWCAP"); + static_assert(CPU_F == HWCAP_ISA_F, "Flag CPU_F must follow Linux HWCAP"); + static_assert(CPU_D == HWCAP_ISA_D, "Flag CPU_D must follow Linux HWCAP"); + static_assert(CPU_C == HWCAP_ISA_C, "Flag CPU_C must follow Linux HWCAP"); + static_assert(CPU_V == HWCAP_ISA_V, "Flag CPU_V must follow Linux HWCAP"); + + // RISC-V has four bit-manipulation ISA-extensions: Zba/Zbb/Zbc/Zbs. + // Availability for those extensions could not be queried from HWCAP. + // TODO: Add proper detection for those extensions. + _features = auxv & ( + HWCAP_ISA_I | + HWCAP_ISA_M | + HWCAP_ISA_A | + HWCAP_ISA_F | + HWCAP_ISA_D | + HWCAP_ISA_C | + HWCAP_ISA_V); + + if (FILE *f = fopen("/proc/cpuinfo", "r")) { + char buf[512], *p; + while (fgets(buf, sizeof (buf), f) != NULL) { + if ((p = strchr(buf, ':')) != NULL) { + if (strncmp(buf, "mmu", sizeof "mmu" - 1) == 0) { + if (_vm_mode[0] != '\0') { + continue; + } + char* vm_mode = os::strdup(p + 2); + vm_mode[strcspn(vm_mode, "\n")] = '\0'; + _vm_mode = vm_mode; + } else if (strncmp(buf, "uarch", sizeof "uarch" - 1) == 0) { + char* uarch = os::strdup(p + 2); + uarch[strcspn(uarch, "\n")] = '\0'; + _uarch = uarch; + break; + } + } + } + fclose(f); + } +} diff --git a/src/hotspot/os_cpu/linux_s390/safefetch_linux_s390.S b/src/hotspot/os_cpu/linux_s390/safefetch_linux_s390.S new file mode 100644 index 0000000000000..47fe82f5a278b --- /dev/null +++ b/src/hotspot/os_cpu/linux_s390/safefetch_linux_s390.S @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2022 SAP SE. All rights reserved. + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + + .globl SafeFetchN_impl + .globl _SafeFetchN_fault + .globl _SafeFetchN_continuation + .globl SafeFetch32_impl + .globl _SafeFetch32_fault + .globl _SafeFetch32_continuation + + # Support for int SafeFetch32(int* address, int defaultval); + # + # r2 : address + # r3 : defaultval + # r2 : retval +SafeFetch32_impl: +_SafeFetch32_fault: + lgf %r2, 0(%r2) + br %r14 +_SafeFetch32_continuation: + lgr %r2, %r3 + br %r14 + + # Support for intptr_t SafeFetchN(intptr_t* address, intptr_t defaultval); + # + # r2 : address + # r3 : defaultval + # r2 : retval +SafeFetchN_impl: +_SafeFetchN_fault: + lg %r2, 0(%r2) + br %r14 +_SafeFetchN_continuation: + lgr %r2, %r3 + br %r14 diff --git a/test/jdk/tools/jpackage/junit/run_junit.sh b/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_32.S similarity index 50% rename from test/jdk/tools/jpackage/junit/run_junit.sh rename to src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_32.S index 55a170f17c0ac..492b1207db6e2 100644 --- a/test/jdk/tools/jpackage/junit/run_junit.sh +++ b/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_32.S @@ -1,6 +1,6 @@ -#!/bin/bash - -# Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved. +# +# Copyright (c) 2022 SAP SE. All rights reserved. +# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -20,35 +20,24 @@ # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA # or visit www.oracle.com if you need additional information or have any # questions. +# + .globl SafeFetch32_impl + .globl _SafeFetch32_fault + .globl _SafeFetch32_continuation -set -x - -set -e -if [ -z "$BASH" ]; then - # The script relies on Bash arrays, rerun in Bash. - /bin/bash $0 $@ - exit -fi - -sources=() -classes=() -for s in $(find "${TESTSRC}" -name "*.java" | grep -v junit.java); do - sources+=( "$s" ) - classes+=( $(echo "$s" | sed -e "s|${TESTSRC}/||" -e 's|/|.|g' -e 's/.java$//') ) -done - -common_args=(\ - --add-modules jdk.jpackage \ - --patch-module jdk.jpackage="${TESTSRC}${PS}${TESTCLASSES}" \ - --add-reads jdk.jpackage=ALL-UNNAMED \ - --add-exports jdk.jpackage/jdk.jpackage.internal=ALL-UNNAMED \ - -classpath "${TESTCLASSPATH}" \ -) - -# Compile classes for junit -"${COMPILEJAVA}/bin/javac" ${TESTTOOLVMOPTS} ${TESTJAVACOPTS} \ - "${common_args[@]}" -d "${TESTCLASSES}" "${sources[@]}" + .text -# Run junit -"${TESTJAVA}/bin/java" ${TESTVMOPTS} ${TESTJAVAOPTS} \ - "${common_args[@]}" org.junit.runner.JUnitCore "${classes[@]}" + # Support for int SafeFetch32(int* address, int defaultval); + # + # 8(%esp) : default value + # 4(%esp) : crash address + # 0(%esp) : return pc + .type _SafeFetch32_impl,@function +SafeFetch32_impl: + movl 4(%esp),%ecx # load address from stack +_SafeFetch32_fault: + movl (%ecx), %eax # load target value, may fault + ret +_SafeFetch32_continuation: + movl 8(%esp),%eax # load default value from stack + ret diff --git a/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_64.S b/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_64.S new file mode 100644 index 0000000000000..617851e8327d4 --- /dev/null +++ b/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_64.S @@ -0,0 +1,58 @@ +# +# Copyright (c) 2022 SAP SE. All rights reserved. +# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# This code is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 only, as +# published by the Free Software Foundation. +# +# This code is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# version 2 for more details (a copy is included in the LICENSE file that +# accompanied this code). +# +# You should have received a copy of the GNU General Public License version +# 2 along with this work; if not, write to the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +# or visit www.oracle.com if you need additional information or have any +# questions. +# + .globl SafeFetch32_impl + .globl SafeFetchN_impl + .globl _SafeFetch32_fault + .globl _SafeFetchN_fault + .globl _SafeFetch32_continuation + .globl _SafeFetchN_continuation + + .text + + + # Support for int SafeFetch32(int* address, int defaultval); + # + # %rdi : address + # %esi : defaultval + .type SafeFetch32_impl,@function +SafeFetch32_impl: +_SafeFetch32_fault: + movl (%rdi), %eax # load target value, may fault + ret +_SafeFetch32_continuation: + movl %esi, %eax # return default + ret + + # Support for intptr_t SafeFetchN(intptr_t* address, intptr_t defaultval); + # + # %rdi : address + # %rsi : defaultval + .type SafeFetchN_impl,@function +SafeFetchN_impl: +_SafeFetchN_fault: + movq (%rdi), %rax # load target value, may fault + ret +_SafeFetchN_continuation: + movq %rsi, %rax # return default + ret diff --git a/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp b/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp index c08f95f8a492c..feace2d6d29da 100644 --- a/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp +++ b/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp @@ -223,7 +223,7 @@ bool os::register_code_area(char *low, char *high) { * loop in vmError.cpp. We need to roll our own loop. */ bool os::platform_print_native_stack(outputStream* st, const void* context, - char *buf, int buf_size) + char *buf, int buf_size, address& lastpc) { CONTEXT ctx; if (context != NULL) { @@ -244,14 +244,14 @@ bool os::platform_print_native_stack(outputStream* st, const void* context, stk.AddrPC.Mode = AddrModeFlat; int count = 0; - address lastpc = 0; + address lastpc_internal = 0; while (count++ < StackPrintLimit) { intptr_t* sp = (intptr_t*)stk.AddrStack.Offset; intptr_t* fp = (intptr_t*)stk.AddrFrame.Offset; // NOT necessarily the same as ctx.Rbp! address pc = (address)stk.AddrPC.Offset; if (pc != NULL) { - if (count == 2 && lastpc == pc) { + if (count == 2 && lastpc_internal == pc) { // Skip it -- StackWalk64() may return the same PC // (but different SP) on the first try. } else { @@ -267,12 +267,13 @@ bool os::platform_print_native_stack(outputStream* st, const void* context, } st->cr(); } - lastpc = pc; + lastpc_internal = pc; } PVOID p = WindowsDbgHelp::symFunctionTableAccess64(GetCurrentProcess(), stk.AddrPC.Offset); if (!p) { // StackWalk64() can't handle this PC. Calling StackWalk64 again may cause crash. + lastpc = lastpc_internal; break; } diff --git a/src/hotspot/os_cpu/windows_x86/os_windows_x86.hpp b/src/hotspot/os_cpu/windows_x86/os_windows_x86.hpp index 7fdd068219ce8..f8af482b72cd1 100644 --- a/src/hotspot/os_cpu/windows_x86/os_windows_x86.hpp +++ b/src/hotspot/os_cpu/windows_x86/os_windows_x86.hpp @@ -40,7 +40,7 @@ #ifdef AMD64 #define PLATFORM_PRINT_NATIVE_STACK 1 static bool platform_print_native_stack(outputStream* st, const void* context, - char *buf, int buf_size); + char *buf, int buf_size, address& lastpc); #endif #endif // OS_CPU_WINDOWS_X86_OS_WINDOWS_X86_HPP diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp index ca30abf7efa48..854b1b310e497 100644 --- a/src/hotspot/share/adlc/formssel.cpp +++ b/src/hotspot/share/adlc/formssel.cpp @@ -4117,6 +4117,7 @@ bool MatchRule::is_ideal_membar() const { !strcmp(_opType,"MemBarReleaseLock") || !strcmp(_opType,"LoadFence" ) || !strcmp(_opType,"StoreFence") || + !strcmp(_opType,"StoreStoreFence") || !strcmp(_opType,"MemBarVolatile") || !strcmp(_opType,"MemBarCPUOrder") || !strcmp(_opType,"MemBarStoreStore") || diff --git a/src/hotspot/share/adlc/output_c.cpp b/src/hotspot/share/adlc/output_c.cpp index a0dbb7b1d28b5..ea070e98b70fa 100644 --- a/src/hotspot/share/adlc/output_c.cpp +++ b/src/hotspot/share/adlc/output_c.cpp @@ -397,7 +397,7 @@ static int pipeline_res_mask_initializer( const uint cyclemasksize = (maxcycleused + 31) >> 5; int i, j; - int element_count = 0; + uint element_count = 0; uint *res_mask = new uint [cyclemasksize]; uint resources_used = 0; uint resources_used_exclusively = 0; @@ -524,12 +524,12 @@ static int pipeline_res_mask_initializer( fprintf(fp_cpp, "static const Pipeline_Use_Element pipeline_res_mask_%03d[%d] = {\n%s};\n\n", ndx+1, element_count, resource_mask); - char* args = new char [9 + 2*masklen + maskdigit]; + // "0x012345678, 0x012345678, 4294967295" + char* args = new char [36 + 1]; - sprintf(args, "0x%0*x, 0x%0*x, %*d", - masklen, resources_used, - masklen, resources_used_exclusively, - maskdigit, element_count); + int printed = sprintf(args, "0x%x, 0x%x, %u", + resources_used, resources_used_exclusively, element_count); + assert(printed <= 36, "overflow"); pipeline_res_args.addName(args); } @@ -1159,11 +1159,10 @@ static void check_peepconstraints(FILE *fp, FormDict &globals, PeepMatch *pmatch case Form::register_interface: { // Check that they are allocated to the same register // Need parameter for index position if not result operand - char left_reg_index[] = ",instXXXX_idxXXXX"; + char left_reg_index[] = ",inst4294967295_idx4294967295"; if( left_op_index != 0 ) { - assert( (left_index <= 9999) && (left_op_index <= 9999), "exceed string size"); // Must have index into operands - sprintf(left_reg_index,",inst%d_idx%d", (int)left_index, left_op_index); + sprintf(left_reg_index,",inst%u_idx%u", (unsigned)left_index, (unsigned)left_op_index); } else { strcpy(left_reg_index, ""); } @@ -1172,11 +1171,10 @@ static void check_peepconstraints(FILE *fp, FormDict &globals, PeepMatch *pmatch fprintf(fp, " == "); if( right_index != -1 ) { - char right_reg_index[18] = ",instXXXX_idxXXXX"; + char right_reg_index[] = ",inst4294967295_idx4294967295"; if( right_op_index != 0 ) { - assert( (right_index <= 9999) && (right_op_index <= 9999), "exceed string size"); // Must have index into operands - sprintf(right_reg_index,",inst%d_idx%d", (int)right_index, right_op_index); + sprintf(right_reg_index,",inst%u_idx%u", (unsigned)right_index, (unsigned)right_op_index); } else { strcpy(right_reg_index, ""); } diff --git a/src/hotspot/share/c1/c1_Compilation.cpp b/src/hotspot/share/c1/c1_Compilation.cpp index aa3857742f93c..a48b16cd355ff 100644 --- a/src/hotspot/share/c1/c1_Compilation.cpp +++ b/src/hotspot/share/c1/c1_Compilation.cpp @@ -314,9 +314,6 @@ void Compilation::emit_code_epilog(LIR_Assembler* assembler) { // Emit the handler to remove the activation from the stack and // dispatch to the caller. offsets()->set_value(CodeOffsets::UnwindHandler, assembler->emit_unwind_handler()); - - // done - masm()->flush(); } diff --git a/src/hotspot/share/c1/c1_Compiler.cpp b/src/hotspot/share/c1/c1_Compiler.cpp index 3795b4e019220..de173c64af16c 100644 --- a/src/hotspot/share/c1/c1_Compiler.cpp +++ b/src/hotspot/share/c1/c1_Compiler.cpp @@ -142,6 +142,7 @@ bool Compiler::is_intrinsic_supported(const methodHandle& method) { // since GC can change its value. case vmIntrinsics::_loadFence: case vmIntrinsics::_storeFence: + case vmIntrinsics::_storeStoreFence: case vmIntrinsics::_fullFence: case vmIntrinsics::_floatToRawIntBits: case vmIntrinsics::_intBitsToFloat: diff --git a/src/hotspot/share/c1/c1_LIR.cpp b/src/hotspot/share/c1/c1_LIR.cpp index 62cff4c750528..308f3a09c1548 100644 --- a/src/hotspot/share/c1/c1_LIR.cpp +++ b/src/hotspot/share/c1/c1_LIR.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -188,6 +188,9 @@ void LIR_Op2::verify() const { #ifdef ASSERT switch (code()) { case lir_cmove: +#ifdef RISCV + assert(false, "lir_cmove is LIR_Op4 on RISCV"); +#endif case lir_xchg: break; @@ -238,8 +241,12 @@ void LIR_Op2::verify() const { LIR_OpBranch::LIR_OpBranch(LIR_Condition cond, BlockBegin* block) +#ifdef RISCV + : LIR_Op2(lir_branch, cond, LIR_OprFact::illegalOpr, LIR_OprFact::illegalOpr, (CodeEmitInfo*)NULL) +#else : LIR_Op(lir_branch, LIR_OprFact::illegalOpr, (CodeEmitInfo*)NULL) , _cond(cond) +#endif , _label(block->label()) , _block(block) , _ublock(NULL) @@ -247,8 +254,12 @@ LIR_OpBranch::LIR_OpBranch(LIR_Condition cond, BlockBegin* block) } LIR_OpBranch::LIR_OpBranch(LIR_Condition cond, CodeStub* stub) : +#ifdef RISCV + LIR_Op2(lir_branch, cond, LIR_OprFact::illegalOpr, LIR_OprFact::illegalOpr, (CodeEmitInfo*)NULL) +#else LIR_Op(lir_branch, LIR_OprFact::illegalOpr, (CodeEmitInfo*)NULL) , _cond(cond) +#endif , _label(stub->entry()) , _block(NULL) , _ublock(NULL) @@ -256,8 +267,12 @@ LIR_OpBranch::LIR_OpBranch(LIR_Condition cond, CodeStub* stub) : } LIR_OpBranch::LIR_OpBranch(LIR_Condition cond, BlockBegin* block, BlockBegin* ublock) +#ifdef RISCV + : LIR_Op2(lir_cond_float_branch, cond, LIR_OprFact::illegalOpr, LIR_OprFact::illegalOpr, (CodeEmitInfo*)NULL) +#else : LIR_Op(lir_cond_float_branch, LIR_OprFact::illegalOpr, (CodeEmitInfo*)NULL) , _cond(cond) +#endif , _label(block->label()) , _block(block) , _ublock(ublock) @@ -279,13 +294,13 @@ void LIR_OpBranch::change_ublock(BlockBegin* b) { } void LIR_OpBranch::negate_cond() { - switch (_cond) { - case lir_cond_equal: _cond = lir_cond_notEqual; break; - case lir_cond_notEqual: _cond = lir_cond_equal; break; - case lir_cond_less: _cond = lir_cond_greaterEqual; break; - case lir_cond_lessEqual: _cond = lir_cond_greater; break; - case lir_cond_greaterEqual: _cond = lir_cond_less; break; - case lir_cond_greater: _cond = lir_cond_lessEqual; break; + switch (cond()) { + case lir_cond_equal: set_cond(lir_cond_notEqual); break; + case lir_cond_notEqual: set_cond(lir_cond_equal); break; + case lir_cond_less: set_cond(lir_cond_greaterEqual); break; + case lir_cond_lessEqual: set_cond(lir_cond_greater); break; + case lir_cond_greaterEqual: set_cond(lir_cond_less); break; + case lir_cond_greater: set_cond(lir_cond_lessEqual); break; default: ShouldNotReachHere(); } } @@ -513,6 +528,15 @@ void LIR_OpVisitState::visit(LIR_Op* op) { assert(op->as_OpBranch() != NULL, "must be"); LIR_OpBranch* opBranch = (LIR_OpBranch*)op; +#ifdef RISCV + assert(opBranch->_tmp1->is_illegal() && opBranch->_tmp2->is_illegal() && + opBranch->_tmp3->is_illegal() && opBranch->_tmp4->is_illegal() && + opBranch->_tmp5->is_illegal(), "not used"); + + if (opBranch->_opr1->is_valid()) do_input(opBranch->_opr1); + if (opBranch->_opr2->is_valid()) do_input(opBranch->_opr2); +#endif + if (opBranch->_info != NULL) do_info(opBranch->_info); assert(opBranch->_result->is_illegal(), "not used"); if (opBranch->_stub != NULL) opBranch->stub()->visit(this); @@ -601,6 +625,21 @@ void LIR_OpVisitState::visit(LIR_Op* op) { // to the result operand, otherwise the backend fails case lir_cmove: { +#ifdef RISCV + assert(op->as_Op4() != NULL, "must be"); + LIR_Op4* op4 = (LIR_Op4*)op; + + assert(op4->_info == NULL && op4->_tmp1->is_illegal() && op4->_tmp2->is_illegal() && + op4->_tmp3->is_illegal() && op4->_tmp4->is_illegal() && op4->_tmp5->is_illegal(), "not used"); + assert(op4->_opr1->is_valid() && op4->_opr2->is_valid() && op4->_result->is_valid(), "used"); + + do_input(op4->_opr1); + do_input(op4->_opr2); + if (op4->_opr3->is_valid()) do_input(op4->_opr3); + if (op4->_opr4->is_valid()) do_input(op4->_opr4); + do_temp(op4->_opr2); + do_output(op4->_result); +#else assert(op->as_Op2() != NULL, "must be"); LIR_Op2* op2 = (LIR_Op2*)op; @@ -612,6 +651,7 @@ void LIR_OpVisitState::visit(LIR_Op* op) { do_input(op2->_opr2); do_temp(op2->_opr2); do_output(op2->_result); +#endif break; } @@ -880,6 +920,19 @@ void LIR_OpVisitState::visit(LIR_Op* op) { break; } +// LIR_OpLoadKlass + case lir_load_klass: + { + LIR_OpLoadKlass* opLoadKlass = op->as_OpLoadKlass(); + assert(opLoadKlass != NULL, "must be"); + + do_input(opLoadKlass->_obj); + do_output(opLoadKlass->_result); + if (opLoadKlass->_info) do_info(opLoadKlass->_info); + break; + } + + // LIR_OpProfileCall: case lir_profile_call: { assert(op->as_OpProfileCall() != NULL, "must be"); @@ -1042,6 +1095,12 @@ void LIR_Op3::emit_code(LIR_Assembler* masm) { masm->emit_op3(this); } +#ifdef RISCV +void LIR_Op4::emit_code(LIR_Assembler* masm) { + masm->emit_op4(this); +} +#endif + void LIR_OpLock::emit_code(LIR_Assembler* masm) { masm->emit_lock(this); if (stub()) { @@ -1049,6 +1108,10 @@ void LIR_OpLock::emit_code(LIR_Assembler* masm) { } } +void LIR_OpLoadKlass::emit_code(LIR_Assembler* masm) { + masm->emit_load_klass(this); +} + #ifdef ASSERT void LIR_OpAssert::emit_code(LIR_Assembler* masm) { masm->emit_assert(this); @@ -1078,6 +1141,10 @@ LIR_List::LIR_List(Compilation* compilation, BlockBegin* block) , _file(NULL) , _line(0) #endif +#ifdef RISCV + , _cmp_opr1(LIR_OprFact::illegalOpr) + , _cmp_opr2(LIR_OprFact::illegalOpr) +#endif { } @@ -1095,6 +1162,38 @@ void LIR_List::set_file_and_line(const char * file, int line) { } #endif +#ifdef RISCV +void LIR_List::set_cmp_oprs(LIR_Op* op) { + switch (op->code()) { + case lir_cmp: + _cmp_opr1 = op->as_Op2()->in_opr1(); + _cmp_opr2 = op->as_Op2()->in_opr2(); + break; + case lir_branch: // fall through + case lir_cond_float_branch: + assert(op->as_OpBranch()->cond() == lir_cond_always || + (_cmp_opr1 != LIR_OprFact::illegalOpr && _cmp_opr2 != LIR_OprFact::illegalOpr), + "conditional branches must have legal operands"); + if (op->as_OpBranch()->cond() != lir_cond_always) { + op->as_Op2()->set_in_opr1(_cmp_opr1); + op->as_Op2()->set_in_opr2(_cmp_opr2); + } + break; + case lir_cmove: + op->as_Op4()->set_in_opr3(_cmp_opr1); + op->as_Op4()->set_in_opr4(_cmp_opr2); + break; +#if INCLUDE_ZGC + case lir_zloadbarrier_test: + _cmp_opr1 = FrameMap::as_opr(t1); + _cmp_opr2 = LIR_OprFact::intConst(0); + break; +#endif + default: + break; + } +} +#endif void LIR_List::append(LIR_InsertionBuffer* buffer) { assert(this == buffer->lir_list(), "wrong lir list"); @@ -1825,6 +1924,10 @@ void LIR_Op1::print_patch_code(outputStream* out, LIR_PatchCode code) { // LIR_OpBranch void LIR_OpBranch::print_instr(outputStream* out) const { print_condition(out, cond()); out->print(" "); +#ifdef RISCV + in_opr1()->print(out); out->print(" "); + in_opr2()->print(out); out->print(" "); +#endif if (block() != NULL) { out->print("[B%d] ", block()->block_id()); } else if (stub() != NULL) { @@ -1911,7 +2014,11 @@ void LIR_OpRoundFP::print_instr(outputStream* out) const { // LIR_Op2 void LIR_Op2::print_instr(outputStream* out) const { +#ifdef RISCV + if (code() == lir_cmp || code() == lir_branch || code() == lir_cond_float_branch) { +#else if (code() == lir_cmove || code() == lir_cmp) { +#endif print_condition(out, condition()); out->print(" "); } in_opr1()->print(out); out->print(" "); @@ -1962,6 +2069,17 @@ void LIR_Op3::print_instr(outputStream* out) const { result_opr()->print(out); } +#ifdef RISCV +// LIR_Op4 +void LIR_Op4::print_instr(outputStream* out) const { + print_condition(out, condition()); out->print(" "); + in_opr1()->print(out); out->print(" "); + in_opr2()->print(out); out->print(" "); + in_opr3()->print(out); out->print(" "); + in_opr4()->print(out); out->print(" "); + result_opr()->print(out); +} +#endif void LIR_OpLock::print_instr(outputStream* out) const { hdr_opr()->print(out); out->print(" "); @@ -1973,6 +2091,11 @@ void LIR_OpLock::print_instr(outputStream* out) const { out->print("[lbl:" INTPTR_FORMAT "]", p2i(stub()->entry())); } +void LIR_OpLoadKlass::print_instr(outputStream* out) const { + obj()->print(out); out->print(" "); + result_opr()->print(out); out->print(" "); +} + #ifdef ASSERT void LIR_OpAssert::print_instr(outputStream* out) const { print_condition(out, condition()); out->print(" "); diff --git a/src/hotspot/share/c1/c1_LIR.hpp b/src/hotspot/share/c1/c1_LIR.hpp index 2342e6117eb4a..717404e97260c 100644 --- a/src/hotspot/share/c1/c1_LIR.hpp +++ b/src/hotspot/share/c1/c1_LIR.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -869,6 +869,9 @@ class LIR_Op2; class LIR_OpDelay; class LIR_Op3; class LIR_OpAllocArray; +#ifdef RISCV +class LIR_Op4; +#endif class LIR_OpCall; class LIR_OpJavaCall; class LIR_OpRTCall; @@ -877,6 +880,7 @@ class LIR_OpUpdateCRC32; class LIR_OpLock; class LIR_OpTypeCheck; class LIR_OpCompareAndSwap; +class LIR_OpLoadKlass; class LIR_OpProfileCall; class LIR_OpProfileType; #ifdef ASSERT @@ -913,8 +917,10 @@ enum LIR_Code { , lir_null_check , lir_return , lir_leal +#ifndef RISCV , lir_branch , lir_cond_float_branch +#endif , lir_move , lir_convert , lir_alloc_object @@ -922,13 +928,20 @@ enum LIR_Code { , lir_roundfp , lir_safepoint , lir_unwind + , lir_load_klass , end_op1 , begin_op2 +#ifdef RISCV + , lir_branch + , lir_cond_float_branch +#endif , lir_cmp , lir_cmp_l2i , lir_ucmp_fd2i , lir_cmp_fd2i +#ifndef RISCV , lir_cmove +#endif , lir_add , lir_sub , lir_mul @@ -956,6 +969,11 @@ enum LIR_Code { , lir_fmad , lir_fmaf , end_op3 +#ifdef RISCV + , begin_op4 + , lir_cmove + , end_op4 +#endif , begin_opJavaCall , lir_static_call , lir_optvirtual_call @@ -992,6 +1010,11 @@ enum LIR_Code { , begin_opAssert , lir_assert , end_opAssert +#if defined(RISCV) && defined(INCLUDE_ZGC) + , begin_opZLoadBarrierTest + , lir_zloadbarrier_test + , end_opZLoadBarrierTest +#endif }; @@ -1128,10 +1151,14 @@ class LIR_Op: public CompilationResourceObj { virtual LIR_Op1* as_Op1() { return NULL; } virtual LIR_Op2* as_Op2() { return NULL; } virtual LIR_Op3* as_Op3() { return NULL; } +#ifdef RISCV + virtual LIR_Op4* as_Op4() { return NULL; } +#endif virtual LIR_OpArrayCopy* as_OpArrayCopy() { return NULL; } virtual LIR_OpUpdateCRC32* as_OpUpdateCRC32() { return NULL; } virtual LIR_OpTypeCheck* as_OpTypeCheck() { return NULL; } virtual LIR_OpCompareAndSwap* as_OpCompareAndSwap() { return NULL; } + virtual LIR_OpLoadKlass* as_OpLoadKlass() { return NULL; } virtual LIR_OpProfileCall* as_OpProfileCall() { return NULL; } virtual LIR_OpProfileType* as_OpProfileType() { return NULL; } #ifdef ASSERT @@ -1399,47 +1426,6 @@ class LIR_OpRTCall: public LIR_OpCall { virtual void verify() const; }; - -class LIR_OpBranch: public LIR_Op { - friend class LIR_OpVisitState; - - private: - LIR_Condition _cond; - Label* _label; - BlockBegin* _block; // if this is a branch to a block, this is the block - BlockBegin* _ublock; // if this is a float-branch, this is the unorderd block - CodeStub* _stub; // if this is a branch to a stub, this is the stub - - public: - LIR_OpBranch(LIR_Condition cond, Label* lbl) - : LIR_Op(lir_branch, LIR_OprFact::illegalOpr, (CodeEmitInfo*) NULL) - , _cond(cond) - , _label(lbl) - , _block(NULL) - , _ublock(NULL) - , _stub(NULL) { } - - LIR_OpBranch(LIR_Condition cond, BlockBegin* block); - LIR_OpBranch(LIR_Condition cond, CodeStub* stub); - - // for unordered comparisons - LIR_OpBranch(LIR_Condition cond, BlockBegin* block, BlockBegin* ublock); - - LIR_Condition cond() const { return _cond; } - Label* label() const { return _label; } - BlockBegin* block() const { return _block; } - BlockBegin* ublock() const { return _ublock; } - CodeStub* stub() const { return _stub; } - - void change_block(BlockBegin* b); - void change_ublock(BlockBegin* b); - void negate_cond(); - - virtual void emit_code(LIR_Assembler* masm); - virtual LIR_OpBranch* as_OpBranch() { return this; } - virtual void print_instr(outputStream* out) const PRODUCT_RETURN; -}; - class LIR_OpReturn: public LIR_Op1 { friend class LIR_OpVisitState; @@ -1612,19 +1598,19 @@ class LIR_Op2: public LIR_Op { void verify() const; public: - LIR_Op2(LIR_Code code, LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, CodeEmitInfo* info = NULL) + LIR_Op2(LIR_Code code, LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, CodeEmitInfo* info = NULL, BasicType type = T_ILLEGAL) : LIR_Op(code, LIR_OprFact::illegalOpr, info) , _fpu_stack_size(0) , _opr1(opr1) , _opr2(opr2) - , _type(T_ILLEGAL) + , _type(type) , _tmp1(LIR_OprFact::illegalOpr) , _tmp2(LIR_OprFact::illegalOpr) , _tmp3(LIR_OprFact::illegalOpr) , _tmp4(LIR_OprFact::illegalOpr) , _tmp5(LIR_OprFact::illegalOpr) , _condition(condition) { - assert(code == lir_cmp || code == lir_assert, "code check"); + assert(code == lir_cmp || code == lir_assert RISCV_ONLY(|| code == lir_branch || code == lir_cond_float_branch), "code check"); } LIR_Op2(LIR_Code code, LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) @@ -1656,7 +1642,7 @@ class LIR_Op2: public LIR_Op { , _tmp4(LIR_OprFact::illegalOpr) , _tmp5(LIR_OprFact::illegalOpr) , _condition(lir_cond_unknown) { - assert(code != lir_cmp && is_in_range(code, begin_op2, end_op2), "code check"); + assert(code != lir_cmp && RISCV_ONLY(code != lir_branch && code != lir_cond_float_branch &&) is_in_range(code, begin_op2, end_op2), "code check"); } LIR_Op2(LIR_Code code, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2 = LIR_OprFact::illegalOpr, @@ -1672,7 +1658,7 @@ class LIR_Op2: public LIR_Op { , _tmp4(tmp4) , _tmp5(tmp5) , _condition(lir_cond_unknown) { - assert(code != lir_cmp && is_in_range(code, begin_op2, end_op2), "code check"); + assert(code != lir_cmp && RISCV_ONLY(code != lir_branch && code != lir_cond_float_branch &&) is_in_range(code, begin_op2, end_op2), "code check"); } LIR_Opr in_opr1() const { return _opr1; } @@ -1684,10 +1670,18 @@ class LIR_Op2: public LIR_Op { LIR_Opr tmp4_opr() const { return _tmp4; } LIR_Opr tmp5_opr() const { return _tmp5; } LIR_Condition condition() const { +#ifdef RISCV + assert(code() == lir_cmp || code() == lir_branch || code() == lir_cond_float_branch || code() == lir_assert, "only valid for branch and assert"); return _condition; +#else assert(code() == lir_cmp || code() == lir_cmove || code() == lir_assert, "only valid for cmp and cmove and assert"); return _condition; +#endif } void set_condition(LIR_Condition condition) { +#ifdef RISCV + assert(code() == lir_cmp || code() == lir_branch || code() == lir_cond_float_branch, "only valid for branch"); _condition = condition; +#else assert(code() == lir_cmp || code() == lir_cmove, "only valid for cmp and cmove"); _condition = condition; +#endif } void set_fpu_stack_size(int size) { _fpu_stack_size = size; } @@ -1701,6 +1695,62 @@ class LIR_Op2: public LIR_Op { virtual void print_instr(outputStream* out) const PRODUCT_RETURN; }; +#ifdef RISCV +class LIR_OpBranch: public LIR_Op2 { +#else +class LIR_OpBranch: public LIR_Op { +#endif + friend class LIR_OpVisitState; + + private: +#ifndef RISCV + LIR_Condition _cond; +#endif + Label* _label; + BlockBegin* _block; // if this is a branch to a block, this is the block + BlockBegin* _ublock; // if this is a float-branch, this is the unordered block + CodeStub* _stub; // if this is a branch to a stub, this is the stub + + public: + LIR_OpBranch(LIR_Condition cond, Label* lbl) +#ifdef RISCV + : LIR_Op2(lir_branch, cond, LIR_OprFact::illegalOpr, LIR_OprFact::illegalOpr, (CodeEmitInfo*) NULL) +#else + : LIR_Op(lir_branch, LIR_OprFact::illegalOpr, (CodeEmitInfo*) NULL) + , _cond(cond) +#endif + , _label(lbl) + , _block(NULL) + , _ublock(NULL) + , _stub(NULL) { } + + LIR_OpBranch(LIR_Condition cond, BlockBegin* block); + LIR_OpBranch(LIR_Condition cond, CodeStub* stub); + + // for unordered comparisons + LIR_OpBranch(LIR_Condition cond, BlockBegin* block, BlockBegin* ublock); + +#ifdef RISCV + LIR_Condition cond() const { return condition(); } + void set_cond(LIR_Condition cond) { set_condition(cond); } +#else + LIR_Condition cond() const { return _cond; } + void set_cond(LIR_Condition cond) { _cond = cond; } +#endif + Label* label() const { return _label; } + BlockBegin* block() const { return _block; } + BlockBegin* ublock() const { return _ublock; } + CodeStub* stub() const { return _stub; } + + void change_block(BlockBegin* b); + void change_ublock(BlockBegin* b); + void negate_cond(); + + virtual void emit_code(LIR_Assembler* masm); + virtual LIR_OpBranch* as_OpBranch() { return this; } + virtual void print_instr(outputStream* out) const PRODUCT_RETURN; +}; + class LIR_OpAllocArray : public LIR_Op { friend class LIR_OpVisitState; @@ -1764,6 +1814,65 @@ class LIR_Op3: public LIR_Op { virtual void print_instr(outputStream* out) const PRODUCT_RETURN; }; +#ifdef RISCV +class LIR_Op4: public LIR_Op { + friend class LIR_OpVisitState; + protected: + LIR_Opr _opr1; + LIR_Opr _opr2; + LIR_Opr _opr3; + LIR_Opr _opr4; + BasicType _type; + LIR_Opr _tmp1; + LIR_Opr _tmp2; + LIR_Opr _tmp3; + LIR_Opr _tmp4; + LIR_Opr _tmp5; + LIR_Condition _condition; + + public: + LIR_Op4(LIR_Code code, LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr opr3, LIR_Opr opr4, + LIR_Opr result, BasicType type) + : LIR_Op(code, result, NULL) + , _opr1(opr1) + , _opr2(opr2) + , _opr3(opr3) + , _opr4(opr4) + , _type(type) + , _tmp1(LIR_OprFact::illegalOpr) + , _tmp2(LIR_OprFact::illegalOpr) + , _tmp3(LIR_OprFact::illegalOpr) + , _tmp4(LIR_OprFact::illegalOpr) + , _tmp5(LIR_OprFact::illegalOpr) + , _condition(condition) { + assert(code == lir_cmove, "code check"); + assert(type != T_ILLEGAL, "cmove should have type"); + } + + LIR_Opr in_opr1() const { return _opr1; } + LIR_Opr in_opr2() const { return _opr2; } + LIR_Opr in_opr3() const { return _opr3; } + LIR_Opr in_opr4() const { return _opr4; } + BasicType type() const { return _type; } + LIR_Opr tmp1_opr() const { return _tmp1; } + LIR_Opr tmp2_opr() const { return _tmp2; } + LIR_Opr tmp3_opr() const { return _tmp3; } + LIR_Opr tmp4_opr() const { return _tmp4; } + LIR_Opr tmp5_opr() const { return _tmp5; } + + LIR_Condition condition() const { return _condition; } + void set_condition(LIR_Condition condition) { _condition = condition; } + + void set_in_opr1(LIR_Opr opr) { _opr1 = opr; } + void set_in_opr2(LIR_Opr opr) { _opr2 = opr; } + void set_in_opr3(LIR_Opr opr) { _opr3 = opr; } + void set_in_opr4(LIR_Opr opr) { _opr4 = opr; } + virtual void emit_code(LIR_Assembler* masm); + virtual LIR_Op4* as_Op4() { return this; } + + virtual void print_instr(outputStream* out) const PRODUCT_RETURN; +}; +#endif //-------------------------------- class LabelObj: public CompilationResourceObj { @@ -1804,6 +1913,23 @@ class LIR_OpLock: public LIR_Op { void print_instr(outputStream* out) const PRODUCT_RETURN; }; +class LIR_OpLoadKlass: public LIR_Op { + friend class LIR_OpVisitState; + + private: + LIR_Opr _obj; + public: + LIR_OpLoadKlass(LIR_Opr obj, LIR_Opr result, CodeEmitInfo* info) + : LIR_Op(lir_load_klass, result, info) + , _obj(obj) + {} + + LIR_Opr obj() const { return _obj; } + + virtual LIR_OpLoadKlass* as_OpLoadKlass() { return this; } + virtual void emit_code(LIR_Assembler* masm); + void print_instr(outputStream* out) const PRODUCT_RETURN; +}; class LIR_OpDelay: public LIR_Op { friend class LIR_OpVisitState; @@ -1986,6 +2112,10 @@ class LIR_List: public CompilationResourceObj { const char * _file; int _line; #endif +#ifdef RISCV + LIR_Opr _cmp_opr1; + LIR_Opr _cmp_opr2; +#endif public: void append(LIR_Op* op) { @@ -1998,6 +2128,12 @@ class LIR_List: public CompilationResourceObj { } #endif // PRODUCT +#ifdef RISCV + set_cmp_oprs(op); + // lir_cmp set cmp oprs only on riscv + if (op->code() == lir_cmp) return; +#endif + _operations.append(op); #ifdef ASSERT @@ -2014,6 +2150,10 @@ class LIR_List: public CompilationResourceObj { void set_file_and_line(const char * file, int line); #endif +#ifdef RISCV + void set_cmp_oprs(LIR_Op* op); +#endif + //---------- accessors --------------- LIR_OpList* instructions_list() { return &_operations; } int length() const { return _operations.length(); } @@ -2133,9 +2273,16 @@ class LIR_List: public CompilationResourceObj { void cmp_mem_int(LIR_Condition condition, LIR_Opr base, int disp, int c, CodeEmitInfo* info); void cmp_reg_mem(LIR_Condition condition, LIR_Opr reg, LIR_Address* addr, CodeEmitInfo* info); +#ifdef RISCV + void cmove(LIR_Condition condition, LIR_Opr src1, LIR_Opr src2, LIR_Opr dst, BasicType type, + LIR_Opr cmp_opr1 = LIR_OprFact::illegalOpr, LIR_Opr cmp_opr2 = LIR_OprFact::illegalOpr) { + append(new LIR_Op4(lir_cmove, condition, src1, src2, cmp_opr1, cmp_opr2, dst, type)); + } +#else void cmove(LIR_Condition condition, LIR_Opr src1, LIR_Opr src2, LIR_Opr dst, BasicType type) { append(new LIR_Op2(lir_cmove, condition, src1, src2, dst, type)); } +#endif void cas_long(LIR_Opr addr, LIR_Opr cmp_value, LIR_Opr new_value, LIR_Opr t1, LIR_Opr t2, LIR_Opr result = LIR_OprFact::illegalOpr); @@ -2249,6 +2396,9 @@ class LIR_List: public CompilationResourceObj { void xadd(LIR_Opr src, LIR_Opr add, LIR_Opr res, LIR_Opr tmp) { append(new LIR_Op2(lir_xadd, src, add, res, tmp)); } void xchg(LIR_Opr src, LIR_Opr set, LIR_Opr res, LIR_Opr tmp) { append(new LIR_Op2(lir_xchg, src, set, res, tmp)); } + + void load_klass(LIR_Opr obj, LIR_Opr result, CodeEmitInfo* info) { append(new LIR_OpLoadKlass(obj, result, info)); } + #ifdef ASSERT void lir_assert(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, const char* msg, bool halt) { append(new LIR_OpAssert(condition, opr1, opr2, msg, halt)); } #endif diff --git a/src/hotspot/share/c1/c1_LIRAssembler.cpp b/src/hotspot/share/c1/c1_LIRAssembler.cpp index 37ce476253d6e..989a6f8ad25e8 100644 --- a/src/hotspot/share/c1/c1_LIRAssembler.cpp +++ b/src/hotspot/share/c1/c1_LIRAssembler.cpp @@ -691,9 +691,11 @@ void LIR_Assembler::emit_op2(LIR_Op2* op) { comp_fl2i(op->code(), op->in_opr1(), op->in_opr2(), op->result_opr(), op); break; +#ifndef RISCV case lir_cmove: cmove(op->condition(), op->in_opr1(), op->in_opr2(), op->result_opr(), op->type()); break; +#endif case lir_shl: case lir_shr: @@ -756,6 +758,19 @@ void LIR_Assembler::emit_op2(LIR_Op2* op) { } } +#ifdef RISCV +void LIR_Assembler::emit_op4(LIR_Op4* op) { + switch(op->code()) { + case lir_cmove: + cmove(op->condition(), op->in_opr1(), op->in_opr2(), op->result_opr(), op->type(), op->in_opr3(), op->in_opr4()); + break; + + default: + Unimplemented(); + break; + } +} +#endif void LIR_Assembler::build_frame() { _masm->build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes()); diff --git a/src/hotspot/share/c1/c1_LIRAssembler.hpp b/src/hotspot/share/c1/c1_LIRAssembler.hpp index 02c79160d0416..c82baa15fe711 100644 --- a/src/hotspot/share/c1/c1_LIRAssembler.hpp +++ b/src/hotspot/share/c1/c1_LIRAssembler.hpp @@ -186,6 +186,9 @@ class LIR_Assembler: public CompilationResourceObj { void emit_op1(LIR_Op1* op); void emit_op2(LIR_Op2* op); void emit_op3(LIR_Op3* op); +#ifdef RISCV + void emit_op4(LIR_Op4* op); +#endif void emit_opBranch(LIR_OpBranch* op); void emit_opLabel(LIR_OpLabel* op); void emit_arraycopy(LIR_OpArrayCopy* op); @@ -197,6 +200,7 @@ class LIR_Assembler: public CompilationResourceObj { void emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null); void emit_compare_and_swap(LIR_OpCompareAndSwap* op); void emit_lock(LIR_OpLock* op); + void emit_load_klass(LIR_OpLoadKlass* op); void emit_call(LIR_OpJavaCall* op); void emit_rtcall(LIR_OpRTCall* op); void emit_profile_call(LIR_OpProfileCall* op); @@ -218,8 +222,12 @@ class LIR_Assembler: public CompilationResourceObj { void volatile_move_op(LIR_Opr src, LIR_Opr result, BasicType type, CodeEmitInfo* info); void comp_mem_op(LIR_Opr src, LIR_Opr result, BasicType type, CodeEmitInfo* info); // info set for null exceptions void comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr result, LIR_Op2* op); +#ifdef RISCV + void cmove(LIR_Condition code, LIR_Opr left, LIR_Opr right, LIR_Opr result, BasicType type, + LIR_Opr cmp_opr1 = LIR_OprFact::illegalOpr, LIR_Opr cmp_opr2 = LIR_OprFact::illegalOpr); +#else void cmove(LIR_Condition code, LIR_Opr left, LIR_Opr right, LIR_Opr result, BasicType type); - +#endif void call( LIR_OpJavaCall* op, relocInfo::relocType rtype); void ic_call( LIR_OpJavaCall* op); void vtable_call( LIR_OpJavaCall* op); diff --git a/src/hotspot/share/c1/c1_LIRGenerator.cpp b/src/hotspot/share/c1/c1_LIRGenerator.cpp index e5e71fab8c17a..97ff08ef7b1ba 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.cpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.cpp @@ -1222,7 +1222,8 @@ void LIRGenerator::do_Reference_get(Intrinsic* x) { LIR_Opr result = rlock_result(x, T_OBJECT); access_load_at(IN_HEAP | ON_WEAK_OOP_REF, T_OBJECT, - reference, LIR_OprFact::intConst(referent_offset), result); + reference, LIR_OprFact::intConst(referent_offset), result, + nullptr, info); } // Example: clazz.isInstance(object) @@ -1255,13 +1256,17 @@ void LIRGenerator::do_isInstance(Intrinsic* x) { __ move(call_result, result); } +void LIRGenerator::load_klass(LIR_Opr obj, LIR_Opr klass, CodeEmitInfo* null_check_info) { + __ load_klass(obj, klass, null_check_info); +} + // Example: object.getClass () void LIRGenerator::do_getClass(Intrinsic* x) { assert(x->number_of_arguments() == 1, "wrong type"); LIRItem rcvr(x->argument_at(0), this); rcvr.load_item(); - LIR_Opr temp = new_register(T_METADATA); + LIR_Opr temp = new_register(T_ADDRESS); LIR_Opr result = rlock_result(x); // need to perform the null check on the rcvr @@ -1270,10 +1275,9 @@ void LIRGenerator::do_getClass(Intrinsic* x) { info = state_for(x); } - // FIXME T_ADDRESS should actually be T_METADATA but it can't because the - // meaning of these two is mixed up (see JDK-8026837). - __ move(new LIR_Address(rcvr.result(), oopDesc::klass_offset_in_bytes(), T_ADDRESS), temp, info); - __ move_wide(new LIR_Address(temp, in_bytes(Klass::java_mirror_offset()), T_ADDRESS), temp); + LIR_Opr klass = new_register(T_METADATA); + load_klass(rcvr.result(), klass, info); + __ move_wide(new LIR_Address(klass, in_bytes(Klass::java_mirror_offset()), T_ADDRESS), temp); // mirror = ((OopHandle)mirror)->resolve(); access_load(IN_NATIVE, T_OBJECT, LIR_OprFact::address(new LIR_Address(temp, T_OBJECT)), result); @@ -1353,7 +1357,7 @@ void LIRGenerator::do_getObjectSize(Intrinsic* x) { value.load_item(); LIR_Opr klass = new_register(T_METADATA); - __ move(new LIR_Address(value.result(), oopDesc::klass_offset_in_bytes(), T_ADDRESS), klass, NULL); + load_klass(value.result(), klass, NULL); LIR_Opr layout = new_register(T_INT); __ move(new LIR_Address(klass, in_bytes(Klass::layout_helper_offset()), T_INT), layout); @@ -3149,6 +3153,9 @@ void LIRGenerator::do_Intrinsic(Intrinsic* x) { case vmIntrinsics::_storeFence: __ membar_release(); break; + case vmIntrinsics::_storeStoreFence: + __ membar_storestore(); + break; case vmIntrinsics::_fullFence : __ membar(); break; @@ -3741,7 +3748,7 @@ LIR_Opr LIRGenerator::mask_boolean(LIR_Opr array, LIR_Opr value, CodeEmitInfo*& __ logical_and(value, LIR_OprFact::intConst(1), value_fixed); } LIR_Opr klass = new_register(T_METADATA); - __ move(new LIR_Address(array, oopDesc::klass_offset_in_bytes(), T_ADDRESS), klass, null_check_info); + load_klass(array, klass, null_check_info); null_check_info = NULL; LIR_Opr layout = new_register(T_INT); __ move(new LIR_Address(klass, in_bytes(Klass::layout_helper_offset()), T_INT), layout); diff --git a/src/hotspot/share/c1/c1_LIRGenerator.hpp b/src/hotspot/share/c1/c1_LIRGenerator.hpp index 67c986cb4e724..3fcaeb58b9d51 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.hpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.hpp @@ -239,6 +239,8 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure { void move_to_phi(PhiResolver* resolver, Value cur_val, Value sux_val); void move_to_phi(ValueStack* cur_state); + void load_klass(LIR_Opr obj, LIR_Opr klass, CodeEmitInfo* null_check_info); + // platform dependent LIR_Opr getThreadPointer(); diff --git a/src/hotspot/share/c1/c1_LinearScan.cpp b/src/hotspot/share/c1/c1_LinearScan.cpp index 84846e8e43d54..71648f62e141d 100644 --- a/src/hotspot/share/c1/c1_LinearScan.cpp +++ b/src/hotspot/share/c1/c1_LinearScan.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1240,8 +1240,13 @@ void LinearScan::add_register_hints(LIR_Op* op) { break; } case lir_cmove: { +#ifdef RISCV + assert(op->as_Op4() != NULL, "lir_cmove must be LIR_Op4"); + LIR_Op4* cmove = (LIR_Op4*)op; +#else assert(op->as_Op2() != NULL, "lir_cmove must be LIR_Op2"); LIR_Op2* cmove = (LIR_Op2*)op; +#endif LIR_Opr move_from = cmove->in_opr1(); LIR_Opr move_to = cmove->result_opr(); @@ -3138,6 +3143,9 @@ void LinearScan::do_linear_scan() { } } +#ifndef RISCV + // Disable these optimizations on riscv temporarily, because it does not + // work when the comparison operands are bound to branches or cmoves. { TIME_LINEAR_SCAN(timer_optimize_lir); EdgeMoveOptimizer::optimize(ir()->code()); @@ -3145,6 +3153,7 @@ void LinearScan::do_linear_scan() { // check that cfg is still correct after optimizations ir()->verify(); } +#endif NOT_PRODUCT(print_lir(1, "Before Code Generation", false)); NOT_PRODUCT(LinearScanStatistic::compute(this, _stat_final)); @@ -6368,14 +6377,23 @@ void ControlFlowOptimizer::delete_unnecessary_jumps(BlockList* code) { // There might be a cmove inserted for profiling which depends on the same // compare. If we change the condition of the respective compare, we have // to take care of this cmove as well. +#ifdef RISCV + LIR_Op4* prev_cmove = NULL; +#else LIR_Op2* prev_cmove = NULL; +#endif for(int j = instructions->length() - 3; j >= 0 && prev_cmp == NULL; j--) { prev_op = instructions->at(j); // check for the cmove if (prev_op->code() == lir_cmove) { +#ifdef RISCV + assert(prev_op->as_Op4() != NULL, "cmove must be of type LIR_Op4"); + prev_cmove = (LIR_Op4*)prev_op; +#else assert(prev_op->as_Op2() != NULL, "cmove must be of type LIR_Op2"); prev_cmove = (LIR_Op2*)prev_op; +#endif assert(prev_branch->cond() == prev_cmove->condition(), "should be the same"); } if (prev_op->code() == lir_cmp) { diff --git a/src/hotspot/share/c1/c1_RangeCheckElimination.cpp b/src/hotspot/share/c1/c1_RangeCheckElimination.cpp index f5275eb9d5c53..fa1678f71c48f 100644 --- a/src/hotspot/share/c1/c1_RangeCheckElimination.cpp +++ b/src/hotspot/share/c1/c1_RangeCheckElimination.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -228,24 +228,23 @@ void RangeCheckEliminator::Visitor::do_ArithmeticOp(ArithmeticOp *ao) { Bound* y_bound = _rce->get_bound(y); if (x_bound->lower() >= 0 && x_bound->lower_instr() == NULL && y->as_ArrayLength() != NULL) { _bound = new Bound(0, NULL, -1, y); - } else if (y->type()->as_IntConstant() && y->type()->as_IntConstant()->value() != 0) { + } else if (x_bound->has_lower() && x_bound->lower() >= 0 && y->type()->as_IntConstant() && + y->type()->as_IntConstant()->value() != 0 && y->type()->as_IntConstant()->value() != min_jint) { // The binary % operator is said to yield the remainder of its operands from an implied division; the // left-hand operand is the dividend and the right-hand operand is the divisor. // - // % operator follows from this rule that the result of the remainder operation can be negative only + // It follows from this rule that the result of the remainder operation can be negative only // if the dividend is negative, and can be positive only if the dividend is positive. Moreover, the - // magnitude of the result is always less than the magnitude of the divisor(See JLS 15.17.3). + // magnitude of the result is always less than the magnitude of the divisor (see JLS 15.17.3). // // So if y is a constant integer and not equal to 0, then we can deduce the bound of remainder operation: // x % -y ==> [0, y - 1] Apply RCE // x % y ==> [0, y - 1] Apply RCE // -x % y ==> [-y + 1, 0] // -x % -y ==> [-y + 1, 0] - if (x_bound->has_lower() && x_bound->lower() >= 0) { - _bound = new Bound(0, NULL, y->type()->as_IntConstant()->value() - 1, NULL); - } else { - _bound = new Bound(); - } + // + // Use the absolute value of y as an upper bound. Skip min_jint because abs(min_jint) is undefined. + _bound = new Bound(0, NULL, abs(y->type()->as_IntConstant()->value()) - 1, NULL); } else { _bound = new Bound(); } @@ -270,17 +269,16 @@ void RangeCheckEliminator::Visitor::do_ArithmeticOp(ArithmeticOp *ao) { Bound * bound = _rce->get_bound(y); if (bound->has_upper() && bound->has_lower()) { - int new_lower = bound->lower() + const_value; - jlong new_lowerl = ((jlong)bound->lower()) + const_value; - int new_upper = bound->upper() + const_value; - jlong new_upperl = ((jlong)bound->upper()) + const_value; - - if (((jlong)new_lower) == new_lowerl && ((jlong)new_upper == new_upperl)) { - Bound *newBound = new Bound(new_lower, bound->lower_instr(), new_upper, bound->upper_instr()); - _bound = newBound; - } else { - // overflow + jint t_lo = bound->lower(); + jint t_hi = bound->upper(); + jint new_lower = java_add(t_lo, const_value); + jint new_upper = java_add(t_hi, const_value); + bool overflow = ((const_value < 0 && (new_lower > t_lo)) || + (const_value > 0 && (new_upper < t_hi))); + if (overflow) { _bound = new Bound(); + } else { + _bound = new Bound(new_lower, bound->lower_instr(), new_upper, bound->upper_instr()); } } else { _bound = new Bound(); @@ -1558,7 +1556,6 @@ void RangeCheckEliminator::Bound::add_assertion(Instruction *instruction, Instru NOT_PRODUCT(ao->set_printable_bci(position->printable_bci())); result = result->insert_after(ao); compare_with = ao; - // TODO: Check that add operation does not overflow! } } assert(compare_with != NULL, "You have to compare with something!"); diff --git a/src/hotspot/share/cds/filemap.cpp b/src/hotspot/share/cds/filemap.cpp index ca9ef15acb4ac..c21625b737815 100644 --- a/src/hotspot/share/cds/filemap.cpp +++ b/src/hotspot/share/cds/filemap.cpp @@ -2347,28 +2347,28 @@ void FileMapInfo::stop_sharing_and_unmap(const char* msg) { ClassPathEntry** FileMapInfo::_classpath_entries_for_jvmti = NULL; ClassPathEntry* FileMapInfo::get_classpath_entry_for_jvmti(int i, TRAPS) { + if (i == 0) { + // index 0 corresponds to the ClassPathImageEntry which is a globally shared object + // and should never be deleted. + return ClassLoader::get_jrt_entry(); + } ClassPathEntry* ent = _classpath_entries_for_jvmti[i]; if (ent == NULL) { - if (i == 0) { - ent = ClassLoader::get_jrt_entry(); - assert(ent != NULL, "must be"); - } else { - SharedClassPathEntry* scpe = shared_path(i); - assert(scpe->is_jar(), "must be"); // other types of scpe will not produce archived classes + SharedClassPathEntry* scpe = shared_path(i); + assert(scpe->is_jar(), "must be"); // other types of scpe will not produce archived classes - const char* path = scpe->name(); - struct stat st; - if (os::stat(path, &st) != 0) { + const char* path = scpe->name(); + struct stat st; + if (os::stat(path, &st) != 0) { + char *msg = NEW_RESOURCE_ARRAY_IN_THREAD(THREAD, char, strlen(path) + 128); + jio_snprintf(msg, strlen(path) + 127, "error in finding JAR file %s", path); + THROW_MSG_(vmSymbols::java_io_IOException(), msg, NULL); + } else { + ent = ClassLoader::create_class_path_entry(THREAD, path, &st, false, false); + if (ent == NULL) { char *msg = NEW_RESOURCE_ARRAY_IN_THREAD(THREAD, char, strlen(path) + 128); - jio_snprintf(msg, strlen(path) + 127, "error in finding JAR file %s", path); + jio_snprintf(msg, strlen(path) + 127, "error in opening JAR file %s", path); THROW_MSG_(vmSymbols::java_io_IOException(), msg, NULL); - } else { - ent = ClassLoader::create_class_path_entry(THREAD, path, &st, false, false); - if (ent == NULL) { - char *msg = NEW_RESOURCE_ARRAY_IN_THREAD(THREAD, char, strlen(path) + 128); - jio_snprintf(msg, strlen(path) + 127, "error in opening JAR file %s", path); - THROW_MSG_(vmSymbols::java_io_IOException(), msg, NULL); - } } } diff --git a/src/hotspot/share/ci/ciReplay.cpp b/src/hotspot/share/ci/ciReplay.cpp index c5e95626b88c9..2476ca5284042 100644 --- a/src/hotspot/share/ci/ciReplay.cpp +++ b/src/hotspot/share/ci/ciReplay.cpp @@ -850,6 +850,7 @@ class CompileReplay : public StackObj { value = oopFactory::new_longArray(length, CHECK); } else if (field_signature[0] == JVM_SIGNATURE_ARRAY && field_signature[1] == JVM_SIGNATURE_CLASS) { + parse_klass(CHECK); // eat up the array class name Klass* kelem = resolve_klass(field_signature + 1, CHECK); value = oopFactory::new_objArray(kelem, length, CHECK); } else { diff --git a/src/hotspot/share/classfile/altHashing.cpp b/src/hotspot/share/classfile/altHashing.cpp index 98c5502fc1fdd..b62456d5ce346 100644 --- a/src/hotspot/share/classfile/altHashing.cpp +++ b/src/hotspot/share/classfile/altHashing.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,9 +31,7 @@ /* SipHash reference C implementation - Copyright (c) 2012-2021 Jean-Philippe Aumasson - - Copyright (c) 2012-2014 Daniel J. Bernstein + Copyright (c) 2016 Jean-Philippe Aumasson To the extent possible under law, the author(s) have dedicated all copyright and related and neighboring rights to this software to the public domain diff --git a/src/hotspot/share/classfile/classLoaderData.cpp b/src/hotspot/share/classfile/classLoaderData.cpp index d164c5f155587..1b5a22e19ea3e 100644 --- a/src/hotspot/share/classfile/classLoaderData.cpp +++ b/src/hotspot/share/classfile/classLoaderData.cpp @@ -821,6 +821,7 @@ void ClassLoaderData::add_to_deallocate_list(Metadata* m) { _deallocate_list = new (ResourceObj::C_HEAP, mtClass) GrowableArray(100, mtClass); } _deallocate_list->append_if_missing(m); + ResourceMark rm; log_debug(class, loader, data)("deallocate added for %s", m->print_value_string()); ClassLoaderDataGraph::set_should_clean_deallocate_lists(); } diff --git a/src/hotspot/share/classfile/classLoaderDataGraph.inline.hpp b/src/hotspot/share/classfile/classLoaderDataGraph.inline.hpp index 9e93c4ef9263c..e5b51ac277cc3 100644 --- a/src/hotspot/share/classfile/classLoaderDataGraph.inline.hpp +++ b/src/hotspot/share/classfile/classLoaderDataGraph.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -30,6 +30,7 @@ #include "classfile/javaClasses.hpp" #include "oops/oop.inline.hpp" #include "runtime/atomic.hpp" +#include "runtime/orderAccess.hpp" inline ClassLoaderData *ClassLoaderDataGraph::find_or_create(Handle loader) { guarantee(loader() != NULL && oopDesc::is_oop(loader()), "Loader must be oop"); @@ -43,29 +44,29 @@ inline ClassLoaderData *ClassLoaderDataGraph::find_or_create(Handle loader) { } size_t ClassLoaderDataGraph::num_instance_classes() { - return _num_instance_classes; + return Atomic::load(&_num_instance_classes); } size_t ClassLoaderDataGraph::num_array_classes() { - return _num_array_classes; + return Atomic::load(&_num_array_classes); } void ClassLoaderDataGraph::inc_instance_classes(size_t count) { - Atomic::add(&_num_instance_classes, count); + Atomic::add(&_num_instance_classes, count, memory_order_relaxed); } void ClassLoaderDataGraph::dec_instance_classes(size_t count) { - assert(count <= _num_instance_classes, "Sanity"); - Atomic::sub(&_num_instance_classes, count); + size_t old_count = Atomic::fetch_and_add(&_num_instance_classes, -count, memory_order_relaxed); + assert(old_count >= count, "Sanity"); } void ClassLoaderDataGraph::inc_array_classes(size_t count) { - Atomic::add(&_num_array_classes, count); + Atomic::add(&_num_array_classes, count, memory_order_relaxed); } void ClassLoaderDataGraph::dec_array_classes(size_t count) { - assert(count <= _num_array_classes, "Sanity"); - Atomic::sub(&_num_array_classes, count); + size_t old_count = Atomic::fetch_and_add(&_num_array_classes, -count, memory_order_relaxed); + assert(old_count >= count, "Sanity"); } bool ClassLoaderDataGraph::should_clean_metaspaces_and_reset() { diff --git a/src/hotspot/share/classfile/javaClasses.cpp b/src/hotspot/share/classfile/javaClasses.cpp index 793beb479ac88..b10131af8772b 100644 --- a/src/hotspot/share/classfile/javaClasses.cpp +++ b/src/hotspot/share/classfile/javaClasses.cpp @@ -1793,11 +1793,18 @@ JavaThread* java_lang_Thread::thread(oop java_thread) { return (JavaThread*)java_thread->address_field(_eetop_offset); } +JavaThread* java_lang_Thread::thread_acquire(oop java_thread) { + return reinterpret_cast(java_thread->address_field_acquire(_eetop_offset)); +} void java_lang_Thread::set_thread(oop java_thread, JavaThread* thread) { java_thread->address_field_put(_eetop_offset, (address)thread); } +void java_lang_Thread::release_set_thread(oop java_thread, JavaThread* thread) { + java_thread->release_address_field_put(_eetop_offset, (address)thread); +} + bool java_lang_Thread::interrupted(oop java_thread) { // Make sure the caller can safely access oops. assert(Thread::current()->is_VM_thread() || @@ -2719,49 +2726,57 @@ void java_lang_Throwable::get_stack_trace_elements(Handle throwable, } } -Handle java_lang_Throwable::get_cause_with_stack_trace(Handle throwable, TRAPS) { - // Call to JVM to fill in the stack trace and clear declaringClassObject to - // not keep classes alive in the stack trace. - // call this: public StackTraceElement[] getStackTrace() +Handle java_lang_Throwable::create_initialization_error(JavaThread* current, Handle throwable) { + // Creates an ExceptionInInitializerError to be recorded as the initialization error when class initialization + // failed due to the passed in 'throwable'. We cannot save 'throwable' directly due to issues with keeping alive + // all objects referenced via its stacktrace. So instead we save a new EIIE instance, with the same message and + // symbolic stacktrace of 'throwable'. assert(throwable.not_null(), "shouldn't be"); - JavaValue result(T_ARRAY); - JavaCalls::call_virtual(&result, throwable, - vmClasses::Throwable_klass(), - vmSymbols::getStackTrace_name(), - vmSymbols::getStackTrace_signature(), - CHECK_NH); - Handle stack_trace(THREAD, result.get_oop()); - assert(stack_trace->is_objArray(), "Should be an array"); - - // Throw ExceptionInInitializerError as the cause with this exception in - // the message and stack trace. - - // Now create the message with the original exception and thread name. + // Now create the message from the original exception and thread name. Symbol* message = java_lang_Throwable::detail_message(throwable()); - ResourceMark rm(THREAD); + ResourceMark rm(current); stringStream st; st.print("Exception %s%s ", throwable()->klass()->name()->as_klass_external_name(), message == nullptr ? "" : ":"); if (message == NULL) { - st.print("[in thread \"%s\"]", THREAD->name()); + st.print("[in thread \"%s\"]", current->name()); } else { - st.print("%s [in thread \"%s\"]", message->as_C_string(), THREAD->name()); + st.print("%s [in thread \"%s\"]", message->as_C_string(), current->name()); } Symbol* exception_name = vmSymbols::java_lang_ExceptionInInitializerError(); - Handle h_cause = Exceptions::new_exception(THREAD, exception_name, st.as_string()); - - // If new_exception returns a different exception while creating the exception, return null. - if (h_cause->klass()->name() != exception_name) { + Handle init_error = Exceptions::new_exception(current, exception_name, st.as_string()); + // If new_exception returns a different exception while creating the exception, + // abandon the attempt to save the initialization error and return null. + if (init_error->klass()->name() != exception_name) { log_info(class, init)("Exception thrown while saving initialization exception %s", - h_cause->klass()->external_name()); + init_error->klass()->external_name()); return Handle(); } - java_lang_Throwable::set_stacktrace(h_cause(), stack_trace()); - // Clear backtrace because the stacktrace should be used instead. - set_backtrace(h_cause(), NULL); - return h_cause; + + // Call to java to fill in the stack trace and clear declaringClassObject to + // not keep classes alive in the stack trace. + // call this: public StackTraceElement[] getStackTrace() + JavaValue result(T_ARRAY); + JavaCalls::call_virtual(&result, throwable, + vmClasses::Throwable_klass(), + vmSymbols::getStackTrace_name(), + vmSymbols::getStackTrace_signature(), + current); + if (!current->has_pending_exception()){ + Handle stack_trace(current, result.get_oop()); + assert(stack_trace->is_objArray(), "Should be an array"); + java_lang_Throwable::set_stacktrace(init_error(), stack_trace()); + // Clear backtrace because the stacktrace should be used instead. + set_backtrace(init_error(), nullptr); + } else { + log_info(class, init)("Exception thrown while getting stack trace for initialization exception %s", + init_error->klass()->external_name()); + current->clear_pending_exception(); + } + + return init_error; } bool java_lang_Throwable::get_top_method_and_bci(oop throwable, Method** method, int* bci) { diff --git a/src/hotspot/share/classfile/javaClasses.hpp b/src/hotspot/share/classfile/javaClasses.hpp index 583e5473c9a1f..4a3d37e50dcdb 100644 --- a/src/hotspot/share/classfile/javaClasses.hpp +++ b/src/hotspot/share/classfile/javaClasses.hpp @@ -420,8 +420,10 @@ class java_lang_Thread : AllStatic { // Returns the JavaThread associated with the thread obj static JavaThread* thread(oop java_thread); + static JavaThread* thread_acquire(oop java_thread); // Set JavaThread for instance static void set_thread(oop java_thread, JavaThread* thread); + static void release_set_thread(oop java_thread, JavaThread* thread); // Interrupted status static bool interrupted(oop java_thread); static void set_interrupted(oop java_thread, bool val); @@ -569,7 +571,7 @@ class java_lang_Throwable: AllStatic { static void get_stack_trace_elements(Handle throwable, objArrayHandle stack_trace, TRAPS); // For recreating class initialization error exceptions. - static Handle get_cause_with_stack_trace(Handle throwable, TRAPS); + static Handle create_initialization_error(JavaThread* current, Handle throwable); // Printing static void print(oop throwable, outputStream* st); @@ -925,6 +927,8 @@ class java_lang_ref_Reference: AllStatic { static bool is_referent_field(oop obj, ptrdiff_t offset); static inline bool is_final(oop ref); static inline bool is_phantom(oop ref); + static inline bool is_weak(oop ref); + static inline bool is_soft(oop ref); static int referent_offset() { CHECK_INIT(_referent_offset); } static int queue_offset() { CHECK_INIT(_queue_offset); } diff --git a/src/hotspot/share/classfile/javaClasses.inline.hpp b/src/hotspot/share/classfile/javaClasses.inline.hpp index 44c0105456615..67c7d121d4316 100644 --- a/src/hotspot/share/classfile/javaClasses.inline.hpp +++ b/src/hotspot/share/classfile/javaClasses.inline.hpp @@ -27,6 +27,7 @@ #include "classfile/javaClasses.hpp" +#include "memory/referenceType.hpp" #include "oops/access.inline.hpp" #include "oops/method.hpp" #include "oops/oop.inline.hpp" @@ -130,10 +131,12 @@ bool java_lang_String::is_instance_inlined(oop obj) { // Accessors oop java_lang_ref_Reference::weak_referent_no_keepalive(oop ref) { + assert(java_lang_ref_Reference::is_weak(ref) || java_lang_ref_Reference::is_soft(ref), "must be Weak or Soft Reference"); return ref->obj_field_access(_referent_offset); } oop java_lang_ref_Reference::phantom_referent_no_keepalive(oop ref) { + assert(java_lang_ref_Reference::is_phantom(ref), "must be Phantom Reference"); return ref->obj_field_access(_referent_offset); } @@ -189,6 +192,14 @@ bool java_lang_ref_Reference::is_phantom(oop ref) { return InstanceKlass::cast(ref->klass())->reference_type() == REF_PHANTOM; } +bool java_lang_ref_Reference::is_weak(oop ref) { + return InstanceKlass::cast(ref->klass())->reference_type() == REF_WEAK; +} + +bool java_lang_ref_Reference::is_soft(oop ref) { + return InstanceKlass::cast(ref->klass())->reference_type() == REF_SOFT; +} + inline void java_lang_invoke_CallSite::set_target_volatile(oop site, oop target) { site->obj_field_put_volatile(_target_offset, target); } diff --git a/src/hotspot/share/classfile/stringTable.cpp b/src/hotspot/share/classfile/stringTable.cpp index ab74df425a708..d2348f16adf2e 100644 --- a/src/hotspot/share/classfile/stringTable.cpp +++ b/src/hotspot/share/classfile/stringTable.cpp @@ -91,6 +91,8 @@ static size_t _current_size = 0; static volatile size_t _items_count = 0; volatile bool _alt_hash = false; + +static bool _rehashed = false; static uint64_t _alt_hash_seed = 0; uintx hash_string(const jchar* s, int len, bool useAlt) { @@ -504,20 +506,46 @@ bool StringTable::do_rehash() { return true; } +bool StringTable::should_grow() { + return get_load_factor() > PREF_AVG_LIST_LEN && !_local_table->is_max_size_reached(); +} + +bool StringTable::rehash_table_expects_safepoint_rehashing() { + // No rehashing required + if (!needs_rehashing()) { + return false; + } + + // Grow instead of rehash + if (should_grow()) { + return false; + } + + // Already rehashed + if (_rehashed) { + return false; + } + + // Resizing in progress + if (!_local_table->is_safepoint_safe()) { + return false; + } + + return true; +} + void StringTable::rehash_table() { - static bool rehashed = false; log_debug(stringtable)("Table imbalanced, rehashing called."); // Grow instead of rehash. - if (get_load_factor() > PREF_AVG_LIST_LEN && - !_local_table->is_max_size_reached()) { + if (should_grow()) { log_debug(stringtable)("Choosing growing over rehashing."); trigger_concurrent_work(); _needs_rehashing = false; return; } // Already rehashed. - if (rehashed) { + if (_rehashed) { log_warning(stringtable)("Rehashing already done, still long lists."); trigger_concurrent_work(); _needs_rehashing = false; @@ -527,7 +555,7 @@ void StringTable::rehash_table() { _alt_hash_seed = AltHashing::compute_seed(); { if (do_rehash()) { - rehashed = true; + _rehashed = true; } else { log_info(stringtable)("Resizes in progress rehashing skipped."); } diff --git a/src/hotspot/share/classfile/stringTable.hpp b/src/hotspot/share/classfile/stringTable.hpp index b849db67101d7..6c6ded24c31f7 100644 --- a/src/hotspot/share/classfile/stringTable.hpp +++ b/src/hotspot/share/classfile/stringTable.hpp @@ -95,6 +95,11 @@ class StringTable : public CHeapObj{ static oop intern(const char *utf8_string, TRAPS); // Rehash the string table if it gets out of balance +private: + static bool should_grow(); + +public: + static bool rehash_table_expects_safepoint_rehashing(); static void rehash_table(); static bool needs_rehashing() { return _needs_rehashing; } static inline void update_needs_rehash(bool rehash) { diff --git a/src/hotspot/share/classfile/symbolTable.cpp b/src/hotspot/share/classfile/symbolTable.cpp index a321d94bbd2b0..46b4a46babc82 100644 --- a/src/hotspot/share/classfile/symbolTable.cpp +++ b/src/hotspot/share/classfile/symbolTable.cpp @@ -103,6 +103,7 @@ static THREAD_LOCAL bool _lookup_shared_first = false; // Static arena for symbols that are not deallocated Arena* SymbolTable::_arena = NULL; +static bool _rehashed = false; static uint64_t _alt_hash_seed = 0; static inline void log_trace_symboltable_helper(Symbol* sym, const char* msg) { @@ -765,13 +766,39 @@ bool SymbolTable::do_rehash() { return true; } +bool SymbolTable::should_grow() { + return get_load_factor() > PREF_AVG_LIST_LEN && !_local_table->is_max_size_reached(); +} + +bool SymbolTable::rehash_table_expects_safepoint_rehashing() { + // No rehashing required + if (!needs_rehashing()) { + return false; + } + + // Grow instead of rehash + if (should_grow()) { + return false; + } + + // Already rehashed + if (_rehashed) { + return false; + } + + // Resizing in progress + if (!_local_table->is_safepoint_safe()) { + return false; + } + + return true; +} + void SymbolTable::rehash_table() { - static bool rehashed = false; log_debug(symboltable)("Table imbalanced, rehashing called."); // Grow instead of rehash. - if (get_load_factor() > PREF_AVG_LIST_LEN && - !_local_table->is_max_size_reached()) { + if (should_grow()) { log_debug(symboltable)("Choosing growing over rehashing."); trigger_cleanup(); _needs_rehashing = false; @@ -779,7 +806,7 @@ void SymbolTable::rehash_table() { } // Already rehashed. - if (rehashed) { + if (_rehashed) { log_warning(symboltable)("Rehashing already done, still long lists."); trigger_cleanup(); _needs_rehashing = false; @@ -789,7 +816,7 @@ void SymbolTable::rehash_table() { _alt_hash_seed = AltHashing::compute_seed(); if (do_rehash()) { - rehashed = true; + _rehashed = true; } else { log_info(symboltable)("Resizes in progress rehashing skipped."); } diff --git a/src/hotspot/share/classfile/symbolTable.hpp b/src/hotspot/share/classfile/symbolTable.hpp index 1ea96733ac5ed..a0b7287f936ab 100644 --- a/src/hotspot/share/classfile/symbolTable.hpp +++ b/src/hotspot/share/classfile/symbolTable.hpp @@ -195,6 +195,11 @@ class SymbolTable : public AllStatic { static Symbol* new_permanent_symbol(const char* name); // Rehash the string table if it gets out of balance +private: + static bool should_grow(); + +public: + static bool rehash_table_expects_safepoint_rehashing(); static void rehash_table(); static bool needs_rehashing() { return _needs_rehashing; } static inline void update_needs_rehash(bool rehash) { diff --git a/src/hotspot/share/classfile/verifier.cpp b/src/hotspot/share/classfile/verifier.cpp index 4c792a86a1ce1..ced7347204065 100644 --- a/src/hotspot/share/classfile/verifier.cpp +++ b/src/hotspot/share/classfile/verifier.cpp @@ -594,7 +594,7 @@ void ErrorContext::stackmap_details(outputStream* ss, const Method* method) cons ClassVerifier::ClassVerifier(JavaThread* current, InstanceKlass* klass) : _thread(current), _previous_symbol(NULL), _symbols(NULL), _exception_type(NULL), - _message(NULL), _method_signatures_table(NULL), _klass(klass) { + _message(NULL), _klass(klass) { _this_type = VerificationType::reference_type(klass->name()); } @@ -625,16 +625,12 @@ void ClassVerifier::verify_class(TRAPS) { // Either verifying both local and remote classes or just remote classes. assert(BytecodeVerificationRemote, "Should not be here"); - // Create hash table containing method signatures. - method_signatures_table_type method_signatures_table; - set_method_signatures_table(&method_signatures_table); - Array* methods = _klass->methods(); int num_methods = methods->length(); for (int index = 0; index < num_methods; index++) { // Check for recursive re-verification before each method. - if (was_recursively_verified()) return; + if (was_recursively_verified()) return; Method* m = methods->at(index); if (m->is_native() || m->is_abstract() || m->is_overpass()) { diff --git a/src/hotspot/share/classfile/verifier.hpp b/src/hotspot/share/classfile/verifier.hpp index a43a14961cf88..999391ae696ff 100644 --- a/src/hotspot/share/classfile/verifier.hpp +++ b/src/hotspot/share/classfile/verifier.hpp @@ -286,7 +286,7 @@ class ClassVerifier : public StackObj { Symbol* _exception_type; char* _message; - method_signatures_table_type* _method_signatures_table; + method_signatures_table_type _method_signatures_table; ErrorContext _error_context; // contains information about an error @@ -438,12 +438,8 @@ class ClassVerifier : public StackObj { Klass* load_class(Symbol* name, TRAPS); - method_signatures_table_type* method_signatures_table() const { - return _method_signatures_table; - } - - void set_method_signatures_table(method_signatures_table_type* method_signatures_table) { - _method_signatures_table = method_signatures_table; + method_signatures_table_type* method_signatures_table() { + return &_method_signatures_table; } int change_sig_to_verificationType( diff --git a/src/hotspot/share/classfile/vmIntrinsics.cpp b/src/hotspot/share/classfile/vmIntrinsics.cpp index 0f9a440b3e632..8ebcf38cb0621 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.cpp +++ b/src/hotspot/share/classfile/vmIntrinsics.cpp @@ -397,6 +397,7 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) { case vmIntrinsics::_compareAndExchangeReference: case vmIntrinsics::_compareAndExchangeReferenceAcquire: case vmIntrinsics::_compareAndExchangeReferenceRelease: + case vmIntrinsics::_allocateInstance: if (!InlineUnsafeOps) return true; break; case vmIntrinsics::_getShortUnaligned: @@ -407,7 +408,6 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) { case vmIntrinsics::_putCharUnaligned: case vmIntrinsics::_putIntUnaligned: case vmIntrinsics::_putLongUnaligned: - case vmIntrinsics::_allocateInstance: if (!InlineUnsafeOps || !UseUnalignedAccesses) return true; break; case vmIntrinsics::_hashCode: diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp index 3422d21128494..50c1968ba2e7d 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.hpp +++ b/src/hotspot/share/classfile/vmIntrinsics.hpp @@ -520,6 +520,9 @@ class methodHandle; do_intrinsic(_storeFence, jdk_internal_misc_Unsafe, storeFence_name, storeFence_signature, F_RN) \ do_name( storeFence_name, "storeFence") \ do_alias( storeFence_signature, void_method_signature) \ + do_intrinsic(_storeStoreFence, jdk_internal_misc_Unsafe, storeStoreFence_name, storeStoreFence_signature, F_R) \ + do_name( storeStoreFence_name, "storeStoreFence") \ + do_alias( storeStoreFence_signature, void_method_signature) \ do_intrinsic(_fullFence, jdk_internal_misc_Unsafe, fullFence_name, fullFence_signature, F_RN) \ do_name( fullFence_name, "fullFence") \ do_alias( fullFence_signature, void_method_signature) \ diff --git a/src/hotspot/share/code/codeBlob.cpp b/src/hotspot/share/code/codeBlob.cpp index 94e26584952f6..ef0ec837a7354 100644 --- a/src/hotspot/share/code/codeBlob.cpp +++ b/src/hotspot/share/code/codeBlob.cpp @@ -653,7 +653,12 @@ void CodeBlob::dump_for_addr(address addr, outputStream* st, bool verbose) const nm->method()->print_value_on(st); } st->cr(); - nm->print_nmethod(verbose); + if (verbose && st == tty) { + // verbose is only ever true when called from findpc in debug.cpp + nm->print_nmethod(true); + } else { + nm->print(st); + } return; } st->print_cr(INTPTR_FORMAT " is at code_begin+%d in ", p2i(addr), (int)(addr - code_begin())); diff --git a/src/hotspot/share/code/codeCache.cpp b/src/hotspot/share/code/codeCache.cpp index 28158adbc5f42..dae813a27b509 100644 --- a/src/hotspot/share/code/codeCache.cpp +++ b/src/hotspot/share/code/codeCache.cpp @@ -609,9 +609,6 @@ void CodeCache::commit(CodeBlob* cb) { if (cb->is_adapter_blob()) { heap->set_adapter_count(heap->adapter_count() + 1); } - - // flush the hardware I-cache - ICache::invalidate_range(cb->content_begin(), cb->content_size()); } bool CodeCache::contains(void *p) { diff --git a/src/hotspot/share/code/codeHeapState.cpp b/src/hotspot/share/code/codeHeapState.cpp index 32bd6ed98b8aa..3fc1e0ddc1083 100644 --- a/src/hotspot/share/code/codeHeapState.cpp +++ b/src/hotspot/share/code/codeHeapState.cpp @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "code/codeHeapState.hpp" +#include "code/codeBlob.hpp" #include "compiler/compileBroker.hpp" #include "runtime/safepoint.hpp" #include "runtime/sweeper.hpp" @@ -1152,10 +1153,9 @@ void CodeHeapState::aggregate(outputStream* out, CodeHeap* heap, size_t granular ast->cr(); int reset_val = NMethodSweeper::hotness_counter_reset_val(); - double reverse_free_ratio = (res_size > size) ? (double)res_size/(double)(res_size-size) : (double)res_size; printBox(ast, '-', "Method hotness information at time of this analysis", NULL); ast->print_cr("Highest possible method temperature: %12d", reset_val); - ast->print_cr("Threshold for method to be considered 'cold': %12.3f", -reset_val + reverse_free_ratio * NmethodSweepActivity); + ast->print_cr("Threshold for method to be considered 'cold': %12.3f", -reset_val + (CodeCache::reverse_free_ratio() * NmethodSweepActivity)); if (n_methods > 0) { avgTemp = hotnessAccumulator/n_methods; ast->print_cr("min. hotness = %6d", minTemp); diff --git a/src/hotspot/share/code/compiledIC.hpp b/src/hotspot/share/code/compiledIC.hpp index 38ec5c5aa6365..ca4190abc37b0 100644 --- a/src/hotspot/share/code/compiledIC.hpp +++ b/src/hotspot/share/code/compiledIC.hpp @@ -338,6 +338,8 @@ class StaticCallInfo { class CompiledStaticCall : public ResourceObj { public: // Code + + // Returns NULL if CodeBuffer::expand fails static address emit_to_interp_stub(CodeBuffer &cbuf, address mark = NULL); static int to_interp_stub_size(); static int to_trampoline_stub_size(); diff --git a/src/hotspot/share/code/icBuffer.cpp b/src/hotspot/share/code/icBuffer.cpp index 1da48f0bd13bd..c30f4a3c3a86e 100644 --- a/src/hotspot/share/code/icBuffer.cpp +++ b/src/hotspot/share/code/icBuffer.cpp @@ -161,6 +161,20 @@ void InlineCacheBuffer::refill_ic_stubs() { VMThread::execute(&ibf); } +bool InlineCacheBuffer::needs_update_inline_caches() { + // Stub removal + if (buffer()->number_of_stubs() > 0) { + return true; + } + + // Release pending CompiledICHolder + if (pending_icholder_count() > 0) { + return true; + } + + return false; +} + void InlineCacheBuffer::update_inline_caches() { if (buffer()->number_of_stubs() > 0) { if (TraceICBuffer) { diff --git a/src/hotspot/share/code/icBuffer.hpp b/src/hotspot/share/code/icBuffer.hpp index a15420e8ddc7d..eb45e043bf19e 100644 --- a/src/hotspot/share/code/icBuffer.hpp +++ b/src/hotspot/share/code/icBuffer.hpp @@ -165,6 +165,7 @@ class InlineCacheBuffer: public AllStatic { static bool contains(address instruction_address); // removes the ICStubs after backpatching + static bool needs_update_inline_caches(); static void update_inline_caches(); static void refill_ic_stubs(); diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp index 7fe1d05ff1c69..5105117079482 100644 --- a/src/hotspot/share/code/nmethod.cpp +++ b/src/hotspot/share/code/nmethod.cpp @@ -67,6 +67,7 @@ #include "runtime/sharedRuntime.hpp" #include "runtime/signature.hpp" #include "runtime/sweeper.hpp" +#include "runtime/threadWXSetters.inline.hpp" #include "runtime/vmThread.hpp" #include "utilities/align.hpp" #include "utilities/copy.hpp" @@ -1657,6 +1658,8 @@ void nmethod::post_compiled_method_load_event(JvmtiThreadState* state) { if (is_not_entrant() && can_convert_to_zombie()) { return; } + // Ensure the sweeper can't collect this nmethod until it become "active" with JvmtiThreadState::nmethods_do. + mark_as_seen_on_stack(); } // This is a bad time for a safepoint. We don't want @@ -1829,7 +1832,8 @@ bool nmethod::is_unloading() { // oops in the CompiledMethod, by calling oops_do on it. state_unloading_cycle = current_cycle; - if (is_zombie() || method()->can_be_allocated_in_NonNMethod_space()) { + Method* m = method(); + if (is_zombie() || (m != nullptr && m->can_be_allocated_in_NonNMethod_space())) { // When the nmethod is in NonNMethod space, we may reach here without IsUnloadingBehaviour. // However, we only allow this for special methods which never get unloaded. @@ -2261,7 +2265,11 @@ PcDesc* PcDescContainer::find_pc_desc_internal(address pc, bool approximate, con if (match_desc(upper, pc_offset, approximate)) { assert(upper == linear_search(search, pc_offset, approximate), "search ok"); - _pc_desc_cache.add_pc_desc(upper); + if (!Thread::current_in_asgct()) { + // we don't want to modify the cache if we're in ASGCT + // which is typically called in a signal handler + _pc_desc_cache.add_pc_desc(upper); + } return upper; } else { assert(NULL == linear_search(search, pc_offset, approximate), "search ok"); @@ -2570,7 +2578,7 @@ void nmethod::print(outputStream* st) const { st->print("(n/a) "); } - print_on(tty, NULL); + print_on(st, NULL); if (WizardMode) { st->print("((nmethod*) " INTPTR_FORMAT ") ", p2i(this)); @@ -2921,6 +2929,9 @@ void nmethod::decode2(outputStream* ost) const { AbstractDisassembler::show_block_comment()); #endif + // Decoding an nmethod can write to a PcDescCache (see PcDescCache::add_pc_desc) + MACOS_AARCH64_ONLY(ThreadWXEnable wx(WXWrite, Thread::current());) + st->cr(); this->print(st); st->cr(); @@ -2930,7 +2941,10 @@ void nmethod::decode2(outputStream* ost) const { //---< Print real disassembly >--- //---------------------------------- if (! use_compressed_format) { + st->print_cr("[Disassembly]"); Disassembler::decode(const_cast(this), st); + st->bol(); + st->print_cr("[/Disassembly]"); return; } #endif diff --git a/src/hotspot/share/compiler/compilationPolicy.hpp b/src/hotspot/share/compiler/compilationPolicy.hpp index 6ec0d70817c82..75571674fe1c4 100644 --- a/src/hotspot/share/compiler/compilationPolicy.hpp +++ b/src/hotspot/share/compiler/compilationPolicy.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2010, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -251,7 +251,6 @@ class CompilationPolicy : AllStatic { static bool can_be_osr_compiled(const methodHandle& m, int comp_level = CompLevel_any); static bool is_compilation_enabled(); - static void do_safepoint_work() { } static CompileTask* select_task_helper(CompileQueue* compile_queue); // Return initial compile level to use with Xcomp (depends on compilation mode). static void reprofile(ScopeDesc* trap_scope, bool is_osr); diff --git a/src/hotspot/share/compiler/compileBroker.cpp b/src/hotspot/share/compiler/compileBroker.cpp index c8c42ecdff4c4..0dd4831220480 100644 --- a/src/hotspot/share/compiler/compileBroker.cpp +++ b/src/hotspot/share/compiler/compileBroker.cpp @@ -878,95 +878,95 @@ void DeoptimizeObjectsALotThread::deoptimize_objects_alot_loop_all() { JavaThread* CompileBroker::make_thread(ThreadType type, jobject thread_handle, CompileQueue* queue, AbstractCompiler* comp, JavaThread* THREAD) { JavaThread* new_thread = NULL; - { - MutexLocker mu(THREAD, Threads_lock); - switch (type) { - case compiler_t: - assert(comp != NULL, "Compiler instance missing."); - if (!InjectCompilerCreationFailure || comp->num_compiler_threads() == 0) { - CompilerCounters* counters = new CompilerCounters(); - new_thread = new CompilerThread(queue, counters); - } - break; - case sweeper_t: - new_thread = new CodeCacheSweeperThread(); - break; + + switch (type) { + case compiler_t: + assert(comp != NULL, "Compiler instance missing."); + if (!InjectCompilerCreationFailure || comp->num_compiler_threads() == 0) { + CompilerCounters* counters = new CompilerCounters(); + new_thread = new CompilerThread(queue, counters); + } + break; + case sweeper_t: + new_thread = new CodeCacheSweeperThread(); + break; #if defined(ASSERT) && COMPILER2_OR_JVMCI - case deoptimizer_t: - new_thread = new DeoptimizeObjectsALotThread(); - break; + case deoptimizer_t: + new_thread = new DeoptimizeObjectsALotThread(); + break; #endif // ASSERT - default: - ShouldNotReachHere(); - } - - // At this point the new CompilerThread data-races with this startup - // thread (which I believe is the primoridal thread and NOT the VM - // thread). This means Java bytecodes being executed at startup can - // queue compile jobs which will run at whatever default priority the - // newly created CompilerThread runs at. - - - // At this point it may be possible that no osthread was created for the - // JavaThread due to lack of memory. We would have to throw an exception - // in that case. However, since this must work and we do not allow - // exceptions anyway, check and abort if this fails. But first release the - // lock. - - if (new_thread != NULL && new_thread->osthread() != NULL) { - - java_lang_Thread::set_thread(JNIHandles::resolve_non_null(thread_handle), new_thread); + default: + ShouldNotReachHere(); + } - // Note that this only sets the JavaThread _priority field, which by - // definition is limited to Java priorities and not OS priorities. - // The os-priority is set in the CompilerThread startup code itself + // At this point the new CompilerThread data-races with this startup + // thread (which is the main thread and NOT the VM thread). + // This means Java bytecodes being executed at startup can + // queue compile jobs which will run at whatever default priority the + // newly created CompilerThread runs at. - java_lang_Thread::set_priority(JNIHandles::resolve_non_null(thread_handle), NearMaxPriority); - // Note that we cannot call os::set_priority because it expects Java - // priorities and we are *explicitly* using OS priorities so that it's - // possible to set the compiler thread priority higher than any Java - // thread. + // At this point it may be possible that no osthread was created for the + // JavaThread due to lack of resources. We will handle that failure below. + // Also check new_thread so that static analysis is happy. + if (new_thread != NULL && new_thread->osthread() != NULL) { + Handle thread_oop(THREAD, JNIHandles::resolve_non_null(thread_handle)); - int native_prio = CompilerThreadPriority; - if (native_prio == -1) { - if (UseCriticalCompilerThreadPriority) { - native_prio = os::java_to_os_priority[CriticalPriority]; - } else { - native_prio = os::java_to_os_priority[NearMaxPriority]; - } - } - os::set_native_priority(new_thread, native_prio); + if (type == compiler_t) { + CompilerThread::cast(new_thread)->set_compiler(comp); + } - java_lang_Thread::set_daemon(JNIHandles::resolve_non_null(thread_handle)); + // Note that we cannot call os::set_priority because it expects Java + // priorities and we are *explicitly* using OS priorities so that it's + // possible to set the compiler thread priority higher than any Java + // thread. - new_thread->set_threadObj(JNIHandles::resolve_non_null(thread_handle)); - if (type == compiler_t) { - CompilerThread::cast(new_thread)->set_compiler(comp); + int native_prio = CompilerThreadPriority; + if (native_prio == -1) { + if (UseCriticalCompilerThreadPriority) { + native_prio = os::java_to_os_priority[CriticalPriority]; + } else { + native_prio = os::java_to_os_priority[NearMaxPriority]; } - Threads::add(new_thread); - Thread::start(new_thread); } - } + os::set_native_priority(new_thread, native_prio); - // First release lock before aborting VM. - if (new_thread == NULL || new_thread->osthread() == NULL) { - if (UseDynamicNumberOfCompilerThreads && type == compiler_t && comp->num_compiler_threads() > 0) { - if (new_thread != NULL) { - new_thread->smr_delete(); - } + // Note that this only sets the JavaThread _priority field, which by + // definition is limited to Java priorities and not OS priorities. + JavaThread::start_internal_daemon(THREAD, new_thread, thread_oop, NearMaxPriority); + + } else { // osthread initialization failure + if (UseDynamicNumberOfCompilerThreads && type == compiler_t + && comp->num_compiler_threads() > 0) { + // The new thread is not known to Thread-SMR yet so we can just delete. + delete new_thread; return NULL; + } else { + vm_exit_during_initialization("java.lang.OutOfMemoryError", + os::native_thread_creation_failed_msg()); } - vm_exit_during_initialization("java.lang.OutOfMemoryError", - os::native_thread_creation_failed_msg()); } - // Let go of Threads_lock before yielding os::naked_yield(); // make sure that the compiler thread is started early (especially helpful on SOLARIS) return new_thread; } +static bool trace_compiler_threads() { + LogTarget(Debug, jit, thread) lt; + return TraceCompilerThreads || lt.is_enabled(); +} + +static void print_compiler_threads(stringStream& msg) { + if (TraceCompilerThreads) { + tty->print_cr("%7d %s", (int)tty->time_stamp().milliseconds(), msg.as_string()); + } + LogTarget(Debug, jit, thread) lt; + if (lt.is_enabled()) { + LogStream ls(lt); + ls.print_cr("%s", msg.as_string()); + } +} void CompileBroker::init_compiler_sweeper_threads() { NMethodSweeper::set_sweep_threshold_bytes(static_cast(SweeperThreshold * ReservedCodeCacheSize / 100.0)); @@ -1009,11 +1009,13 @@ void CompileBroker::init_compiler_sweeper_threads() { JavaThread *ct = make_thread(compiler_t, thread_handle, _c2_compile_queue, _compilers[1], THREAD); assert(ct != NULL, "should have been handled for initial thread"); _compilers[1]->set_num_compiler_threads(i + 1); - if (TraceCompilerThreads) { + if (trace_compiler_threads()) { ResourceMark rm; ThreadsListHandle tlh; // get_thread_name() depends on the TLH. assert(tlh.includes(ct), "ct=" INTPTR_FORMAT " exited unexpectedly.", p2i(ct)); - tty->print_cr("Added initial compiler thread %s", ct->get_thread_name()); + stringStream msg; + msg.print("Added initial compiler thread %s", ct->get_thread_name()); + print_compiler_threads(msg); } } } @@ -1030,11 +1032,13 @@ void CompileBroker::init_compiler_sweeper_threads() { JavaThread *ct = make_thread(compiler_t, thread_handle, _c1_compile_queue, _compilers[0], THREAD); assert(ct != NULL, "should have been handled for initial thread"); _compilers[0]->set_num_compiler_threads(i + 1); - if (TraceCompilerThreads) { + if (trace_compiler_threads()) { ResourceMark rm; ThreadsListHandle tlh; // get_thread_name() depends on the TLH. assert(tlh.includes(ct), "ct=" INTPTR_FORMAT " exited unexpectedly.", p2i(ct)); - tty->print_cr("Added initial compiler thread %s", ct->get_thread_name()); + stringStream msg; + msg.print("Added initial compiler thread %s", ct->get_thread_name()); + print_compiler_threads(msg); } } } @@ -1099,10 +1103,12 @@ void CompileBroker::possibly_add_compiler_threads(JavaThread* THREAD) { thread_oop = create_thread_oop(name_buffer, THREAD); } if (HAS_PENDING_EXCEPTION) { - if (TraceCompilerThreads) { + if (trace_compiler_threads()) { ResourceMark rm; - tty->print_cr("JVMCI compiler thread creation failed:"); - PENDING_EXCEPTION->print(); + stringStream msg; + msg.print_cr("JVMCI compiler thread creation failed:"); + PENDING_EXCEPTION->print_on(&msg); + print_compiler_threads(msg); } CLEAR_PENDING_EXCEPTION; break; @@ -1117,12 +1123,14 @@ void CompileBroker::possibly_add_compiler_threads(JavaThread* THREAD) { JavaThread *ct = make_thread(compiler_t, compiler2_object(i), _c2_compile_queue, _compilers[1], THREAD); if (ct == NULL) break; _compilers[1]->set_num_compiler_threads(i + 1); - if (TraceCompilerThreads) { + if (trace_compiler_threads()) { ResourceMark rm; ThreadsListHandle tlh; // get_thread_name() depends on the TLH. assert(tlh.includes(ct), "ct=" INTPTR_FORMAT " exited unexpectedly.", p2i(ct)); - tty->print_cr("Added compiler thread %s (available memory: %dMB, available non-profiled code cache: %dMB)", - ct->get_thread_name(), (int)(available_memory/M), (int)(available_cc_np/M)); + stringStream msg; + msg.print("Added compiler thread %s (available memory: %dMB, available non-profiled code cache: %dMB)", + ct->get_thread_name(), (int)(available_memory/M), (int)(available_cc_np/M)); + print_compiler_threads(msg); } } } @@ -1138,12 +1146,14 @@ void CompileBroker::possibly_add_compiler_threads(JavaThread* THREAD) { JavaThread *ct = make_thread(compiler_t, compiler1_object(i), _c1_compile_queue, _compilers[0], THREAD); if (ct == NULL) break; _compilers[0]->set_num_compiler_threads(i + 1); - if (TraceCompilerThreads) { + if (trace_compiler_threads()) { ResourceMark rm; ThreadsListHandle tlh; // get_thread_name() depends on the TLH. assert(tlh.includes(ct), "ct=" INTPTR_FORMAT " exited unexpectedly.", p2i(ct)); - tty->print_cr("Added compiler thread %s (available memory: %dMB, available profiled code cache: %dMB)", - ct->get_thread_name(), (int)(available_memory/M), (int)(available_cc_p/M)); + stringStream msg; + msg.print("Added compiler thread %s (available memory: %dMB, available profiled code cache: %dMB)", + ct->get_thread_name(), (int)(available_memory/M), (int)(available_cc_p/M)); + print_compiler_threads(msg); } } } @@ -1959,9 +1969,12 @@ void CompileBroker::compiler_thread_loop() { // Access compiler_count under lock to enforce consistency. MutexLocker only_one(CompileThread_lock); if (can_remove(thread, true)) { - if (TraceCompilerThreads) { - tty->print_cr("Removing compiler thread %s after " JLONG_FORMAT " ms idle time", - thread->name(), thread->idle_time_millis()); + if (trace_compiler_threads()) { + ResourceMark rm; + stringStream msg; + msg.print("Removing compiler thread %s after " JLONG_FORMAT " ms idle time", + thread->name(), thread->idle_time_millis()); + print_compiler_threads(msg); } // Free buffer blob, if allocated if (thread->get_buffer_blob() != NULL) { @@ -2905,13 +2918,13 @@ void CompileBroker::print_heapinfo(outputStream* out, const char* function, size ts.update(); // record starting point MutexLocker mu11(function_lock_1, Mutex::_safepoint_check_flag); MutexLocker mu22(function_lock_2, Mutex::_no_safepoint_check_flag); - if ((function_lock_1 != NULL) || (function_lock_1 != NULL)) { + if ((function_lock_1 != NULL) || (function_lock_2 != NULL)) { out->print_cr("\n__ Compile & CodeCache (function) lock wait took %10.3f seconds _________\n", ts.seconds()); } ts.update(); // record starting point CodeCache::aggregate(out, granularity); - if ((function_lock_1 != NULL) || (function_lock_1 != NULL)) { + if ((function_lock_1 != NULL) || (function_lock_2 != NULL)) { out->print_cr("\n__ Compile & CodeCache (function) lock hold took %10.3f seconds _________\n", ts.seconds()); } } diff --git a/src/hotspot/share/compiler/compilerDefinitions.cpp b/src/hotspot/share/compiler/compilerDefinitions.cpp index 33fbe16a8480d..421997081a111 100644 --- a/src/hotspot/share/compiler/compilerDefinitions.cpp +++ b/src/hotspot/share/compiler/compilerDefinitions.cpp @@ -394,13 +394,7 @@ void CompilerConfig::set_compilation_policy_flags() { if (CompilerConfig::is_tiered() && CompilerConfig::is_c2_enabled()) { #ifdef COMPILER2 // Some inlining tuning -#ifdef X86 - if (FLAG_IS_DEFAULT(InlineSmallCode)) { - FLAG_SET_DEFAULT(InlineSmallCode, 2500); - } -#endif - -#if defined AARCH64 +#if defined(X86) || defined(AARCH64) || defined(RISCV64) if (FLAG_IS_DEFAULT(InlineSmallCode)) { FLAG_SET_DEFAULT(InlineSmallCode, 2500); } diff --git a/src/hotspot/share/compiler/compilerOracle.cpp b/src/hotspot/share/compiler/compilerOracle.cpp index 69a3278736ced..5149121c5bc79 100644 --- a/src/hotspot/share/compiler/compilerOracle.cpp +++ b/src/hotspot/share/compiler/compilerOracle.cpp @@ -308,6 +308,8 @@ static void register_command(TypedMethodOptionMatcher* matcher, if (option == CompileCommand::Blackhole && !UnlockExperimentalVMOptions) { warning("Blackhole compile option is experimental and must be enabled via -XX:+UnlockExperimentalVMOptions"); + // Delete matcher as we don't keep it + delete matcher; return; } diff --git a/src/hotspot/share/compiler/disassembler.cpp b/src/hotspot/share/compiler/disassembler.cpp index bd9272a8e6c90..4cc2f8781c336 100644 --- a/src/hotspot/share/compiler/disassembler.cpp +++ b/src/hotspot/share/compiler/disassembler.cpp @@ -887,7 +887,7 @@ void Disassembler::decode(CodeBlob* cb, outputStream* st) { if (cb->is_nmethod()) { // If we have an nmethod at hand, // call the specialized decoder directly. - decode((nmethod*)cb, st); + ((nmethod*)cb)->decode2(st); return; } diff --git a/src/hotspot/share/gc/epsilon/epsilonHeap.hpp b/src/hotspot/share/gc/epsilon/epsilonHeap.hpp index 2d90439a1eeba..66a668a531741 100644 --- a/src/hotspot/share/gc/epsilon/epsilonHeap.hpp +++ b/src/hotspot/share/gc/epsilon/epsilonHeap.hpp @@ -52,7 +52,7 @@ class EpsilonHeap : public CollectedHeap { static EpsilonHeap* heap(); EpsilonHeap() : - _memory_manager("Epsilon Heap", ""), + _memory_manager("Epsilon Heap"), _space(NULL) {}; virtual Name kind() const { diff --git a/src/hotspot/share/gc/g1/g1BarrierSet.cpp b/src/hotspot/share/gc/g1/g1BarrierSet.cpp index d6e103d478e5c..d22dc6fb601f1 100644 --- a/src/hotspot/share/gc/g1/g1BarrierSet.cpp +++ b/src/hotspot/share/gc/g1/g1BarrierSet.cpp @@ -62,21 +62,18 @@ G1BarrierSet::G1BarrierSet(G1CardTable* card_table) : _shared_dirty_card_queue(&_dirty_card_queue_set) {} -void G1BarrierSet::enqueue(oop pre_val) { - // Nulls should have been already filtered. - assert(oopDesc::is_oop(pre_val, true), "Error"); - SATBMarkQueue& queue = G1ThreadLocalData::satb_mark_queue(Thread::current()); - G1BarrierSet::satb_mark_queue_set().enqueue(queue, pre_val); -} - template void G1BarrierSet::write_ref_array_pre_work(T* dst, size_t count) { - if (!_satb_mark_queue_set.is_active()) return; + G1SATBMarkQueueSet& queue_set = G1BarrierSet::satb_mark_queue_set(); + if (!queue_set.is_active()) return; + + SATBMarkQueue& queue = G1ThreadLocalData::satb_mark_queue(Thread::current()); + T* elem_ptr = dst; for (size_t i = 0; i < count; i++, elem_ptr++) { T heap_oop = RawAccess<>::oop_load(elem_ptr); if (!CompressedOops::is_null(heap_oop)) { - enqueue(CompressedOops::decode_not_null(heap_oop)); + queue_set.enqueue_known_active(queue, CompressedOops::decode_not_null(heap_oop)); } } } diff --git a/src/hotspot/share/gc/g1/g1BarrierSet.hpp b/src/hotspot/share/gc/g1/g1BarrierSet.hpp index 8d009a9e19f3f..c60628fe7ff4f 100644 --- a/src/hotspot/share/gc/g1/g1BarrierSet.hpp +++ b/src/hotspot/share/gc/g1/g1BarrierSet.hpp @@ -58,10 +58,12 @@ class G1BarrierSet: public CardTableBarrierSet { } // Add "pre_val" to a set of objects that may have been disconnected from the - // pre-marking object graph. - static void enqueue(oop pre_val); + // pre-marking object graph. Prefer the version that takes location, as it + // can avoid touching the heap unnecessarily. + template static void enqueue(T* dst); + static void enqueue_preloaded(oop pre_val); - static void enqueue_if_weak(DecoratorSet decorators, oop value); + static void enqueue_preloaded_if_weak(DecoratorSet decorators, oop value); template void write_ref_array_pre_work(T* dst, size_t count); virtual void write_ref_array_pre(oop* dst, size_t count, bool dest_uninitialized); diff --git a/src/hotspot/share/gc/g1/g1BarrierSet.inline.hpp b/src/hotspot/share/gc/g1/g1BarrierSet.inline.hpp index 8fa29544ca433..f7a6fd3ee5a37 100644 --- a/src/hotspot/share/gc/g1/g1BarrierSet.inline.hpp +++ b/src/hotspot/share/gc/g1/g1BarrierSet.inline.hpp @@ -28,11 +28,35 @@ #include "gc/g1/g1BarrierSet.hpp" #include "gc/g1/g1CardTable.hpp" +#include "gc/g1/g1ThreadLocalData.hpp" #include "gc/shared/accessBarrierSupport.inline.hpp" #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" #include "oops/oop.hpp" +inline void G1BarrierSet::enqueue_preloaded(oop pre_val) { + // Nulls should have been already filtered. + assert(oopDesc::is_oop(pre_val, true), "Error"); + + G1SATBMarkQueueSet& queue_set = G1BarrierSet::satb_mark_queue_set(); + if (!queue_set.is_active()) return; + + SATBMarkQueue& queue = G1ThreadLocalData::satb_mark_queue(Thread::current()); + queue_set.enqueue_known_active(queue, pre_val); +} + +template +inline void G1BarrierSet::enqueue(T* dst) { + G1SATBMarkQueueSet& queue_set = G1BarrierSet::satb_mark_queue_set(); + if (!queue_set.is_active()) return; + + T heap_oop = RawAccess::oop_load(dst); + if (!CompressedOops::is_null(heap_oop)) { + SATBMarkQueue& queue = G1ThreadLocalData::satb_mark_queue(Thread::current()); + queue_set.enqueue_known_active(queue, CompressedOops::decode_not_null(heap_oop)); + } +} + template inline void G1BarrierSet::write_ref_field_pre(T* field) { if (HasDecorator::value || @@ -40,10 +64,7 @@ inline void G1BarrierSet::write_ref_field_pre(T* field) { return; } - T heap_oop = RawAccess::oop_load(field); - if (!CompressedOops::is_null(heap_oop)) { - enqueue(CompressedOops::decode_not_null(heap_oop)); - } + enqueue(field); } template @@ -55,7 +76,7 @@ inline void G1BarrierSet::write_ref_field_post(T* field, oop new_val) { } } -inline void G1BarrierSet::enqueue_if_weak(DecoratorSet decorators, oop value) { +inline void G1BarrierSet::enqueue_preloaded_if_weak(DecoratorSet decorators, oop value) { assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known"); // Loading from a weak or phantom reference needs enqueueing, as // the object may not have been reachable (part of the snapshot) @@ -65,7 +86,7 @@ inline void G1BarrierSet::enqueue_if_weak(DecoratorSet decorators, oop value) { const bool needs_enqueue = (!peek && !on_strong_oop_ref); if (needs_enqueue && value != NULL) { - enqueue(value); + enqueue_preloaded(value); } } @@ -74,7 +95,7 @@ template inline oop G1BarrierSet::AccessBarrier:: oop_load_not_in_heap(T* addr) { oop value = ModRef::oop_load_not_in_heap(addr); - enqueue_if_weak(decorators, value); + enqueue_preloaded_if_weak(decorators, value); return value; } @@ -83,7 +104,7 @@ template inline oop G1BarrierSet::AccessBarrier:: oop_load_in_heap(T* addr) { oop value = ModRef::oop_load_in_heap(addr); - enqueue_if_weak(decorators, value); + enqueue_preloaded_if_weak(decorators, value); return value; } @@ -91,7 +112,7 @@ template inline oop G1BarrierSet::AccessBarrier:: oop_load_in_heap_at(oop base, ptrdiff_t offset) { oop value = ModRef::oop_load_in_heap_at(base, offset); - enqueue_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset), value); + enqueue_preloaded_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset), value); return value; } diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp index 07edbe49817f8..2f9af5ed4f21e 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp @@ -2267,7 +2267,7 @@ void G1CollectedHeap::object_iterate(ObjectClosure* cl) { heap_region_iterate(&blk); } -class G1ParallelObjectIterator : public ParallelObjectIterator { +class G1ParallelObjectIterator : public ParallelObjectIteratorImpl { private: G1CollectedHeap* _heap; HeapRegionClaimer _claimer; @@ -2282,7 +2282,7 @@ class G1ParallelObjectIterator : public ParallelObjectIterator { } }; -ParallelObjectIterator* G1CollectedHeap::parallel_object_iterator(uint thread_num) { +ParallelObjectIteratorImpl* G1CollectedHeap::parallel_object_iterator(uint thread_num) { return new G1ParallelObjectIterator(thread_num); } @@ -2292,7 +2292,7 @@ void G1CollectedHeap::object_iterate_parallel(ObjectClosure* cl, uint worker_id, } void G1CollectedHeap::keep_alive(oop obj) { - G1BarrierSet::enqueue(obj); + G1BarrierSet::enqueue_preloaded(obj); } void G1CollectedHeap::heap_region_iterate(HeapRegionClosure* cl) const { @@ -3201,6 +3201,31 @@ class G1CopyingKeepAliveClosure: public OopClosure { } }; +// Special closure for enqueuing discovered fields: during enqueue the card table +// may not be in shape to properly handle normal barrier calls (e.g. card marks +// in regions that failed evacuation, scribbling of various values by card table +// scan code). Additionally the regular barrier enqueues into the "global" +// DCQS, but during GC we need these to-be-refined entries in the GC local queue +// so that after clearing the card table, the redirty cards phase will properly +// mark all dirty cards to be picked up by refinement. +class G1EnqueueDiscoveredFieldClosure : public EnqueueDiscoveredFieldClosure { + G1CollectedHeap* _g1h; + G1ParScanThreadState* _pss; + +public: + G1EnqueueDiscoveredFieldClosure(G1CollectedHeap* g1h, G1ParScanThreadState* pss) : _g1h(g1h), _pss(pss) { } + + virtual void enqueue(HeapWord* discovered_field_addr, oop value) { + assert(_g1h->is_in(discovered_field_addr), PTR_FORMAT " is not in heap ", p2i(discovered_field_addr)); + // Store the value first, whatever it is. + RawAccess<>::oop_store(discovered_field_addr, value); + if (value == NULL) { + return; + } + _pss->write_ref_field_post(discovered_field_addr, value); + } +}; + // Serial drain queue closure. Called as the 'complete_gc' // closure for each discovered list in some of the // reference processing phases. @@ -3245,7 +3270,8 @@ class G1STWRefProcProxyTask : public RefProcProxyTask { G1STWIsAliveClosure is_alive(&_g1h); G1CopyingKeepAliveClosure keep_alive(&_g1h, _pss.state_for_worker(index)); G1ParEvacuateFollowersClosure complete_gc(&_g1h, _pss.state_for_worker(index), &_task_queues, _tm == RefProcThreadModel::Single ? nullptr : &_terminator, G1GCPhaseTimes::ObjCopy); - _rp_task->rp_work(worker_id, &is_alive, &keep_alive, &complete_gc); + G1EnqueueDiscoveredFieldClosure enqueue(&_g1h, _pss.state_for_worker(index)); + _rp_task->rp_work(worker_id, &is_alive, &keep_alive, &enqueue, &complete_gc); } void prepare_run_task_hook() override { diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp index 2068236265e3d..d2e90a864240b 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp @@ -1205,7 +1205,7 @@ class G1CollectedHeap : public CollectedHeap { // Iterate over all objects, calling "cl.do_object" on each. virtual void object_iterate(ObjectClosure* cl); - virtual ParallelObjectIterator* parallel_object_iterator(uint thread_num); + virtual ParallelObjectIteratorImpl* parallel_object_iterator(uint thread_num); // Keep alive an object that was loaded with AS_NO_KEEPALIVE. virtual void keep_alive(oop obj); diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp index f07223ddc684a..a2f08726b29a9 100644 --- a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp +++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp @@ -1478,8 +1478,9 @@ class G1CMRefProcProxyTask : public RefProcProxyTask { G1CMIsAliveClosure is_alive(&_g1h); uint index = (_tm == RefProcThreadModel::Single) ? 0 : worker_id; G1CMKeepAliveAndDrainClosure keep_alive(&_cm, _cm.task(index), _tm == RefProcThreadModel::Single); + BarrierEnqueueDiscoveredFieldClosure enqueue; G1CMDrainMarkingStackClosure complete_gc(&_cm, _cm.task(index), _tm == RefProcThreadModel::Single); - _rp_task->rp_work(worker_id, &is_alive, &keep_alive, &complete_gc); + _rp_task->rp_work(worker_id, &is_alive, &keep_alive, &enqueue, &complete_gc); } void prepare_run_task_hook() override { @@ -1628,10 +1629,10 @@ void G1ConcurrentMark::report_object_count(bool mark_completed) { // using either the next or prev bitmap. if (mark_completed) { G1ObjectCountIsAliveClosure is_alive(_g1h); - _gc_tracer_cm->report_object_count_after_gc(&is_alive); + _gc_tracer_cm->report_object_count_after_gc(&is_alive, _g1h->workers()); } else { G1CMIsAliveClosure is_alive(_g1h); - _gc_tracer_cm->report_object_count_after_gc(&is_alive); + _gc_tracer_cm->report_object_count_after_gc(&is_alive, _g1h->workers()); } } diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.cpp index f03bbb42a7b9e..948e564e50e78 100644 --- a/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.cpp +++ b/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.cpp @@ -56,26 +56,6 @@ G1ConcurrentMarkThread::G1ConcurrentMarkThread(G1ConcurrentMark* cm) : create_and_start(); } -class CMRemark : public VoidClosure { - G1ConcurrentMark* _cm; -public: - CMRemark(G1ConcurrentMark* cm) : _cm(cm) {} - - void do_void(){ - _cm->remark(); - } -}; - -class CMCleanup : public VoidClosure { - G1ConcurrentMark* _cm; -public: - CMCleanup(G1ConcurrentMark* cm) : _cm(cm) {} - - void do_void(){ - _cm->cleanup(); - } -}; - double G1ConcurrentMarkThread::mmu_delay_end(G1Policy* policy, bool remark) { // There are 3 reasons to use SuspendibleThreadSetJoiner. // 1. To avoid concurrency problem. @@ -239,8 +219,7 @@ bool G1ConcurrentMarkThread::subphase_delay_to_keep_mmu_before_remark() { bool G1ConcurrentMarkThread::subphase_remark() { ConcurrentGCBreakpoints::at("BEFORE MARKING COMPLETED"); - CMRemark cl(_cm); - VM_G1Concurrent op(&cl, "Pause Remark"); + VM_G1PauseRemark op; VMThread::execute(&op); return _cm->has_aborted(); } @@ -257,8 +236,7 @@ bool G1ConcurrentMarkThread::phase_delay_to_keep_mmu_before_cleanup() { } bool G1ConcurrentMarkThread::phase_cleanup() { - CMCleanup cl(_cm); - VM_G1Concurrent op(&cl, "Pause Cleanup"); + VM_G1PauseCleanup op; VMThread::execute(&op); return _cm->has_aborted(); } diff --git a/src/hotspot/share/gc/g1/g1FullCollector.cpp b/src/hotspot/share/gc/g1/g1FullCollector.cpp index a48c043e850a1..f4c58c90afa50 100644 --- a/src/hotspot/share/gc/g1/g1FullCollector.cpp +++ b/src/hotspot/share/gc/g1/g1FullCollector.cpp @@ -259,8 +259,9 @@ class G1FullGCRefProcProxyTask : public RefProcProxyTask { G1IsAliveClosure is_alive(&_collector); uint index = (_tm == RefProcThreadModel::Single) ? 0 : worker_id; G1FullKeepAliveClosure keep_alive(_collector.marker(index)); + BarrierEnqueueDiscoveredFieldClosure enqueue; G1FollowStackClosure* complete_gc = _collector.marker(index)->stack_closure(); - _rp_task->rp_work(worker_id, &is_alive, &keep_alive, complete_gc); + _rp_task->rp_work(worker_id, &is_alive, &keep_alive, &enqueue, complete_gc); } }; @@ -303,7 +304,10 @@ void G1FullCollector::phase1_mark_live_objects() { _heap->complete_cleaning(&_is_alive, purged_class); } - scope()->tracer()->report_object_count_after_gc(&_is_alive); + { + GCTraceTime(Debug, gc, phases) debug("Report Object Count", scope()->timer()); + scope()->tracer()->report_object_count_after_gc(&_is_alive, _heap->workers()); + } } void G1FullCollector::phase2_prepare_compaction() { diff --git a/src/hotspot/share/gc/g1/g1FullGCMarker.hpp b/src/hotspot/share/gc/g1/g1FullGCMarker.hpp index ec7f029b7cee7..2d935d863c5de 100644 --- a/src/hotspot/share/gc/g1/g1FullGCMarker.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCMarker.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -71,8 +71,6 @@ class G1FullGCMarker : public CHeapObj { G1RegionMarkStatsCache _mark_stats_cache; inline bool is_empty(); - inline bool pop_object(oop& obj); - inline bool pop_objarray(ObjArrayTask& array); inline void push_objarray(oop obj, size_t index); inline bool mark_object(oop obj); @@ -80,6 +78,14 @@ class G1FullGCMarker : public CHeapObj { inline void follow_object(oop obj); inline void follow_array(objArrayOop array); inline void follow_array_chunk(objArrayOop array, int index); + + inline void drain_oop_stack(); + // Transfer contents from the objArray task queue overflow stack to the shared + // objArray stack. + // Returns true and a valid task if there has not been enough space in the shared + // objArray stack, otherwise the task is invalid. + inline bool transfer_objArray_overflow_stack(ObjArrayTask& task); + public: G1FullGCMarker(G1FullCollector* collector, uint worker_id, diff --git a/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp b/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp index 992cbb2321c83..6441ea9c2d455 100644 --- a/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -93,20 +93,12 @@ inline bool G1FullGCMarker::is_empty() { return _oop_stack.is_empty() && _objarray_stack.is_empty(); } -inline bool G1FullGCMarker::pop_object(oop& oop) { - return _oop_stack.pop_overflow(oop) || _oop_stack.pop_local(oop); -} - inline void G1FullGCMarker::push_objarray(oop obj, size_t index) { ObjArrayTask task(obj, index); assert(task.is_valid(), "bad ObjArrayTask"); _objarray_stack.push(task); } -inline bool G1FullGCMarker::pop_objarray(ObjArrayTask& arr) { - return _objarray_stack.pop_overflow(arr) || _objarray_stack.pop_local(arr); -} - inline void G1FullGCMarker::follow_array(objArrayOop array) { follow_klass(array->klass()); // Don't push empty arrays to avoid unnecessary work. @@ -161,16 +153,40 @@ inline void G1FullGCMarker::follow_object(oop obj) { } } -void G1FullGCMarker::drain_stack() { - do { - oop obj; - while (pop_object(obj)) { +inline void G1FullGCMarker::drain_oop_stack() { + oop obj; + while (_oop_stack.pop_overflow(obj)) { + if (!_oop_stack.try_push_to_taskqueue(obj)) { assert(_bitmap->is_marked(obj), "must be marked"); follow_object(obj); } - // Process ObjArrays one at a time to avoid marking stack bloat. + } + while (_oop_stack.pop_local(obj)) { + assert(_bitmap->is_marked(obj), "must be marked"); + follow_object(obj); + } +} + +inline bool G1FullGCMarker::transfer_objArray_overflow_stack(ObjArrayTask& task) { + // It is desirable to move as much as possible work from the overflow queue to + // the shared queue as quickly as possible. + while (_objarray_stack.pop_overflow(task)) { + if (!_objarray_stack.try_push_to_taskqueue(task)) { + return true; + } + } + return false; +} + +void G1FullGCMarker::drain_stack() { + do { + // First, drain regular oop stack. + drain_oop_stack(); + + // Then process ObjArrays one at a time to avoid marking stack bloat. ObjArrayTask task; - if (pop_objarray(task)) { + if (transfer_objArray_overflow_stack(task) || + _objarray_stack.pop_local(task)) { follow_array_chunk(objArrayOop(task.obj()), task.index()); } } while (!is_empty()); diff --git a/src/hotspot/share/gc/g1/g1MonitoringSupport.cpp b/src/hotspot/share/gc/g1/g1MonitoringSupport.cpp index fd21cc1f47bec..7e9005068b92a 100644 --- a/src/hotspot/share/gc/g1/g1MonitoringSupport.cpp +++ b/src/hotspot/share/gc/g1/g1MonitoringSupport.cpp @@ -86,8 +86,8 @@ class G1OldGenerationCounters : public G1GenerationCounters { G1MonitoringSupport::G1MonitoringSupport(G1CollectedHeap* g1h) : _g1h(g1h), - _incremental_memory_manager("G1 Young Generation", "end of minor GC"), - _full_gc_memory_manager("G1 Old Generation", "end of major GC"), + _incremental_memory_manager("G1 Young Generation"), + _full_gc_memory_manager("G1 Old Generation"), _eden_space_pool(NULL), _survivor_space_pool(NULL), _old_gen_pool(NULL), @@ -345,5 +345,5 @@ MemoryUsage G1MonitoringSupport::old_gen_memory_usage(size_t initial_size, size_ G1MonitoringScope::G1MonitoringScope(G1MonitoringSupport* g1mm, bool full_gc, bool all_memory_pools_affected) : _tcs(full_gc ? g1mm->_full_collection_counters : g1mm->_incremental_collection_counters), _tms(full_gc ? &g1mm->_full_gc_memory_manager : &g1mm->_incremental_memory_manager, - G1CollectedHeap::heap()->gc_cause(), all_memory_pools_affected) { + G1CollectedHeap::heap()->gc_cause(), full_gc ? "end of major GC" : "end of minor GC", all_memory_pools_affected) { } diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp index 9eebe411936f9..896c891ae74b4 100644 --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp @@ -203,14 +203,7 @@ void G1ParScanThreadState::do_oop_evac(T* p) { } RawAccess::oop_store(p, obj); - assert(obj != NULL, "Must be"); - if (HeapRegion::is_in_same_region(p, obj)) { - return; - } - HeapRegion* from = _g1h->heap_region_containing(p); - if (!from->is_young()) { - enqueue_card_if_tracked(_g1h->region_attr(obj), p, obj); - } + write_ref_field_post(p, obj); } MAYBE_INLINE_EVACUATION diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp index 48ba8de20e05c..94f91ff2a1c52 100644 --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp @@ -128,6 +128,12 @@ class G1ParScanThreadState : public CHeapObj { void push_on_queue(ScannerTask task); + // Apply the post barrier to the given reference field. Enqueues the card of p + // if the barrier does not filter out the reference for some reason (e.g. + // p and q are in the same region, p is in survivor, p is in collection set) + // To be called during GC if nothing particular about p and obj are known. + template void write_ref_field_post(T* p, oop obj); + template void enqueue_card_if_tracked(G1HeapRegionAttr region_attr, T* p, oop o) { assert(!HeapRegion::is_in_same_region(p, o), "Should have filtered out cross-region references already."); assert(!_g1h->heap_region_containing(p)->is_young(), "Should have filtered out from-young references already."); diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.inline.hpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.inline.hpp index db117c50611e6..f094410881003 100644 --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.inline.hpp +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.inline.hpp @@ -95,4 +95,16 @@ G1OopStarChunkedList* G1ParScanThreadState::oops_into_optional_region(const Heap return &_oops_into_optional_regions[hr->index_in_opt_cset()]; } +template void G1ParScanThreadState::write_ref_field_post(T* p, oop obj) { + assert(obj != NULL, "Must be"); + if (HeapRegion::is_in_same_region(p, obj)) { + return; + } + HeapRegion* from = _g1h->heap_region_containing(p); + if (!from->is_young()) { + enqueue_card_if_tracked(_g1h->region_attr(obj), p, obj); + } +} + + #endif // SHARE_GC_G1_G1PARSCANTHREADSTATE_INLINE_HPP diff --git a/src/hotspot/share/gc/g1/g1ServiceThread.cpp b/src/hotspot/share/gc/g1/g1ServiceThread.cpp index 7b39b7d2808ff..cb6b897fcb04d 100644 --- a/src/hotspot/share/gc/g1/g1ServiceThread.cpp +++ b/src/hotspot/share/gc/g1/g1ServiceThread.cpp @@ -112,12 +112,10 @@ void G1ServiceThread::notify() { } void G1ServiceThread::sleep_before_next_cycle() { + MonitorLocker ml(&_monitor, Mutex::_no_safepoint_check_flag); if (should_terminate()) { return; - } - - MonitorLocker ml(&_monitor, Mutex::_no_safepoint_check_flag); - if (_task_queue.is_empty()) { + } else if (_task_queue.is_empty()) { // Sleep until new task is registered if no tasks available. log_trace(gc, task)("G1 Service Thread (wait for new tasks)"); ml.wait(0); diff --git a/src/hotspot/share/gc/g1/g1VMOperations.cpp b/src/hotspot/share/gc/g1/g1VMOperations.cpp index a3663301a3f12..59004af9a908c 100644 --- a/src/hotspot/share/gc/g1/g1VMOperations.cpp +++ b/src/hotspot/share/gc/g1/g1VMOperations.cpp @@ -159,7 +159,7 @@ void VM_G1CollectForAllocation::doit() { } } -void VM_G1Concurrent::doit() { +void VM_G1PauseConcurrent::doit() { GCIdMark gc_id_mark(_gc_id); GCTraceCPUTime tcpu; G1CollectedHeap* g1h = G1CollectedHeap::heap(); @@ -173,17 +173,28 @@ void VM_G1Concurrent::doit() { TraceCollectorStats tcs(g1h->g1mm()->conc_collection_counters()); SvcGCMarker sgcm(SvcGCMarker::CONCURRENT); IsGCActiveMark x; - _cl->do_void(); + + work(); } -bool VM_G1Concurrent::doit_prologue() { +bool VM_G1PauseConcurrent::doit_prologue() { Heap_lock->lock(); return true; } -void VM_G1Concurrent::doit_epilogue() { +void VM_G1PauseConcurrent::doit_epilogue() { if (Universe::has_reference_pending_list()) { Heap_lock->notify_all(); } Heap_lock->unlock(); } + +void VM_G1PauseRemark::work() { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + g1h->concurrent_mark()->remark(); +} + +void VM_G1PauseCleanup::work() { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + g1h->concurrent_mark()->cleanup(); +} diff --git a/src/hotspot/share/gc/g1/g1VMOperations.hpp b/src/hotspot/share/gc/g1/g1VMOperations.hpp index 317ab76a5a9da..b5bd8eb916628 100644 --- a/src/hotspot/share/gc/g1/g1VMOperations.hpp +++ b/src/hotspot/share/gc/g1/g1VMOperations.hpp @@ -84,18 +84,33 @@ class VM_G1CollectForAllocation : public VM_CollectForAllocation { }; // Concurrent G1 stop-the-world operations such as remark and cleanup. -class VM_G1Concurrent : public VM_Operation { - VoidClosure* _cl; - const char* _message; +class VM_G1PauseConcurrent : public VM_Operation { uint _gc_id; + const char* _message; + +protected: + VM_G1PauseConcurrent(const char* message) : + _gc_id(GCId::current()), _message(message) { } + virtual void work() = 0; public: - VM_G1Concurrent(VoidClosure* cl, const char* message) : - _cl(cl), _message(message), _gc_id(GCId::current()) { } - virtual VMOp_Type type() const { return VMOp_G1Concurrent; } - virtual void doit(); - virtual bool doit_prologue(); - virtual void doit_epilogue(); + bool doit_prologue() override; + void doit_epilogue() override; + void doit() override; +}; + +class VM_G1PauseRemark : public VM_G1PauseConcurrent { +public: + VM_G1PauseRemark() : VM_G1PauseConcurrent("Pause Remark") { } + VMOp_Type type() const override { return VMOp_G1PauseRemark; } + void work() override; +}; + +class VM_G1PauseCleanup : public VM_G1PauseConcurrent { +public: + VM_G1PauseCleanup() : VM_G1PauseConcurrent("Pause Cleanup") { } + VMOp_Type type() const override { return VMOp_G1PauseCleanup; } + void work() override; }; #endif // SHARE_GC_G1_G1VMOPERATIONS_HPP diff --git a/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp b/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp index 272c1e28c3a5f..68d0865f75d38 100644 --- a/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp +++ b/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp @@ -150,8 +150,8 @@ void ParallelScavengeHeap::initialize_serviceability() { "PS Old Gen", true /* support_usage_threshold */); - _young_manager = new GCMemoryManager("PS Scavenge", "end of minor GC"); - _old_manager = new GCMemoryManager("PS MarkSweep", "end of major GC"); + _young_manager = new GCMemoryManager("PS Scavenge"); + _old_manager = new GCMemoryManager("PS MarkSweep"); _old_manager->add_pool(_eden_pool); _old_manager->add_pool(_survivor_pool); @@ -588,7 +588,7 @@ void ParallelScavengeHeap::object_iterate_parallel(ObjectClosure* cl, } } -class PSScavengeParallelObjectIterator : public ParallelObjectIterator { +class PSScavengeParallelObjectIterator : public ParallelObjectIteratorImpl { private: ParallelScavengeHeap* _heap; HeapBlockClaimer _claimer; @@ -603,7 +603,7 @@ class PSScavengeParallelObjectIterator : public ParallelObjectIterator { } }; -ParallelObjectIterator* ParallelScavengeHeap::parallel_object_iterator(uint thread_num) { +ParallelObjectIteratorImpl* ParallelScavengeHeap::parallel_object_iterator(uint thread_num) { return new PSScavengeParallelObjectIterator(); } diff --git a/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp b/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp index 689400fbe040e..e8b04b3090bc3 100644 --- a/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp +++ b/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp @@ -211,7 +211,7 @@ class ParallelScavengeHeap : public CollectedHeap { void object_iterate(ObjectClosure* cl); void object_iterate_parallel(ObjectClosure* cl, HeapBlockClaimer* claimer); - virtual ParallelObjectIterator* parallel_object_iterator(uint thread_num); + virtual ParallelObjectIteratorImpl* parallel_object_iterator(uint thread_num); HeapWord* block_start(const void* addr) const; bool block_is_obj(const HeapWord* addr) const; diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.cpp b/src/hotspot/share/gc/parallel/psParallelCompact.cpp index ca85e21ba80ad..69b7dc9881a1a 100644 --- a/src/hotspot/share/gc/parallel/psParallelCompact.cpp +++ b/src/hotspot/share/gc/parallel/psParallelCompact.cpp @@ -1773,7 +1773,7 @@ bool PSParallelCompact::invoke_no_policy(bool maximum_heap_compaction) { heap->pre_full_gc_dump(&_gc_timer); TraceCollectorStats tcs(counters()); - TraceMemoryManagerStats tms(heap->old_gc_manager(), gc_cause); + TraceMemoryManagerStats tms(heap->old_gc_manager(), gc_cause, "end of major GC"); if (log_is_enabled(Debug, gc, heap, exit)) { accumulated_time()->start(); @@ -2067,8 +2067,9 @@ class ParallelCompactRefProcProxyTask : public RefProcProxyTask { assert(worker_id < _max_workers, "sanity"); ParCompactionManager* cm = (_tm == RefProcThreadModel::Single) ? ParCompactionManager::get_vmthread_cm() : ParCompactionManager::gc_thread_compaction_manager(worker_id); PCMarkAndPushClosure keep_alive(cm); + BarrierEnqueueDiscoveredFieldClosure enqueue; ParCompactionManager::FollowStackClosure complete_gc(cm, (_tm == RefProcThreadModel::Single) ? nullptr : &_terminator, worker_id); - _rp_task->rp_work(worker_id, PSParallelCompact::is_alive_closure(), &keep_alive, &complete_gc); + _rp_task->rp_work(worker_id, PSParallelCompact::is_alive_closure(), &keep_alive, &enqueue, &complete_gc); } void prepare_run_task_hook() override { @@ -2136,7 +2137,10 @@ void PSParallelCompact::marking_phase(ParCompactionManager* cm, JVMCI_ONLY(JVMCI::do_unloading(purged_class)); } - _gc_tracer.report_object_count_after_gc(is_alive_closure()); + { + GCTraceTime(Debug, gc, phases) tm("Report Object Count", &_gc_timer); + _gc_tracer.report_object_count_after_gc(is_alive_closure(), &ParallelScavengeHeap::heap()->workers()); + } } #ifdef ASSERT diff --git a/src/hotspot/share/gc/parallel/psScavenge.cpp b/src/hotspot/share/gc/parallel/psScavenge.cpp index 078dcdbeed174..20d5caaa84deb 100644 --- a/src/hotspot/share/gc/parallel/psScavenge.cpp +++ b/src/hotspot/share/gc/parallel/psScavenge.cpp @@ -210,9 +210,10 @@ class ParallelScavengeRefProcProxyTask : public RefProcProxyTask { assert(worker_id < _max_workers, "sanity"); PSPromotionManager* promotion_manager = (_tm == RefProcThreadModel::Single) ? PSPromotionManager::vm_thread_promotion_manager() : PSPromotionManager::gc_thread_promotion_manager(worker_id); PSIsAliveClosure is_alive; - PSKeepAliveClosure keep_alive(promotion_manager);; + PSKeepAliveClosure keep_alive(promotion_manager); + BarrierEnqueueDiscoveredFieldClosure enqueue; PSEvacuateFollowersClosure complete_gc(promotion_manager, (_marks_oops_alive && _tm == RefProcThreadModel::Multi) ? &_terminator : nullptr, worker_id);; - _rp_task->rp_work(worker_id, &is_alive, &keep_alive, &complete_gc); + _rp_task->rp_work(worker_id, &is_alive, &keep_alive, &enqueue, &complete_gc); } void prepare_run_task_hook() override { @@ -419,7 +420,7 @@ bool PSScavenge::invoke_no_policy() { GCTraceCPUTime tcpu; GCTraceTime(Info, gc) tm("Pause Young", NULL, gc_cause, true); TraceCollectorStats tcs(counters()); - TraceMemoryManagerStats tms(heap->young_gc_manager(), gc_cause); + TraceMemoryManagerStats tms(heap->young_gc_manager(), gc_cause, "end of minor GC"); if (log_is_enabled(Debug, gc, heap, exit)) { accumulated_time()->start(); diff --git a/src/hotspot/share/gc/parallel/spaceCounters.cpp b/src/hotspot/share/gc/parallel/spaceCounters.cpp index 08e1d13b17c63..588c743d86587 100644 --- a/src/hotspot/share/gc/parallel/spaceCounters.cpp +++ b/src/hotspot/share/gc/parallel/spaceCounters.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -33,9 +33,9 @@ #include "utilities/macros.hpp" SpaceCounters::SpaceCounters(const char* name, int ordinal, size_t max_size, - MutableSpace* m, GenerationCounters* gc) : - _object_space(m) { - + MutableSpace* m, GenerationCounters* gc) + : _last_used_in_bytes(0), _object_space(m) +{ if (UsePerfData) { EXCEPTION_MARK; ResourceMark rm; @@ -55,13 +55,14 @@ SpaceCounters::SpaceCounters(const char* name, int ordinal, size_t max_size, cname = PerfDataManager::counter_name(_name_space, "capacity"); _capacity = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, - _object_space->capacity_in_bytes(), CHECK); + PerfData::U_Bytes, + _object_space->capacity_in_bytes(), + CHECK); cname = PerfDataManager::counter_name(_name_space, "used"); _used = PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, - new MutableSpaceUsedHelper(_object_space), - CHECK); + new UsedHelper(this), + CHECK); cname = PerfDataManager::counter_name(_name_space, "initCapacity"); PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, @@ -73,23 +74,22 @@ SpaceCounters::~SpaceCounters() { FREE_C_HEAP_ARRAY(char, _name_space); } -static volatile size_t last_used_in_bytes = 0; - void SpaceCounters::update_used() { size_t new_used = _object_space->used_in_bytes(); - Atomic::store(&last_used_in_bytes, new_used); + Atomic::store(&_last_used_in_bytes, new_used); _used->set_value(new_used); } -jlong MutableSpaceUsedHelper::take_sample() { +jlong SpaceCounters::UsedHelper::take_sample() { // Sampling may occur during GC, possibly while GC is updating the space. // The space can be in an inconsistent state during such an update. We // don't want to block sampling for the duration of a GC. Instead, skip // sampling in that case, using the last recorded value. assert(!Heap_lock->owned_by_self(), "precondition"); if (Heap_lock->try_lock()) { - Atomic::store(&last_used_in_bytes, _m->used_in_bytes()); + size_t new_used = _counters->_object_space->used_in_bytes(); + Atomic::store(&_counters->_last_used_in_bytes, new_used); Heap_lock->unlock(); } - return Atomic::load(&last_used_in_bytes); + return Atomic::load(&_counters->_last_used_in_bytes); } diff --git a/src/hotspot/share/gc/parallel/spaceCounters.hpp b/src/hotspot/share/gc/parallel/spaceCounters.hpp index d86762f6fca5b..3d0c1ff7b5015 100644 --- a/src/hotspot/share/gc/parallel/spaceCounters.hpp +++ b/src/hotspot/share/gc/parallel/spaceCounters.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -39,6 +39,7 @@ class SpaceCounters: public CHeapObj { private: PerfVariable* _capacity; PerfVariable* _used; + volatile size_t _last_used_in_bytes; // Constant PerfData types don't need to retain a reference. // However, it's a good idea to document them here. @@ -47,6 +48,8 @@ class SpaceCounters: public CHeapObj { MutableSpace* _object_space; char* _name_space; + class UsedHelper; + public: SpaceCounters(const char* name, int ordinal, size_t max_size, @@ -68,14 +71,14 @@ class SpaceCounters: public CHeapObj { const char* name_space() const { return _name_space; } }; -class MutableSpaceUsedHelper: public PerfLongSampleHelper { - private: - MutableSpace* _m; +class SpaceCounters::UsedHelper: public PerfLongSampleHelper { + private: + SpaceCounters* _counters; - public: - MutableSpaceUsedHelper(MutableSpace* m) : _m(m) { } + public: + UsedHelper(SpaceCounters* counters) : _counters(counters) { } - jlong take_sample() override; + jlong take_sample() override; }; #endif // SHARE_GC_PARALLEL_SPACECOUNTERS_HPP diff --git a/src/hotspot/share/gc/serial/cSpaceCounters.cpp b/src/hotspot/share/gc/serial/cSpaceCounters.cpp index 4bb6ecc7d62c8..6a9b51245783a 100644 --- a/src/hotspot/share/gc/serial/cSpaceCounters.cpp +++ b/src/hotspot/share/gc/serial/cSpaceCounters.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,9 +28,9 @@ #include "memory/resourceArea.hpp" CSpaceCounters::CSpaceCounters(const char* name, int ordinal, size_t max_size, - ContiguousSpace* s, GenerationCounters* gc) : - _space(s) { - + ContiguousSpace* s, GenerationCounters* gc) + : _last_used_in_bytes(0), _space(s) +{ if (UsePerfData) { EXCEPTION_MARK; ResourceMark rm; @@ -45,18 +45,21 @@ CSpaceCounters::CSpaceCounters(const char* name, int ordinal, size_t max_size, PerfDataManager::create_string_constant(SUN_GC, cname, name, CHECK); cname = PerfDataManager::counter_name(_name_space, "maxCapacity"); - _max_capacity = PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, - (jlong)max_size, CHECK); + _max_capacity = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, + (jlong)max_size, + CHECK); cname = PerfDataManager::counter_name(_name_space, "capacity"); _capacity = PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, - _space->capacity(), CHECK); + _space->capacity(), + CHECK); cname = PerfDataManager::counter_name(_name_space, "used"); _used = PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, - new ContiguousSpaceUsedHelper(_space), - CHECK); + new UsedHelper(this), + CHECK); cname = PerfDataManager::counter_name(_name_space, "initCapacity"); PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, @@ -72,11 +75,9 @@ void CSpaceCounters::update_capacity() { _capacity->set_value(_space->capacity()); } -static volatile size_t last_used_in_bytes = 0; - void CSpaceCounters::update_used() { size_t new_used = _space->used(); - Atomic::store(&last_used_in_bytes, new_used); + Atomic::store(&_last_used_in_bytes, new_used); _used->set_value(new_used); } @@ -85,15 +86,16 @@ void CSpaceCounters::update_all() { update_capacity(); } -jlong ContiguousSpaceUsedHelper::take_sample(){ +jlong CSpaceCounters::UsedHelper::take_sample(){ // Sampling may occur during GC, possibly while GC is updating the space. // The space can be in an inconsistent state during such an update. We // don't want to block sampling for the duration of a GC. Instead, skip // sampling in that case, using the last recorded value. assert(!Heap_lock->owned_by_self(), "precondition"); if (Heap_lock->try_lock()) { - Atomic::store(&last_used_in_bytes, _space->used()); + size_t new_used = _counters->_space->used(); + Atomic::store(&_counters->_last_used_in_bytes, new_used); Heap_lock->unlock(); } - return Atomic::load(&last_used_in_bytes); + return Atomic::load(&_counters->_last_used_in_bytes); } diff --git a/src/hotspot/share/gc/serial/cSpaceCounters.hpp b/src/hotspot/share/gc/serial/cSpaceCounters.hpp index fa3ee42140c08..334f649382bfe 100644 --- a/src/hotspot/share/gc/serial/cSpaceCounters.hpp +++ b/src/hotspot/share/gc/serial/cSpaceCounters.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -39,6 +39,7 @@ class CSpaceCounters: public CHeapObj { PerfVariable* _capacity; PerfVariable* _used; PerfVariable* _max_capacity; + volatile size_t _last_used_in_bytes; // Constant PerfData types don't need to retain a reference. // However, it's a good idea to document them here. @@ -47,6 +48,8 @@ class CSpaceCounters: public CHeapObj { ContiguousSpace* _space; char* _name_space; + class UsedHelper; + public: CSpaceCounters(const char* name, int ordinal, size_t max_size, @@ -61,14 +64,14 @@ class CSpaceCounters: public CHeapObj { const char* name_space() const { return _name_space; } }; -class ContiguousSpaceUsedHelper : public PerfLongSampleHelper { - private: - ContiguousSpace* _space; +class CSpaceCounters::UsedHelper : public PerfLongSampleHelper { + private: + CSpaceCounters* _counters; - public: - ContiguousSpaceUsedHelper(ContiguousSpace* space) : _space(space) { } + public: + UsedHelper(CSpaceCounters* counters) : _counters(counters) { } - jlong take_sample() override; + jlong take_sample() override; }; #endif // SHARE_GC_SERIAL_CSPACECOUNTERS_HPP diff --git a/src/hotspot/share/gc/serial/defNewGeneration.cpp b/src/hotspot/share/gc/serial/defNewGeneration.cpp index 1a24a86dcf61b..7ff1f97d6fe9d 100644 --- a/src/hotspot/share/gc/serial/defNewGeneration.cpp +++ b/src/hotspot/share/gc/serial/defNewGeneration.cpp @@ -198,6 +198,10 @@ void DefNewGeneration::compute_space_boundaries(uintx minimum_eden_size, uintx size = _virtual_space.committed_size(); uintx survivor_size = compute_survivor_size(size, SpaceAlignment); uintx eden_size = size - (2*survivor_size); + if (eden_size > max_eden_size()) { + eden_size = max_eden_size(); + survivor_size = (size - eden_size)/2; + } assert(eden_size > 0 && survivor_size <= eden_size, "just checking"); if (eden_size < minimum_eden_size) { diff --git a/src/hotspot/share/gc/serial/genMarkSweep.cpp b/src/hotspot/share/gc/serial/genMarkSweep.cpp index 38ebfef280393..f9ab5c2c15ff4 100644 --- a/src/hotspot/share/gc/serial/genMarkSweep.cpp +++ b/src/hotspot/share/gc/serial/genMarkSweep.cpp @@ -230,7 +230,10 @@ void GenMarkSweep::mark_sweep_phase1(bool clear_all_softrefs) { JVMCI_ONLY(JVMCI::do_unloading(purged_class)); } - gc_tracer()->report_object_count_after_gc(&is_alive); + { + GCTraceTime(Debug, gc, phases) tm_m("Report Object Count", gc_timer()); + gc_tracer()->report_object_count_after_gc(&is_alive, nullptr); + } } diff --git a/src/hotspot/share/gc/serial/serialGcRefProcProxyTask.hpp b/src/hotspot/share/gc/serial/serialGcRefProcProxyTask.hpp index 912a11181db1c..80006a4e8a4ca 100644 --- a/src/hotspot/share/gc/serial/serialGcRefProcProxyTask.hpp +++ b/src/hotspot/share/gc/serial/serialGcRefProcProxyTask.hpp @@ -41,7 +41,8 @@ class SerialGCRefProcProxyTask : public RefProcProxyTask { void work(uint worker_id) override { assert(worker_id < _max_workers, "sanity"); - _rp_task->rp_work(worker_id, &_is_alive, &_keep_alive, &_complete_gc); + BarrierEnqueueDiscoveredFieldClosure enqueue; + _rp_task->rp_work(worker_id, &_is_alive, &_keep_alive, &enqueue, &_complete_gc); } }; diff --git a/src/hotspot/share/gc/serial/serialHeap.cpp b/src/hotspot/share/gc/serial/serialHeap.cpp index d9a8812d580ed..0ea8e85e5ad04 100644 --- a/src/hotspot/share/gc/serial/serialHeap.cpp +++ b/src/hotspot/share/gc/serial/serialHeap.cpp @@ -42,8 +42,8 @@ SerialHeap::SerialHeap() : _eden_pool(NULL), _survivor_pool(NULL), _old_pool(NULL) { - _young_manager = new GCMemoryManager("Copy", "end of minor GC"); - _old_manager = new GCMemoryManager("MarkSweepCompact", "end of major GC"); + _young_manager = new GCMemoryManager("Copy"); + _old_manager = new GCMemoryManager("MarkSweepCompact"); } void SerialHeap::initialize_serviceability() { diff --git a/src/hotspot/share/gc/shared/c1/barrierSetC1.cpp b/src/hotspot/share/gc/shared/c1/barrierSetC1.cpp index 663ff91372b58..b1828fcdb178a 100644 --- a/src/hotspot/share/gc/shared/c1/barrierSetC1.cpp +++ b/src/hotspot/share/gc/shared/c1/barrierSetC1.cpp @@ -334,7 +334,7 @@ void BarrierSetC1::generate_referent_check(LIRAccess& access, LabelObj* cont) { if (gen_type_check) { // We have determined that offset == referent_offset && src != null. // if (src->_klass->_reference_type == REF_NONE) -> continue - __ move(new LIR_Address(base_reg, oopDesc::klass_offset_in_bytes(), T_ADDRESS), src_klass); + gen->load_klass(base_reg, src_klass, NULL); LIR_Address* reference_type_addr = new LIR_Address(src_klass, in_bytes(InstanceKlass::reference_type_offset()), T_BYTE); LIR_Opr reference_type = gen->new_register(T_INT); __ move(reference_type_addr, reference_type); diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp index b64a4bf9b9884..d5d4e57596aab 100644 --- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp +++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp @@ -290,7 +290,7 @@ class BarrierSetC2: public CHeapObj { virtual void verify_gc_barriers(Compile* compile, CompilePhase phase) const {} #endif - virtual bool final_graph_reshaping(Compile* compile, Node* n, uint opcode) const { return false; } + virtual bool final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const { return false; } virtual bool escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const { return false; } virtual bool escape_add_final_edges(ConnectionGraph* conn_graph, PhaseGVN* gvn, Node* n, uint opcode) const { return false; } diff --git a/src/hotspot/share/gc/shared/collectedHeap.cpp b/src/hotspot/share/gc/shared/collectedHeap.cpp index 96b3eb94681b3..2f90992f6ee1e 100644 --- a/src/hotspot/share/gc/shared/collectedHeap.cpp +++ b/src/hotspot/share/gc/shared/collectedHeap.cpp @@ -110,6 +110,18 @@ void GCHeapLog::log_heap(CollectedHeap* heap, bool before) { st.print_cr("}"); } +ParallelObjectIterator::ParallelObjectIterator(uint thread_num) : + _impl(Universe::heap()->parallel_object_iterator(thread_num)) +{} + +ParallelObjectIterator::~ParallelObjectIterator() { + delete _impl; +} + +void ParallelObjectIterator::object_iterate(ObjectClosure* cl, uint worker_id) { + _impl->object_iterate(cl, worker_id); +} + size_t CollectedHeap::unused() const { MutexLocker ml(Heap_lock); return capacity() - used(); @@ -371,17 +383,6 @@ void CollectedHeap::set_gc_cause(GCCause::Cause v) { _gc_cause = v; } -#ifndef PRODUCT -void CollectedHeap::check_for_non_bad_heap_word_value(HeapWord* addr, size_t size) { - if (CheckMemoryInitialization && ZapUnusedHeapArea) { - // please note mismatch between size (in 32/64 bit words), and ju_addr that always point to a 32 bit word - for (juint* ju_addr = reinterpret_cast(addr); ju_addr < reinterpret_cast(addr + size); ++ju_addr) { - assert(*ju_addr == badHeapWordVal, "Found non badHeapWordValue in pre-allocation check"); - } - } -} -#endif // PRODUCT - size_t CollectedHeap::max_tlab_size() const { // TLABs can't be bigger than we can fill with a int[Integer.MAX_VALUE]. // This restriction could be removed by enabling filling with multiple arrays. diff --git a/src/hotspot/share/gc/shared/collectedHeap.hpp b/src/hotspot/share/gc/shared/collectedHeap.hpp index e4d444b8c95fc..cf1a6b6ff316b 100644 --- a/src/hotspot/share/gc/shared/collectedHeap.hpp +++ b/src/hotspot/share/gc/shared/collectedHeap.hpp @@ -62,10 +62,23 @@ class VirtualSpaceSummary; class WorkGang; class nmethod; -class ParallelObjectIterator : public CHeapObj { +class ParallelObjectIteratorImpl : public CHeapObj { public: + virtual ~ParallelObjectIteratorImpl() {} virtual void object_iterate(ObjectClosure* cl, uint worker_id) = 0; - virtual ~ParallelObjectIterator() {} +}; + +// User facing parallel object iterator. This is a StackObj, which ensures that +// the _impl is allocated and deleted in the scope of this object. This ensures +// the life cycle of the implementation is as required by ThreadsListHandle, +// which is sometimes used by the root iterators. +class ParallelObjectIterator : public StackObj { + ParallelObjectIteratorImpl* _impl; + +public: + ParallelObjectIterator(uint thread_num); + ~ParallelObjectIterator(); + void object_iterate(ObjectClosure* cl, uint worker_id); }; // @@ -82,6 +95,7 @@ class CollectedHeap : public CHeapObj { friend class JVMCIVMStructs; friend class IsGCActiveMark; // Block structured external access to _is_gc_active friend class MemAllocator; + friend class ParallelObjectIterator; private: GCHeapLog* _gc_heap_log; @@ -156,8 +170,6 @@ class CollectedHeap : public CHeapObj { virtual void trace_heap(GCWhen::Type when, const GCTracer* tracer); // Verification functions - virtual void check_for_non_bad_heap_word_value(HeapWord* addr, size_t size) - PRODUCT_RETURN; debug_only(static void check_for_valid_allocation_state();) public: @@ -384,10 +396,12 @@ class CollectedHeap : public CHeapObj { // Iterate over all objects, calling "cl.do_object" on each. virtual void object_iterate(ObjectClosure* cl) = 0; - virtual ParallelObjectIterator* parallel_object_iterator(uint thread_num) { + protected: + virtual ParallelObjectIteratorImpl* parallel_object_iterator(uint thread_num) { return NULL; } + public: // Keep alive an object that was loaded with AS_NO_KEEPALIVE. virtual void keep_alive(oop obj) {} diff --git a/src/hotspot/share/gc/shared/gcTrace.cpp b/src/hotspot/share/gc/shared/gcTrace.cpp index aeddbee841c7b..126cb464b8e15 100644 --- a/src/hotspot/share/gc/shared/gcTrace.cpp +++ b/src/hotspot/share/gc/shared/gcTrace.cpp @@ -92,7 +92,7 @@ class ObjectCountEventSenderClosure : public KlassInfoClosure { } }; -void GCTracer::report_object_count_after_gc(BoolObjectClosure* is_alive_cl) { +void GCTracer::report_object_count_after_gc(BoolObjectClosure* is_alive_cl, WorkGang* workers) { assert(is_alive_cl != NULL, "Must supply function to check liveness"); if (ObjectCountEventSender::should_send_event()) { @@ -101,7 +101,7 @@ void GCTracer::report_object_count_after_gc(BoolObjectClosure* is_alive_cl) { KlassInfoTable cit(false); if (!cit.allocation_failed()) { HeapInspection hi; - hi.populate_table(&cit, is_alive_cl); + hi.populate_table(&cit, is_alive_cl, workers); ObjectCountEventSenderClosure event_sender(cit.size_of_instances_in_words(), Ticks::now()); cit.iterate(&event_sender); } diff --git a/src/hotspot/share/gc/shared/gcTrace.hpp b/src/hotspot/share/gc/shared/gcTrace.hpp index 75273418cff2a..6a00b7c71bea0 100644 --- a/src/hotspot/share/gc/shared/gcTrace.hpp +++ b/src/hotspot/share/gc/shared/gcTrace.hpp @@ -30,6 +30,7 @@ #include "gc/shared/gcId.hpp" #include "gc/shared/gcName.hpp" #include "gc/shared/gcWhen.hpp" +#include "gc/shared/workgroup.hpp" #include "memory/metaspace.hpp" #include "memory/referenceType.hpp" #include "utilities/macros.hpp" @@ -101,7 +102,7 @@ class GCTracer : public ResourceObj { void report_gc_heap_summary(GCWhen::Type when, const GCHeapSummary& heap_summary) const; void report_metaspace_summary(GCWhen::Type when, const MetaspaceSummary& metaspace_summary) const; void report_gc_reference_stats(const ReferenceProcessorStats& rp) const; - void report_object_count_after_gc(BoolObjectClosure* object_filter) NOT_SERVICES_RETURN; + void report_object_count_after_gc(BoolObjectClosure* object_filter, WorkGang* workers) NOT_SERVICES_RETURN; protected: GCTracer(GCName name) : _shared_gc_info(name) {} diff --git a/src/hotspot/share/gc/shared/gcVMOperations.cpp b/src/hotspot/share/gc/shared/gcVMOperations.cpp index f73974981c9cf..711565c4ecfc6 100644 --- a/src/hotspot/share/gc/shared/gcVMOperations.cpp +++ b/src/hotspot/share/gc/shared/gcVMOperations.cpp @@ -166,7 +166,16 @@ void VM_GC_HeapInspection::doit() { } } HeapInspection inspect; - inspect.heap_inspection(_out, _parallel_thread_num); + WorkGang* workers = Universe::heap()->safepoint_workers(); + if (workers != nullptr) { + // The GC provided a WorkGang to be used during a safepoint. + // Can't run with more threads than provided by the WorkGang. + const uint capped_parallel_thread_num = MIN2(_parallel_thread_num, workers->total_workers()); + WithUpdatedActiveWorkers with_active_workers(workers, capped_parallel_thread_num); + inspect.heap_inspection(_out, workers); + } else { + inspect.heap_inspection(_out, nullptr); + } } diff --git a/src/hotspot/share/gc/shared/genCollectedHeap.cpp b/src/hotspot/share/gc/shared/genCollectedHeap.cpp index 528aacf3b5b9d..411c1dc06ba7d 100644 --- a/src/hotspot/share/gc/shared/genCollectedHeap.cpp +++ b/src/hotspot/share/gc/shared/genCollectedHeap.cpp @@ -444,7 +444,7 @@ void GenCollectedHeap::collect_generation(Generation* gen, bool full, size_t siz FormatBuffer<> title("Collect gen: %s", gen->short_name()); GCTraceTime(Trace, gc, phases) t1(title); TraceCollectorStats tcs(gen->counters()); - TraceMemoryManagerStats tmms(gen->gc_manager(), gc_cause()); + TraceMemoryManagerStats tmms(gen->gc_manager(), gc_cause(), heap()->is_young_gen(gen) ? "end of minor GC" : "end of major GC"); gen->stat_record()->invocations++; gen->stat_record()->accumulated_time.start(); diff --git a/src/hotspot/share/gc/shared/memAllocator.cpp b/src/hotspot/share/gc/shared/memAllocator.cpp index a23a2e70bf6a0..a68a3604a6a20 100644 --- a/src/hotspot/share/gc/shared/memAllocator.cpp +++ b/src/hotspot/share/gc/shared/memAllocator.cpp @@ -59,7 +59,6 @@ class MemAllocator::Allocation: StackObj { void notify_allocation_low_memory_detector(); void notify_allocation_jfr_sampler(); void notify_allocation_dtrace_sampler(); - void check_for_bad_heap_word_value() const; #ifdef ASSERT void check_for_valid_allocation_state() const; #endif @@ -81,7 +80,6 @@ class MemAllocator::Allocation: StackObj { ~Allocation() { if (!check_out_of_memory()) { - verify_after(); notify_allocation(); } } @@ -148,22 +146,6 @@ void MemAllocator::Allocation::verify_before() { assert(!Universe::heap()->is_gc_active(), "Allocation during gc not allowed"); } -void MemAllocator::Allocation::verify_after() { - NOT_PRODUCT(check_for_bad_heap_word_value();) -} - -void MemAllocator::Allocation::check_for_bad_heap_word_value() const { - MemRegion obj_range = _allocator.obj_memory_range(obj()); - HeapWord* addr = obj_range.start(); - size_t size = obj_range.word_size(); - if (CheckMemoryInitialization && ZapUnusedHeapArea) { - for (size_t slot = 0; slot < size; slot += 1) { - assert((*(intptr_t*) (addr + slot)) != ((intptr_t) badHeapWordVal), - "Found badHeapWordValue in post-allocation check"); - } - } -} - #ifdef ASSERT void MemAllocator::Allocation::check_for_valid_allocation_state() const { // How to choose between a pending exception and a potential @@ -258,7 +240,6 @@ HeapWord* MemAllocator::allocate_outside_tlab(Allocation& allocation) const { return mem; } - NOT_PRODUCT(Universe::heap()->check_for_non_bad_heap_word_value(mem, _word_size)); size_t size_in_bytes = _word_size * HeapWordSize; _thread->incr_allocated_bytes(size_in_bytes); @@ -400,15 +381,6 @@ oop ObjAllocator::initialize(HeapWord* mem) const { return finish(mem); } -MemRegion ObjArrayAllocator::obj_memory_range(oop obj) const { - if (_do_zero) { - return MemAllocator::obj_memory_range(obj); - } - ArrayKlass* array_klass = ArrayKlass::cast(_klass); - const size_t hs = arrayOopDesc::header_size(array_klass->element_type()); - return MemRegion(cast_from_oop(obj) + hs, _word_size - hs); -} - oop ObjArrayAllocator::initialize(HeapWord* mem) const { // Set array length before setting the _klass field because a // non-NULL klass field indicates that the object is parsable by diff --git a/src/hotspot/share/gc/shared/memAllocator.hpp b/src/hotspot/share/gc/shared/memAllocator.hpp index 6df76dca2ff15..2c93114135fa9 100644 --- a/src/hotspot/share/gc/shared/memAllocator.hpp +++ b/src/hotspot/share/gc/shared/memAllocator.hpp @@ -66,10 +66,6 @@ class MemAllocator: StackObj { // back to calling CollectedHeap::mem_allocate(). HeapWord* mem_allocate(Allocation& allocation) const; - virtual MemRegion obj_memory_range(oop obj) const { - return MemRegion(cast_from_oop(obj), _word_size); - } - public: oop allocate() const; virtual oop initialize(HeapWord* mem) const = 0; @@ -85,8 +81,6 @@ class ObjAllocator: public MemAllocator { class ObjArrayAllocator: public MemAllocator { const int _length; const bool _do_zero; -protected: - virtual MemRegion obj_memory_range(oop obj) const; public: ObjArrayAllocator(Klass* klass, size_t word_size, int length, bool do_zero, diff --git a/src/hotspot/share/gc/shared/oopStorage.cpp b/src/hotspot/share/gc/shared/oopStorage.cpp index 4869b6f03b5c1..e06ac1bf6be7d 100644 --- a/src/hotspot/share/gc/shared/oopStorage.cpp +++ b/src/hotspot/share/gc/shared/oopStorage.cpp @@ -36,7 +36,7 @@ #include "runtime/mutexLocker.hpp" #include "runtime/orderAccess.hpp" #include "runtime/os.hpp" -#include "runtime/safefetch.inline.hpp" +#include "runtime/safefetch.hpp" #include "runtime/safepoint.hpp" #include "runtime/thread.hpp" #include "services/memTracker.hpp" @@ -300,7 +300,6 @@ void OopStorage::Block::set_active_index(size_t index) { size_t OopStorage::Block::active_index_safe(const Block* block) { STATIC_ASSERT(sizeof(intptr_t) == sizeof(block->_active_index)); - assert(CanUseSafeFetchN(), "precondition"); return SafeFetchN((intptr_t*)&block->_active_index, 0); } @@ -366,7 +365,6 @@ void OopStorage::Block::delete_block(const Block& block) { // require additional validation of the result. OopStorage::Block* OopStorage::Block::block_for_ptr(const OopStorage* owner, const oop* ptr) { - assert(CanUseSafeFetchN(), "precondition"); STATIC_ASSERT(_data_pos == 0); // Const-ness of ptr is not related to const-ness of containing block. // Blocks are allocated section-aligned, so get the containing section. diff --git a/src/hotspot/share/gc/shared/referenceProcessor.cpp b/src/hotspot/share/gc/shared/referenceProcessor.cpp index 0db8cfee6cdc4..c32a41be797b5 100644 --- a/src/hotspot/share/gc/shared/referenceProcessor.cpp +++ b/src/hotspot/share/gc/shared/referenceProcessor.cpp @@ -245,6 +245,12 @@ ReferenceProcessorStats ReferenceProcessor::process_discovered_references(RefPro return stats; } +void BarrierEnqueueDiscoveredFieldClosure::enqueue(HeapWord* discovered_field_addr, oop value) { + assert(Universe::heap()->is_in(discovered_field_addr), PTR_FORMAT " not in heap", p2i(discovered_field_addr)); + HeapAccess::oop_store(discovered_field_addr, + value); +} + void DiscoveredListIterator::load_ptrs(DEBUG_ONLY(bool allow_null_referent)) { _current_discovered_addr = java_lang_ref_Reference::discovered_addr_raw(_current_discovered); oop discovered = java_lang_ref_Reference::discovered(_current_discovered); @@ -304,12 +310,12 @@ void DiscoveredListIterator::enqueue() { } void DiscoveredListIterator::complete_enqueue() { - if (_prev_discovered != NULL) { + if (_prev_discovered != nullptr) { // This is the last object. // Swap refs_list into pending list and set obj's // discovered to what we read from the pending list. oop old = Universe::swap_reference_pending_list(_refs_list.head()); - HeapAccess::oop_store_at(_prev_discovered, java_lang_ref_Reference::discovered_offset(), old); + _enqueue->enqueue(java_lang_ref_Reference::discovered_addr_raw(_prev_discovered), old); } } @@ -337,7 +343,7 @@ size_t ReferenceProcessor::process_soft_ref_reconsider_work(DiscoveredList& r OopClosure* keep_alive, VoidClosure* complete_gc) { assert(policy != NULL, "Must have a non-NULL policy"); - DiscoveredListIterator iter(refs_list, keep_alive, is_alive); + DiscoveredListIterator iter(refs_list, keep_alive, is_alive, NULL /* enqueue */); // Decide which softly reachable refs should be kept alive. while (iter.has_next()) { iter.load_ptrs(DEBUG_ONLY(!discovery_is_atomic() /* allow_null_referent */)); @@ -365,8 +371,9 @@ size_t ReferenceProcessor::process_soft_ref_reconsider_work(DiscoveredList& r size_t ReferenceProcessor::process_soft_weak_final_refs_work(DiscoveredList& refs_list, BoolObjectClosure* is_alive, OopClosure* keep_alive, + EnqueueDiscoveredFieldClosure* enqueue, bool do_enqueue_and_clear) { - DiscoveredListIterator iter(refs_list, keep_alive, is_alive); + DiscoveredListIterator iter(refs_list, keep_alive, is_alive, enqueue); while (iter.has_next()) { iter.load_ptrs(DEBUG_ONLY(!discovery_is_atomic() /* allow_null_referent */)); if (iter.referent() == NULL) { @@ -409,8 +416,9 @@ size_t ReferenceProcessor::process_soft_weak_final_refs_work(DiscoveredList& size_t ReferenceProcessor::process_final_keep_alive_work(DiscoveredList& refs_list, OopClosure* keep_alive, - VoidClosure* complete_gc) { - DiscoveredListIterator iter(refs_list, keep_alive, NULL); + VoidClosure* complete_gc, + EnqueueDiscoveredFieldClosure* enqueue) { + DiscoveredListIterator iter(refs_list, keep_alive, NULL, enqueue); while (iter.has_next()) { iter.load_ptrs(DEBUG_ONLY(false /* allow_null_referent */)); // keep the referent and followers around @@ -436,8 +444,9 @@ size_t ReferenceProcessor::process_final_keep_alive_work(DiscoveredList& refs_li size_t ReferenceProcessor::process_phantom_refs_work(DiscoveredList& refs_list, BoolObjectClosure* is_alive, OopClosure* keep_alive, - VoidClosure* complete_gc) { - DiscoveredListIterator iter(refs_list, keep_alive, is_alive); + VoidClosure* complete_gc, + EnqueueDiscoveredFieldClosure* enqueue) { + DiscoveredListIterator iter(refs_list, keep_alive, is_alive, enqueue); while (iter.has_next()) { iter.load_ptrs(DEBUG_ONLY(!discovery_is_atomic() /* allow_null_referent */)); @@ -509,8 +518,6 @@ size_t ReferenceProcessor::total_reference_count(ReferenceType type) const { return total_count(list); } - - class RefProcPhase1Task : public RefProcTask { public: RefProcPhase1Task(ReferenceProcessor& ref_processor, @@ -523,6 +530,7 @@ class RefProcPhase1Task : public RefProcTask { void rp_work(uint worker_id, BoolObjectClosure* is_alive, OopClosure* keep_alive, + EnqueueDiscoveredFieldClosure* enqueue, VoidClosure* complete_gc) override { ResourceMark rm; RefProcSubPhasesWorkerTimeTracker tt(ReferenceProcessor::SoftRefSubPhase1, _phase_times, worker_id); @@ -543,11 +551,13 @@ class RefProcPhase2Task: public RefProcTask { DiscoveredList list[], BoolObjectClosure* is_alive, OopClosure* keep_alive, + EnqueueDiscoveredFieldClosure* enqueue, bool do_enqueue_and_clear, ReferenceType ref_type) { size_t const removed = _ref_processor.process_soft_weak_final_refs_work(list[worker_id], is_alive, keep_alive, + enqueue, do_enqueue_and_clear); _phase_times->add_ref_cleared(ref_type, removed); } @@ -561,20 +571,21 @@ class RefProcPhase2Task: public RefProcTask { void rp_work(uint worker_id, BoolObjectClosure* is_alive, OopClosure* keep_alive, + EnqueueDiscoveredFieldClosure* enqueue, VoidClosure* complete_gc) override { ResourceMark rm; RefProcWorkerTimeTracker t(_phase_times->phase2_worker_time_sec(), worker_id); { RefProcSubPhasesWorkerTimeTracker tt(ReferenceProcessor::SoftRefSubPhase2, _phase_times, worker_id); - run_phase2(worker_id, _ref_processor._discoveredSoftRefs, is_alive, keep_alive, true /* do_enqueue_and_clear */, REF_SOFT); + run_phase2(worker_id, _ref_processor._discoveredSoftRefs, is_alive, keep_alive, enqueue, true /* do_enqueue_and_clear */, REF_SOFT); } { RefProcSubPhasesWorkerTimeTracker tt(ReferenceProcessor::WeakRefSubPhase2, _phase_times, worker_id); - run_phase2(worker_id, _ref_processor._discoveredWeakRefs, is_alive, keep_alive, true /* do_enqueue_and_clear */, REF_WEAK); + run_phase2(worker_id, _ref_processor._discoveredWeakRefs, is_alive, keep_alive, enqueue, true /* do_enqueue_and_clear */, REF_WEAK); } { RefProcSubPhasesWorkerTimeTracker tt(ReferenceProcessor::FinalRefSubPhase2, _phase_times, worker_id); - run_phase2(worker_id, _ref_processor._discoveredFinalRefs, is_alive, keep_alive, false /* do_enqueue_and_clear */, REF_FINAL); + run_phase2(worker_id, _ref_processor._discoveredFinalRefs, is_alive, keep_alive, enqueue, false /* do_enqueue_and_clear */, REF_FINAL); } // Close the reachable set; needed for collectors which keep_alive_closure do // not immediately complete their work. @@ -592,10 +603,11 @@ class RefProcPhase3Task: public RefProcTask { void rp_work(uint worker_id, BoolObjectClosure* is_alive, OopClosure* keep_alive, + EnqueueDiscoveredFieldClosure* enqueue, VoidClosure* complete_gc) override { ResourceMark rm; RefProcSubPhasesWorkerTimeTracker tt(ReferenceProcessor::FinalRefSubPhase3, _phase_times, worker_id); - _ref_processor.process_final_keep_alive_work(_ref_processor._discoveredFinalRefs[worker_id], keep_alive, complete_gc); + _ref_processor.process_final_keep_alive_work(_ref_processor._discoveredFinalRefs[worker_id], keep_alive, complete_gc, enqueue); } }; @@ -609,13 +621,15 @@ class RefProcPhase4Task: public RefProcTask { void rp_work(uint worker_id, BoolObjectClosure* is_alive, OopClosure* keep_alive, + EnqueueDiscoveredFieldClosure* enqueue, VoidClosure* complete_gc) override { ResourceMark rm; RefProcSubPhasesWorkerTimeTracker tt(ReferenceProcessor::PhantomRefSubPhase4, _phase_times, worker_id); size_t const removed = _ref_processor.process_phantom_refs_work(_ref_processor._discoveredPhantomRefs[worker_id], is_alive, keep_alive, - complete_gc); + complete_gc, + enqueue); _phase_times->add_ref_cleared(REF_PHANTOM, removed); } }; @@ -1259,7 +1273,7 @@ bool ReferenceProcessor::preclean_discovered_reflist(DiscoveredList& refs_lis OopClosure* keep_alive, VoidClosure* complete_gc, YieldClosure* yield) { - DiscoveredListIterator iter(refs_list, keep_alive, is_alive); + DiscoveredListIterator iter(refs_list, keep_alive, is_alive, NULL /* enqueue */); while (iter.has_next()) { if (yield->should_return_fine_grain()) { return true; diff --git a/src/hotspot/share/gc/shared/referenceProcessor.hpp b/src/hotspot/share/gc/shared/referenceProcessor.hpp index b90beee0057f6..0d303a0a1739e 100644 --- a/src/hotspot/share/gc/shared/referenceProcessor.hpp +++ b/src/hotspot/share/gc/shared/referenceProcessor.hpp @@ -38,6 +38,28 @@ class ReferenceProcessorPhaseTimes; class RefProcTask; class RefProcProxyTask; +// Provides a callback to the garbage collector to set the given value to the +// discovered field of the j.l.ref.Reference instance. This is called during STW +// reference processing when iterating over the discovered lists for all +// discovered references. +// Typically garbage collectors may just call the barrier, but for some garbage +// collectors the barrier environment (e.g. card table) may not be set up correctly +// at the point of invocation. +class EnqueueDiscoveredFieldClosure { +public: + // For the given j.l.ref.Reference discovered field address, set the discovered + // field to value and apply any barriers to it. + virtual void enqueue(HeapWord* discovered_field_addr, oop value) = 0; + +}; + +// EnqueueDiscoveredFieldClosure that executes the default barrier on the discovered +// field of the j.l.ref.Reference with the given value. +class BarrierEnqueueDiscoveredFieldClosure : public EnqueueDiscoveredFieldClosure { +public: + void enqueue(HeapWord* discovered_field_addr, oop value) override; +}; + // List of discovered references. class DiscoveredList { public: @@ -66,7 +88,6 @@ class DiscoveredList { // Iterator for the list of discovered references. class DiscoveredListIterator { -private: DiscoveredList& _refs_list; HeapWord* _prev_discovered_addr; oop _prev_discovered; @@ -78,6 +99,7 @@ class DiscoveredListIterator { OopClosure* _keep_alive; BoolObjectClosure* _is_alive; + EnqueueDiscoveredFieldClosure* _enqueue; DEBUG_ONLY( oop _first_seen; // cyclic linked list check @@ -89,7 +111,8 @@ class DiscoveredListIterator { public: inline DiscoveredListIterator(DiscoveredList& refs_list, OopClosure* keep_alive, - BoolObjectClosure* is_alive); + BoolObjectClosure* is_alive, + EnqueueDiscoveredFieldClosure* enqueue); // End Of List. inline bool has_next() const { return _current_discovered != NULL; } @@ -274,18 +297,21 @@ class ReferenceProcessor : public ReferenceDiscoverer { size_t process_soft_weak_final_refs_work(DiscoveredList& refs_list, BoolObjectClosure* is_alive, OopClosure* keep_alive, + EnqueueDiscoveredFieldClosure* enqueue, bool do_enqueue_and_clear); // Keep alive followers of referents for FinalReferences. Must only be called for // those. size_t process_final_keep_alive_work(DiscoveredList& refs_list, OopClosure* keep_alive, - VoidClosure* complete_gc); + VoidClosure* complete_gc, + EnqueueDiscoveredFieldClosure* enqueue); size_t process_phantom_refs_work(DiscoveredList& refs_list, BoolObjectClosure* is_alive, OopClosure* keep_alive, - VoidClosure* complete_gc); + VoidClosure* complete_gc, + EnqueueDiscoveredFieldClosure* enqueue); public: static int number_of_subclasses_of_ref() { return (REF_PHANTOM - REF_OTHER); } @@ -610,6 +636,7 @@ class RefProcTask : StackObj { virtual void rp_work(uint worker_id, BoolObjectClosure* is_alive, OopClosure* keep_alive, + EnqueueDiscoveredFieldClosure* enqueue, VoidClosure* complete_gc) = 0; }; diff --git a/src/hotspot/share/gc/shared/referenceProcessor.inline.hpp b/src/hotspot/share/gc/shared/referenceProcessor.inline.hpp index 85d83298c43be..67b6c81007459 100644 --- a/src/hotspot/share/gc/shared/referenceProcessor.inline.hpp +++ b/src/hotspot/share/gc/shared/referenceProcessor.inline.hpp @@ -60,7 +60,8 @@ void DiscoveredList::clear() { DiscoveredListIterator::DiscoveredListIterator(DiscoveredList& refs_list, OopClosure* keep_alive, - BoolObjectClosure* is_alive): + BoolObjectClosure* is_alive, + EnqueueDiscoveredFieldClosure* enqueue): _refs_list(refs_list), _prev_discovered_addr(refs_list.adr_head()), _prev_discovered(NULL), @@ -70,6 +71,7 @@ DiscoveredListIterator::DiscoveredListIterator(DiscoveredList& refs_list, _referent(NULL), _keep_alive(keep_alive), _is_alive(is_alive), + _enqueue(enqueue), #ifdef ASSERT _first_seen(refs_list.head()), #endif diff --git a/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp b/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp index 8bdcbec6f72f3..9d9a61572ca81 100644 --- a/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp +++ b/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp @@ -213,9 +213,11 @@ void ThreadLocalAllocBuffer::initialize() { set_desired_size(initial_desired_size()); size_t capacity = Universe::heap()->tlab_capacity(thread()) / HeapWordSize; - // Keep alloc_frac as float and not double to avoid the double to float conversion - float alloc_frac = desired_size() * target_refills() / (float) capacity; - _allocation_fraction.sample(alloc_frac); + if (capacity > 0) { + // Keep alloc_frac as float and not double to avoid the double to float conversion + float alloc_frac = desired_size() * target_refills() / (float)capacity; + _allocation_fraction.sample(alloc_frac); + } set_refill_waste_limit(initial_refill_waste_limit()); diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp index b18e4c310d70c..c10fcf18f9b42 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp @@ -723,7 +723,7 @@ bool ShenandoahBarrierSetC2::is_gc_pre_barrier_node(Node* node) const { // Support for GC barriers emitted during parsing bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const { - if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) return true; + if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier || node->Opcode() == Op_ShenandoahIUBarrier) return true; if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) { return false; } @@ -1124,7 +1124,7 @@ bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) { return n->outcnt() > 0; } -bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const { +bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const { switch (opcode) { case Op_CallLeaf: case Op_CallLeafNoFP: { diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp index 656b701f278fa..add752a413a04 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp @@ -139,7 +139,7 @@ class ShenandoahBarrierSetC2 : public BarrierSetC2 { #endif virtual Node* ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const; - virtual bool final_graph_reshaping(Compile* compile, Node* n, uint opcode) const; + virtual bool final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const; virtual bool escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const; virtual bool escape_add_final_edges(ConnectionGraph* conn_graph, PhaseGVN* gvn, Node* n, uint opcode) const; diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp index 2867cbd2a4ef7..ef848e7ba9548 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp @@ -54,7 +54,6 @@ bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) { PhaseIdealLoop::optimize(igvn, LoopOptsShenandoahExpand); if (C->failing()) return false; PhaseIdealLoop::verify(igvn); - DEBUG_ONLY(verify_raw_mem(C->root());) if (attempt_more_loopopts) { C->set_major_progress(); if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) { @@ -964,18 +963,11 @@ void ShenandoahBarrierC2Support::test_in_cset(Node*& ctrl, Node*& not_cset_ctrl, phase->register_new_node(cset_bool, old_ctrl); } -void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, +void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, DecoratorSet decorators, PhaseIdealLoop* phase) { IdealLoopTree*loop = phase->get_loop(ctrl); const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr(); - // The slow path stub consumes and produces raw memory in addition - // to the existing memory edges - Node* base = find_bottom_mem(ctrl, phase); - MergeMemNode* mm = MergeMemNode::make(base); - mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); - phase->register_new_node(mm, ctrl); - address calladdr = NULL; const char* name = NULL; bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators); @@ -1013,7 +1005,7 @@ void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* lo call->init_req(TypeFunc::Control, ctrl); call->init_req(TypeFunc::I_O, phase->C->top()); - call->init_req(TypeFunc::Memory, mm); + call->init_req(TypeFunc::Memory, phase->C->top()); call->init_req(TypeFunc::FramePtr, phase->C->top()); call->init_req(TypeFunc::ReturnAdr, phase->C->top()); call->init_req(TypeFunc::Parms, val); @@ -1021,8 +1013,6 @@ void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* lo phase->register_control(call, loop, ctrl); ctrl = new ProjNode(call, TypeFunc::Control); phase->register_control(ctrl, loop, call); - result_mem = new ProjNode(call, TypeFunc::Memory); - phase->register_new_node(result_mem, call); val = new ProjNode(call, TypeFunc::Parms); phase->register_new_node(val, call); val = new CheckCastPPNode(ctrl, val, obj_type); @@ -1341,11 +1331,9 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) { Node* ctrl = phase->get_ctrl(lrb); Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn); - Node* orig_ctrl = ctrl; Node* raw_mem = fixer.find_mem(ctrl, lrb); - Node* init_raw_mem = raw_mem; Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL); IdealLoopTree *loop = phase->get_loop(ctrl); @@ -1359,7 +1347,6 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) { enum { _heap_stable = 1, _evac_path, _not_cset, PATH_LIMIT }; Node* region = new RegionNode(PATH_LIMIT); Node* val_phi = new PhiNode(region, val->bottom_type()->is_oopptr()); - Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); // Stable path. int flags = ShenandoahHeap::HAS_FORWARDED; @@ -1372,7 +1359,6 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) { // Heap stable case region->init_req(_heap_stable, heap_stable_ctrl); val_phi->init_req(_heap_stable, val); - raw_mem_phi->init_req(_heap_stable, raw_mem); // Test for in-cset, unless it's a native-LRB. Native LRBs need to return NULL // even for non-cset objects to prevent ressurrection of such objects. @@ -1384,11 +1370,9 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) { if (not_cset_ctrl != NULL) { region->init_req(_not_cset, not_cset_ctrl); val_phi->init_req(_not_cset, val); - raw_mem_phi->init_req(_not_cset, raw_mem); } else { region->del_req(_not_cset); val_phi->del_req(_not_cset); - raw_mem_phi->del_req(_not_cset); } // Resolve object when orig-value is in cset. @@ -1429,15 +1413,13 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) { } } } - call_lrb_stub(ctrl, val, addr, result_mem, raw_mem, lrb->decorators(), phase); + call_lrb_stub(ctrl, val, addr, lrb->decorators(), phase); region->init_req(_evac_path, ctrl); val_phi->init_req(_evac_path, val); - raw_mem_phi->init_req(_evac_path, result_mem); phase->register_control(region, loop, heap_stable_iff); Node* out_val = val_phi; phase->register_new_node(val_phi, region); - phase->register_new_node(raw_mem_phi, region); fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase); @@ -1450,18 +1432,11 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) { for(uint next = 0; next < uses.size(); next++ ) { Node *n = uses.at(next); assert(phase->get_ctrl(n) == ctrl, "bad control"); - assert(n != init_raw_mem, "should leave input raw mem above the barrier"); + assert(n != raw_mem, "should leave input raw mem above the barrier"); phase->set_ctrl(n, region); follow_barrier_uses(n, ctrl, uses, phase); } - - // The slow path call produces memory: hook the raw memory phi - // from the expanded load reference barrier with the rest of the graph - // which may require adding memory phis at every post dominated - // region and at enclosing loop heads. Use the memory state - // collected in memory_nodes to fix the memory graph. Update that - // memory state as we go. - fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses); + fixer.record_new_ctrl(ctrl, region, raw_mem, raw_mem_for_ctrl); } // Done expanding load-reference-barriers. assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced"); @@ -1902,105 +1877,6 @@ void ShenandoahBarrierC2Support::optimize_after_expansion(VectorSet &visited, No } } -#ifdef ASSERT -void ShenandoahBarrierC2Support::verify_raw_mem(RootNode* root) { - const bool trace = false; - ResourceMark rm; - Unique_Node_List nodes; - Unique_Node_List controls; - Unique_Node_List memories; - - nodes.push(root); - for (uint next = 0; next < nodes.size(); next++) { - Node *n = nodes.at(next); - if (ShenandoahBarrierSetC2::is_shenandoah_lrb_call(n)) { - controls.push(n); - if (trace) { tty->print("XXXXXX verifying"); n->dump(); } - for (uint next2 = 0; next2 < controls.size(); next2++) { - Node *m = controls.at(next2); - for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { - Node* u = m->fast_out(i); - if (u->is_CFG() && !u->is_Root() && - !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) && - !(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) { - if (trace) { tty->print("XXXXXX pushing control"); u->dump(); } - controls.push(u); - } - } - } - memories.push(n->as_Call()->proj_out(TypeFunc::Memory)); - for (uint next2 = 0; next2 < memories.size(); next2++) { - Node *m = memories.at(next2); - assert(m->bottom_type() == Type::MEMORY, ""); - for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { - Node* u = m->fast_out(i); - if (u->bottom_type() == Type::MEMORY && (u->is_Mem() || u->is_ClearArray())) { - if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } - memories.push(u); - } else if (u->is_LoadStore()) { - if (trace) { tty->print("XXXXXX pushing memory"); u->find_out_with(Op_SCMemProj)->dump(); } - memories.push(u->find_out_with(Op_SCMemProj)); - } else if (u->is_MergeMem() && u->as_MergeMem()->memory_at(Compile::AliasIdxRaw) == m) { - if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } - memories.push(u); - } else if (u->is_Phi()) { - assert(u->bottom_type() == Type::MEMORY, ""); - if (u->adr_type() == TypeRawPtr::BOTTOM || u->adr_type() == TypePtr::BOTTOM) { - assert(controls.member(u->in(0)), ""); - if (trace) { tty->print("XXXXXX pushing memory"); u->dump(); } - memories.push(u); - } - } else if (u->is_SafePoint() || u->is_MemBar()) { - for (DUIterator_Fast jmax, j = u->fast_outs(jmax); j < jmax; j++) { - Node* uu = u->fast_out(j); - if (uu->bottom_type() == Type::MEMORY) { - if (trace) { tty->print("XXXXXX pushing memory"); uu->dump(); } - memories.push(uu); - } - } - } - } - } - for (uint next2 = 0; next2 < controls.size(); next2++) { - Node *m = controls.at(next2); - if (m->is_Region()) { - bool all_in = true; - for (uint i = 1; i < m->req(); i++) { - if (!controls.member(m->in(i))) { - all_in = false; - break; - } - } - if (trace) { tty->print("XXX verifying %s", all_in ? "all in" : ""); m->dump(); } - bool found_phi = false; - for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax && !found_phi; j++) { - Node* u = m->fast_out(j); - if (u->is_Phi() && memories.member(u)) { - found_phi = true; - for (uint i = 1; i < u->req() && found_phi; i++) { - Node* k = u->in(i); - if (memories.member(k) != controls.member(m->in(i))) { - found_phi = false; - } - } - } - } - assert(found_phi || all_in, ""); - } - } - controls.clear(); - memories.clear(); - } - for( uint i = 0; i < n->len(); ++i ) { - Node *m = n->in(i); - if (m != NULL) { - nodes.push(m); - } - } - } -} -#endif - ShenandoahIUBarrierNode::ShenandoahIUBarrierNode(Node* val) : Node(NULL, val) { ShenandoahBarrierSetC2::bsc2()->state()->add_iu_barrier(this); } @@ -2796,6 +2672,13 @@ void MemoryGraphFixer::fix_mem(Node* ctrl, Node* new_ctrl, Node* mem, Node* mem_ #endif } +void MemoryGraphFixer::record_new_ctrl(Node* ctrl, Node* new_ctrl, Node* mem, Node* mem_for_ctrl) { + if (mem_for_ctrl != mem && new_ctrl != ctrl) { + _memory_nodes.map(ctrl->_idx, mem); + _memory_nodes.map(new_ctrl->_idx, mem_for_ctrl); + } +} + MergeMemNode* MemoryGraphFixer::allocate_merge_mem(Node* mem, Node* rep_proj, Node* rep_ctrl) const { MergeMemNode* mm = MergeMemNode::make(mem); mm->set_memory_at(_alias, rep_proj); diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp index 6632e42b36f07..c9404af5ff8cc 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp @@ -50,7 +50,6 @@ class ShenandoahBarrierC2Support : public AllStatic { static bool verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used); static void report_verify_failure(const char* msg, Node* n1 = NULL, Node* n2 = NULL); - static void verify_raw_mem(RootNode* root); #endif static Node* dom_mem(Node* mem, Node* ctrl, int alias, Node*& mem_ctrl, PhaseIdealLoop* phase); static Node* no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase); @@ -61,7 +60,7 @@ class ShenandoahBarrierC2Support : public AllStatic { static void test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase); static void test_gc_state(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl, PhaseIdealLoop* phase, int flags); - static void call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, + static void call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, DecoratorSet decorators, PhaseIdealLoop* phase); static void test_in_cset(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase); static void move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase); @@ -133,6 +132,8 @@ class MemoryGraphFixer : public ResourceObj { int alias() const { return _alias; } Node* collect_memory_for_infinite_loop(const Node* in); + + void record_new_ctrl(Node* ctrl, Node* region, Node* mem, Node* mem_for_ctrl); }; class ShenandoahCompareAndSwapPNode : public CompareAndSwapPNode { diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahIUMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahIUMode.cpp index 0c42824616997..cc9d8c26d44a5 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahIUMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahIUMode.cpp @@ -40,7 +40,6 @@ void ShenandoahIUMode::initialize_flags() const { FLAG_SET_DEFAULT(ClassUnloadingWithConcurrentMark, false); if (ClassUnloading) { - FLAG_SET_DEFAULT(ShenandoahSuspendibleWorkers, true); FLAG_SET_DEFAULT(VerifyBeforeExit, false); } diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp index 06ed25e59c62b..b1770fa0b71e7 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp @@ -35,7 +35,6 @@ void ShenandoahSATBMode::initialize_flags() const { if (ClassUnloading) { - FLAG_SET_DEFAULT(ShenandoahSuspendibleWorkers, true); FLAG_SET_DEFAULT(VerifyBeforeExit, false); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp index c7e0c9b0cd9d7..7d31ff02e1add 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -35,7 +35,7 @@ #include "utilities/defaultStream.hpp" void ShenandoahArguments::initialize() { -#if !(defined AARCH64 || defined AMD64 || defined IA32 || defined PPC64) +#if !(defined AARCH64 || defined AMD64 || defined IA32 || defined PPC64 || defined RISCV64) vm_exit_during_initialization("Shenandoah GC is not supported on this platform."); #endif diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp index a4353df545b95..1d4dd7b9e3e5f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp @@ -133,6 +133,10 @@ class ShenandoahBarrierSet: public BarrierSet { class AccessBarrier: public BarrierSet::AccessBarrier { typedef BarrierSet::AccessBarrier Raw; + private: + template + static void oop_store_common(T* addr, oop value); + public: // Heap oop accesses. These accessors get resolved when // IN_HEAP is set (e.g. when using the HeapAccess API), it is @@ -174,7 +178,6 @@ class ShenandoahBarrierSet: public BarrierSet { template static oop oop_atomic_xchg_not_in_heap(T* addr, oop new_value); - }; }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp index 3cb596a887d9a..02da87ae9165a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @@ -70,7 +70,7 @@ inline oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj, T* load if (load_addr != NULL && fwd != obj) { // Since we are here and we know the load address, update the reference. - ShenandoahHeap::cas_oop(fwd, load_addr, obj); + ShenandoahHeap::atomic_update_oop(fwd, load_addr, obj); } return fwd; @@ -125,7 +125,7 @@ inline oop ShenandoahBarrierSet::load_reference_barrier(DecoratorSet decorators, oop fwd = load_reference_barrier(obj); if (ShenandoahSelfFixing && load_addr != NULL && fwd != obj) { // Since we are here and we know the load address, update the reference. - ShenandoahHeap::cas_oop(fwd, load_addr, obj); + ShenandoahHeap::atomic_update_oop(fwd, load_addr, obj); } return fwd; @@ -241,7 +241,7 @@ inline oop ShenandoahBarrierSet::AccessBarrier::oop_loa template template -inline void ShenandoahBarrierSet::AccessBarrier::oop_store_not_in_heap(T* addr, oop value) { +inline void ShenandoahBarrierSet::AccessBarrier::oop_store_common(T* addr, oop value) { shenandoah_assert_marked_if(NULL, value, !CompressedOops::is_null(value) && ShenandoahHeap::heap()->is_evacuation_in_progress()); shenandoah_assert_not_in_cset_if(addr, value, value != NULL && !ShenandoahHeap::heap()->cancelled_gc()); ShenandoahBarrierSet* const bs = ShenandoahBarrierSet::barrier_set(); @@ -250,14 +250,19 @@ inline void ShenandoahBarrierSet::AccessBarrier::oop_st Raw::oop_store(addr, value); } +template +template +inline void ShenandoahBarrierSet::AccessBarrier::oop_store_not_in_heap(T* addr, oop value) { + oop_store_common(addr, value); +} + template template inline void ShenandoahBarrierSet::AccessBarrier::oop_store_in_heap(T* addr, oop value) { shenandoah_assert_not_in_cset_loc_except(addr, ShenandoahHeap::heap()->cancelled_gc()); shenandoah_assert_not_forwarded_except (addr, value, value == NULL || ShenandoahHeap::heap()->cancelled_gc() || !ShenandoahHeap::heap()->is_concurrent_mark_in_progress()); - shenandoah_assert_not_in_cset_except (addr, value, value == NULL || ShenandoahHeap::heap()->cancelled_gc() || !ShenandoahHeap::heap()->is_concurrent_mark_in_progress()); - oop_store_not_in_heap(addr, value); + oop_store_common(addr, value); } template @@ -353,7 +358,7 @@ void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) { fwd = _heap->evacuate_object(obj, thread); } assert(obj != fwd || _heap->cancelled_gc(), "must be forwarded"); - oop witness = ShenandoahHeap::cas_oop(fwd, elem_ptr, o); + ShenandoahHeap::atomic_update_oop(fwd, elem_ptr, o); obj = fwd; } if (ENQUEUE && !ctx->is_marked_strong(obj)) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp index 69785a6106021..71cafcd49dc59 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp @@ -54,7 +54,7 @@ class ShenandoahUpdateRefsForOopClosure: public BasicOopIterateClosure { fwd = _heap->evacuate_object(obj, _thread); } assert(obj != fwd || _heap->cancelled_gc(), "must be forwarded"); - ShenandoahHeap::cas_oop(fwd, p, o); + ShenandoahHeap::atomic_update_oop(fwd, p, o); obj = fwd; } if (ENQUEUE) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp index de59519d605ce..d03c029a0da44 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp @@ -161,7 +161,7 @@ void ShenandoahEvacuateUpdateRootsClosure::do_oop_work(T* p, Thread* t) { if (resolved == obj) { resolved = _heap->evacuate_object(obj, t); } - _heap->cas_oop(resolved, p, o); + ShenandoahHeap::atomic_update_oop(resolved, p, o); } } } @@ -207,7 +207,7 @@ void ShenandoahCleanUpdateWeakOopsClosure::do_oo _keep_alive->do_oop(p); } else { if (CONCURRENT) { - Atomic::cmpxchg(p, obj, oop()); + ShenandoahHeap::atomic_clear_oop(p, obj); } else { RawAccess::oop_store(p, oop()); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 4dae246302aa1..817f43e693246 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -698,13 +698,13 @@ void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) { if (!CompressedOops::is_null(obj)) { if (!_mark_context->is_marked(obj)) { shenandoah_assert_correct(p, obj); - Atomic::cmpxchg(p, obj, oop(NULL)); + ShenandoahHeap::atomic_clear_oop(p, obj); } else if (_evac_in_progress && _heap->in_collection_set(obj)) { oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); if (resolved == obj) { resolved = _heap->evacuate_object(obj, _thread); } - Atomic::cmpxchg(p, obj, resolved); + ShenandoahHeap::atomic_update_oop(resolved, p, obj); assert(_heap->cancelled_gc() || _mark_context->is_marked(resolved) && !_heap->in_collection_set(resolved), "Sanity"); @@ -766,6 +766,7 @@ class ShenandoahConcurrentWeakRootsEvacUpdateTask : public AbstractGangTask { void work(uint worker_id) { ShenandoahConcurrentWorkerSession worker_session(worker_id); + ShenandoahSuspendibleThreadSetJoiner sts_join; { ShenandoahEvacOOMScope oom; // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp index 87c3ee8dc2090..214ab73a2a2c5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp @@ -86,9 +86,11 @@ class ShenandoahConcurrentMarkingTask : public AbstractGangTask { ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id); ShenandoahReferenceProcessor* rp = heap->ref_processor(); assert(rp != NULL, "need reference processor"); + StringDedup::Requests requests; _cm->mark_loop(worker_id, _terminator, rp, true /*cancellable*/, - ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP); + ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP, + &requests); } }; @@ -134,6 +136,7 @@ class ShenandoahFinalMarkingTask : public AbstractGangTask { ShenandoahParallelWorkerSession worker_session(worker_id); ShenandoahReferenceProcessor* rp = heap->ref_processor(); + StringDedup::Requests requests; // First drain remaining SATB buffers. { @@ -144,14 +147,15 @@ class ShenandoahFinalMarkingTask : public AbstractGangTask { while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {} assert(!heap->has_forwarded_objects(), "Not expected"); - ShenandoahMarkRefsClosure mark_cl(q, rp); + ShenandoahMarkRefsClosure mark_cl(q, rp); ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set, ShenandoahIUBarrier ? &mark_cl : NULL); Threads::threads_do(&tc); } _cm->mark_loop(worker_id, _terminator, rp, false /*not cancellable*/, - _dedup_string ? ENQUEUE_DEDUP : NO_DEDUP); + _dedup_string ? ENQUEUE_DEDUP : NO_DEDUP, + &requests); assert(_cm->task_queues()->is_empty(), "Should be empty"); } }; @@ -189,9 +193,7 @@ ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahO void ShenandoahMarkConcurrentRootsTask::work(uint worker_id) { ShenandoahConcurrentWorkerSession worker_session(worker_id); ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id); - // Cannot enable string deduplication during root scanning. Otherwise, - // may result lock inversion between stack watermark and string dedup queue lock. - ShenandoahMarkRefsClosure cl(q, _rp); + ShenandoahMarkRefsClosure cl(q, _rp); _root_scanner.roots_do(&cl, worker_id); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 81371c398d725..245feffa0d56d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -53,7 +53,7 @@ ShenandoahControlThread::ShenandoahControlThread() : _requested_gc_cause(GCCause::_no_cause_specified), _degen_point(ShenandoahGC::_degenerated_outside_cycle), _allocs_seen(0) { - + set_name("Shenandoah Control Thread"); reset_gc_id(); create_and_start(); _periodic_task.enroll(); @@ -479,10 +479,11 @@ void ShenandoahControlThread::request_gc(GCCause::Cause cause) { GCCause::is_serviceability_requested_gc(cause) || cause == GCCause::_metadata_GC_clear_soft_refs || cause == GCCause::_full_gc_alot || + cause == GCCause::_wb_young_gc || cause == GCCause::_wb_full_gc || cause == GCCause::_wb_breakpoint || cause == GCCause::_scavenge_alot, - "only requested GCs here"); + "only requested GCs here: %s", GCCause::to_string(cause)); if (is_explicit_gc(cause)) { if (!DisableExplicitGC) { @@ -623,16 +624,6 @@ size_t ShenandoahControlThread::get_gc_id() { return Atomic::load(&_gc_id); } -void ShenandoahControlThread::print() const { - print_on(tty); -} - -void ShenandoahControlThread::print_on(outputStream* st) const { - st->print("Shenandoah Concurrent Thread"); - Thread::print_on(st); - st->cr(); -} - void ShenandoahControlThread::start() { create_and_start(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp index 84ffa11e8f01d..782b134a4495f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp @@ -142,12 +142,6 @@ class ShenandoahControlThread: public ConcurrentGCThread { void start(); void prepare_for_graceful_shutdown(); bool in_graceful_shutdown(); - - char* name() const { return (char*)"ShenandoahControlThread";} - - // Printing - void print_on(outputStream* st) const; - void print() const; }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCONTROLTHREAD_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 0b3128580595d..1bf24c7617bc5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -475,8 +475,8 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _phase_timings(NULL), _monitoring_support(NULL), _memory_pool(NULL), - _stw_memory_manager("Shenandoah Pauses", "end of GC pause"), - _cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"), + _stw_memory_manager("Shenandoah Pauses"), + _cycle_memory_manager("Shenandoah Cycles"), _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()), _soft_ref_policy(), _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes), @@ -595,6 +595,7 @@ void ShenandoahHeap::print_on(outputStream* st) const { MetaspaceUtils::print_on(st); if (Verbose) { + st->cr(); print_heap_regions_on(st); } } @@ -1023,10 +1024,13 @@ void ShenandoahHeap::trash_cset_regions() { void ShenandoahHeap::print_heap_regions_on(outputStream* st) const { st->print_cr("Heap Regions:"); - st->print_cr("EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HC=humongous continuation, CS=collection set, T=trash, P=pinned"); - st->print_cr("BTE=bottom/top/end, U=used, T=TLAB allocs, G=GCLAB allocs, S=shared allocs, L=live data"); - st->print_cr("R=root, CP=critical pins, TAMS=top-at-mark-start, UWM=update watermark"); - st->print_cr("SN=alloc sequence number"); + st->print_cr("Region state: EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HP=pinned humongous start"); + st->print_cr(" HC=humongous continuation, CS=collection set, TR=trash, P=pinned, CSP=pinned collection set"); + st->print_cr("BTE=bottom/top/end, TAMS=top-at-mark-start"); + st->print_cr("UWM=update watermark, U=used"); + st->print_cr("T=TLAB allocs, G=GCLAB allocs"); + st->print_cr("S=shared allocs, L=live data"); + st->print_cr("CP=critical pins"); for (size_t i = 0; i < num_regions(); i++) { get_region(i)->print_on(st); @@ -1182,6 +1186,7 @@ void ShenandoahHeap::prepare_for_verify() { } void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const { + tcl->do_thread(_control_thread); workers()->threads_do(tcl); if (_safepoint_workers != NULL) { _safepoint_workers->threads_do(tcl); @@ -1239,7 +1244,7 @@ class ObjectIterateScanRootClosure : public BasicOopIterateClosure { // There may be dead oops in weak roots in concurrent root phase, do not touch them. return; } - obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); + obj = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(obj); assert(oopDesc::is_oop(obj), "must be a valid oop"); if (!_bitmap->is_marked(obj)) { @@ -1348,7 +1353,7 @@ class ShenandoahObjectIterateParScanClosure : public BasicOopIterateClosure { // There may be dead oops in weak roots in concurrent root phase, do not touch them. return; } - obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); + obj = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(obj); assert(oopDesc::is_oop(obj), "Must be a valid oop"); if (_bitmap->par_mark(obj)) { @@ -1369,7 +1374,7 @@ class ShenandoahObjectIterateParScanClosure : public BasicOopIterateClosure { // parallel marking queues. // Every worker processes it's own marking queue. work-stealing is used // to balance workload. -class ShenandoahParallelObjectIterator : public ParallelObjectIterator { +class ShenandoahParallelObjectIterator : public ParallelObjectIteratorImpl { private: uint _num_workers; bool _init_ready; @@ -1469,7 +1474,7 @@ class ShenandoahParallelObjectIterator : public ParallelObjectIterator { } }; -ParallelObjectIterator* ShenandoahHeap::parallel_object_iterator(uint workers) { +ParallelObjectIteratorImpl* ShenandoahHeap::parallel_object_iterator(uint workers) { return new ShenandoahParallelObjectIterator(workers, &_aux_bit_map); } @@ -1740,20 +1745,8 @@ size_t ShenandoahHeap::tlab_used(Thread* thread) const { } bool ShenandoahHeap::try_cancel_gc() { - while (true) { - jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE); - if (prev == CANCELLABLE) return true; - else if (prev == CANCELLED) return false; - assert(ShenandoahSuspendibleWorkers, "should not get here when not using suspendible workers"); - assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED"); - Thread* thread = Thread::current(); - if (thread->is_Java_thread()) { - // We need to provide a safepoint here, otherwise we might - // spin forever if a SP is pending. - ThreadBlockInVM sp(thread->as_Java_thread()); - SpinPause(); - } - } + jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE); + return prev == CANCELLABLE; } void ShenandoahHeap::cancel_gc(GCCause::Cause cause) { @@ -2135,6 +2128,7 @@ void ShenandoahHeap::rebuild_free_set(bool concurrent) { void ShenandoahHeap::print_extended_on(outputStream *st) const { print_on(st); + st->cr(); print_heap_regions_on(st); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 12e863584b262..588cb13076fc3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -326,12 +326,7 @@ class ShenandoahHeap : public CollectedHeap { // GC has been cancelled. Worker threads can not suspend for // safepoint but must finish their work as soon as possible. - CANCELLED, - - // GC has not been cancelled and must not be cancelled. At least - // one worker thread checks for pending safepoint and may suspend - // if a safepoint is pending. - NOT_CANCELLED + CANCELLED }; ShenandoahSharedEnumFlag _cancelled_gc; @@ -484,7 +479,7 @@ class ShenandoahHeap : public CollectedHeap { // Used for native heap walkers: heap dumpers, mostly void object_iterate(ObjectClosure* cl); // Parallel heap iteration support - virtual ParallelObjectIterator* parallel_object_iterator(uint workers); + virtual ParallelObjectIteratorImpl* parallel_object_iterator(uint workers); // Keep alive an object that was loaded with AS_NO_KEEPALIVE. void keep_alive(oop obj); @@ -635,9 +630,17 @@ class ShenandoahHeap : public CollectedHeap { template inline void update_with_forwarded(T* p); - static inline oop cas_oop(oop n, narrowOop* addr, oop c); - static inline oop cas_oop(oop n, oop* addr, oop c); - static inline oop cas_oop(oop n, narrowOop* addr, narrowOop c); + static inline void atomic_update_oop(oop update, oop* addr, oop compare); + static inline void atomic_update_oop(oop update, narrowOop* addr, oop compare); + static inline void atomic_update_oop(oop update, narrowOop* addr, narrowOop compare); + + static inline bool atomic_update_oop_check(oop update, oop* addr, oop compare); + static inline bool atomic_update_oop_check(oop update, narrowOop* addr, oop compare); + static inline bool atomic_update_oop_check(oop update, narrowOop* addr, narrowOop compare); + + static inline void atomic_clear_oop( oop* addr, oop compare); + static inline void atomic_clear_oop(narrowOop* addr, oop compare); + static inline void atomic_clear_oop(narrowOop* addr, narrowOop compare); void trash_humongous_region_at(ShenandoahHeapRegion *r); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index f300233d7e28d..1bd3857b8b425 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -132,29 +132,110 @@ inline void ShenandoahHeap::conc_update_with_forwarded(T* p) { // Either we succeed in updating the reference, or something else gets in our way. // We don't care if that is another concurrent GC update, or another mutator update. - // We only check that non-NULL store still updated with non-forwarded reference. - oop witness = cas_oop(fwd, p, obj); - shenandoah_assert_not_forwarded_except(p, witness, (witness == NULL) || (witness == obj)); + atomic_update_oop(fwd, p, obj); } } } -inline oop ShenandoahHeap::cas_oop(oop n, oop* addr, oop c) { +// Atomic updates of heap location. This is only expected to work with updating the same +// logical object with its forwardee. The reason why we need stronger-than-relaxed memory +// ordering has to do with coordination with GC barriers and mutator accesses. +// +// In essence, stronger CAS access is required to maintain the transitive chains that mutator +// accesses build by themselves. To illustrate this point, consider the following example. +// +// Suppose "o" is the object that has a field "x" and the reference to "o" is stored +// to field at "addr", which happens to be Java volatile field. Normally, the accesses to volatile +// field at "addr" would be matched with release/acquire barriers. This changes when GC moves +// the object under mutator feet. +// +// Thread 1 (Java) +// // --- previous access starts here +// ... +// T1.1: store(&o.x, 1, mo_relaxed) +// T1.2: store(&addr, o, mo_release) // volatile store +// +// // --- new access starts here +// // LRB: copy and install the new copy to fwdptr +// T1.3: var copy = copy(o) +// T1.4: cas(&fwd, t, copy, mo_release) // pointer-mediated publication +// +// +// Thread 2 (GC updater) +// T2.1: var f = load(&fwd, mo_{consume|acquire}) // pointer-mediated acquisition +// T2.2: cas(&addr, o, f, mo_release) // this method +// +// Thread 3 (Java) +// T3.1: var o = load(&addr, mo_acquire) // volatile read +// T3.2: if (o != null) +// T3.3: var r = load(&o.x, mo_relaxed) +// +// r is guaranteed to contain "1". +// +// Without GC involvement, there is synchronizes-with edge from T1.2 to T3.1, +// which guarantees this. With GC involvement, when LRB copies the object and +// another thread updates the reference to it, we need to have the transitive edge +// from T1.4 to T2.1 (that one is guaranteed by forwarding accesses), plus the edge +// from T2.2 to T3.1 (which is brought by this CAS). +// +// Note that we do not need to "acquire" in these methods, because we do not read the +// failure witnesses contents on any path, and "release" is enough. +// + +inline void ShenandoahHeap::atomic_update_oop(oop update, oop* addr, oop compare) { assert(is_aligned(addr, HeapWordSize), "Address should be aligned: " PTR_FORMAT, p2i(addr)); - return (oop) Atomic::cmpxchg(addr, c, n); + Atomic::cmpxchg(addr, compare, update, memory_order_release); } -inline oop ShenandoahHeap::cas_oop(oop n, narrowOop* addr, narrowOop c) { +inline void ShenandoahHeap::atomic_update_oop(oop update, narrowOop* addr, narrowOop compare) { assert(is_aligned(addr, sizeof(narrowOop)), "Address should be aligned: " PTR_FORMAT, p2i(addr)); - narrowOop val = CompressedOops::encode(n); - return CompressedOops::decode(Atomic::cmpxchg(addr, c, val)); + narrowOop u = CompressedOops::encode(update); + Atomic::cmpxchg(addr, compare, u, memory_order_release); } -inline oop ShenandoahHeap::cas_oop(oop n, narrowOop* addr, oop c) { +inline void ShenandoahHeap::atomic_update_oop(oop update, narrowOop* addr, oop compare) { assert(is_aligned(addr, sizeof(narrowOop)), "Address should be aligned: " PTR_FORMAT, p2i(addr)); - narrowOop cmp = CompressedOops::encode(c); - narrowOop val = CompressedOops::encode(n); - return CompressedOops::decode(Atomic::cmpxchg(addr, cmp, val)); + narrowOop c = CompressedOops::encode(compare); + narrowOop u = CompressedOops::encode(update); + Atomic::cmpxchg(addr, c, u, memory_order_release); +} + +inline bool ShenandoahHeap::atomic_update_oop_check(oop update, oop* addr, oop compare) { + assert(is_aligned(addr, HeapWordSize), "Address should be aligned: " PTR_FORMAT, p2i(addr)); + return (oop) Atomic::cmpxchg(addr, compare, update, memory_order_release) == compare; +} + +inline bool ShenandoahHeap::atomic_update_oop_check(oop update, narrowOop* addr, narrowOop compare) { + assert(is_aligned(addr, sizeof(narrowOop)), "Address should be aligned: " PTR_FORMAT, p2i(addr)); + narrowOop u = CompressedOops::encode(update); + return (narrowOop) Atomic::cmpxchg(addr, compare, u, memory_order_release) == compare; +} + +inline bool ShenandoahHeap::atomic_update_oop_check(oop update, narrowOop* addr, oop compare) { + assert(is_aligned(addr, sizeof(narrowOop)), "Address should be aligned: " PTR_FORMAT, p2i(addr)); + narrowOop c = CompressedOops::encode(compare); + narrowOop u = CompressedOops::encode(update); + return CompressedOops::decode(Atomic::cmpxchg(addr, c, u, memory_order_release)) == compare; +} + +// The memory ordering discussion above does not apply for methods that store NULLs: +// then, there is no transitive reads in mutator (as we see NULLs), and we can do +// relaxed memory ordering there. + +inline void ShenandoahHeap::atomic_clear_oop(oop* addr, oop compare) { + assert(is_aligned(addr, HeapWordSize), "Address should be aligned: " PTR_FORMAT, p2i(addr)); + Atomic::cmpxchg(addr, compare, oop(), memory_order_relaxed); +} + +inline void ShenandoahHeap::atomic_clear_oop(narrowOop* addr, oop compare) { + assert(is_aligned(addr, sizeof(narrowOop)), "Address should be aligned: " PTR_FORMAT, p2i(addr)); + narrowOop cmp = CompressedOops::encode(compare); + Atomic::cmpxchg(addr, cmp, narrowOop(), memory_order_relaxed); +} + +inline void ShenandoahHeap::atomic_clear_oop(narrowOop* addr, narrowOop compare) { + assert(is_aligned(addr, sizeof(narrowOop)), "Address should be aligned: " PTR_FORMAT, p2i(addr)); + Atomic::cmpxchg(addr, compare, narrowOop(), memory_order_relaxed); } inline bool ShenandoahHeap::cancelled_gc() const { @@ -162,25 +243,12 @@ inline bool ShenandoahHeap::cancelled_gc() const { } inline bool ShenandoahHeap::check_cancelled_gc_and_yield(bool sts_active) { - if (! (sts_active && ShenandoahSuspendibleWorkers)) { - return cancelled_gc(); - } - - jbyte prev = _cancelled_gc.cmpxchg(NOT_CANCELLED, CANCELLABLE); - if (prev == CANCELLABLE || prev == NOT_CANCELLED) { + if (sts_active && ShenandoahSuspendibleWorkers && !cancelled_gc()) { if (SuspendibleThreadSet::should_yield()) { SuspendibleThreadSet::yield(); } - - // Back to CANCELLABLE. The thread that poked NOT_CANCELLED first gets - // to restore to CANCELLABLE. - if (prev == CANCELLABLE) { - _cancelled_gc.set(CANCELLABLE); - } - return false; - } else { - return true; } + return cancelled_gc(); } inline void ShenandoahHeap::clear_cancelled_gc() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index a49aa215e3b0b..8c9784d577869 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -351,7 +351,7 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const { st->print("|CS "); break; case _trash: - st->print("|T "); + st->print("|TR "); break; case _pinned: st->print("|P "); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp index 76edf786018e9..7fb77f381df3e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp @@ -36,7 +36,6 @@ ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : MetadataVisitingOopIterateClosure(rp), - _stringDedup_requests(), _queue(q), _mark_context(ShenandoahHeap::heap()->marking_context()), _weak(false) @@ -56,7 +55,7 @@ void ShenandoahMark::clear() { } template -void ShenandoahMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahReferenceProcessor *rp) { +void ShenandoahMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahReferenceProcessor *rp, StringDedup::Requests* const req) { ShenandoahObjToScanQueue* q = get_queue(w); ShenandoahHeap* const heap = ShenandoahHeap::heap(); @@ -66,23 +65,23 @@ void ShenandoahMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahRefe // play nice with specialized_oop_iterators. if (heap->unload_classes()) { if (heap->has_forwarded_objects()) { - using Closure = ShenandoahMarkUpdateRefsMetadataClosure; + using Closure = ShenandoahMarkUpdateRefsMetadataClosure; Closure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + mark_loop_work(&cl, ld, w, t, req); } else { - using Closure = ShenandoahMarkRefsMetadataClosure; + using Closure = ShenandoahMarkRefsMetadataClosure; Closure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + mark_loop_work(&cl, ld, w, t, req); } } else { if (heap->has_forwarded_objects()) { - using Closure = ShenandoahMarkUpdateRefsClosure; + using Closure = ShenandoahMarkUpdateRefsClosure; Closure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + mark_loop_work(&cl, ld, w, t, req); } else { - using Closure = ShenandoahMarkRefsClosure; + using Closure = ShenandoahMarkRefsClosure; Closure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + mark_loop_work(&cl, ld, w, t, req); } } @@ -90,36 +89,36 @@ void ShenandoahMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahRefe } void ShenandoahMark::mark_loop(uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp, - bool cancellable, StringDedupMode dedup_mode) { + bool cancellable, StringDedupMode dedup_mode, StringDedup::Requests* const req) { if (cancellable) { switch(dedup_mode) { case NO_DEDUP: - mark_loop_prework(worker_id, terminator, rp); + mark_loop_prework(worker_id, terminator, rp, req); break; case ENQUEUE_DEDUP: - mark_loop_prework(worker_id, terminator, rp); + mark_loop_prework(worker_id, terminator, rp, req); break; case ALWAYS_DEDUP: - mark_loop_prework(worker_id, terminator, rp); + mark_loop_prework(worker_id, terminator, rp, req); break; } } else { switch(dedup_mode) { case NO_DEDUP: - mark_loop_prework(worker_id, terminator, rp); + mark_loop_prework(worker_id, terminator, rp, req); break; case ENQUEUE_DEDUP: - mark_loop_prework(worker_id, terminator, rp); + mark_loop_prework(worker_id, terminator, rp, req); break; case ALWAYS_DEDUP: - mark_loop_prework(worker_id, terminator, rp); + mark_loop_prework(worker_id, terminator, rp, req); break; } } } -template -void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint worker_id, TaskTerminator *terminator) { +template +void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint worker_id, TaskTerminator *terminator, StringDedup::Requests* const req) { uintx stride = ShenandoahMarkLoopStride; ShenandoahHeap* heap = ShenandoahHeap::heap(); @@ -147,7 +146,7 @@ void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint w for (uint i = 0; i < stride; i++) { if (q->pop(t)) { - do_task(q, cl, live_data, &t); + do_task(q, cl, live_data, req, &t); } else { assert(q->is_empty(), "Must be empty"); q = queues->claim_next(); @@ -176,7 +175,7 @@ void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint w for (uint i = 0; i < stride; i++) { if (q->pop(t) || queues->steal(worker_id, t)) { - do_task(q, cl, live_data, &t); + do_task(q, cl, live_data, req, &t); work++; } else { break; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp index 6b31fabee25fa..5b29fa54e35d6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp @@ -45,8 +45,8 @@ class ShenandoahMark: public StackObj { ShenandoahMark(); public: - template - static inline void mark_through_ref(T* p, ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context, StringDedup::Requests* const req, bool weak); + template + static inline void mark_through_ref(T* p, ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context, bool weak); static void clear(); @@ -56,8 +56,8 @@ class ShenandoahMark: public StackObj { // ---------- Marking loop and tasks private: - template - inline void do_task(ShenandoahObjToScanQueue* q, T* cl, ShenandoahLiveData* live_data, ShenandoahMarkTask* task); + template + inline void do_task(ShenandoahObjToScanQueue* q, T* cl, ShenandoahLiveData* live_data, StringDedup::Requests* const req, ShenandoahMarkTask* task); template inline void do_chunked_array_start(ShenandoahObjToScanQueue* q, T* cl, oop array, bool weak); @@ -67,15 +67,17 @@ class ShenandoahMark: public StackObj { inline void count_liveness(ShenandoahLiveData* live_data, oop obj); - template - void mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint worker_id, TaskTerminator *t); + template + void mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint worker_id, TaskTerminator *t, StringDedup::Requests* const req); template - void mark_loop_prework(uint worker_id, TaskTerminator *terminator, ShenandoahReferenceProcessor *rp); + void mark_loop_prework(uint worker_id, TaskTerminator *terminator, ShenandoahReferenceProcessor *rp, StringDedup::Requests* const req); + template + inline void dedup_string(oop obj, StringDedup::Requests* const req); protected: void mark_loop(uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp, - bool cancellable, StringDedupMode dedup_mode); + bool cancellable, StringDedupMode dedup_mode, StringDedup::Requests* const req); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHMARK_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp index 929965ce8eceb..cfafd623d8e75 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp @@ -40,8 +40,22 @@ #include "runtime/prefetch.inline.hpp" #include "utilities/powerOfTwo.hpp" -template -void ShenandoahMark::do_task(ShenandoahObjToScanQueue* q, T* cl, ShenandoahLiveData* live_data, ShenandoahMarkTask* task) { +template +void ShenandoahMark::dedup_string(oop obj, StringDedup::Requests* const req) { + if (STRING_DEDUP == ENQUEUE_DEDUP) { + if (ShenandoahStringDedup::is_candidate(obj)) { + req->add(obj); + } + } else if (STRING_DEDUP == ALWAYS_DEDUP) { + if (ShenandoahStringDedup::is_string_candidate(obj) && + !ShenandoahStringDedup::dedup_requested(obj)) { + req->add(obj); + } + } +} + +template +void ShenandoahMark::do_task(ShenandoahObjToScanQueue* q, T* cl, ShenandoahLiveData* live_data, StringDedup::Requests* const req, ShenandoahMarkTask* task) { oop obj = task->obj(); shenandoah_assert_not_forwarded(NULL, obj); @@ -56,6 +70,7 @@ void ShenandoahMark::do_task(ShenandoahObjToScanQueue* q, T* cl, ShenandoahLiveD if (obj->is_instance()) { // Case 1: Normal oop, process as usual. obj->oop_iterate(cl); + dedup_string(obj, req); } else if (obj->is_objArray()) { // Case 2: Object array instance and no chunk is set. Must be the first // time we visit it, start the chunked processing. @@ -208,7 +223,6 @@ inline void ShenandoahMark::do_chunked_array(ShenandoahObjToScanQueue* q, T* cl, class ShenandoahSATBBufferClosure : public SATBBufferClosure { private: - StringDedup::Requests _stringdedup_requests; ShenandoahObjToScanQueue* _queue; ShenandoahHeap* _heap; ShenandoahMarkingContext* const _mark_context; @@ -222,24 +236,15 @@ class ShenandoahSATBBufferClosure : public SATBBufferClosure { void do_buffer(void **buffer, size_t size) { assert(size == 0 || !_heap->has_forwarded_objects(), "Forwarded objects are not expected here"); - if (ShenandoahStringDedup::is_enabled()) { - do_buffer_impl(buffer, size); - } else { - do_buffer_impl(buffer, size); - } - } - - template - void do_buffer_impl(void **buffer, size_t size) { for (size_t i = 0; i < size; ++i) { oop *p = (oop *) &buffer[i]; - ShenandoahMark::mark_through_ref(p, _queue, _mark_context, &_stringdedup_requests, false); + ShenandoahMark::mark_through_ref(p, _queue, _mark_context, false); } } }; -template -inline void ShenandoahMark::mark_through_ref(T* p, ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context, StringDedup::Requests* const req, bool weak) { +template +inline void ShenandoahMark::mark_through_ref(T* p, ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context, bool weak) { T o = RawAccess<>::oop_load(p); if (!CompressedOops::is_null(o)) { oop obj = CompressedOops::decode_not_null(o); @@ -257,16 +262,6 @@ inline void ShenandoahMark::mark_through_ref(T* p, ShenandoahObjToScanQueue* q, if (marked) { bool pushed = q->push(ShenandoahMarkTask(obj, skip_live, weak)); assert(pushed, "overflow queue should always succeed pushing"); - - if ((STRING_DEDUP == ENQUEUE_DEDUP) && ShenandoahStringDedup::is_candidate(obj)) { - assert(ShenandoahStringDedup::is_enabled(), "Must be enabled"); - req->add(obj); - } else if ((STRING_DEDUP == ALWAYS_DEDUP) && - ShenandoahStringDedup::is_string_candidate(obj) && - !ShenandoahStringDedup::dedup_requested(obj)) { - assert(ShenandoahStringDedup::is_enabled(), "Must be enabled"); - req->add(obj); - } } shenandoah_assert_marked(p, obj); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp index 33d5df425b35b..c0fc3aca5a7c6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp @@ -56,7 +56,7 @@ void HdrSeq::add(double val) { int mag; if (v > 0) { mag = 0; - while (v > 1) { + while (v >= 1) { mag++; v /= 10; } @@ -71,7 +71,7 @@ void HdrSeq::add(double val) { int bucket = -MagMinimum + mag; int sub_bucket = (int) (v * ValBuckets); - // Defensively saturate for product bits: + // Defensively saturate for product bits if (bucket < 0) { assert (false, "bucket index (%d) underflow for value (%8.2f)", bucket, val); bucket = 0; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp index 9f9cb2ecaaf05..42f91f6a9b8b3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp @@ -55,7 +55,7 @@ class HdrSeq: public NumberSeq { // Binary magnitude sequence stores the power-of-two histogram. // It has very low memory requirements, and is thread-safe. When accuracy // is not needed, it is preferred over HdrSeq. -class BinaryMagnitudeSeq { +class BinaryMagnitudeSeq : public CHeapObj { private: size_t _sum; size_t* _mags; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp index fe3cb0fe77b2d..5bd42916a2f55 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp @@ -40,13 +40,12 @@ enum StringDedupMode { class ShenandoahMarkRefsSuperClosure : public MetadataVisitingOopIterateClosure { private: - StringDedup::Requests _stringDedup_requests; ShenandoahObjToScanQueue* _queue; ShenandoahMarkingContext* const _mark_context; bool _weak; protected: - template + template void work(T *p); public: @@ -65,7 +64,7 @@ class ShenandoahMarkUpdateRefsSuperClosure : public ShenandoahMarkRefsSuperClosu protected: ShenandoahHeap* const _heap; - template + template inline void work(T* p); public: @@ -76,11 +75,10 @@ class ShenandoahMarkUpdateRefsSuperClosure : public ShenandoahMarkRefsSuperClosu }; }; -template class ShenandoahMarkUpdateRefsClosure : public ShenandoahMarkUpdateRefsSuperClosure { private: template - inline void do_oop_work(T* p) { work(p); } + inline void do_oop_work(T* p) { work(p); } public: ShenandoahMarkUpdateRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : @@ -91,11 +89,10 @@ class ShenandoahMarkUpdateRefsClosure : public ShenandoahMarkUpdateRefsSuperClos virtual bool do_metadata() { return false; } }; -template class ShenandoahMarkUpdateRefsMetadataClosure : public ShenandoahMarkUpdateRefsSuperClosure { private: template - inline void do_oop_work(T* p) { work(p); } + inline void do_oop_work(T* p) { work(p); } public: ShenandoahMarkUpdateRefsMetadataClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : @@ -107,11 +104,10 @@ class ShenandoahMarkUpdateRefsMetadataClosure : public ShenandoahMarkUpdateRefsS }; -template class ShenandoahMarkRefsClosure : public ShenandoahMarkRefsSuperClosure { private: template - inline void do_oop_work(T* p) { work(p); } + inline void do_oop_work(T* p) { work(p); } public: ShenandoahMarkRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : @@ -123,11 +119,10 @@ class ShenandoahMarkRefsClosure : public ShenandoahMarkRefsSuperClosure { }; -template class ShenandoahMarkRefsMetadataClosure : public ShenandoahMarkRefsSuperClosure { private: template - inline void do_oop_work(T* p) { work(p); } + inline void do_oop_work(T* p) { work(p); } public: ShenandoahMarkRefsMetadataClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp index 47bd5cac5c1e7..1812b4e8f0577 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp @@ -30,18 +30,18 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahMark.inline.hpp" -template +template inline void ShenandoahMarkRefsSuperClosure::work(T* p) { - ShenandoahMark::mark_through_ref(p, _queue, _mark_context, &_stringDedup_requests, _weak); + ShenandoahMark::mark_through_ref(p, _queue, _mark_context, _weak); } -template +template inline void ShenandoahMarkUpdateRefsSuperClosure::work(T* p) { // Update the location _heap->update_with_forwarded(p); // ...then do the usual thing - ShenandoahMarkRefsSuperClosure::work(p); + ShenandoahMarkRefsSuperClosure::work(p); } template diff --git a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp index 6ac55d60c8094..e843f3b250b71 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp @@ -117,20 +117,9 @@ void reference_set_discovered(oop reference, oop discovered) { } template -static bool reference_cas_discovered(oop reference, oop discovered); - -template<> -bool reference_cas_discovered(oop reference, oop discovered) { - volatile narrowOop* addr = reinterpret_cast(java_lang_ref_Reference::discovered_addr_raw(reference)); - narrowOop compare = CompressedOops::encode(NULL); - narrowOop exchange = CompressedOops::encode(discovered); - return Atomic::cmpxchg(addr, compare, exchange) == compare; -} - -template<> -bool reference_cas_discovered(oop reference, oop discovered) { - volatile oop* addr = reinterpret_cast(java_lang_ref_Reference::discovered_addr_raw(reference)); - return Atomic::cmpxchg(addr, oop(NULL), discovered) == NULL; +static bool reference_cas_discovered(oop reference, oop discovered) { + T* addr = reinterpret_cast(java_lang_ref_Reference::discovered_addr_raw(reference)); + return ShenandoahHeap::atomic_update_oop_check(discovered, addr, NULL); } template diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp index 1d4d46dc5383c..09fea5b848779 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp @@ -57,7 +57,7 @@ ShenandoahInitMarkRootsClosure::ShenandoahInitMarkRootsClosure(ShenandoahObjToSc template void ShenandoahInitMarkRootsClosure::do_oop_work(T* p) { - ShenandoahMark::mark_through_ref(p, _queue, _mark_context, NULL, false); + ShenandoahMark::mark_through_ref(p, _queue, _mark_context, false); } class ShenandoahSTWMarkTask : public AbstractGangTask { @@ -131,9 +131,10 @@ void ShenandoahSTWMark::finish_mark(uint worker_id) { ShenandoahPhaseTimings::Phase phase = _full_gc ? ShenandoahPhaseTimings::full_gc_mark : ShenandoahPhaseTimings::degen_gc_stw_mark; ShenandoahWorkerTimingsTracker timer(phase, ShenandoahPhaseTimings::ParallelMark, worker_id); ShenandoahReferenceProcessor* rp = ShenandoahHeap::heap()->ref_processor(); + StringDedup::Requests requests; mark_loop(worker_id, &_terminator, rp, false /* not cancellable */, - ShenandoahStringDedup::is_enabled() ? ALWAYS_DEDUP : NO_DEDUP); + ShenandoahStringDedup::is_enabled() ? ALWAYS_DEDUP : NO_DEDUP, &requests); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahStringDedup.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahStringDedup.inline.hpp index 57b30358e94b2..e26ef79f603f8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahStringDedup.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahStringDedup.inline.hpp @@ -45,14 +45,15 @@ bool ShenandoahStringDedup::is_candidate(oop obj) { return false; } - if (StringDedup::is_below_threshold_age(obj->age())) { - const markWord mark = obj->mark(); - // Having/had displaced header, too risk to deal with them, skip - if (mark == markWord::INFLATING() || mark.has_displaced_mark_helper()) { - return false; - } + const markWord mark = obj->mark(); + + // Having/had displaced header, too risky to deal with them, skip + if (mark == markWord::INFLATING() || mark.has_displaced_mark_helper()) { + return false; + } - // Increase string age and enqueue it when it rearches age threshold + if (StringDedup::is_below_threshold_age(mark.age())) { + // Increase string age and enqueue it when it reaches age threshold markWord new_mark = mark.incr_age(); if (mark == obj->cas_set_mark(new_mark, mark)) { return StringDedup::is_threshold_age(new_mark.age()) && diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp index 9a7213241b764..675bbe2f8d6da 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp @@ -52,6 +52,7 @@ ShenandoahGCSession::ShenandoahGCSession(GCCause::Cause cause) : _heap->shenandoah_policy()->record_cycle_start(); _heap->heuristics()->record_cycle_start(); _trace_cycle.initialize(_heap->cycle_memory_manager(), cause, + "end of GC cycle", /* allMemoryPoolsAffected */ true, /* recordGCBeginTime = */ true, /* recordPreGCUsage = */ true, @@ -73,9 +74,10 @@ ShenandoahGCSession::~ShenandoahGCSession() { _heap->set_gc_cause(GCCause::_no_gc); } -ShenandoahGCPauseMark::ShenandoahGCPauseMark(uint gc_id, SvcGCMarker::reason_type type) : +ShenandoahGCPauseMark::ShenandoahGCPauseMark(uint gc_id, const char* notification_message, SvcGCMarker::reason_type type) : _heap(ShenandoahHeap::heap()), _gc_id_mark(gc_id), _svc_gc_mark(type), _is_gc_active_mark() { _trace_pause.initialize(_heap->stw_memory_manager(), _heap->gc_cause(), + notification_message, /* allMemoryPoolsAffected */ true, /* recordGCBeginTime = */ true, /* recordPreGCUsage = */ false, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp index 49ff6ca28649f..f543800131c2b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp @@ -135,7 +135,7 @@ class ShenandoahGCPauseMark : public StackObj { TraceMemoryManagerStats _trace_pause; public: - ShenandoahGCPauseMark(uint gc_id, SvcGCMarker::reason_type type); + ShenandoahGCPauseMark(uint gc_id, const char* notification_action, SvcGCMarker::reason_type type); }; class ShenandoahSafepoint : public AllStatic { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp index 18bd4d6e6aa21..d6be092055820 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp @@ -47,36 +47,36 @@ void VM_ShenandoahReferenceOperation::doit_epilogue() { } void VM_ShenandoahInitMark::doit() { - ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::CONCURRENT); + ShenandoahGCPauseMark mark(_gc_id, "Init Mark", SvcGCMarker::CONCURRENT); _gc->entry_init_mark(); } void VM_ShenandoahFinalMarkStartEvac::doit() { - ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::CONCURRENT); + ShenandoahGCPauseMark mark(_gc_id, "Final Mark", SvcGCMarker::CONCURRENT); _gc->entry_final_mark(); } void VM_ShenandoahFullGC::doit() { - ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::FULL); + ShenandoahGCPauseMark mark(_gc_id, "Full GC", SvcGCMarker::FULL); _full_gc->entry_full(_gc_cause); } void VM_ShenandoahDegeneratedGC::doit() { - ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::CONCURRENT); + ShenandoahGCPauseMark mark(_gc_id, "Degenerated GC", SvcGCMarker::CONCURRENT); _gc->entry_degenerated(); } void VM_ShenandoahInitUpdateRefs::doit() { - ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::CONCURRENT); + ShenandoahGCPauseMark mark(_gc_id, "Init Update Refs", SvcGCMarker::CONCURRENT); _gc->entry_init_updaterefs(); } void VM_ShenandoahFinalUpdateRefs::doit() { - ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::CONCURRENT); + ShenandoahGCPauseMark mark(_gc_id, "Final Update Refs", SvcGCMarker::CONCURRENT); _gc->entry_final_updaterefs(); } void VM_ShenandoahFinalRoots::doit() { - ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::CONCURRENT); + ShenandoahGCPauseMark mark(_gc_id, "Final Roots", SvcGCMarker::CONCURRENT); _gc->entry_final_roots(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index 5a920fb884bc1..88c460b5f19bc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -334,7 +334,7 @@ "How many times to maximum attempt to flush SATB buffers at the " \ "end of concurrent marking.") \ \ - product(bool, ShenandoahSuspendibleWorkers, false, EXPERIMENTAL, \ + product(bool, ShenandoahSuspendibleWorkers, true, EXPERIMENTAL, \ "Suspend concurrent GC worker threads at safepoints") \ \ product(bool, ShenandoahSATBBarrier, true, DIAGNOSTIC, \ diff --git a/src/hotspot/share/gc/z/c1/zBarrierSetC1.cpp b/src/hotspot/share/gc/z/c1/zBarrierSetC1.cpp index 4f2e36a8304ce..0e99bf107c1e0 100644 --- a/src/hotspot/share/gc/z/c1/zBarrierSetC1.cpp +++ b/src/hotspot/share/gc/z/c1/zBarrierSetC1.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -94,7 +94,11 @@ class LIR_OpZLoadBarrierTest : public LIR_Op { public: LIR_OpZLoadBarrierTest(LIR_Opr opr) : +#ifdef RISCV + LIR_Op(lir_zloadbarrier_test, LIR_OprFact::illegalOpr, NULL), +#else LIR_Op(), +#endif _opr(opr) {} virtual void visit(LIR_OpVisitState* state) { diff --git a/src/hotspot/share/gc/z/zCollectedHeap.cpp b/src/hotspot/share/gc/z/zCollectedHeap.cpp index 3ea6e7a9bcb86..cfbb06798f86d 100644 --- a/src/hotspot/share/gc/z/zCollectedHeap.cpp +++ b/src/hotspot/share/gc/z/zCollectedHeap.cpp @@ -243,7 +243,7 @@ void ZCollectedHeap::object_iterate(ObjectClosure* cl) { _heap.object_iterate(cl, true /* visit_weaks */); } -ParallelObjectIterator* ZCollectedHeap::parallel_object_iterator(uint nworkers) { +ParallelObjectIteratorImpl* ZCollectedHeap::parallel_object_iterator(uint nworkers) { return _heap.parallel_object_iterator(nworkers, true /* visit_weaks */); } diff --git a/src/hotspot/share/gc/z/zCollectedHeap.hpp b/src/hotspot/share/gc/z/zCollectedHeap.hpp index 91c78420995b2..cb0ce3013f1a9 100644 --- a/src/hotspot/share/gc/z/zCollectedHeap.hpp +++ b/src/hotspot/share/gc/z/zCollectedHeap.hpp @@ -95,7 +95,7 @@ class ZCollectedHeap : public CollectedHeap { virtual GrowableArray memory_pools(); virtual void object_iterate(ObjectClosure* cl); - virtual ParallelObjectIterator* parallel_object_iterator(uint nworkers); + virtual ParallelObjectIteratorImpl* parallel_object_iterator(uint nworkers); virtual void keep_alive(oop obj); diff --git a/src/hotspot/share/gc/z/zHeap.cpp b/src/hotspot/share/gc/z/zHeap.cpp index 3bf3cf50ed8ee..b937a645f6264 100644 --- a/src/hotspot/share/gc/z/zHeap.cpp +++ b/src/hotspot/share/gc/z/zHeap.cpp @@ -439,7 +439,7 @@ void ZHeap::object_iterate(ObjectClosure* cl, bool visit_weaks) { iter.object_iterate(cl, 0 /* worker_id */); } -ParallelObjectIterator* ZHeap::parallel_object_iterator(uint nworkers, bool visit_weaks) { +ParallelObjectIteratorImpl* ZHeap::parallel_object_iterator(uint nworkers, bool visit_weaks) { assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); return new ZHeapIterator(nworkers, visit_weaks); } diff --git a/src/hotspot/share/gc/z/zHeap.hpp b/src/hotspot/share/gc/z/zHeap.hpp index f4e36ad738e0b..723ec52d39cd6 100644 --- a/src/hotspot/share/gc/z/zHeap.hpp +++ b/src/hotspot/share/gc/z/zHeap.hpp @@ -141,7 +141,7 @@ class ZHeap { // Iteration void object_iterate(ObjectClosure* cl, bool visit_weaks); - ParallelObjectIterator* parallel_object_iterator(uint nworkers, bool visit_weaks); + ParallelObjectIteratorImpl* parallel_object_iterator(uint nworkers, bool visit_weaks); void pages_do(ZPageClosure* cl); // Serviceability diff --git a/src/hotspot/share/gc/z/zHeapIterator.hpp b/src/hotspot/share/gc/z/zHeapIterator.hpp index bbd3eb203c374..5c3a82d8bb7d9 100644 --- a/src/hotspot/share/gc/z/zHeapIterator.hpp +++ b/src/hotspot/share/gc/z/zHeapIterator.hpp @@ -42,7 +42,7 @@ using ZHeapIteratorQueues = GenericTaskQueueSet; using ZHeapIteratorArrayQueue = OverflowTaskQueue; using ZHeapIteratorArrayQueues = GenericTaskQueueSet; -class ZHeapIterator : public ParallelObjectIterator { +class ZHeapIterator : public ParallelObjectIteratorImpl { friend class ZHeapIteratorContext; private: diff --git a/src/hotspot/share/gc/z/zServiceability.cpp b/src/hotspot/share/gc/z/zServiceability.cpp index 744ad7270e39b..c708d0d0c479d 100644 --- a/src/hotspot/share/gc/z/zServiceability.cpp +++ b/src/hotspot/share/gc/z/zServiceability.cpp @@ -110,9 +110,8 @@ MemoryUsage ZServiceabilityMemoryPool::get_memory_usage() { } ZServiceabilityMemoryManager::ZServiceabilityMemoryManager(const char* name, - const char* end_message, ZServiceabilityMemoryPool* pool) : - GCMemoryManager(name, end_message) { + GCMemoryManager(name) { add_pool(pool); } @@ -120,8 +119,8 @@ ZServiceability::ZServiceability(size_t min_capacity, size_t max_capacity) : _min_capacity(min_capacity), _max_capacity(max_capacity), _memory_pool(_min_capacity, _max_capacity), - _cycle_memory_manager("ZGC Cycles", "end of GC cycle", &_memory_pool), - _pause_memory_manager("ZGC Pauses", "end of GC pause", &_memory_pool), + _cycle_memory_manager("ZGC Cycles", &_memory_pool), + _pause_memory_manager("ZGC Pauses", &_memory_pool), _counters(NULL) {} void ZServiceability::initialize() { @@ -147,6 +146,7 @@ ZServiceabilityCounters* ZServiceability::counters() { ZServiceabilityCycleTracer::ZServiceabilityCycleTracer() : _memory_manager_stats(ZHeap::heap()->serviceability_cycle_memory_manager(), ZCollectedHeap::heap()->gc_cause(), + "end of GC cycle", true /* allMemoryPoolsAffected */, true /* recordGCBeginTime */, true /* recordPreGCUsage */, @@ -161,6 +161,7 @@ ZServiceabilityPauseTracer::ZServiceabilityPauseTracer() : _counters_stats(ZHeap::heap()->serviceability_counters()->collector_counters()), _memory_manager_stats(ZHeap::heap()->serviceability_pause_memory_manager(), ZCollectedHeap::heap()->gc_cause(), + "end of GC pause", true /* allMemoryPoolsAffected */, true /* recordGCBeginTime */, false /* recordPreGCUsage */, diff --git a/src/hotspot/share/gc/z/zServiceability.hpp b/src/hotspot/share/gc/z/zServiceability.hpp index 3947038c4ceaa..5d03184d244f9 100644 --- a/src/hotspot/share/gc/z/zServiceability.hpp +++ b/src/hotspot/share/gc/z/zServiceability.hpp @@ -44,7 +44,6 @@ class ZServiceabilityMemoryPool : public CollectedMemoryPool { class ZServiceabilityMemoryManager : public GCMemoryManager { public: ZServiceabilityMemoryManager(const char* name, - const char* end_message, ZServiceabilityMemoryPool* pool); }; diff --git a/src/hotspot/share/include/jvm.h b/src/hotspot/share/include/jvm.h index f12a4bec3d024..c38684cee428b 100644 --- a/src/hotspot/share/include/jvm.h +++ b/src/hotspot/share/include/jvm.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -257,9 +257,6 @@ JVM_StartThread(JNIEnv *env, jobject thread); JNIEXPORT void JNICALL JVM_StopThread(JNIEnv *env, jobject thread, jobject exception); -JNIEXPORT jboolean JNICALL -JVM_IsThreadAlive(JNIEnv *env, jobject thread); - JNIEXPORT void JNICALL JVM_SuspendThread(JNIEnv *env, jobject thread); diff --git a/src/hotspot/share/interpreter/oopMapCache.cpp b/src/hotspot/share/interpreter/oopMapCache.cpp index 8799275683ae9..09314df40c9b6 100644 --- a/src/hotspot/share/interpreter/oopMapCache.cpp +++ b/src/hotspot/share/interpreter/oopMapCache.cpp @@ -536,6 +536,7 @@ void OopMapCache::lookup(const methodHandle& method, // at this time. We give the caller of lookup() a copy of the // interesting info via parameter entry_for, but we don't add it to // the cache. See the gory details in Method*.cpp. + tmp->flush(); FREE_C_HEAP_OBJ(tmp); return; } @@ -606,5 +607,6 @@ void OopMapCache::compute_one_oop_map(const methodHandle& method, int bci, Inter tmp->initialize(); tmp->fill(method, bci); entry->resource_copy(tmp); + tmp->flush(); FREE_C_HEAP_OBJ(tmp); } diff --git a/src/hotspot/share/jfr/dcmd/jfrDcmds.cpp b/src/hotspot/share/jfr/dcmd/jfrDcmds.cpp index 875e7f8e475a3..a21a4fa7d4823 100644 --- a/src/hotspot/share/jfr/dcmd/jfrDcmds.cpp +++ b/src/hotspot/share/jfr/dcmd/jfrDcmds.cpp @@ -30,6 +30,7 @@ #include "jfr/jni/jfrJavaSupport.hpp" #include "jfr/recorder/jfrRecorder.hpp" #include "jfr/recorder/service/jfrOptionSet.hpp" +#include "jfr/support/jfrThreadLocal.hpp" #include "logging/log.hpp" #include "logging/logConfiguration.hpp" #include "logging/logMessage.hpp" @@ -294,12 +295,10 @@ static void initialize_dummy_descriptors(GrowableArray* array // we keep them in a thread local arena. The arena is reset between invocations. static THREAD_LOCAL Arena* dcmd_arena = NULL; -static void prepare_dcmd_string_arena() { - if (dcmd_arena == NULL) { - dcmd_arena = new (mtTracing) Arena(mtTracing); - } else { - dcmd_arena->destruct_contents(); // will grow on next allocation - } +static void prepare_dcmd_string_arena(JavaThread* jt) { + dcmd_arena = JfrThreadLocal::dcmd_arena(jt); + assert(dcmd_arena != nullptr, "invariant"); + dcmd_arena->destruct_contents(); // will grow on next allocation } static char* dcmd_arena_allocate(size_t size) { @@ -307,7 +306,7 @@ static char* dcmd_arena_allocate(size_t size) { return (char*)dcmd_arena->Amalloc(size); } -static const char* get_as_dcmd_arena_string(oop string, JavaThread* t) { +static const char* get_as_dcmd_arena_string(oop string) { char* str = NULL; const typeArrayOop value = java_lang_String::value(string); if (value != NULL) { @@ -328,7 +327,7 @@ static const char* read_string_field(oop argument, const char* field_name, TRAPS args.set_receiver(argument); JfrJavaSupport::get_field(&args, THREAD); const oop string_oop = result.get_oop(); - return string_oop != NULL ? get_as_dcmd_arena_string(string_oop, (JavaThread*)THREAD) : NULL; + return string_oop != NULL ? get_as_dcmd_arena_string(string_oop) : NULL; } static bool read_boolean_field(oop argument, const char* field_name, TRAPS) { @@ -377,7 +376,7 @@ GrowableArray* JfrDCmd::argument_info_array() const { assert(arguments->is_array(), "must be array"); const int num_arguments = arguments->length(); assert(num_arguments == _num_arguments, "invariant"); - prepare_dcmd_string_arena(); + prepare_dcmd_string_arena(thread); for (int i = 0; i < num_arguments; ++i) { DCmdArgumentInfo* const dai = create_info(arguments->obj_at(i), thread); assert(dai != NULL, "invariant"); diff --git a/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp b/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp index 5645ed396d8a1..1ff65aaf5c955 100644 --- a/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp +++ b/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp @@ -726,20 +726,21 @@ static bool check_exclusion_state_on_thread_start(JavaThread* jt) { return true; } -static JavaThread* get_native(jobject thread) { - ThreadsListHandle tlh; +static JavaThread* get_native(ThreadsListHandle& tlh, jobject thread) { JavaThread* native_thread = NULL; (void)tlh.cv_internal_thread_to_JavaThread(thread, &native_thread, NULL); return native_thread; } jlong JfrJavaSupport::jfr_thread_id(jobject thread) { - JavaThread* native_thread = get_native(thread); + ThreadsListHandle tlh; + JavaThread* native_thread = get_native(tlh, thread); return native_thread != NULL ? JFR_THREAD_ID(native_thread) : 0; } void JfrJavaSupport::exclude(jobject thread) { - JavaThread* native_thread = get_native(thread); + ThreadsListHandle tlh; + JavaThread* native_thread = get_native(tlh, thread); if (native_thread != NULL) { JfrThreadLocal::exclude(native_thread); } else { @@ -749,7 +750,8 @@ void JfrJavaSupport::exclude(jobject thread) { } void JfrJavaSupport::include(jobject thread) { - JavaThread* native_thread = get_native(thread); + ThreadsListHandle tlh; + JavaThread* native_thread = get_native(tlh, thread); if (native_thread != NULL) { JfrThreadLocal::include(native_thread); } else { @@ -759,7 +761,8 @@ void JfrJavaSupport::include(jobject thread) { } bool JfrJavaSupport::is_excluded(jobject thread) { - JavaThread* native_thread = get_native(thread); + ThreadsListHandle tlh; + JavaThread* native_thread = get_native(tlh, thread); return native_thread != NULL ? native_thread->jfr_thread_local()->is_excluded() : is_thread_excluded(thread); } diff --git a/src/hotspot/share/jfr/jni/jfrJniMethod.cpp b/src/hotspot/share/jfr/jni/jfrJniMethod.cpp index 54a4680ced84b..1ed22eea9b3c0 100644 --- a/src/hotspot/share/jfr/jni/jfrJniMethod.cpp +++ b/src/hotspot/share/jfr/jni/jfrJniMethod.cpp @@ -362,3 +362,7 @@ JVM_END JVM_ENTRY_NO_ENV(jboolean, jfr_set_handler(JNIEnv * env, jobject jvm, jobject clazz, jobject handler)) return JfrJavaSupport::set_handler(clazz, handler, thread); JVM_END + +JVM_ENTRY_NO_ENV(void, jfr_emit_data_loss(JNIEnv* env, jclass jvm, jlong bytes)) + EventDataLoss::commit(bytes, min_jlong); +JVM_END diff --git a/src/hotspot/share/jfr/jni/jfrJniMethod.hpp b/src/hotspot/share/jfr/jni/jfrJniMethod.hpp index 19a676c4a22b9..105bad87f2beb 100644 --- a/src/hotspot/share/jfr/jni/jfrJniMethod.hpp +++ b/src/hotspot/share/jfr/jni/jfrJniMethod.hpp @@ -152,6 +152,8 @@ jboolean JNICALL jfr_set_handler(JNIEnv* env, jobject jvm, jobject clazz, jobjec jlong JNICALL jfr_get_type_id_from_string(JNIEnv* env, jobject jvm, jstring type); +void JNICALL jfr_emit_data_loss(JNIEnv* env, jclass jvm, jlong bytes); + #ifdef __cplusplus } #endif diff --git a/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp b/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp index db137776f6544..7db414b80e1ac 100644 --- a/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp +++ b/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp @@ -90,7 +90,8 @@ JfrJniMethodRegistration::JfrJniMethodRegistration(JNIEnv* env) { (char*)"getChunkStartNanos", (char*)"()J", (void*)jfr_chunk_start_nanos, (char*)"getHandler", (char*)"(Ljava/lang/Class;)Ljava/lang/Object;", (void*)jfr_get_handler, (char*)"setHandler", (char*)"(Ljava/lang/Class;Ljdk/jfr/internal/handlers/EventHandler;)Z", (void*)jfr_set_handler, - (char*)"getTypeId", (char*)"(Ljava/lang/String;)J", (void*)jfr_get_type_id_from_string + (char*)"getTypeId", (char*)"(Ljava/lang/String;)J", (void*)jfr_get_type_id_from_string, + (char*)"emitDataLoss", (char*)"(J)V", (void*)jfr_emit_data_loss }; const size_t method_array_length = sizeof(method) / sizeof(JNINativeMethod); diff --git a/src/hotspot/share/jfr/leakprofiler/chains/dfsClosure.cpp b/src/hotspot/share/jfr/leakprofiler/chains/dfsClosure.cpp index 8161e172215f4..c6d368a902042 100644 --- a/src/hotspot/share/jfr/leakprofiler/chains/dfsClosure.cpp +++ b/src/hotspot/share/jfr/leakprofiler/chains/dfsClosure.cpp @@ -80,24 +80,23 @@ void DFSClosure::closure_impl(UnifiedOopRef reference, const oop pointee) { if (GranularTimer::is_finished()) { return; } + if (_depth == 0 && _ignore_root_set) { // Root set is already marked, but we want // to continue, so skip is_marked check. assert(_mark_bits->is_marked(pointee), "invariant"); - } else { + _reference_stack[_depth] = reference; + } else { if (_mark_bits->is_marked(pointee)) { return; } + _mark_bits->mark_obj(pointee); + _reference_stack[_depth] = reference; + // is the pointee a sample object? + if (pointee->mark().is_marked()) { + add_chain(); + } } - _reference_stack[_depth] = reference; - _mark_bits->mark_obj(pointee); - assert(_mark_bits->is_marked(pointee), "invariant"); - - // is the pointee a sample object? - if (pointee->mark().is_marked()) { - add_chain(); - } - assert(_max_depth >= 1, "invariant"); if (_depth < _max_depth - 1) { _depth++; diff --git a/src/hotspot/share/jfr/leakprofiler/chains/edgeStore.cpp b/src/hotspot/share/jfr/leakprofiler/chains/edgeStore.cpp index 011dce70da4f4..c9bee7d1ab6d5 100644 --- a/src/hotspot/share/jfr/leakprofiler/chains/edgeStore.cpp +++ b/src/hotspot/share/jfr/leakprofiler/chains/edgeStore.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,8 +25,10 @@ #include "precompiled.hpp" #include "jfr/leakprofiler/chains/edgeStore.hpp" #include "jfr/leakprofiler/chains/edgeUtils.hpp" +#include "jfr/leakprofiler/sampling/objectSample.hpp" #include "jfr/leakprofiler/utilities/unifiedOopRef.inline.hpp" #include "oops/oop.inline.hpp" +#include "runtime/safepoint.hpp" StoredEdge::StoredEdge(const Edge* parent, UnifiedOopRef reference) : Edge(parent, reference), _gc_root_id(0), _skip_length(0) {} @@ -36,15 +38,6 @@ StoredEdge::StoredEdge(const StoredEdge& edge) : Edge(edge), _gc_root_id(edge._g traceid EdgeStore::_edge_id_counter = 0; -EdgeStore::EdgeStore() : _edges(NULL) { - _edges = new EdgeHashTable(this); -} - -EdgeStore::~EdgeStore() { - assert(_edges != NULL, "invariant"); - delete _edges; -} - bool EdgeStore::is_empty() const { return !_edges->has_entries(); } @@ -224,15 +217,91 @@ bool EdgeStore::put_edges(StoredEdge** previous, const Edge** current, size_t li return NULL == *current; } -// Install the immediate edge into the mark word of the leak candidate object +static GrowableArray* _leak_context_edges = nullptr; + +EdgeStore::EdgeStore() : _edges(new EdgeHashTable(this)) {} + +EdgeStore::~EdgeStore() { + assert(_edges != NULL, "invariant"); + delete _edges; + delete _leak_context_edges; + _leak_context_edges = nullptr; +} + +static int leak_context_edge_idx(const ObjectSample* sample) { + assert(sample != nullptr, "invariant"); + return static_cast(sample->object()->mark().value()) >> markWord::lock_bits; +} + +bool EdgeStore::has_leak_context(const ObjectSample* sample) const { + const int idx = leak_context_edge_idx(sample); + if (idx == 0) { + return false; + } + assert(idx > 0, "invariant"); + assert(_leak_context_edges != nullptr, "invariant"); + assert(idx < _leak_context_edges->length(), "invariant"); + assert(_leak_context_edges->at(idx) != nullptr, "invariant"); + return true; +} + +const StoredEdge* EdgeStore::get(const ObjectSample* sample) const { + assert(sample != nullptr, "invariant"); + if (_leak_context_edges != nullptr) { + assert(SafepointSynchronize::is_at_safepoint(), "invariant"); + const int idx = leak_context_edge_idx(sample); + if (idx > 0) { + assert(idx < _leak_context_edges->length(), "invariant"); + const StoredEdge* const edge =_leak_context_edges->at(idx); + assert(edge != nullptr, "invariant"); + return edge; + } + } + return get(UnifiedOopRef::encode_in_native(sample->object_addr())); +} + +#ifdef ASSERT +// max_idx to ensure idx fit in lower 32-bits of markword together with lock bits. +static constexpr const int max_idx = right_n_bits(32 - markWord::lock_bits); + +static void store_idx_precondition(oop sample_object, int idx) { + assert(sample_object != NULL, "invariant"); + assert(sample_object->mark().is_marked(), "invariant"); + assert(idx > 0, "invariant"); + assert(idx <= max_idx, "invariant"); +} +#endif + +static void store_idx_in_markword(oop sample_object, int idx) { + DEBUG_ONLY(store_idx_precondition(sample_object, idx);) + const markWord idx_mark_word(sample_object->mark().value() | idx << markWord::lock_bits); + sample_object->set_mark(idx_mark_word); + assert(sample_object->mark().is_marked(), "must still be marked"); +} + +static const int initial_size = 64; + +static int save(const StoredEdge* edge) { + assert(edge != nullptr, "invariant"); + if (_leak_context_edges == nullptr) { + _leak_context_edges = new (ResourceObj::C_HEAP, mtTracing)GrowableArray(initial_size, mtTracing); + _leak_context_edges->append(nullptr); // next idx now at 1, for disambiguation in markword. + } + return _leak_context_edges->append(edge); +} + +// We associate the leak context edge with the leak candidate object by saving the +// edge in an array and storing the array idx (shifted) into the markword of the candidate object. +static void associate_with_candidate(const StoredEdge* leak_context_edge) { + assert(leak_context_edge != nullptr, "invariant"); + store_idx_in_markword(leak_context_edge->pointee(), save(leak_context_edge)); +} + StoredEdge* EdgeStore::associate_leak_context_with_candidate(const Edge* edge) { assert(edge != NULL, "invariant"); assert(!contains(edge->reference()), "invariant"); StoredEdge* const leak_context_edge = put(edge->reference()); - oop sample_object = edge->pointee(); - assert(sample_object != NULL, "invariant"); - assert(sample_object->mark().is_marked(), "invariant"); - sample_object->set_mark(markWord::from_pointer(leak_context_edge)); + associate_with_candidate(leak_context_edge); return leak_context_edge; } diff --git a/src/hotspot/share/jfr/leakprofiler/chains/edgeStore.hpp b/src/hotspot/share/jfr/leakprofiler/chains/edgeStore.hpp index f948525a5b980..e920fd64ea99f 100644 --- a/src/hotspot/share/jfr/leakprofiler/chains/edgeStore.hpp +++ b/src/hotspot/share/jfr/leakprofiler/chains/edgeStore.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,6 +31,7 @@ #include "memory/allocation.hpp" typedef u8 traceid; +class ObjectSample; class StoredEdge : public Edge { private: @@ -79,6 +80,7 @@ class EdgeStore : public CHeapObj { void on_unlink(EdgeEntry* entry); StoredEdge* get(UnifiedOopRef reference) const; + const StoredEdge* get(const ObjectSample* sample) const; StoredEdge* put(UnifiedOopRef reference); traceid gc_root_id(const Edge* edge) const; @@ -90,6 +92,7 @@ class EdgeStore : public CHeapObj { void store_gc_root_id_in_leak_context_edge(StoredEdge* leak_context_edge, const Edge* root) const; StoredEdge* link_new_edge(StoredEdge** previous, const Edge** current); void link_with_existing_chain(const StoredEdge* current_stored, StoredEdge** previous, size_t previous_length); + bool has_leak_context(const ObjectSample* sample) const; template void iterate(T& functor) const { _edges->iterate_value(functor); } diff --git a/src/hotspot/share/jfr/leakprofiler/checkpoint/eventEmitter.cpp b/src/hotspot/share/jfr/leakprofiler/checkpoint/eventEmitter.cpp index 0cb3841a64f5e..2fa5a99619464 100644 --- a/src/hotspot/share/jfr/leakprofiler/checkpoint/eventEmitter.cpp +++ b/src/hotspot/share/jfr/leakprofiler/checkpoint/eventEmitter.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -120,7 +120,7 @@ void EventEmitter::link_sample_with_edge(const ObjectSample* sample, EdgeStore* assert(!sample->is_dead(), "invariant"); assert(edge_store != NULL, "invariant"); if (SafepointSynchronize::is_at_safepoint()) { - if (!sample->object()->mark().is_marked()) { + if (edge_store->has_leak_context(sample)) { // Associated with an edge (chain) already during heap traversal. return; } @@ -137,21 +137,12 @@ void EventEmitter::write_event(const ObjectSample* sample, EdgeStore* edge_store assert(edge_store != NULL, "invariant"); assert(_jfr_thread_local != NULL, "invariant"); - traceid gc_root_id = 0; - const Edge* edge = NULL; - if (SafepointSynchronize::is_at_safepoint()) { - if (!sample->object()->mark().is_marked()) { - edge = (const Edge*)(sample->object())->mark().to_pointer(); - } - } - if (edge == NULL) { - edge = edge_store->get(UnifiedOopRef::encode_in_native(sample->object_addr())); - } else { - gc_root_id = edge_store->gc_root_id(edge); - } + const StoredEdge* const edge = edge_store->get(sample); assert(edge != NULL, "invariant"); + assert(edge->pointee() == sample->object(), "invariant"); const traceid object_id = edge_store->get_id(edge); assert(object_id != 0, "invariant"); + const traceid gc_root_id = edge->gc_root_id(); Tickspan object_age = Ticks(_start_time.value()) - sample->allocation_time(); diff --git a/src/hotspot/share/jfr/metadata/metadata.xml b/src/hotspot/share/jfr/metadata/metadata.xml index 6d7e5cf6b3fdd..f75df93e6735a 100644 --- a/src/hotspot/share/jfr/metadata/metadata.xml +++ b/src/hotspot/share/jfr/metadata/metadata.xml @@ -349,7 +349,7 @@ - + @@ -709,7 +709,9 @@ - + @@ -717,14 +719,18 @@ - + - + diff --git a/src/hotspot/share/jfr/periodic/jfrOSInterface.cpp b/src/hotspot/share/jfr/periodic/jfrOSInterface.cpp index 85f6614ff5e9d..a0aa9154ff5c7 100644 --- a/src/hotspot/share/jfr/periodic/jfrOSInterface.cpp +++ b/src/hotspot/share/jfr/periodic/jfrOSInterface.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -253,6 +253,8 @@ const char* JfrOSInterface::virtualization_name() { VirtualizationType vrt = VM_Version::get_detected_virtualization(); if (vrt == XenHVM) { return "Xen hardware-assisted virtualization"; + } else if (vrt == XenPVHVM) { + return "Xen optimized paravirtualization"; } else if (vrt == KVM) { return "KVM virtualization"; } else if (vrt == VMWare) { diff --git a/src/hotspot/share/jfr/recorder/service/jfrRecorderService.cpp b/src/hotspot/share/jfr/recorder/service/jfrRecorderService.cpp index 7a14b082dfece..0627593611cd3 100644 --- a/src/hotspot/share/jfr/recorder/service/jfrRecorderService.cpp +++ b/src/hotspot/share/jfr/recorder/service/jfrRecorderService.cpp @@ -313,18 +313,20 @@ static size_t write_storage(JfrStorage& storage, JfrChunkWriter& chunkwriter) { return invoke(fs); } -typedef Content StringPool; -typedef WriteCheckpointEvent WriteStringPool; +typedef Content FlushStringPoolFunctor; +typedef Content WriteStringPoolFunctor; +typedef WriteCheckpointEvent FlushStringPool; +typedef WriteCheckpointEvent WriteStringPool; static u4 flush_stringpool(JfrStringPool& string_pool, JfrChunkWriter& chunkwriter) { - StringPool sp(string_pool); - WriteStringPool wsp(chunkwriter, sp, TYPE_STRING); - return invoke(wsp); + FlushStringPoolFunctor fspf(string_pool); + FlushStringPool fsp(chunkwriter, fspf, TYPE_STRING); + return invoke(fsp); } static u4 write_stringpool(JfrStringPool& string_pool, JfrChunkWriter& chunkwriter) { - StringPool sp(string_pool); - WriteStringPool wsp(chunkwriter, sp, TYPE_STRING); + WriteStringPoolFunctor wspf(string_pool); + WriteStringPool wsp(chunkwriter, wspf, TYPE_STRING); return invoke(wsp); } @@ -435,7 +437,6 @@ void JfrRecorderService::clear() { } void JfrRecorderService::pre_safepoint_clear() { - _string_pool.clear(); _storage.clear(); JfrStackTraceRepository::clear(); } @@ -449,7 +450,6 @@ void JfrRecorderService::invoke_safepoint_clear() { void JfrRecorderService::safepoint_clear() { assert(SafepointSynchronize::is_at_safepoint(), "invariant"); _checkpoint_manager.begin_epoch_shift(); - _string_pool.clear(); _storage.clear(); _chunkwriter.set_time_stamp(); JfrStackTraceRepository::clear(); @@ -457,6 +457,7 @@ void JfrRecorderService::safepoint_clear() { } void JfrRecorderService::post_safepoint_clear() { + _string_pool.clear(); _checkpoint_manager.clear(); } @@ -541,9 +542,6 @@ void JfrRecorderService::pre_safepoint_write() { // The sampler is released (unlocked) later in post_safepoint_write. ObjectSampleCheckpoint::on_rotation(ObjectSampler::acquire()); } - if (_string_pool.is_modified()) { - write_stringpool(_string_pool, _chunkwriter); - } write_storage(_storage, _chunkwriter); if (_stack_trace_repository.is_modified()) { write_stacktrace(_stack_trace_repository, _chunkwriter, false); @@ -561,9 +559,6 @@ void JfrRecorderService::safepoint_write() { assert(SafepointSynchronize::is_at_safepoint(), "invariant"); _checkpoint_manager.begin_epoch_shift(); JfrStackTraceRepository::clear_leak_profiler(); - if (_string_pool.is_modified()) { - write_stringpool(_string_pool, _chunkwriter); - } _checkpoint_manager.on_rotation(); _storage.write_at_safepoint(); _chunkwriter.set_time_stamp(); @@ -577,6 +572,7 @@ void JfrRecorderService::post_safepoint_write() { // Type tagging is epoch relative which entails we are able to write out the // already tagged artifacts for the previous epoch. We can accomplish this concurrently // with threads now tagging artifacts in relation to the new, now updated, epoch and remain outside of a safepoint. + write_stringpool(_string_pool, _chunkwriter); _checkpoint_manager.write_type_set(); if (LeakProfiler::is_running()) { // The object sampler instance was exclusively acquired and locked in pre_safepoint_write. diff --git a/src/hotspot/share/jfr/recorder/service/jfrRecorderThread.cpp b/src/hotspot/share/jfr/recorder/service/jfrRecorderThread.cpp index 0ccc2dbf17999..00f504a81e5df 100644 --- a/src/hotspot/share/jfr/recorder/service/jfrRecorderThread.cpp +++ b/src/hotspot/share/jfr/recorder/service/jfrRecorderThread.cpp @@ -46,29 +46,18 @@ static Thread* start_thread(instanceHandle thread_oop, ThreadFunction proc, TRAP assert(thread_oop.not_null(), "invariant"); assert(proc != NULL, "invariant"); - bool allocation_failed = false; - JavaThread* new_thread = NULL; - { - MutexLocker mu(THREAD, Threads_lock); - new_thread = new JavaThread(proc); - // At this point it may be possible that no - // osthread was created for the JavaThread due to lack of memory. - if (new_thread == NULL || new_thread->osthread() == NULL) { - delete new_thread; - allocation_failed = true; - } else { - java_lang_Thread::set_thread(thread_oop(), new_thread); - java_lang_Thread::set_priority(thread_oop(), NormPriority); - java_lang_Thread::set_daemon(thread_oop()); - new_thread->set_threadObj(thread_oop()); - Threads::add(new_thread); - } - } - if (allocation_failed) { - JfrJavaSupport::throw_out_of_memory_error("Unable to create native recording thread for JFR", CHECK_NULL); + JavaThread* new_thread = new JavaThread(proc); + + // At this point it may be possible that no + // osthread was created for the JavaThread due to lack of resources. + if (new_thread->osthread() == NULL) { + delete new_thread; + JfrJavaSupport::throw_out_of_memory_error("Unable to create native recording thread for JFR", THREAD); + return NULL; + } else { + JavaThread::start_internal_daemon(THREAD, new_thread, thread_oop, NormPriority); + return new_thread; } - Thread::start(new_thread); - return new_thread; } JfrPostBox* JfrRecorderThread::_post_box = NULL; diff --git a/src/hotspot/share/jfr/recorder/storage/jfrStorageUtils.hpp b/src/hotspot/share/jfr/recorder/storage/jfrStorageUtils.hpp index 36ae38b6c2b6e..5c9cefa4dbcef 100644 --- a/src/hotspot/share/jfr/recorder/storage/jfrStorageUtils.hpp +++ b/src/hotspot/share/jfr/recorder/storage/jfrStorageUtils.hpp @@ -214,4 +214,11 @@ class EpochDispatchOp { size_t elements() const { return _elements; } }; +template +class ReinitializationOp { + public: + typedef T Type; + bool process(Type* t); +}; + #endif // SHARE_JFR_RECORDER_STORAGE_JFRSTORAGEUTILS_HPP diff --git a/src/hotspot/share/jfr/recorder/storage/jfrStorageUtils.inline.hpp b/src/hotspot/share/jfr/recorder/storage/jfrStorageUtils.inline.hpp index 4ed8000d18275..4323bba92e894 100644 --- a/src/hotspot/share/jfr/recorder/storage/jfrStorageUtils.inline.hpp +++ b/src/hotspot/share/jfr/recorder/storage/jfrStorageUtils.inline.hpp @@ -168,4 +168,13 @@ size_t EpochDispatchOp::dispatch(bool previous_epoch, const u1* eleme return elements; } +template +bool ReinitializationOp::process(T* t) { + assert(t != nullptr, "invariant"); + assert(t->identity() != nullptr, "invariant"); + t->reinitialize(); + t->release(); + return true; +} + #endif // SHARE_JFR_RECORDER_STORAGE_JFRSTORAGEUTILS_INLINE_HPP diff --git a/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.cpp b/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.cpp index 7001f0f0a0272..132295230ac5a 100644 --- a/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.cpp +++ b/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.cpp @@ -77,10 +77,17 @@ static const size_t string_pool_buffer_size = 512 * K; bool JfrStringPool::initialize() { assert(_mspace == NULL, "invariant"); _mspace = create_mspace(string_pool_buffer_size, - string_pool_cache_count, // cache limit - string_pool_cache_count, // cache preallocate count - false, // preallocate_to_free_list (== preallocate directly to live list) + 0, + 0, // cache preallocate count + false, this); + + // preallocate buffer count to each of the epoch live lists + for (size_t i = 0; i < string_pool_cache_count * 2; ++i) { + Buffer* const buffer = mspace_allocate(string_pool_buffer_size, _mspace); + _mspace->add_to_live_list(buffer, i % 2 == 0); + } + assert(_mspace->free_list_is_empty(), "invariant"); return _mspace != NULL; } @@ -95,11 +102,7 @@ static void release(BufferPtr buffer, Thread* thread) { assert(buffer->lease(), "invariant"); assert(buffer->acquired_by_self(), "invariant"); buffer->clear_lease(); - if (buffer->transient()) { - buffer->set_retired(); - } else { - buffer->release(); - } + buffer->release(); } BufferPtr JfrStringPool::flush(BufferPtr old, size_t used, size_t requested, Thread* thread) { @@ -180,8 +183,10 @@ typedef StringPoolOp WriteOperation; typedef StringPoolOp DiscardOperation; typedef ExclusiveOp ExclusiveWriteOperation; typedef ExclusiveOp ExclusiveDiscardOperation; +typedef ReinitializationOp ReinitializationOperation; typedef ReleaseWithExcisionOp ReleaseOperation; typedef CompositeOperation WriteReleaseOperation; +typedef CompositeOperation WriteReinitializeOperation; typedef CompositeOperation DiscardReleaseOperation; size_t JfrStringPool::write() { @@ -189,10 +194,22 @@ size_t JfrStringPool::write() { WriteOperation wo(_chunkwriter, thread); ExclusiveWriteOperation ewo(wo); assert(_mspace->free_list_is_empty(), "invariant"); - ReleaseOperation ro(_mspace, _mspace->live_list()); + ReleaseOperation ro(_mspace, _mspace->live_list(true)); // previous epoch list WriteReleaseOperation wro(&ewo, &ro); assert(_mspace->live_list_is_nonempty(), "invariant"); - process_live_list(wro, _mspace); + process_live_list(wro, _mspace, true); // previous epoch list + return wo.processed(); +} + +size_t JfrStringPool::flush() { + Thread* const thread = Thread::current(); + WriteOperation wo(_chunkwriter, thread); + ExclusiveWriteOperation ewo(wo); + ReinitializationOperation rio; + WriteReinitializeOperation wro(&ewo, &rio); + assert(_mspace->free_list_is_empty(), "invariant"); + assert(_mspace->live_list_is_nonempty(), "invariant"); + process_live_list(wro, _mspace); // current epoch list return wo.processed(); } @@ -200,10 +217,10 @@ size_t JfrStringPool::clear() { DiscardOperation discard_operation; ExclusiveDiscardOperation edo(discard_operation); assert(_mspace->free_list_is_empty(), "invariant"); - ReleaseOperation ro(_mspace, _mspace->live_list()); + ReleaseOperation ro(_mspace, _mspace->live_list(true)); // previous epoch list DiscardReleaseOperation discard_op(&edo, &ro); assert(_mspace->live_list_is_nonempty(), "invariant"); - process_live_list(discard_op, _mspace); + process_live_list(discard_op, _mspace, true); // previous epoch list return discard_operation.processed(); } diff --git a/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.hpp b/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.hpp index 3d437a06506b6..c27575d38ccdf 100644 --- a/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.hpp +++ b/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -35,7 +35,7 @@ class JavaThread; class JfrChunkWriter; class JfrStringPool; -typedef JfrMemorySpace > JfrStringPoolMspace; +typedef JfrMemorySpace, JfrLinkedList, true > JfrStringPoolMspace; // // Although called JfrStringPool, a more succinct description would be @@ -45,8 +45,10 @@ typedef JfrMemorySpaceset_retired(); _load_barrier_buffer_epoch_1 = NULL; } + if (_dcmd_arena != nullptr) { + delete _dcmd_arena; + _dcmd_arena = nullptr; + } } void JfrThreadLocal::release(JfrThreadLocal* tl, Thread* t) { @@ -218,3 +224,15 @@ void JfrThreadLocal::include(Thread* t) { u4 JfrThreadLocal::stackdepth() const { return _stackdepth != 0 ? _stackdepth : (u4)JfrOptionSet::stackdepth(); } + +Arena* JfrThreadLocal::dcmd_arena(JavaThread* jt) { + assert(jt != nullptr, "invariant"); + JfrThreadLocal* tl = jt->jfr_thread_local(); + Arena* arena = tl->_dcmd_arena; + if (arena != nullptr) { + return arena; + } + arena = new (mtTracing) Arena(mtTracing); + tl->_dcmd_arena = arena; + return arena; +} diff --git a/src/hotspot/share/jfr/support/jfrThreadLocal.hpp b/src/hotspot/share/jfr/support/jfrThreadLocal.hpp index 7e4a0819cf200..ab7d6170cdb6d 100644 --- a/src/hotspot/share/jfr/support/jfrThreadLocal.hpp +++ b/src/hotspot/share/jfr/support/jfrThreadLocal.hpp @@ -28,6 +28,7 @@ #include "jfr/utilities/jfrBlob.hpp" #include "jfr/utilities/jfrTypes.hpp" +class Arena; class JavaThread; class JfrBuffer; class JfrStackFrame; @@ -42,6 +43,7 @@ class JfrThreadLocal { JfrBuffer* _load_barrier_buffer_epoch_0; JfrBuffer* _load_barrier_buffer_epoch_1; mutable JfrStackFrame* _stackframes; + Arena* _dcmd_arena; mutable traceid _trace_id; JfrBlobHandle _thread; u8 _data_lost; @@ -219,6 +221,8 @@ class JfrThreadLocal { return _dead; } + static Arena* dcmd_arena(JavaThread* jt); + bool has_thread_blob() const; void set_thread_blob(const JfrBlobHandle& handle); const JfrBlobHandle& thread_blob() const; diff --git a/src/hotspot/share/jfr/utilities/jfrBigEndian.hpp b/src/hotspot/share/jfr/utilities/jfrBigEndian.hpp index db2e7ebaad2f9..597ddb3800ff6 100644 --- a/src/hotspot/share/jfr/utilities/jfrBigEndian.hpp +++ b/src/hotspot/share/jfr/utilities/jfrBigEndian.hpp @@ -102,7 +102,7 @@ inline T JfrBigEndian::read_unaligned(const address location) { inline bool JfrBigEndian::platform_supports_unaligned_reads(void) { #if defined(IA32) || defined(AMD64) || defined(PPC) || defined(S390) return true; -#elif defined(ARM) || defined(AARCH64) +#elif defined(ARM) || defined(AARCH64) || defined(RISCV) return false; #else #warning "Unconfigured platform" diff --git a/src/hotspot/share/jvmci/jvmciCodeInstaller.cpp b/src/hotspot/share/jvmci/jvmciCodeInstaller.cpp index 1eeb48a9a5fde..3c8abb36ef11d 100644 --- a/src/hotspot/share/jvmci/jvmciCodeInstaller.cpp +++ b/src/hotspot/share/jvmci/jvmciCodeInstaller.cpp @@ -1147,7 +1147,9 @@ void CodeInstaller::site_Call(CodeBuffer& buffer, jint pc_offset, JVMCIObject si CodeInstaller::pd_relocate_JavaMethod(buffer, hotspot_method, pc_offset, JVMCI_CHECK); if (_next_call_type == INVOKESTATIC || _next_call_type == INVOKESPECIAL) { // Need a static call stub for transitions from compiled to interpreted. - CompiledStaticCall::emit_to_interp_stub(buffer, _instructions->start() + pc_offset); + if (CompiledStaticCall::emit_to_interp_stub(buffer, _instructions->start() + pc_offset) == nullptr) { + JVMCI_ERROR("could not emit to_interp stub - code cache is full"); + } } } diff --git a/src/hotspot/share/logging/logAsyncWriter.cpp b/src/hotspot/share/logging/logAsyncWriter.cpp index e7c9ff8c7f9ce..e5099b708583e 100644 --- a/src/hotspot/share/logging/logAsyncWriter.cpp +++ b/src/hotspot/share/logging/logAsyncWriter.cpp @@ -25,6 +25,7 @@ #include "logging/logAsyncWriter.hpp" #include "logging/logConfiguration.hpp" #include "logging/logFileOutput.hpp" +#include "logging/logFileStreamOutput.hpp" #include "logging/logHandle.hpp" #include "runtime/atomic.hpp" #include "runtime/os.inline.hpp" @@ -56,7 +57,7 @@ void AsyncLogWriter::enqueue_locked(const AsyncLogMessage& msg) { _lock.notify(); } -void AsyncLogWriter::enqueue(LogFileOutput& output, const LogDecorations& decorations, const char* msg) { +void AsyncLogWriter::enqueue(LogFileStreamOutput& output, const LogDecorations& decorations, const char* msg) { AsyncLogMessage m(&output, decorations, os::strdup(msg)); { // critical area @@ -67,7 +68,7 @@ void AsyncLogWriter::enqueue(LogFileOutput& output, const LogDecorations& decora // LogMessageBuffer consists of a multiple-part/multiple-line messsage. // The lock here guarantees its integrity. -void AsyncLogWriter::enqueue(LogFileOutput& output, LogMessageBuffer::Iterator msg_iterator) { +void AsyncLogWriter::enqueue(LogFileStreamOutput& output, LogMessageBuffer::Iterator msg_iterator) { AsyncLogLocker locker; for (; !msg_iterator.is_at_end(); msg_iterator++) { @@ -95,7 +96,7 @@ class AsyncLogMapIterator { public: AsyncLogMapIterator(AsyncLogBuffer& logs) :_logs(logs) {} - bool do_entry(LogFileOutput* output, uint32_t* counter) { + bool do_entry(LogFileStreamOutput* output, uint32_t* counter) { using none = LogTagSetMapping; if (*counter > 0) { diff --git a/src/hotspot/share/logging/logAsyncWriter.hpp b/src/hotspot/share/logging/logAsyncWriter.hpp index 76166f9a25610..858dd6ddbac44 100644 --- a/src/hotspot/share/logging/logAsyncWriter.hpp +++ b/src/hotspot/share/logging/logAsyncWriter.hpp @@ -25,10 +25,10 @@ #define SHARE_LOGGING_LOGASYNCWRITER_HPP #include "logging/log.hpp" #include "logging/logDecorations.hpp" -#include "logging/logFileOutput.hpp" #include "logging/logMessageBuffer.hpp" #include "memory/resourceArea.hpp" #include "runtime/nonJavaThread.hpp" +#include "runtime/semaphore.hpp" #include "utilities/hashtable.hpp" #include "utilities/linkedlist.hpp" @@ -90,25 +90,28 @@ class LinkedListDeque : private LinkedListImpl { } }; +// Forward declaration +class LogFileStreamOutput; + class AsyncLogMessage { - LogFileOutput* _output; + LogFileStreamOutput* _output; const LogDecorations _decorations; char* _message; public: - AsyncLogMessage(LogFileOutput* output, const LogDecorations& decorations, char* msg) + AsyncLogMessage(LogFileStreamOutput* output, const LogDecorations& decorations, char* msg) : _output(output), _decorations(decorations), _message(msg) {} // placeholder for LinkedListImpl. bool equals(const AsyncLogMessage& o) const { return false; } - LogFileOutput* output() const { return _output; } + LogFileStreamOutput* output() const { return _output; } const LogDecorations& decorations() const { return _decorations; } char* message() const { return _message; } }; typedef LinkedListDeque AsyncLogBuffer; -typedef KVHashtable AsyncLogMap; +typedef KVHashtable AsyncLogMap; // // ASYNC LOGGING SUPPORT @@ -156,7 +159,6 @@ class AsyncLogWriter : public NonJavaThread { log_debug(logging, thread)("starting AsyncLog Thread tid = " INTX_FORMAT, os::current_thread_id()); } char* name() const override { return (char*)"AsyncLog Thread"; } - bool is_Named_thread() const override { return true; } void print_on(outputStream* st) const override { st->print("\"%s\" ", name()); Thread::print_on(st); @@ -164,8 +166,8 @@ class AsyncLogWriter : public NonJavaThread { } public: - void enqueue(LogFileOutput& output, const LogDecorations& decorations, const char* msg); - void enqueue(LogFileOutput& output, LogMessageBuffer::Iterator msg_iterator); + void enqueue(LogFileStreamOutput& output, const LogDecorations& decorations, const char* msg); + void enqueue(LogFileStreamOutput& output, LogMessageBuffer::Iterator msg_iterator); static AsyncLogWriter* instance(); static void initialize(); diff --git a/src/hotspot/share/logging/logFileOutput.cpp b/src/hotspot/share/logging/logFileOutput.cpp index 856907f8be412..4b8065f686e7b 100644 --- a/src/hotspot/share/logging/logFileOutput.cpp +++ b/src/hotspot/share/logging/logFileOutput.cpp @@ -305,7 +305,9 @@ int LogFileOutput::write_blocking(const LogDecorations& decorations, const char* return 0; } - int written = LogFileStreamOutput::write(decorations, msg); + int written = write_internal(decorations, msg); + // Need to flush to the filesystem before should_rotate() + written = flush() ? written : -1; if (written > 0) { _current_size += written; diff --git a/src/hotspot/share/logging/logFileOutput.hpp b/src/hotspot/share/logging/logFileOutput.hpp index 9b22969368ed5..0932a9402c20f 100644 --- a/src/hotspot/share/logging/logFileOutput.hpp +++ b/src/hotspot/share/logging/logFileOutput.hpp @@ -85,7 +85,7 @@ class LogFileOutput : public LogFileStreamOutput { virtual bool initialize(const char* options, outputStream* errstream); virtual int write(const LogDecorations& decorations, const char* msg); virtual int write(LogMessageBuffer::Iterator msg_iterator); - int write_blocking(const LogDecorations& decorations, const char* msg); + virtual int write_blocking(const LogDecorations& decorations, const char* msg); virtual void force_rotate(); virtual void describe(outputStream* out); diff --git a/src/hotspot/share/logging/logFileStreamOutput.cpp b/src/hotspot/share/logging/logFileStreamOutput.cpp index 8ce465ed1ed8c..b1133c30dd347 100644 --- a/src/hotspot/share/logging/logFileStreamOutput.cpp +++ b/src/hotspot/share/logging/logFileStreamOutput.cpp @@ -23,6 +23,7 @@ */ #include "precompiled.hpp" #include "jvm.h" +#include "logging/logAsyncWriter.hpp" #include "logging/logDecorators.hpp" #include "logging/logDecorations.hpp" #include "logging/logFileStreamOutput.hpp" @@ -117,31 +118,47 @@ bool LogFileStreamOutput::flush() { total += result; \ } -int LogFileStreamOutput::write(const LogDecorations& decorations, const char* msg) { +int LogFileStreamOutput::write_internal(const LogDecorations& decorations, const char* msg) { + int written = 0; const bool use_decorations = !_decorators.is_empty(); - int written = 0; - FileLocker flocker(_stream); if (use_decorations) { WRITE_LOG_WITH_RESULT_CHECK(write_decorations(decorations), written); WRITE_LOG_WITH_RESULT_CHECK(jio_fprintf(_stream, " "), written); } WRITE_LOG_WITH_RESULT_CHECK(jio_fprintf(_stream, "%s\n", msg), written); + return written; +} + +int LogFileStreamOutput::write_blocking(const LogDecorations& decorations, const char* msg) { + int written = write_internal(decorations, msg); + return flush() ? written : -1; +} + +int LogFileStreamOutput::write(const LogDecorations& decorations, const char* msg) { + AsyncLogWriter* aio_writer = AsyncLogWriter::instance(); + if (aio_writer != nullptr) { + aio_writer->enqueue(*this, decorations, msg); + return 0; + } + + FileLocker flocker(_stream); + int written = write_internal(decorations, msg); return flush() ? written : -1; } int LogFileStreamOutput::write(LogMessageBuffer::Iterator msg_iterator) { - const bool use_decorations = !_decorators.is_empty(); + AsyncLogWriter* aio_writer = AsyncLogWriter::instance(); + if (aio_writer != nullptr) { + aio_writer->enqueue(*this, msg_iterator); + return 0; + } int written = 0; FileLocker flocker(_stream); for (; !msg_iterator.is_at_end(); msg_iterator++) { - if (use_decorations) { - WRITE_LOG_WITH_RESULT_CHECK(write_decorations(msg_iterator.decorations()), written); - WRITE_LOG_WITH_RESULT_CHECK(jio_fprintf(_stream, " "), written); - } - WRITE_LOG_WITH_RESULT_CHECK(jio_fprintf(_stream, "%s\n", msg_iterator.message()), written); + written += write_internal(msg_iterator.decorations(), msg_iterator.message()); } return flush() ? written : -1; diff --git a/src/hotspot/share/logging/logFileStreamOutput.hpp b/src/hotspot/share/logging/logFileStreamOutput.hpp index c4a0db7355d2c..cc0f5f31c7d30 100644 --- a/src/hotspot/share/logging/logFileStreamOutput.hpp +++ b/src/hotspot/share/logging/logFileStreamOutput.hpp @@ -42,6 +42,7 @@ static LogFileStreamInitializer log_stream_initializer; class LogFileStreamOutput : public LogOutput { private: bool _write_error_is_shown; + protected: FILE* _stream; size_t _decorator_padding[LogDecorators::Count]; @@ -53,11 +54,14 @@ class LogFileStreamOutput : public LogOutput { } int write_decorations(const LogDecorations& decorations); + int write_internal(const LogDecorations& decorations, const char* msg); bool flush(); public: virtual int write(const LogDecorations& decorations, const char* msg); virtual int write(LogMessageBuffer::Iterator msg_iterator); + // Write API used by AsyncLogWriter + virtual int write_blocking(const LogDecorations& decorations, const char* msg); }; class LogStdoutOutput : public LogFileStreamOutput { diff --git a/src/hotspot/share/memory/allocation.cpp b/src/hotspot/share/memory/allocation.cpp index 241213d0b2c02..89be338dadc73 100644 --- a/src/hotspot/share/memory/allocation.cpp +++ b/src/hotspot/share/memory/allocation.cpp @@ -49,7 +49,7 @@ char* AllocateHeap(size_t size, char* AllocateHeap(size_t size, MEMFLAGS flags, AllocFailType alloc_failmode /* = AllocFailStrategy::EXIT_OOM*/) { - return AllocateHeap(size, flags, CALLER_PC); + return AllocateHeap(size, flags, CALLER_PC, alloc_failmode); } char* ReallocateHeap(char *old, diff --git a/src/hotspot/share/memory/heapInspection.cpp b/src/hotspot/share/memory/heapInspection.cpp index 6a21a3c417c49..a98bdeced25e1 100644 --- a/src/hotspot/share/memory/heapInspection.cpp +++ b/src/hotspot/share/memory/heapInspection.cpp @@ -571,32 +571,16 @@ void ParHeapInspectTask::work(uint worker_id) { } } -uintx HeapInspection::populate_table(KlassInfoTable* cit, BoolObjectClosure *filter, uint parallel_thread_num) { - +uintx HeapInspection::populate_table(KlassInfoTable* cit, BoolObjectClosure *filter, WorkGang* workers) { // Try parallel first. - if (parallel_thread_num > 1) { + if (workers != nullptr) { ResourceMark rm; - - WorkGang* gang = Universe::heap()->safepoint_workers(); - if (gang != NULL) { - // The GC provided a WorkGang to be used during a safepoint. - - // Can't run with more threads than provided by the WorkGang. - WithUpdatedActiveWorkers update_and_restore(gang, parallel_thread_num); - - ParallelObjectIterator* poi = Universe::heap()->parallel_object_iterator(gang->active_workers()); - if (poi != NULL) { - // The GC supports parallel object iteration. - - ParHeapInspectTask task(poi, cit, filter); - // Run task with the active workers. - gang->run_task(&task); - - delete poi; - if (task.success()) { - return task.missed_count(); - } - } + ParallelObjectIterator poi(workers->active_workers()); + ParHeapInspectTask task(&poi, cit, filter); + // Run task with the active workers. + workers->run_task(&task); + if (task.success()) { + return task.missed_count(); } } @@ -607,13 +591,13 @@ uintx HeapInspection::populate_table(KlassInfoTable* cit, BoolObjectClosure *fil return ric.missed_count(); } -void HeapInspection::heap_inspection(outputStream* st, uint parallel_thread_num) { +void HeapInspection::heap_inspection(outputStream* st, WorkGang* workers) { ResourceMark rm; KlassInfoTable cit(false); if (!cit.allocation_failed()) { // populate table with object allocation info - uintx missed_count = populate_table(&cit, NULL, parallel_thread_num); + uintx missed_count = populate_table(&cit, NULL, workers); if (missed_count != 0) { log_info(gc, classhisto)("WARNING: Ran out of C-heap; undercounted " UINTX_FORMAT " total instances in data below", diff --git a/src/hotspot/share/memory/heapInspection.hpp b/src/hotspot/share/memory/heapInspection.hpp index a7b2d6fa3f063..bdf57809d459c 100644 --- a/src/hotspot/share/memory/heapInspection.hpp +++ b/src/hotspot/share/memory/heapInspection.hpp @@ -216,8 +216,8 @@ class KlassInfoClosure; class HeapInspection : public StackObj { public: - void heap_inspection(outputStream* st, uint parallel_thread_num = 1) NOT_SERVICES_RETURN; - uintx populate_table(KlassInfoTable* cit, BoolObjectClosure* filter = NULL, uint parallel_thread_num = 1) NOT_SERVICES_RETURN_(0); + void heap_inspection(outputStream* st, WorkGang* workers) NOT_SERVICES_RETURN; + uintx populate_table(KlassInfoTable* cit, BoolObjectClosure* filter, WorkGang* workers) NOT_SERVICES_RETURN_(0); static void find_instances_at_safepoint(Klass* k, GrowableArray* result) NOT_SERVICES_RETURN; private: void iterate_over_heap(KlassInfoTable* cit, BoolObjectClosure* filter = NULL); diff --git a/src/hotspot/share/memory/metaspace.cpp b/src/hotspot/share/memory/metaspace.cpp index 2c42c0135603c..3f29a72a86d75 100644 --- a/src/hotspot/share/memory/metaspace.cpp +++ b/src/hotspot/share/memory/metaspace.cpp @@ -590,8 +590,8 @@ ReservedSpace Metaspace::reserve_address_space_for_compressed_classes(size_t siz #if defined(AARCH64) || defined(PPC64) const size_t alignment = Metaspace::reserve_alignment(); - // AArch64: Try to align metaspace so that we can decode a compressed - // klass with a single MOVK instruction. We can do this iff the + // AArch64: Try to align metaspace class space so that we can decode a + // compressed klass with a single MOVK instruction. We can do this iff the // compressed class base is a multiple of 4G. // Additionally, above 32G, ensure the lower LogKlassAlignmentInBytes bits // of the upper 32-bits of the address are zero so we can handle a shift @@ -614,19 +614,39 @@ ReservedSpace Metaspace::reserve_address_space_for_compressed_classes(size_t siz { NULL, NULL, 0 } }; + // Calculate a list of all possible values for the starting address for the + // compressed class space. + ResourceMark rm; + GrowableArray
list(36); for (int i = 0; search_ranges[i].from != NULL; i ++) { address a = search_ranges[i].from; assert(CompressedKlassPointers::is_valid_base(a), "Sanity"); while (a < search_ranges[i].to) { - ReservedSpace rs(size, Metaspace::reserve_alignment(), - os::vm_page_size(), (char*)a); - if (rs.is_reserved()) { - assert(a == (address)rs.base(), "Sanity"); - return rs; - } + list.append(a); a += search_ranges[i].increment; } } + + int len = list.length(); + int r = 0; + if (!DumpSharedSpaces) { + // Starting from a random position in the list. If the address cannot be reserved + // (the OS already assigned it for something else), go to the next position, wrapping + // around if necessary, until we exhaust all the items. + os::init_random((int)os::javaTimeNanos()); + r = os::random(); + log_info(metaspace)("Randomizing compressed class space: start from %d out of %d locations", + r % len, len); + } + for (int i = 0; i < len; i++) { + address a = list.at((i + r) % len); + ReservedSpace rs(size, Metaspace::reserve_alignment(), + os::vm_page_size(), (char*)a); + if (rs.is_reserved()) { + assert(a == (address)rs.base(), "Sanity"); + return rs; + } + } #endif // defined(AARCH64) || defined(PPC64) #ifdef AARCH64 diff --git a/src/hotspot/share/memory/resourceArea.hpp b/src/hotspot/share/memory/resourceArea.hpp index ff3effe758921..d8ec3580b88bb 100644 --- a/src/hotspot/share/memory/resourceArea.hpp +++ b/src/hotspot/share/memory/resourceArea.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -117,16 +117,34 @@ class ResourceArea: public Arena { size_in_bytes(), state._size_in_bytes); set_size_in_bytes(state._size_in_bytes); state._chunk->next_chop(); + assert(_hwm != state._hwm, "Sanity check: HWM moves when we have later chunks"); } else { assert(size_in_bytes() == state._size_in_bytes, "Sanity check"); } - _chunk = state._chunk; // Roll back to saved chunk. - _hwm = state._hwm; - _max = state._max; - // Clear out this chunk (to detect allocation bugs) - if (ZapResourceArea) { - memset(state._hwm, badResourceValue, state._max - state._hwm); + if (_hwm != state._hwm) { + // HWM moved: resource area was used. Roll back! + + char* replaced_hwm = _hwm; + + _chunk = state._chunk; + _hwm = state._hwm; + _max = state._max; + + // Clear out this chunk (to detect allocation bugs). + // If current chunk contains the replaced HWM, this means we are + // doing the rollback within the same chunk, and we only need to + // clear up to replaced HWM. + if (ZapResourceArea) { + char* limit = _chunk->contains(replaced_hwm) ? replaced_hwm : _max; + assert(limit >= _hwm, "Sanity check: non-negative memset size"); + memset(_hwm, badResourceValue, limit - _hwm); + } + } else { + // No allocations. Nothing to rollback. Check it. + assert(_chunk == state._chunk, "Sanity check: idempotence"); + assert(_hwm == state._hwm, "Sanity check: idempotence"); + assert(_max == state._max, "Sanity check: idempotence"); } } }; diff --git a/src/hotspot/share/oops/constantPool.cpp b/src/hotspot/share/oops/constantPool.cpp index c7dc3499f61ae..a97f2fe8d02ec 100644 --- a/src/hotspot/share/oops/constantPool.cpp +++ b/src/hotspot/share/oops/constantPool.cpp @@ -2244,9 +2244,10 @@ void ConstantPool::print_on(outputStream* st) const { st->print_cr(" - holder: " INTPTR_FORMAT, p2i(pool_holder())); } st->print_cr(" - cache: " INTPTR_FORMAT, p2i(cache())); - st->print_cr(" - resolved_references: " INTPTR_FORMAT, p2i(resolved_references())); + st->print_cr(" - resolved_references: " INTPTR_FORMAT, p2i(resolved_references_or_null())); st->print_cr(" - reference_map: " INTPTR_FORMAT, p2i(reference_map())); st->print_cr(" - resolved_klasses: " INTPTR_FORMAT, p2i(resolved_klasses())); + st->print_cr(" - cp length: %d", length()); for (int index = 1; index < length(); index++) { // Index 0 is unused ((ConstantPool*)this)->print_entry_on(index, st); diff --git a/src/hotspot/share/oops/instanceKlass.cpp b/src/hotspot/share/oops/instanceKlass.cpp index bf9416915ce36..5c71a49c30b96 100644 --- a/src/hotspot/share/oops/instanceKlass.cpp +++ b/src/hotspot/share/oops/instanceKlass.cpp @@ -1035,18 +1035,18 @@ void InstanceKlass::add_initialization_error(JavaThread* current, Handle excepti // If the initialization error is OOM, this might not work, but if GC kicks in // this would be still be helpful. JavaThread* THREAD = current; - Handle cause = java_lang_Throwable::get_cause_with_stack_trace(exception, THREAD); - if (HAS_PENDING_EXCEPTION || cause.is_null()) { - CLEAR_PENDING_EXCEPTION; + Handle init_error = java_lang_Throwable::create_initialization_error(current, exception); + ResourceMark rm(THREAD); + if (init_error.is_null()) { + log_trace(class, init)("Initialization error is null for class %s", external_name()); return; } MutexLocker ml(THREAD, ClassInitError_lock); - OopHandle elem = OopHandle(Universe::vm_global(), cause()); - bool created = false; + OopHandle elem = OopHandle(Universe::vm_global(), init_error()); + bool created; _initialization_error_table.put_if_absent(this, elem, &created); assert(created, "Initialization is single threaded"); - ResourceMark rm(THREAD); log_trace(class, init)("Initialization error added for class %s", external_name()); } diff --git a/src/hotspot/share/oops/klass.cpp b/src/hotspot/share/oops/klass.cpp index 59038cdd80f88..5c73c4634f95c 100644 --- a/src/hotspot/share/oops/klass.cpp +++ b/src/hotspot/share/oops/klass.cpp @@ -27,7 +27,7 @@ #include "cds/heapShared.hpp" #include "classfile/classLoaderData.inline.hpp" #include "classfile/classLoaderDataGraph.inline.hpp" -#include "classfile/javaClasses.hpp" +#include "classfile/javaClasses.inline.hpp" #include "classfile/moduleEntry.hpp" #include "classfile/systemDictionary.hpp" #include "classfile/systemDictionaryShared.hpp" @@ -63,10 +63,6 @@ oop Klass::java_mirror_no_keepalive() const { return _java_mirror.peek(); } -void Klass::replace_java_mirror(oop mirror) { - _java_mirror.replace(mirror); -} - bool Klass::is_cloneable() const { return _access_flags.is_cloneable_fast() || is_subtype_of(vmClasses::Cloneable_klass()); @@ -786,7 +782,7 @@ void Klass::verify_on(outputStream* st) { } if (java_mirror_no_keepalive() != NULL) { - guarantee(oopDesc::is_oop(java_mirror_no_keepalive()), "should be instance"); + guarantee(java_lang_Class::is_instance(java_mirror_no_keepalive()), "should be instance"); } } diff --git a/src/hotspot/share/oops/klass.hpp b/src/hotspot/share/oops/klass.hpp index 1d58cc3ec2bde..9c7b2a2453257 100644 --- a/src/hotspot/share/oops/klass.hpp +++ b/src/hotspot/share/oops/klass.hpp @@ -268,7 +268,8 @@ class Klass : public Metadata { void set_archived_java_mirror(oop m) NOT_CDS_JAVA_HEAP_RETURN; // Temporary mirror switch used by RedefineClasses - void replace_java_mirror(oop mirror); + OopHandle java_mirror_handle() const { return _java_mirror; } + void swap_java_mirror_handle(OopHandle& mirror) { _java_mirror.swap(mirror); } // Set java mirror OopHandle to NULL for CDS // This leaves the OopHandle in the CLD, but that's ok, you can't release them. diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp index c77b09adf0d38..04354d7e8aee0 100644 --- a/src/hotspot/share/oops/oop.hpp +++ b/src/hotspot/share/oops/oop.hpp @@ -59,6 +59,7 @@ class oopDesc { public: inline markWord mark() const; + inline markWord mark_acquire() const; inline markWord* mark_addr() const; inline void set_mark(markWord m); diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index 0b706f1aa3bb0..b9684d98880b9 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -45,8 +45,11 @@ // We need a separate file to avoid circular references markWord oopDesc::mark() const { - uintptr_t v = HeapAccess::load_at(as_oop(), mark_offset_in_bytes()); - return markWord(v); + return Atomic::load(&_mark); +} + +markWord oopDesc::mark_acquire() const { + return Atomic::load_acquire(&_mark); } markWord* oopDesc::mark_addr() const { @@ -54,7 +57,7 @@ markWord* oopDesc::mark_addr() const { } void oopDesc::set_mark(markWord m) { - HeapAccess::store_at(as_oop(), mark_offset_in_bytes(), m.value()); + Atomic::store(&_mark, m); } void oopDesc::set_mark(HeapWord* mem, markWord m) { @@ -62,12 +65,11 @@ void oopDesc::set_mark(HeapWord* mem, markWord m) { } void oopDesc::release_set_mark(markWord m) { - HeapAccess::store_at(as_oop(), mark_offset_in_bytes(), m.value()); + Atomic::release_store(&_mark, m); } markWord oopDesc::cas_set_mark(markWord new_mark, markWord old_mark) { - uintptr_t v = HeapAccess<>::atomic_cmpxchg_at(as_oop(), mark_offset_in_bytes(), old_mark.value(), new_mark.value()); - return markWord(v); + return Atomic::cmpxchg(&_mark, old_mark, new_mark); } markWord oopDesc::cas_set_mark(markWord new_mark, markWord old_mark, atomic_memory_order order) { diff --git a/src/hotspot/share/oops/oopHandle.hpp b/src/hotspot/share/oops/oopHandle.hpp index 4a304b968f236..fd4c9679219e7 100644 --- a/src/hotspot/share/oops/oopHandle.hpp +++ b/src/hotspot/share/oops/oopHandle.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -55,6 +55,10 @@ class OopHandle { return *this; } + void swap(OopHandle& copy) { + ::swap(_obj, copy._obj); + } + inline oop resolve() const; inline oop peek() const; diff --git a/src/hotspot/share/oops/symbol.cpp b/src/hotspot/share/oops/symbol.cpp index 3884c79f29046..581a5f8f6dd83 100644 --- a/src/hotspot/share/oops/symbol.cpp +++ b/src/hotspot/share/oops/symbol.cpp @@ -310,10 +310,8 @@ bool Symbol::try_increment_refcount() { // this caller. void Symbol::increment_refcount() { if (!try_increment_refcount()) { -#ifdef ASSERT print(); fatal("refcount has gone to zero"); -#endif } #ifndef PRODUCT if (refcount() != PERM_REFCOUNT) { // not a permanent symbol @@ -333,10 +331,8 @@ void Symbol::decrement_refcount() { if (refc == PERM_REFCOUNT) { return; // refcount is permanent, permanent is sticky } else if (refc == 0) { -#ifdef ASSERT print(); fatal("refcount underflow"); -#endif return; } else { found = Atomic::cmpxchg(&_hash_and_refcount, old_value, old_value - 1); @@ -356,10 +352,8 @@ void Symbol::make_permanent() { if (refc == PERM_REFCOUNT) { return; // refcount is permanent, permanent is sticky } else if (refc == 0) { -#ifdef ASSERT print(); fatal("refcount underflow"); -#endif return; } else { int hash = extract_hash(old_value); diff --git a/src/hotspot/share/oops/typeArrayKlass.cpp b/src/hotspot/share/oops/typeArrayKlass.cpp index 57bec046822b1..d15afe8537d80 100644 --- a/src/hotspot/share/oops/typeArrayKlass.cpp +++ b/src/hotspot/share/oops/typeArrayKlass.cpp @@ -230,7 +230,7 @@ Klass* TypeArrayKlass::array_klass_or_null() { int TypeArrayKlass::oop_size(oop obj) const { assert(obj->is_typeArray(),"must be a type array"); typeArrayOop t = typeArrayOop(obj); - return t->object_size(); + return t->object_size(this); } void TypeArrayKlass::initialize(TRAPS) { diff --git a/src/hotspot/share/oops/typeArrayOop.hpp b/src/hotspot/share/oops/typeArrayOop.hpp index dd6e718703ca6..2af7be778a4a2 100644 --- a/src/hotspot/share/oops/typeArrayOop.hpp +++ b/src/hotspot/share/oops/typeArrayOop.hpp @@ -131,7 +131,7 @@ class typeArrayOopDesc : public arrayOopDesc { } public: - inline int object_size(); + inline int object_size(const TypeArrayKlass* tk) const; }; #endif // SHARE_OOPS_TYPEARRAYOOP_HPP diff --git a/src/hotspot/share/oops/typeArrayOop.inline.hpp b/src/hotspot/share/oops/typeArrayOop.inline.hpp index 9d2e7ea0fd24e..cf4cfc6995ecc 100644 --- a/src/hotspot/share/oops/typeArrayOop.inline.hpp +++ b/src/hotspot/share/oops/typeArrayOop.inline.hpp @@ -31,8 +31,7 @@ #include "oops/oop.inline.hpp" #include "oops/arrayOop.hpp" -int typeArrayOopDesc::object_size() { - TypeArrayKlass* tk = TypeArrayKlass::cast(klass()); +int typeArrayOopDesc::object_size(const TypeArrayKlass* tk) const { return object_size(tk->layout_helper(), length()); } diff --git a/src/hotspot/share/opto/addnode.cpp b/src/hotspot/share/opto/addnode.cpp index 6f7ca9df7b80d..959f059fa6b38 100644 --- a/src/hotspot/share/opto/addnode.cpp +++ b/src/hotspot/share/opto/addnode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -142,7 +142,7 @@ Node *AddNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Check for commutative operation desired if (commute(phase, this)) return this; - AddNode *progress = NULL; // Progress flag + AddNode *progress = nullptr; // Progress flag // Convert "(x+1)+2" into "x+(1+2)". If the right input is a // constant, and the left input is an add of a constant, flatten the @@ -246,7 +246,7 @@ const Type *AddNode::add_of_identity( const Type *t1, const Type *t2 ) const { if( t1->higher_equal( zero ) ) return t2; if( t2->higher_equal( zero ) ) return t1; - return NULL; + return nullptr; } AddNode* AddNode::make(Node* in1, Node* in2, BasicType bt) { @@ -258,7 +258,7 @@ AddNode* AddNode::make(Node* in1, Node* in2, BasicType bt) { default: fatal("Not implemented for %s", type2name(bt)); } - return NULL; + return nullptr; } //============================================================================= @@ -286,7 +286,7 @@ Node *AddINode::Ideal(PhaseGVN *phase, bool can_reshape) { // Check for dead cycle: d = (a-b)+(c-d) assert( in1->in(2) != this && in2->in(2) != this, "dead loop in AddINode::Ideal" ); - Node *sub = new SubINode(NULL, NULL); + Node *sub = new SubINode(nullptr, nullptr); sub->init_req(1, phase->transform(new AddINode(in1->in(1), in2->in(1) ) )); sub->init_req(2, phase->transform(new AddINode(in1->in(2), in2->in(2) ) )); return sub; @@ -349,14 +349,14 @@ Node *AddINode::Ideal(PhaseGVN *phase, bool can_reshape) { // Convert (x >>> rshift) + (x << lshift) into RotateRight(x, rshift) if (Matcher::match_rule_supported(Op_RotateRight) && ((op1 == Op_URShiftI && op2 == Op_LShiftI) || (op1 == Op_LShiftI && op2 == Op_URShiftI)) && - in1->in(1) != NULL && in1->in(1) == in2->in(1)) { + in1->in(1) != nullptr && in1->in(1) == in2->in(1)) { Node* rshift = op1 == Op_URShiftI ? in1->in(2) : in2->in(2); Node* lshift = op1 == Op_URShiftI ? in2->in(2) : in1->in(2); - if (rshift != NULL && lshift != NULL) { + if (rshift != nullptr && lshift != nullptr) { const TypeInt* rshift_t = phase->type(rshift)->isa_int(); const TypeInt* lshift_t = phase->type(lshift)->isa_int(); - if (lshift_t != NULL && lshift_t->is_con() && - rshift_t != NULL && rshift_t->is_con() && + if (lshift_t != nullptr && lshift_t->is_con() && + rshift_t != nullptr && rshift_t->is_con() && ((lshift_t->get_con() & 0x1F) == (32 - (rshift_t->get_con() & 0x1F)))) { return new RotateRightNode(in1->in(1), phase->intcon(rshift_t->get_con() & 0x1F), TypeInt::INT); } @@ -441,7 +441,7 @@ Node *AddLNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Check for dead cycle: d = (a-b)+(c-d) assert( in1->in(2) != this && in2->in(2) != this, "dead loop in AddLNode::Ideal" ); - Node *sub = new SubLNode(NULL, NULL); + Node *sub = new SubLNode(nullptr, nullptr); sub->init_req(1, phase->transform(new AddLNode(in1->in(1), in2->in(1) ) )); sub->init_req(2, phase->transform(new AddLNode(in1->in(2), in2->in(2) ) )); return sub; @@ -479,14 +479,14 @@ Node *AddLNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Convert (x >>> rshift) + (x << lshift) into RotateRight(x, rshift) if (Matcher::match_rule_supported(Op_RotateRight) && ((op1 == Op_URShiftL && op2 == Op_LShiftL) || (op1 == Op_LShiftL && op2 == Op_URShiftL)) && - in1->in(1) != NULL && in1->in(1) == in2->in(1)) { + in1->in(1) != nullptr && in1->in(1) == in2->in(1)) { Node* rshift = op1 == Op_URShiftL ? in1->in(2) : in2->in(2); Node* lshift = op1 == Op_URShiftL ? in2->in(2) : in1->in(2); - if (rshift != NULL && lshift != NULL) { + if (rshift != nullptr && lshift != nullptr) { const TypeInt* rshift_t = phase->type(rshift)->isa_int(); const TypeInt* lshift_t = phase->type(lshift)->isa_int(); - if (lshift_t != NULL && lshift_t->is_con() && - rshift_t != NULL && rshift_t->is_con() && + if (lshift_t != nullptr && lshift_t->is_con() && + rshift_t != nullptr && rshift_t->is_con() && ((lshift_t->get_con() & 0x3F) == (64 - (rshift_t->get_con() & 0x3F)))) { return new RotateRightNode(in1->in(1), phase->intcon(rshift_t->get_con() & 0x3F), TypeLong::LONG); } @@ -558,7 +558,7 @@ const Type *AddFNode::add_of_identity( const Type *t1, const Type *t2 ) const { // if( t1->higher_equal( zero ) ) return t2; // if( t2->higher_equal( zero ) ) return t1; - return NULL; + return nullptr; } //------------------------------add_ring--------------------------------------- @@ -573,7 +573,7 @@ const Type *AddFNode::add_ring( const Type *t0, const Type *t1 ) const { //------------------------------Ideal------------------------------------------ Node *AddFNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Floating point additions are not associative because of boundary conditions (infinity) - return commute(phase, this) ? this : NULL; + return commute(phase, this) ? this : nullptr; } @@ -590,7 +590,7 @@ const Type *AddDNode::add_of_identity( const Type *t1, const Type *t2 ) const { // if( t1->higher_equal( zero ) ) return t2; // if( t2->higher_equal( zero ) ) return t1; - return NULL; + return nullptr; } //------------------------------add_ring--------------------------------------- // Supplied function returns the sum of the inputs. @@ -604,7 +604,7 @@ const Type *AddDNode::add_ring( const Type *t0, const Type *t1 ) const { //------------------------------Ideal------------------------------------------ Node *AddDNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Floating point additions are not associative because of boundary conditions (infinity) - return commute(phase, this) ? this : NULL; + return commute(phase, this) ? this : nullptr; } @@ -618,7 +618,7 @@ Node* AddPNode::Identity(PhaseGVN* phase) { //------------------------------Idealize--------------------------------------- Node *AddPNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Bail out if dead inputs - if( phase->type( in(Address) ) == Type::TOP ) return NULL; + if( phase->type( in(Address) ) == Type::TOP ) return nullptr; // If the left input is an add of a constant, flatten the expression tree. const Node *n = in(Address); @@ -629,12 +629,12 @@ Node *AddPNode::Ideal(PhaseGVN *phase, bool can_reshape) { "dead loop in AddPNode::Ideal" ); // Type of left input's right input const Type *t = phase->type( addp->in(Offset) ); - if( t == Type::TOP ) return NULL; + if( t == Type::TOP ) return nullptr; const TypeX *t12 = t->is_intptr_t(); if( t12->is_con() ) { // Left input is an add of a constant? // If the right input is a constant, combine constants const Type *temp_t2 = phase->type( in(Offset) ); - if( temp_t2 == Type::TOP ) return NULL; + if( temp_t2 == Type::TOP ) return nullptr; const TypeX *t2 = temp_t2->is_intptr_t(); Node* address; Node* offset; @@ -655,7 +655,7 @@ Node *AddPNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Raw pointers? if( in(Base)->bottom_type() == Type::TOP ) { - // If this is a NULL+long form (from unsafe accesses), switch to a rawptr. + // If this is a null+long form (from unsafe accesses), switch to a rawptr. if (phase->type(in(Address)) == TypePtr::NULL_PTR) { Node* offset = in(Offset); return new CastX2PNode(offset); @@ -681,13 +681,13 @@ Node *AddPNode::Ideal(PhaseGVN *phase, bool can_reshape) { } } - return NULL; // No progress + return nullptr; // No progress } //------------------------------bottom_type------------------------------------ // Bottom-type is the pointer-type with unknown offset. const Type *AddPNode::bottom_type() const { - if (in(Address) == NULL) return TypePtr::BOTTOM; + if (in(Address) == nullptr) return TypePtr::BOTTOM; const TypePtr *tp = in(Address)->bottom_type()->isa_ptr(); if( !tp ) return Type::TOP; // TOP input means TOP output assert( in(Offset)->Opcode() != Op_ConP, "" ); @@ -725,7 +725,7 @@ const Type* AddPNode::Value(PhaseGVN* phase) const { //------------------------Ideal_base_and_offset-------------------------------- // Split an oop pointer into a base and offset. // (The offset might be Type::OffsetBot in the case of an array.) -// Return the base, or NULL if failure. +// Return the base, or null if failure. Node* AddPNode::Ideal_base_and_offset(Node* ptr, PhaseTransform* phase, // second return value: intptr_t& offset) { @@ -741,7 +741,7 @@ Node* AddPNode::Ideal_base_and_offset(Node* ptr, PhaseTransform* phase, } } offset = Type::OffsetBot; - return NULL; + return nullptr; } //------------------------------unpack_offsets---------------------------------- @@ -791,20 +791,20 @@ Node* rotate_shift(PhaseGVN* phase, Node* lshift, Node* rshift, int mask) { // val << norm_con_shift | val >> ({32|64} - norm_con_shift) => rotate_left val, norm_con_shift const TypeInt* lshift_t = phase->type(lshift)->isa_int(); const TypeInt* rshift_t = phase->type(rshift)->isa_int(); - if (lshift_t != NULL && lshift_t->is_con() && - rshift_t != NULL && rshift_t->is_con() && + if (lshift_t != nullptr && lshift_t->is_con() && + rshift_t != nullptr && rshift_t->is_con() && ((lshift_t->get_con() & mask) == ((mask + 1) - (rshift_t->get_con() & mask)))) { return phase->intcon(lshift_t->get_con() & mask); } // val << var_shift | val >> ({0|32|64} - var_shift) => rotate_left val, var_shift if (rshift->Opcode() == Op_SubI && rshift->in(2) == lshift && rshift->in(1)->is_Con()){ const TypeInt* shift_t = phase->type(rshift->in(1))->isa_int(); - if (shift_t != NULL && shift_t->is_con() && + if (shift_t != nullptr && shift_t->is_con() && (shift_t->get_con() == 0 || shift_t->get_con() == (mask + 1))) { return lshift; } } - return NULL; + return nullptr; } Node* OrINode::Ideal(PhaseGVN* phase, bool can_reshape) { @@ -815,21 +815,21 @@ Node* OrINode::Ideal(PhaseGVN* phase, bool can_reshape) { Node* lshift = in(1)->in(2); Node* rshift = in(2)->in(2); Node* shift = rotate_shift(phase, lshift, rshift, 0x1F); - if (shift != NULL) { + if (shift != nullptr) { return new RotateLeftNode(in(1)->in(1), shift, TypeInt::INT); } - return NULL; + return nullptr; } if (Matcher::match_rule_supported(Op_RotateRight) && lopcode == Op_URShiftI && ropcode == Op_LShiftI && in(1)->in(1) == in(2)->in(1)) { Node* rshift = in(1)->in(2); Node* lshift = in(2)->in(2); Node* shift = rotate_shift(phase, rshift, lshift, 0x1F); - if (shift != NULL) { + if (shift != nullptr) { return new RotateRightNode(in(1)->in(1), shift, TypeInt::INT); } } - return NULL; + return nullptr; } //------------------------------add_ring--------------------------------------- @@ -881,21 +881,21 @@ Node* OrLNode::Ideal(PhaseGVN* phase, bool can_reshape) { Node* lshift = in(1)->in(2); Node* rshift = in(2)->in(2); Node* shift = rotate_shift(phase, lshift, rshift, 0x3F); - if (shift != NULL) { + if (shift != nullptr) { return new RotateLeftNode(in(1)->in(1), shift, TypeLong::LONG); } - return NULL; + return nullptr; } if (Matcher::match_rule_supported(Op_RotateRight) && lopcode == Op_URShiftL && ropcode == Op_LShiftL && in(1)->in(1) == in(2)->in(1)) { Node* rshift = in(1)->in(2); Node* lshift = in(2)->in(2); Node* shift = rotate_shift(phase, rshift, lshift, 0x3F); - if (shift != NULL) { + if (shift != nullptr) { return new RotateRightNode(in(1)->in(1), shift, TypeLong::LONG); } } - return NULL; + return nullptr; } //------------------------------add_ring--------------------------------------- @@ -1038,15 +1038,15 @@ const Type* XorLNode::Value(PhaseGVN* phase) const { Node* MaxNode::build_min_max(Node* a, Node* b, bool is_max, bool is_unsigned, const Type* t, PhaseGVN& gvn) { bool is_int = gvn.type(a)->isa_int(); assert(is_int || gvn.type(a)->isa_long(), "int or long inputs"); - assert(is_int == (gvn.type(b)->isa_int() != NULL), "inconsistent inputs"); - Node* hook = NULL; + assert(is_int == (gvn.type(b)->isa_int() != nullptr), "inconsistent inputs"); + Node* hook = nullptr; if (gvn.is_IterGVN()) { // Make sure a and b are not destroyed hook = new Node(2); hook->init_req(0, a); hook->init_req(1, b); } - Node* res = NULL; + Node* res = nullptr; if (!is_unsigned) { if (is_max) { if (is_int) { @@ -1090,7 +1090,7 @@ Node* MaxNode::build_min_max(Node* a, Node* b, bool is_max, bool is_unsigned, co } } } - if (hook != NULL) { + if (hook != nullptr) { hook->destruct(&gvn); } return res; @@ -1099,21 +1099,21 @@ Node* MaxNode::build_min_max(Node* a, Node* b, bool is_max, bool is_unsigned, co Node* MaxNode::build_min_max_diff_with_zero(Node* a, Node* b, bool is_max, const Type* t, PhaseGVN& gvn) { bool is_int = gvn.type(a)->isa_int(); assert(is_int || gvn.type(a)->isa_long(), "int or long inputs"); - assert(is_int == (gvn.type(b)->isa_int() != NULL), "inconsistent inputs"); - Node* zero = NULL; + assert(is_int == (gvn.type(b)->isa_int() != nullptr), "inconsistent inputs"); + Node* zero = nullptr; if (is_int) { zero = gvn.intcon(0); } else { zero = gvn.longcon(0); } - Node* hook = NULL; + Node* hook = nullptr; if (gvn.is_IterGVN()) { // Make sure a and b are not destroyed hook = new Node(2); hook->init_req(0, a); hook->init_req(1, b); } - Node* res = NULL; + Node* res = nullptr; if (is_max) { if (is_int) { Node* cmp = gvn.transform(new CmpINode(a, b)); @@ -1139,7 +1139,7 @@ Node* MaxNode::build_min_max_diff_with_zero(Node* a, Node* b, bool is_max, const res = gvn.transform(new CMoveLNode(bol, sub, zero, t->is_long())); } } - if (hook != NULL) { + if (hook != nullptr) { hook->destruct(&gvn); } return res; @@ -1169,7 +1169,7 @@ static bool can_overflow(const TypeInt* t, jint c) { // MINs show up in range-check loop limit calculations. Look for // "MIN2(x+c0,MIN2(y,x+c1))". Pick the smaller constant: "MIN2(x+c0,y)" Node *MinINode::Ideal(PhaseGVN *phase, bool can_reshape) { - Node *progress = NULL; + Node *progress = nullptr; // Force a right-spline graph Node *l = in(1); Node *r = in(2); @@ -1190,7 +1190,7 @@ Node *MinINode::Ideal(PhaseGVN *phase, bool can_reshape) { if( x->Opcode() == Op_AddI && // Check for "x+c0" and collect constant x->in(2)->is_Con() ) { const Type *t = x->in(2)->bottom_type(); - if( t == Type::TOP ) return NULL; // No progress + if( t == Type::TOP ) return nullptr; // No progress x_off = t->is_int()->get_con(); x = x->in(1); } @@ -1202,7 +1202,7 @@ Node *MinINode::Ideal(PhaseGVN *phase, bool can_reshape) { if( y->Opcode() == Op_AddI && // Check for "y+c1" and collect constant y->in(2)->is_Con() ) { const Type *t = y->in(2)->bottom_type(); - if( t == Type::TOP ) return NULL; // No progress + if( t == Type::TOP ) return nullptr; // No progress y_off = t->is_int()->get_con(); y = y->in(1); } @@ -1220,7 +1220,7 @@ Node *MinINode::Ideal(PhaseGVN *phase, bool can_reshape) { if( y->Opcode() == Op_AddI &&// Check for "y+c1" and collect constant y->in(2)->is_Con() ) { const Type *t = y->in(2)->bottom_type(); - if( t == Type::TOP ) return NULL; // No progress + if( t == Type::TOP ) return nullptr; // No progress y_off = t->is_int()->get_con(); y = y->in(1); } @@ -1230,7 +1230,7 @@ Node *MinINode::Ideal(PhaseGVN *phase, bool can_reshape) { // Transform MIN2(x + c0, MIN2(x + c1, z)) into MIN2(x + MIN2(c0, c1), z) // if x == y and the additions can't overflow. - if (x == y && tx != NULL && + if (x == y && tx != nullptr && !can_overflow(tx, x_off) && !can_overflow(tx, y_off)) { return new MinINode(phase->transform(new AddINode(x, phase->intcon(MIN2(x_off, y_off)))), r->in(2)); @@ -1238,13 +1238,13 @@ Node *MinINode::Ideal(PhaseGVN *phase, bool can_reshape) { } else { // Transform MIN2(x + c0, y + c1) into x + MIN2(c0, c1) // if x == y and the additions can't overflow. - if (x == y && tx != NULL && + if (x == y && tx != nullptr && !can_overflow(tx, x_off) && !can_overflow(tx, y_off)) { return new AddINode(x,phase->intcon(MIN2(x_off,y_off))); } } - return NULL; + return nullptr; } //------------------------------add_ring--------------------------------------- diff --git a/src/hotspot/share/opto/arraycopynode.cpp b/src/hotspot/share/opto/arraycopynode.cpp index b0f30703f91f0..d6cc405751162 100644 --- a/src/hotspot/share/opto/arraycopynode.cpp +++ b/src/hotspot/share/opto/arraycopynode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -34,7 +34,7 @@ #include "utilities/powerOfTwo.hpp" ArrayCopyNode::ArrayCopyNode(Compile* C, bool alloc_tightly_coupled, bool has_negative_length_guard) - : CallNode(arraycopy_type(), NULL, TypePtr::BOTTOM), + : CallNode(arraycopy_type(), nullptr, TypePtr::BOTTOM), _kind(None), _alloc_tightly_coupled(alloc_tightly_coupled), _has_negative_length_guard(has_negative_length_guard), @@ -131,7 +131,7 @@ int ArrayCopyNode::get_count(PhaseGVN *phase) const { return nb_fields; } else { const TypeAryPtr* ary_src = src_type->isa_aryptr(); - assert (ary_src != NULL, "not an array or instance?"); + assert (ary_src != nullptr, "not an array or instance?"); // clone passes a length as a rounded number of longs. If we're // cloning an array we'll do it element by element. If the // length input to ArrayCopyNode is constant, length of input @@ -174,7 +174,7 @@ void ArrayCopyNode::store(BarrierSetC2* bs, PhaseGVN *phase, Node*& ctl, MergeMe Node* ArrayCopyNode::try_clone_instance(PhaseGVN *phase, bool can_reshape, int count) { if (!is_clonebasic()) { - return NULL; + return nullptr; } Node* base_src = in(ArrayCopyNode::Src); @@ -184,8 +184,8 @@ Node* ArrayCopyNode::try_clone_instance(PhaseGVN *phase, bool can_reshape, int c const Type* src_type = phase->type(base_src); const TypeInstPtr* inst_src = src_type->isa_instptr(); - if (inst_src == NULL) { - return NULL; + if (inst_src == nullptr) { + return nullptr; } MergeMemNode* mem = phase->transform(MergeMemNode::make(in_mem))->as_MergeMem(); @@ -318,7 +318,7 @@ bool ArrayCopyNode::prepare_array_copy(PhaseGVN *phase, bool can_reshape, copy_type = dest_elem; } else { - assert(ary_src != NULL, "should be a clone"); + assert(ary_src != nullptr, "should be a clone"); assert(is_clonebasic(), "should be"); disjoint_bases = true; @@ -366,7 +366,7 @@ void ArrayCopyNode::array_copy_test_overlap(PhaseGVN *phase, bool can_reshape, b if (!disjoint_bases && count > 1) { Node* src_offset = in(ArrayCopyNode::SrcPos); Node* dest_offset = in(ArrayCopyNode::DestPos); - assert(src_offset != NULL && dest_offset != NULL, "should be"); + assert(src_offset != nullptr && dest_offset != nullptr, "should be"); Node* cmp = phase->transform(new CmpINode(src_offset, dest_offset)); Node *bol = phase->transform(new BoolNode(cmp, BoolTest::lt)); IfNode *iff = new IfNode(ctl, bol, PROB_FAIR, COUNT_UNKNOWN); @@ -483,13 +483,13 @@ bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape, CallProjections callprojs; extract_projections(&callprojs, true, false); - if (callprojs.fallthrough_ioproj != NULL) { + if (callprojs.fallthrough_ioproj != nullptr) { igvn->replace_node(callprojs.fallthrough_ioproj, in(TypeFunc::I_O)); } - if (callprojs.fallthrough_memproj != NULL) { + if (callprojs.fallthrough_memproj != nullptr) { igvn->replace_node(callprojs.fallthrough_memproj, mem); } - if (callprojs.fallthrough_catchproj != NULL) { + if (callprojs.fallthrough_catchproj != nullptr) { igvn->replace_node(callprojs.fallthrough_catchproj, ctl); } @@ -519,7 +519,7 @@ Node *ArrayCopyNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (StressArrayCopyMacroNode && !can_reshape) { phase->record_for_igvn(this); - return NULL; + return nullptr; } // See if it's a small array copy and we can inline it as @@ -531,51 +531,51 @@ Node *ArrayCopyNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (!is_clonebasic() && !is_arraycopy_validated() && !is_copyofrange_validated() && !is_copyof_validated()) { - return NULL; + return nullptr; } - assert(in(TypeFunc::Control) != NULL && - in(TypeFunc::Memory) != NULL && - in(ArrayCopyNode::Src) != NULL && - in(ArrayCopyNode::Dest) != NULL && - in(ArrayCopyNode::Length) != NULL && - in(ArrayCopyNode::SrcPos) != NULL && - in(ArrayCopyNode::DestPos) != NULL, "broken inputs"); + assert(in(TypeFunc::Control) != nullptr && + in(TypeFunc::Memory) != nullptr && + in(ArrayCopyNode::Src) != nullptr && + in(ArrayCopyNode::Dest) != nullptr && + in(ArrayCopyNode::Length) != nullptr && + in(ArrayCopyNode::SrcPos) != nullptr && + in(ArrayCopyNode::DestPos) != nullptr, "broken inputs"); if (in(TypeFunc::Control)->is_top() || in(TypeFunc::Memory)->is_top() || phase->type(in(ArrayCopyNode::Src)) == Type::TOP || phase->type(in(ArrayCopyNode::Dest)) == Type::TOP || - (in(ArrayCopyNode::SrcPos) != NULL && in(ArrayCopyNode::SrcPos)->is_top()) || - (in(ArrayCopyNode::DestPos) != NULL && in(ArrayCopyNode::DestPos)->is_top())) { - return NULL; + (in(ArrayCopyNode::SrcPos) != nullptr && in(ArrayCopyNode::SrcPos)->is_top()) || + (in(ArrayCopyNode::DestPos) != nullptr && in(ArrayCopyNode::DestPos)->is_top())) { + return nullptr; } int count = get_count(phase); if (count < 0 || count > ArrayCopyLoadStoreMaxElem) { - return NULL; + return nullptr; } Node* mem = try_clone_instance(phase, can_reshape, count); - if (mem != NULL) { - return (mem == NodeSentinel) ? NULL : mem; + if (mem != nullptr) { + return (mem == NodeSentinel) ? nullptr : mem; } - Node* adr_src = NULL; - Node* base_src = NULL; - Node* adr_dest = NULL; - Node* base_dest = NULL; + Node* adr_src = nullptr; + Node* base_src = nullptr; + Node* adr_dest = nullptr; + Node* base_dest = nullptr; BasicType copy_type = T_ILLEGAL; - const Type* value_type = NULL; + const Type* value_type = nullptr; bool disjoint_bases = false; if (!prepare_array_copy(phase, can_reshape, adr_src, base_src, adr_dest, base_dest, copy_type, value_type, disjoint_bases)) { - assert(adr_src == NULL, "no node can be left behind"); - assert(adr_dest == NULL, "no node can be left behind"); - return NULL; + assert(adr_src == nullptr, "no node can be left behind"); + assert(adr_dest == nullptr, "no node can be left behind"); + return nullptr; } Node* src = in(ArrayCopyNode::Src); @@ -605,7 +605,7 @@ Node *ArrayCopyNode::Ideal(PhaseGVN *phase, bool can_reshape) { adr_src, base_src, adr_dest, base_dest, copy_type, value_type, count); - Node* ctl = NULL; + Node* ctl = nullptr; if (!forward_ctl->is_top() && !backward_ctl->is_top()) { ctl = new RegionNode(3); ctl->init_req(1, forward_ctl); @@ -642,7 +642,7 @@ Node *ArrayCopyNode::Ideal(PhaseGVN *phase, bool can_reshape) { // put in worklist, so that if it happens to be dead it is removed phase->is_IterGVN()->_worklist.push(mem); } - return NULL; + return nullptr; } return mem; @@ -667,7 +667,7 @@ bool ArrayCopyNode::may_modify(const TypeOopPtr *t_oop, PhaseTransform *phase) { } bool ArrayCopyNode::may_modify_helper(const TypeOopPtr *t_oop, Node* n, PhaseTransform *phase, CallNode*& call) { - if (n != NULL && + if (n != nullptr && n->is_Call() && n->as_Call()->may_modify(t_oop, phase) && (n->as_Call()->is_ArrayCopy() || n->as_Call()->is_call_to_arraycopystub())) { @@ -685,11 +685,11 @@ bool ArrayCopyNode::may_modify(const TypeOopPtr *t_oop, MemBarNode* mb, PhaseTra // step over g1 gc barrier if we're at e.g. a clone with ReduceInitialCardMarks off c = bs->step_over_gc_barrier(c); - CallNode* call = NULL; - guarantee(c != NULL, "step_over_gc_barrier failed, there must be something to step to."); + CallNode* call = nullptr; + guarantee(c != nullptr, "step_over_gc_barrier failed, there must be something to step to."); if (c->is_Region()) { for (uint i = 1; i < c->req(); i++) { - if (c->in(i) != NULL) { + if (c->in(i) != nullptr) { Node* n = c->in(i)->in(0); if (may_modify_helper(t_oop, n, phase, call)) { ac = call->isa_ArrayCopy(); @@ -703,7 +703,7 @@ bool ArrayCopyNode::may_modify(const TypeOopPtr *t_oop, MemBarNode* mb, PhaseTra #ifdef ASSERT bool use_ReduceInitialCardMarks = BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) && static_cast(bs)->use_ReduceInitialCardMarks(); - assert(c == mb->in(0) || (ac != NULL && ac->is_clonebasic() && !use_ReduceInitialCardMarks), "only for clone"); + assert(c == mb->in(0) || (ac != nullptr && ac->is_clonebasic() && !use_ReduceInitialCardMarks), "only for clone"); #endif return true; } else if (mb->trailing_partial_array_copy()) { @@ -730,7 +730,7 @@ bool ArrayCopyNode::modifies(intptr_t offset_lo, intptr_t offset_hi, PhaseTransf const TypeInt *len_t = phase->type(len)->isa_int(); const TypeAryPtr* ary_t = phase->type(dest)->isa_aryptr(); - if (dest_pos_t == NULL || len_t == NULL || ary_t == NULL) { + if (dest_pos_t == nullptr || len_t == nullptr || ary_t == nullptr) { return !must_modify; } diff --git a/src/hotspot/share/opto/arraycopynode.hpp b/src/hotspot/share/opto/arraycopynode.hpp index ba04e43970fbc..3b4c4930cb0ab 100644 --- a/src/hotspot/share/opto/arraycopynode.hpp +++ b/src/hotspot/share/opto/arraycopynode.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -142,8 +142,8 @@ class ArrayCopyNode : public CallNode { Node* length, bool alloc_tightly_coupled, bool has_negative_length_guard, - Node* src_klass = NULL, Node* dest_klass = NULL, - Node* src_length = NULL, Node* dest_length = NULL); + Node* src_klass = nullptr, Node* dest_klass = nullptr, + Node* src_length = nullptr, Node* dest_length = nullptr); void connect_outputs(GraphKit* kit, bool deoptimize_on_exception = false); diff --git a/src/hotspot/share/opto/block.cpp b/src/hotspot/share/opto/block.cpp index 648ac5c671b21..be54ac64e3c06 100644 --- a/src/hotspot/share/opto/block.cpp +++ b/src/hotspot/share/opto/block.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -45,7 +45,7 @@ void Block_Array::grow( uint i ) { if( !_size ) { _size = 1; _blocks = (Block**)_arena->Amalloc( _size * sizeof(Block*) ); - _blocks[0] = NULL; + _blocks[0] = nullptr; } uint old = _size; _size = next_power_of_2(i); @@ -313,7 +313,7 @@ void Block::dump_head(const PhaseCFG* cfg, outputStream* st) const { st->print("in( "); for (uint i=1; iget_block_for_node(s); p->dump_pred(cfg, p, st); } else { @@ -332,7 +332,7 @@ void Block::dump_head(const PhaseCFG* cfg, outputStream* st) const { const Block *bhead = this; // Head of self-loop Node *bh = bhead->head(); - if ((cfg != NULL) && bh->is_Loop() && !head()->is_Root()) { + if ((cfg != nullptr) && bh->is_Loop() && !head()->is_Root()) { LoopNode *loop = bh->as_Loop(); const Block *bx = cfg->get_block_for_node(loop->in(LoopNode::LoopBackControl)); while (bx->is_connector()) { @@ -359,7 +359,7 @@ void Block::dump_head(const PhaseCFG* cfg, outputStream* st) const { } void Block::dump() const { - dump(NULL); + dump(nullptr); } void Block::dump(const PhaseCFG* cfg) const { @@ -375,11 +375,11 @@ PhaseCFG::PhaseCFG(Arena* arena, RootNode* root, Matcher& matcher) : Phase(CFG) , _root(root) , _block_arena(arena) -, _regalloc(NULL) +, _regalloc(nullptr) , _scheduling_for_pressure(false) , _matcher(matcher) , _node_to_block_mapping(arena) -, _node_latency(NULL) +, _node_latency(nullptr) #ifndef PRODUCT , _trace_opto_pipelining(C->directive()->TraceOptoPipeliningOption) #endif @@ -391,10 +391,10 @@ PhaseCFG::PhaseCFG(Arena* arena, RootNode* root, Matcher& matcher) // I'll need a few machine-specific GotoNodes. Make an Ideal GotoNode, // then Match it into a machine-specific Node. Then clone the machine // Node on demand. - Node *x = new GotoNode(NULL); + Node *x = new GotoNode(nullptr); x->init_req(0, x); _goto = matcher.match_tree(x); - assert(_goto != NULL, ""); + assert(_goto != nullptr, ""); _goto->set_req(0,_goto); // Build the CFG in Reverse Post Order @@ -427,7 +427,7 @@ uint PhaseCFG::build_cfg() { const Node *x = proj->is_block_proj(); // Does the block end with a proper block-ending Node? One of Return, // If or Goto? (This check should be done for visited nodes also). - if (x == NULL) { // Does not end right... + if (x == nullptr) { // Does not end right... Node *g = _goto->clone(); // Force it to end in a Goto g->set_req(0, proj); np->set_req(idx, g); @@ -661,7 +661,7 @@ void PhaseCFG::convert_NeverBranch_to_Goto(Block *b) { // Helper function to move block bx to the slot following b_index. Return // true if the move is successful, otherwise false bool PhaseCFG::move_to_next(Block* bx, uint b_index) { - if (bx == NULL) return false; + if (bx == nullptr) return false; // Return false if bx is already scheduled. uint bx_index = bx->_pre_order; @@ -848,7 +848,7 @@ void PhaseCFG::fixup_flow() { } assert(block->is_Empty() != Block::completely_empty, "Empty blocks should be connectors"); - Block* bnext = (i < number_of_blocks() - 1) ? get_block(i + 1) : NULL; + Block* bnext = (i < number_of_blocks() - 1) ? get_block(i + 1) : nullptr; Block* bs0 = block->non_connector_successor(0); // Check for multi-way branches where I cannot negate the test to @@ -1207,7 +1207,7 @@ void PhaseCFG::postalloc_expand(PhaseRegAlloc* _ra) { uint index = b->find_node(n); // Insert new nodes into block and map them in nodes->blocks array // and remember last node in n2. - Node *n2 = NULL; + Node *n2 = nullptr; for (int k = 0; k < new_nodes.length(); ++k) { n2 = new_nodes.at(k); b->insert_node(n2, ++index); @@ -1236,7 +1236,7 @@ void PhaseCFG::postalloc_expand(PhaseRegAlloc* _ra) { assert(remove.at(k)->is_Proj() && (remove.at(k)->in(0)->is_MachBranch()), ""); } } - // If anything has been inserted (n2 != NULL), continue after last node inserted. + // If anything has been inserted (n2 != nullptr), continue after last node inserted. // This does not always work. Some postalloc expands don't insert any nodes, if they // do optimizations (e.g., max(x,x)). In this case we decrement j accordingly. j = n2 ? b->find_node(n2) : j; @@ -1295,7 +1295,7 @@ void PhaseCFG::dump( ) const { void PhaseCFG::dump_headers() { for (uint i = 0; i < number_of_blocks(); i++) { Block* block = get_block(i); - if (block != NULL) { + if (block != nullptr) { block->dump_head(this); } } @@ -1315,7 +1315,7 @@ void PhaseCFG::verify_memory_writer_placement(const Block* b, const Node* n) con break; } home_or_ancestor = home_or_ancestor->parent(); - } while (home_or_ancestor != NULL); + } while (home_or_ancestor != nullptr); assert(found, "block b is not in n's home loop or an ancestor of it"); } @@ -1368,7 +1368,7 @@ void PhaseCFG::verify() const { // when CreateEx node is moved in build_ifg_physical(). if (def_block == block && !(block->head()->is_Loop() && n->is_Phi()) && // See (+++) comment in reg_split.cpp - !(n->jvms() != NULL && n->jvms()->is_monitor_use(k))) { + !(n->jvms() != nullptr && n->jvms()->is_monitor_use(k))) { bool is_loop = false; if (n->is_Phi()) { for (uint l = 1; l < def->req(); l++) { @@ -1378,7 +1378,13 @@ void PhaseCFG::verify() const { } } } - assert(is_loop || block->find_node(def) < j, "uses must follow definitions"); + // Uses must be before definition, except if: + // - We are in some kind of loop we already detected + // - We are in infinite loop, where Region may not have been turned into LoopNode + assert(block->find_node(def) < j || + is_loop || + (n->is_Phi() && block->head()->as_Region()->is_in_infinite_subgraph()), + "uses must follow definitions (except in loops)"); } } } @@ -1386,7 +1392,7 @@ void PhaseCFG::verify() const { assert(j >= 1, "a projection cannot be the first instruction in a block"); Node* pred = block->get_node(j - 1); Node* parent = n->in(0); - assert(parent != NULL, "projections must have a parent"); + assert(parent != nullptr, "projections must have a parent"); assert(pred == parent || (pred->is_Proj() && pred->in(0) == parent), "projections must follow their parents or other sibling projections"); } @@ -1480,7 +1486,7 @@ void UnionFind::Union( uint idx1, uint idx2 ) { #ifndef PRODUCT void Trace::dump( ) const { tty->print_cr("Trace (freq %f)", first_block()->_freq); - for (Block *b = first_block(); b != NULL; b = next(b)) { + for (Block *b = first_block(); b != nullptr; b = next(b)) { tty->print(" B%d", b->_pre_order); if (b->head()->is_Loop()) { tty->print(" (L%d)", b->compute_loop_alignment()); @@ -1558,7 +1564,7 @@ extern "C" int trace_frequency_order(const void *p0, const void *p1) { void PhaseBlockLayout::find_edges() { // Walk the blocks, creating edges and Traces uint i; - Trace *tr = NULL; + Trace *tr = nullptr; for (i = 0; i < _cfg.number_of_blocks(); i++) { Block* b = _cfg.get_block(i); tr = new Trace(b, next, prev); @@ -1587,7 +1593,7 @@ void PhaseBlockLayout::find_edges() { assert(n == _cfg.get_block(i), "expecting next block"); tr->append(n); uf->map(n->_pre_order, tr->id()); - traces[n->_pre_order] = NULL; + traces[n->_pre_order] = nullptr; nfallthru = b->num_fall_throughs(); b = n; } @@ -1613,7 +1619,7 @@ void PhaseBlockLayout::find_edges() { assert(b->is_connector(), "connector blocks at the end"); tr->append(b); uf->map(b->_pre_order, tr->id()); - traces[b->_pre_order] = NULL; + traces[b->_pre_order] = nullptr; } } @@ -1639,7 +1645,7 @@ void PhaseBlockLayout::union_traces(Trace* updated_trace, Trace* old_trace) { // Union the lower with the higher and remove the pointer // to the higher. uf->Union(lo_id, hi_id); - traces[hi_id] = NULL; + traces[hi_id] = nullptr; } // Append traces together via the most frequently executed edges @@ -1770,7 +1776,7 @@ void PhaseBlockLayout::reorder_traces(int count) { // Compact the traces. for (int i = 0; i < count; i++) { Trace *tr = traces[i]; - if (tr != NULL) { + if (tr != nullptr) { new_traces[new_count++] = tr; } } @@ -1786,7 +1792,7 @@ void PhaseBlockLayout::reorder_traces(int count) { _cfg.clear_blocks(); for (int i = 0; i < new_count; i++) { Trace *tr = new_traces[i]; - if (tr != NULL) { + if (tr != nullptr) { tr->fixup_blocks(_cfg); } } @@ -1850,13 +1856,13 @@ bool Trace::backedge(CFGEdge *e) { // Find the last block in the trace that has a conditional // branch. Block *b; - for (b = last_block(); b != NULL; b = prev(b)) { + for (b = last_block(); b != nullptr; b = prev(b)) { if (b->num_fall_throughs() == 2) { break; } } - if (b != last_block() && b != NULL) { + if (b != last_block() && b != nullptr) { loop_rotated = true; // Rotate the loop by doing two-part linked-list surgery. @@ -1868,7 +1874,7 @@ bool Trace::backedge(CFGEdge *e) { // Backbranch to the top of a trace // Scroll forward through the trace from the targ_block. If we find // a loop head before another loop top, use the the loop head alignment. - for (Block *b = targ_block; b != NULL; b = next(b)) { + for (Block *b = targ_block; b != nullptr; b = next(b)) { if (b->has_loop_alignment()) { break; } diff --git a/src/hotspot/share/opto/block.hpp b/src/hotspot/share/opto/block.hpp index 9eefde00fe068..5ccb46548d4bb 100644 --- a/src/hotspot/share/opto/block.hpp +++ b/src/hotspot/share/opto/block.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -43,7 +43,7 @@ struct Tarjan; //------------------------------Block_Array------------------------------------ // Map dense integer indices to Blocks. Uses classic doubling-array trick. -// Abstractly provides an infinite array of Block*'s, initialized to NULL. +// Abstractly provides an infinite array of Block*'s, initialized to null. // Note that the constructor just zeros things, and since I use Arena // allocation I do not need a destructor to reclaim storage. class Block_Array : public ResourceObj { @@ -60,11 +60,11 @@ class Block_Array : public ResourceObj { debug_only(_limit=0); _blocks = NEW_ARENA_ARRAY( a, Block *, OptoBlockListSize ); for( int i = 0; i < OptoBlockListSize; i++ ) { - _blocks[i] = NULL; + _blocks[i] = nullptr; } } - Block *lookup( uint i ) const // Lookup, or NULL for not mapped - { return (i_idx, NULL); + _node_to_block_mapping.map(node->_idx, nullptr); } // get the block in which this node resides @@ -582,7 +582,7 @@ class PhaseCFG : public Phase { // does this node reside in a block; return true bool has_block(const Node* node) const { - return (_node_to_block_mapping.lookup(node->_idx) != NULL); + return (_node_to_block_mapping.lookup(node->_idx) != nullptr); } // Use frequency calculations and code shape to predict if the block @@ -685,7 +685,7 @@ class BlockProbPair { Block* _target; // block target double _prob; // probability of edge to block public: - BlockProbPair() : _target(NULL), _prob(0.0) {} + BlockProbPair() : _target(nullptr), _prob(0.0) {} BlockProbPair(Block* b, double p) : _target(b), _prob(p) {} Block* get_target() const { return _target; } @@ -710,9 +710,9 @@ class CFGLoop : public CFGElement { CFGElement(), _id(id), _depth(0), - _parent(NULL), - _sibling(NULL), - _child(NULL), + _parent(nullptr), + _sibling(nullptr), + _child(nullptr), _exit_prob(1.0f) {} CFGLoop* parent() { return _parent; } void push_pred(Block* blk, int i, Block_List& worklist, PhaseCFG* cfg); @@ -725,7 +725,7 @@ class CFGLoop : public CFGElement { assert(hd->head()->is_Loop(), "must begin with loop head node"); return hd; } - Block* backedge_block(); // Return the block on the backedge of the loop (else NULL) + Block* backedge_block(); // Return the block on the backedge of the loop (else null) void compute_loop_depth(int depth); void compute_freq(); // compute frequency with loop assuming head freq 1.0f void scale_freq(); // scale frequency by loop trip count (including outer loops) @@ -813,8 +813,8 @@ class Trace : public ResourceObj { void break_loop_after(Block *b) { _last = b; _first = next(b); - set_prev(_first, NULL); - set_next(_last, NULL); + set_prev(_first, nullptr); + set_next(_last, nullptr); } public: @@ -825,8 +825,8 @@ class Trace : public ResourceObj { _prev_list(prev_list), _first(b), _last(b) { - set_next(b, NULL); - set_prev(b, NULL); + set_next(b, nullptr); + set_prev(b, nullptr); }; // Return the id number @@ -842,7 +842,7 @@ class Trace : public ResourceObj { // Insert a trace in the middle of this one after b void insert_after(Block *b, Trace *tr) { set_next(tr->last_block(), next(b)); - if (next(b) != NULL) { + if (next(b) != nullptr) { set_prev(next(b), tr->last_block()); } @@ -856,7 +856,7 @@ class Trace : public ResourceObj { void insert_before(Block *b, Trace *tr) { Block *p = prev(b); - assert(p != NULL, "use append instead"); + assert(p != nullptr, "use append instead"); insert_after(p, tr); } diff --git a/src/hotspot/share/opto/buildOopMap.cpp b/src/hotspot/share/opto/buildOopMap.cpp index 46ccd0a13f03b..f1b5e822e1c4d 100644 --- a/src/hotspot/share/opto/buildOopMap.cpp +++ b/src/hotspot/share/opto/buildOopMap.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -71,7 +71,7 @@ // an array of structs, but the struct-of-arrays is generally a little more // efficient). The arrays are indexed by register number (including // stack-slots as registers) and so is bounded by 200 to 300 elements in -// practice. One array will map to a reaching def Node (or NULL for +// practice. One array will map to a reaching def Node (or null for // conflict/dead). The other array will map to a callee-saved register or // OptoReg::Bad for not-callee-saved. @@ -80,16 +80,16 @@ struct OopFlow : public ResourceObj { short *_callees; // Array mapping register to callee-saved Node **_defs; // array mapping register to reaching def - // or NULL if dead/conflict + // or null if dead/conflict // OopFlow structs, when not being actively modified, describe the _end_ of // this block. Block *_b; // Block for this struct OopFlow *_next; // Next free OopFlow - // or NULL if dead/conflict + // or null if dead/conflict Compile* C; OopFlow( short *callees, Node **defs, Compile* c ) : _callees(callees), _defs(defs), - _b(NULL), _next(NULL), C(c) { } + _b(nullptr), _next(nullptr), C(c) { } // Given reaching-defs for this block start, compute it for this block end void compute_reach( PhaseRegAlloc *regalloc, int max_reg, Dict *safehash ); @@ -166,19 +166,19 @@ void OopFlow::compute_reach( PhaseRegAlloc *regalloc, int max_reg, Dict *safehas // Merge the given flow into the 'this' flow void OopFlow::merge( OopFlow *flow, int max_reg ) { - assert( _b == NULL, "merging into a happy flow" ); + assert( _b == nullptr, "merging into a happy flow" ); assert( flow->_b, "this flow is still alive" ); assert( flow != this, "no self flow" ); // Do the merge. If there are any differences, drop to 'bottom' which - // is OptoReg::Bad or NULL depending. + // is OptoReg::Bad or null depending. for( int i=0; i_callees[i] ) _callees[i] = OptoReg::Bad; // Merge the reaching defs if( _defs[i] != flow->_defs[i] ) - _defs[i] = NULL; + _defs[i] = nullptr; } } @@ -214,7 +214,7 @@ OopMap *OopFlow::build_oop_map( Node *n, int max_reg, PhaseRegAlloc *regalloc, i memset(dup_check,0,OptoReg::stack0()) ); OopMap *omap = new OopMap( framesize, max_inarg_slot ); - MachCallNode *mcall = n->is_MachCall() ? n->as_MachCall() : NULL; + MachCallNode *mcall = n->is_MachCall() ? n->as_MachCall() : nullptr; JVMState* jvms = n->jvms(); // For all registers do... @@ -456,7 +456,7 @@ static void do_liveness(PhaseRegAlloc* regalloc, PhaseCFG* cfg, Block_List* work if( OptoReg::is_valid(first) ) clr_live_bit(tmp_live,first); if( OptoReg::is_valid(second) ) clr_live_bit(tmp_live,second); - MachNode *m = n->is_Mach() ? n->as_Mach() : NULL; + MachNode *m = n->is_Mach() ? n->as_Mach() : nullptr; // Check if m is potentially a CISC alternate instruction (i.e, possibly // synthesized by RegAlloc from a conventional instruction and a @@ -479,7 +479,7 @@ static void do_liveness(PhaseRegAlloc* regalloc, PhaseCFG* cfg, Block_List* work // for this stack location, and set the appropriate bit in the // live vector 4987749. if (is_cisc_alternate && def == fp) { - const TypePtr *adr_type = NULL; + const TypePtr *adr_type = nullptr; intptr_t offset; const Node* base = m->get_base_and_disp(offset, adr_type); if (base == NodeSentinel) { @@ -490,7 +490,7 @@ static void do_liveness(PhaseRegAlloc* regalloc, PhaseCFG* cfg, Block_List* work assert(!def->bottom_type()->isa_oop_ptr(), "expecting non-oop mem input"); } else if (base != fp || offset == Type::OffsetBot) { // Do nothing: the fp operand is either not from a memory use - // (base == NULL) OR the fp is used in a non-memory context + // (base == nullptr) OR the fp is used in a non-memory context // (base is some other register) OR the offset is not constant, // so it is not a stack slot. } else { @@ -546,7 +546,7 @@ static void do_liveness(PhaseRegAlloc* regalloc, PhaseCFG* cfg, Block_List* work Block* block = cfg->get_block(i); uint j; for (j = 1; j < block->number_of_nodes(); j++) { - if (block->get_node(j)->jvms() && (*safehash)[block->get_node(j)] == NULL) { + if (block->get_node(j)->jvms() && (*safehash)[block->get_node(j)] == nullptr) { break; } } @@ -581,12 +581,12 @@ void PhaseOutput::BuildOopMaps() { Block_List worklist; // Worklist of pending blocks int max_reg_ints = align_up(max_reg, BitsPerInt)>>LogBitsPerInt; - Dict *safehash = NULL; // Used for assert only + Dict *safehash = nullptr; // Used for assert only // Compute a backwards liveness per register. Needs a bitarray of // #blocks x (#registers, rounded up to ints) safehash = new Dict(cmpkey,hashkey,A); do_liveness( C->regalloc(), C->cfg(), &worklist, max_reg_ints, A, safehash ); - OopFlow *free_list = NULL; // Free, unused + OopFlow *free_list = nullptr; // Free, unused // Array mapping blocks to completed oopflows OopFlow **flows = NEW_ARENA_ARRAY(A, OopFlow*, C->cfg()->number_of_blocks()); @@ -630,7 +630,7 @@ void PhaseOutput::BuildOopMaps() { // If this block has a visited predecessor AND that predecessor has this // last block as his only undone child, we can move the OopFlow from the // pred to this block. Otherwise we have to grab a new OopFlow. - OopFlow *flow = NULL; // Flag for finding optimized flow + OopFlow *flow = nullptr; // Flag for finding optimized flow Block *pred = (Block*)((intptr_t)0xdeadbeef); // Scan this block's preds to find a done predecessor for (uint j = 1; j < b->num_preds(); j++) { @@ -664,9 +664,9 @@ void PhaseOutput::BuildOopMaps() { if( !free_list ) free_list = OopFlow::make(A,max_reg,C); flow = free_list; - assert( flow->_b == NULL, "oopFlow is not free" ); + assert( flow->_b == nullptr, "oopFlow is not free" ); free_list = flow->_next; - flow->_next = NULL; + flow->_next = nullptr; // Copy/clone over the data flow->clone(flows[pred->_pre_order], max_reg); @@ -676,7 +676,7 @@ void PhaseOutput::BuildOopMaps() { // because after the first time they are guarded from entering // this code again. assert( flow->_b == pred, "have some prior flow" ); - flow->_b = NULL; + flow->_b = nullptr; // Now push flow forward flows[b->_pre_order] = flow;// Mark flow for this block diff --git a/src/hotspot/share/opto/bytecodeInfo.cpp b/src/hotspot/share/opto/bytecodeInfo.cpp index 1e252a87c01f8..659d562429ee4 100644 --- a/src/hotspot/share/opto/bytecodeInfo.cpp +++ b/src/hotspot/share/opto/bytecodeInfo.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -43,30 +43,30 @@ InlineTree::InlineTree(Compile* c, JVMState* caller_jvms, int caller_bci, int max_inline_level) : C(c), - _caller_jvms(NULL), + _caller_jvms(nullptr), _method(callee), _caller_tree((InlineTree*) caller_tree), _count_inline_bcs(method()->code_size_for_inlining()), _max_inline_level(max_inline_level), - _subtrees(c->comp_arena(), 2, 0, NULL), - _msg(NULL) + _subtrees(c->comp_arena(), 2, 0, nullptr), + _msg(nullptr) { #ifndef PRODUCT _count_inlines = 0; _forced_inline = false; #endif - if (caller_jvms != NULL) { + if (caller_jvms != nullptr) { // Keep a private copy of the caller_jvms: _caller_jvms = new (C) JVMState(caller_jvms->method(), caller_tree->caller_jvms()); _caller_jvms->set_bci(caller_jvms->bci()); assert(!caller_jvms->should_reexecute(), "there should be no reexecute bytecode with inlining"); assert(_caller_jvms->same_calls_as(caller_jvms), "consistent JVMS"); } - assert((caller_tree == NULL ? 0 : caller_tree->stack_depth() + 1) == stack_depth(), "correct (redundant) depth parameter"); + assert((caller_tree == nullptr ? 0 : caller_tree->stack_depth() + 1) == stack_depth(), "correct (redundant) depth parameter"); assert(caller_bci == this->caller_bci(), "correct (redundant) bci parameter"); // Update hierarchical counts, count_inline_bcs() and count_inlines() InlineTree *caller = (InlineTree *)caller_tree; - for( ; caller != NULL; caller = ((InlineTree *)(caller->caller_tree())) ) { + for( ; caller != nullptr; caller = ((InlineTree *)(caller->caller_tree())) ) { caller->_count_inline_bcs += count_inline_bcs(); NOT_PRODUCT(caller->_count_inlines++;) } @@ -197,7 +197,7 @@ bool InlineTree::should_not_inline(ciMethod *callee_method, ciMethod* caller_method, JVMState* jvms) { - const char* fail_msg = NULL; + const char* fail_msg = nullptr; // First check all inlining restrictions which are required for correctness if (callee_method->is_abstract()) { @@ -213,11 +213,11 @@ bool InlineTree::should_not_inline(ciMethod *callee_method, } // one more inlining restriction - if (fail_msg == NULL && callee_method->has_unloaded_classes_in_signature()) { + if (fail_msg == nullptr && callee_method->has_unloaded_classes_in_signature()) { fail_msg = "unloaded signature classes"; } - if (fail_msg != NULL) { + if (fail_msg != nullptr) { set_msg(fail_msg); return true; } @@ -272,10 +272,10 @@ bool InlineTree::should_not_inline(ciMethod *callee_method, // don't inline exception code unless the top method belongs to an // exception class - if (caller_tree() != NULL && + if (caller_tree() != nullptr && callee_method->holder()->is_subclass_of(C->env()->Throwable_klass())) { const InlineTree *top = this; - while (top->caller_tree() != NULL) top = top->caller_tree(); + while (top->caller_tree() != nullptr) top = top->caller_tree(); ciInstanceKlass* k = top->method()->holder(); if (!k->is_subclass_of(C->env()->Throwable_klass())) { set_msg("exception method"); @@ -436,8 +436,8 @@ bool InlineTree::try_to_inline(ciMethod* callee_method, ciMethod* caller_method, } } // count callers of current method and callee - Node* callee_argument0 = is_compiled_lambda_form ? jvms->map()->argument(jvms, 0)->uncast() : NULL; - for (JVMState* j = jvms->caller(); j != NULL && j->has_method(); j = j->caller()) { + Node* callee_argument0 = is_compiled_lambda_form ? jvms->map()->argument(jvms, 0)->uncast() : nullptr; + for (JVMState* j = jvms->caller(); j != nullptr && j->has_method(); j = j->caller()) { if (j->method() == callee_method) { if (is_compiled_lambda_form) { // Since compiled lambda forms are heavily reused we allow recursive inlining. If it is truly @@ -476,7 +476,7 @@ bool InlineTree::try_to_inline(ciMethod* callee_method, ciMethod* caller_method, //------------------------------pass_initial_checks---------------------------- bool InlineTree::pass_initial_checks(ciMethod* caller_method, int caller_bci, ciMethod* callee_method) { // Check if a callee_method was suggested - if (callee_method == NULL) { + if (callee_method == nullptr) { return false; } ciInstanceKlass *callee_holder = callee_method->holder(); @@ -518,15 +518,15 @@ const char* InlineTree::check_can_parse(ciMethod* callee) { if (!callee->has_balanced_monitors()) return "not compilable (unbalanced monitors)"; if ( callee->get_flow_analysis()->failing()) return "not compilable (flow analysis failed)"; if (!callee->can_be_parsed()) return "cannot be parsed"; - return NULL; + return nullptr; } //------------------------------print_inlining--------------------------------- void InlineTree::print_inlining(ciMethod* callee_method, int caller_bci, ciMethod* caller_method, bool success) const { const char* inline_msg = msg(); - assert(inline_msg != NULL, "just checking"); - if (C->log() != NULL) { + assert(inline_msg != nullptr, "just checking"); + if (C->log() != nullptr) { if (success) { C->log()->inline_success(inline_msg); } else { @@ -537,10 +537,10 @@ void InlineTree::print_inlining(ciMethod* callee_method, int caller_bci, caller_bci, inline_msg); if (C->print_inlining()) { C->print_inlining(callee_method, inline_level(), caller_bci, inline_msg); - guarantee(callee_method != NULL, "would crash in CompilerEvent::InlineEvent::post"); + guarantee(callee_method != nullptr, "would crash in CompilerEvent::InlineEvent::post"); if (Verbose) { const InlineTree *top = this; - while (top->caller_tree() != NULL) { top = top->caller_tree(); } + while (top->caller_tree() != nullptr) { top = top->caller_tree(); } //tty->print(" bcs: %d+%d invoked: %d", top->count_inline_bcs(), callee_method->code_size(), callee_method->interpreter_invocation_count()); } } @@ -553,13 +553,13 @@ void InlineTree::print_inlining(ciMethod* callee_method, int caller_bci, //------------------------------ok_to_inline----------------------------------- bool InlineTree::ok_to_inline(ciMethod* callee_method, JVMState* jvms, ciCallProfile& profile, bool& should_delay) { - assert(callee_method != NULL, "caller checks for optimized virtual!"); + assert(callee_method != nullptr, "caller checks for optimized virtual!"); assert(!should_delay, "should be initialized to false"); #ifdef ASSERT // Make sure the incoming jvms has the same information content as me. // This means that we can eventually make this whole class AllStatic. - if (jvms->caller() == NULL) { - assert(_caller_jvms == NULL, "redundant instance state"); + if (jvms->caller() == nullptr) { + assert(_caller_jvms == nullptr, "redundant instance state"); } else { assert(_caller_jvms->same_calls_as(jvms->caller()), "redundant instance state"); } @@ -577,7 +577,7 @@ bool InlineTree::ok_to_inline(ciMethod* callee_method, JVMState* jvms, ciCallPro // Do some parse checks. set_msg(check_can_parse(callee_method)); - if (msg() != NULL) { + if (msg() != nullptr) { print_inlining(callee_method, caller_bci, caller_method, false /* !success */); return false; } @@ -587,7 +587,7 @@ bool InlineTree::ok_to_inline(ciMethod* callee_method, JVMState* jvms, ciCallPro should_delay); // out if (success) { // Inline! - if (msg() == NULL) { + if (msg() == nullptr) { set_msg("inline (hot)"); } print_inlining(callee_method, caller_bci, caller_method, true /* success */); @@ -595,7 +595,7 @@ bool InlineTree::ok_to_inline(ciMethod* callee_method, JVMState* jvms, ciCallPro return true; } else { // Do not inline - if (msg() == NULL) { + if (msg() == nullptr) { set_msg("too cold to inline"); } print_inlining(callee_method, caller_bci, caller_method, false /* !success */ ); @@ -607,11 +607,11 @@ bool InlineTree::ok_to_inline(ciMethod* callee_method, JVMState* jvms, ciCallPro InlineTree *InlineTree::build_inline_tree_for_callee( ciMethod* callee_method, JVMState* caller_jvms, int caller_bci) { // Attempt inlining. InlineTree* old_ilt = callee_at(caller_bci, callee_method); - if (old_ilt != NULL) { + if (old_ilt != nullptr) { return old_ilt; } int max_inline_level_adjust = 0; - if (caller_jvms->method() != NULL) { + if (caller_jvms->method() != nullptr) { if (caller_jvms->method()->is_compiled_lambda_form()) { max_inline_level_adjust += 1; // don't count actions in MH or indy adapter frames } else if (callee_method->is_method_handle_intrinsic() || @@ -646,7 +646,7 @@ InlineTree *InlineTree::callee_at(int bci, ciMethod* callee) const { return sub; } } - return NULL; + return nullptr; } @@ -655,7 +655,7 @@ InlineTree *InlineTree::build_inline_tree_root() { Compile* C = Compile::current(); // Root of inline tree - InlineTree* ilt = new InlineTree(C, NULL, C->method(), NULL, -1, MaxInlineLevel); + InlineTree* ilt = new InlineTree(C, nullptr, C->method(), nullptr, -1, MaxInlineLevel); return ilt; } @@ -674,11 +674,11 @@ InlineTree* InlineTree::find_subtree_from_root(InlineTree* root, JVMState* jvms, assert(jvmsp->method() == iltp->method(), "tree still in sync"); ciMethod* d_callee = (d == depth) ? callee : jvms->of_depth(d+1)->method(); InlineTree* sub = iltp->callee_at(jvmsp->bci(), d_callee); - if (sub == NULL) { + if (sub == nullptr) { if (d == depth) { sub = iltp->build_inline_tree_for_callee(d_callee, jvmsp, jvmsp->bci()); } - guarantee(sub != NULL, "should be a sub-ilt here"); + guarantee(sub != nullptr, "should be a sub-ilt here"); return sub; } iltp = sub; diff --git a/src/hotspot/share/opto/c2_globals.hpp b/src/hotspot/share/opto/c2_globals.hpp index f6b0ba8da0ec4..a8ef7ecd044c1 100644 --- a/src/hotspot/share/opto/c2_globals.hpp +++ b/src/hotspot/share/opto/c2_globals.hpp @@ -380,7 +380,7 @@ notproduct(ccstr, PrintIdealGraphAddress, "127.0.0.1", \ "IP address to connect to visualizer") \ \ - notproduct(ccstr, PrintIdealGraphFile, NULL, \ + notproduct(ccstr, PrintIdealGraphFile, nullptr, \ "File to dump ideal graph to. If set overrides the " \ "use of the network") \ \ diff --git a/src/hotspot/share/opto/c2compiler.cpp b/src/hotspot/share/opto/c2compiler.cpp index f939145e19858..473684107198b 100644 --- a/src/hotspot/share/opto/c2compiler.cpp +++ b/src/hotspot/share/opto/c2compiler.cpp @@ -107,7 +107,7 @@ void C2Compiler::compile_method(ciEnv* env, ciMethod* target, int entry_bci, boo Compile C(env, target, entry_bci, subsume_loads, do_escape_analysis, eliminate_boxing, do_locks_coarsening, install_code, directive); // Check result and retry if appropriate. - if (C.failure_reason() != NULL) { + if (C.failure_reason() != nullptr) { if (C.failure_reason_is(retry_class_loading_during_parsing())) { env->report_failure(C.failure_reason()); continue; // retry @@ -214,7 +214,7 @@ bool C2Compiler::is_intrinsic_supported(const methodHandle& method, bool is_virt if (!Matcher::match_rule_supported(Op_AryEq)) return false; break; case vmIntrinsics::_copyMemory: - if (StubRoutines::unsafe_arraycopy() == NULL) return false; + if (StubRoutines::unsafe_arraycopy() == nullptr) return false; break; case vmIntrinsics::_encodeAsciiArray: if (!Matcher::match_rule_supported(Op_EncodeISOArray) || !Matcher::supports_encode_ascii_array) return false; @@ -427,7 +427,7 @@ bool C2Compiler::is_intrinsic_supported(const methodHandle& method, bool is_virt if (!Matcher::match_rule_supported(Op_MulHiL)) return false; break; case vmIntrinsics::_getCallerClass: - if (vmClasses::reflect_CallerSensitive_klass() == NULL) return false; + if (vmClasses::reflect_CallerSensitive_klass() == nullptr) return false; break; case vmIntrinsics::_onSpinWait: if (!Matcher::match_rule_supported(Op_OnSpinWait)) return false; @@ -601,6 +601,7 @@ bool C2Compiler::is_intrinsic_supported(const methodHandle& method, bool is_virt case vmIntrinsics::_putLongUnaligned: case vmIntrinsics::_loadFence: case vmIntrinsics::_storeFence: + case vmIntrinsics::_storeStoreFence: case vmIntrinsics::_fullFence: case vmIntrinsics::_currentThread: #ifdef JFR_HAVE_INTRINSICS diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index ecc3594d5cab1..e856ed2814d5e 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -74,7 +74,7 @@ class ParseGenerator : public InlineCallGenerator { { _is_osr = is_osr; _expected_uses = expected_uses; - assert(InlineTree::check_can_parse(method) == NULL, "parse must be possible"); + assert(InlineTree::check_can_parse(method) == nullptr, "parse must be possible"); } virtual bool is_parse() const { return true; } @@ -93,7 +93,7 @@ JVMState* ParseGenerator::generate(JVMState* jvms) { } if (C->failing()) { - return NULL; // bailing out of the compile; do not try to parse + return nullptr; // bailing out of the compile; do not try to parse } Parse parser(jvms, method(), _expected_uses); @@ -101,8 +101,8 @@ JVMState* ParseGenerator::generate(JVMState* jvms) { GraphKit& exits = parser.exits(); if (C->failing()) { - while (exits.pop_exception_state() != NULL) ; - return NULL; + while (exits.pop_exception_state() != nullptr) ; + return nullptr; } assert(exits.jvms()->same_calls_as(jvms), "sanity"); @@ -147,7 +147,7 @@ JVMState* DirectCallGenerator::generate(JVMState* jvms) { address target = is_static ? SharedRuntime::get_resolve_static_call_stub() : SharedRuntime::get_resolve_opt_virtual_call_stub(); - if (kit.C->log() != NULL) { + if (kit.C->log() != nullptr) { kit.C->log()->elem("direct_call bci='%d'", jvms->bci()); } @@ -195,7 +195,7 @@ class VirtualCallGenerator : public CallGenerator { public: VirtualCallGenerator(ciMethod* method, int vtable_index, bool separate_io_proj) - : CallGenerator(method), _vtable_index(vtable_index), _separate_io_proj(separate_io_proj), _call_node(NULL) + : CallGenerator(method), _vtable_index(vtable_index), _separate_io_proj(separate_io_proj), _call_node(nullptr) { assert(vtable_index == Method::invalid_vtable_index || vtable_index >= 0, "either invalid or usable"); @@ -219,7 +219,7 @@ JVMState* VirtualCallGenerator::generate(JVMState* jvms) { kit.C->print_inlining_update(this); - if (kit.C->log() != NULL) { + if (kit.C->log() != nullptr) { kit.C->log()->elem("virtual_call bci='%d'", jvms->bci()); } @@ -235,7 +235,7 @@ JVMState* VirtualCallGenerator::generate(JVMState* jvms) { kit.inc_sp(arg_size); // restore arguments kit.uncommon_trap(Deoptimization::Reason_null_check, Deoptimization::Action_none, - NULL, "null receiver"); + nullptr, "null receiver"); return kit.transfer_exceptions_into_jvms(); } @@ -244,7 +244,7 @@ JVMState* VirtualCallGenerator::generate(JVMState* jvms) { // However currently the conversion to implicit null checks in // Block::implicit_null_check() only looks for loads and stores, not calls. ciMethod *caller = kit.method(); - ciMethodData *caller_md = (caller == NULL) ? NULL : caller->method_data(); + ciMethodData *caller_md = (caller == nullptr) ? nullptr : caller->method_data(); if (!UseInlineCaches || !ImplicitNullChecks || !os::zero_page_read_protected() || ((ImplicitNullCheckThreshold > 0) && caller_md && (caller_md->trap_count(Deoptimization::Reason_null_check) @@ -288,7 +288,7 @@ JVMState* VirtualCallGenerator::generate(JVMState* jvms) { } CallGenerator* CallGenerator::for_inline(ciMethod* m, float expected_uses) { - if (InlineTree::check_can_parse(m) != NULL) return NULL; + if (InlineTree::check_can_parse(m) != nullptr) return nullptr; return new ParseGenerator(m, expected_uses); } @@ -296,7 +296,7 @@ CallGenerator* CallGenerator::for_inline(ciMethod* m, float expected_uses) { // for the method execution already in progress, not just the JVMS // of the caller. Thus, this CallGenerator cannot be mixed with others! CallGenerator* CallGenerator::for_osr(ciMethod* m, int osr_bci) { - if (InlineTree::check_can_parse(m) != NULL) return NULL; + if (InlineTree::check_can_parse(m) != nullptr) return nullptr; float past_uses = m->interpreter_invocation_count(); float expected_uses = past_uses; return new ParseGenerator(m, expected_uses, true); @@ -388,7 +388,7 @@ class LateInlineMHCallGenerator : public LateInlineCallGenerator { public: LateInlineMHCallGenerator(ciMethod* caller, ciMethod* callee, bool input_not_const) : - LateInlineCallGenerator(callee, NULL), _caller(caller), _input_not_const(input_not_const) {} + LateInlineCallGenerator(callee, nullptr), _caller(caller), _input_not_const(input_not_const) {} virtual bool is_mh_late_inline() const { return true; } @@ -429,7 +429,7 @@ bool LateInlineMHCallGenerator::do_late_inline_check(Compile* C, JVMState* jvms) CallGenerator* cg = for_method_handle_inline(jvms, _caller, method(), allow_inline, input_not_const); assert(!input_not_const, "sanity"); // shouldn't have been scheduled for inlining in the first place - if (cg != NULL) { + if (cg != nullptr) { assert(!cg->is_late_inline() || cg->is_mh_late_inline() || AlwaysIncrementalInline, "we're doing late inlining"); _inline_cg = cg; C->dec_number_of_mh_late_inlines(); @@ -466,7 +466,7 @@ class LateInlineVirtualCallGenerator : public VirtualCallGenerator { public: LateInlineVirtualCallGenerator(ciMethod* method, int vtable_index, float prof_factor) : VirtualCallGenerator(method, vtable_index, true /*separate_io_projs*/), - _unique_id(0), _inline_cg(NULL), _callee(NULL), _is_pure_call(false), _prof_factor(prof_factor) { + _unique_id(0), _inline_cg(nullptr), _callee(nullptr), _is_pure_call(false), _prof_factor(prof_factor) { assert(IncrementalInlineVirtual, "required"); } @@ -478,7 +478,7 @@ class LateInlineVirtualCallGenerator : public VirtualCallGenerator { virtual void do_late_inline(); virtual void set_callee_method(ciMethod* m) { - assert(_callee == NULL, "repeated inlining attempt"); + assert(_callee == nullptr, "repeated inlining attempt"); _callee = m; } @@ -488,7 +488,7 @@ class LateInlineVirtualCallGenerator : public VirtualCallGenerator { // through and exceptional uses of the memory and io projections // as is done for allocations and macro expansion. JVMState* new_jvms = VirtualCallGenerator::generate(jvms); - if (call_node() != NULL) { + if (call_node() != nullptr) { call_node()->set_generator(this); } return new_jvms; @@ -548,10 +548,10 @@ bool LateInlineVirtualCallGenerator::do_late_inline_check(Compile* C, JVMState* jvms, allow_inline, _prof_factor, - NULL /*speculative_receiver_type*/, + nullptr /*speculative_receiver_type*/, true /*allow_intrinsics*/); - if (cg != NULL) { + if (cg != nullptr) { assert(!cg->is_late_inline() || cg->is_mh_late_inline() || AlwaysIncrementalInline, "we're doing late inlining"); _inline_cg = cg; return true; @@ -578,7 +578,7 @@ void LateInlineMHCallGenerator::do_late_inline() { } void LateInlineVirtualCallGenerator::do_late_inline() { - assert(_callee != NULL, "required"); // set up in CallDynamicJavaNode::Ideal + assert(_callee != nullptr, "required"); // set up in CallDynamicJavaNode::Ideal CallGenerator::do_late_inline_helper(); } @@ -643,8 +643,8 @@ void CallGenerator::do_late_inline_helper() { // Can't inline it CallNode* call = call_node(); - if (call == NULL || call->outcnt() == 0 || - call->in(0) == NULL || call->in(0)->is_top()) { + if (call == nullptr || call->outcnt() == 0 || + call->in(0) == nullptr || call->in(0)->is_top()) { return; } @@ -676,8 +676,8 @@ void CallGenerator::do_late_inline_helper() { (callprojs.catchall_memproj == call->in(TypeFunc::Memory)) || (callprojs.fallthrough_ioproj == call->in(TypeFunc::I_O)) || (callprojs.catchall_ioproj == call->in(TypeFunc::I_O)) || - (callprojs.resproj != NULL && call->find_edge(callprojs.resproj) != -1) || - (callprojs.exobj != NULL && call->find_edge(callprojs.exobj) != -1)) { + (callprojs.resproj != nullptr && call->find_edge(callprojs.resproj) != -1) || + (callprojs.exobj != nullptr && call->find_edge(callprojs.exobj) != -1)) { return; } @@ -701,7 +701,7 @@ void CallGenerator::do_late_inline_helper() { // The call is marked as pure (no important side effects), but result isn't used. // It's safe to remove the call. - result_not_used = (callprojs.resproj == NULL || callprojs.resproj->outcnt() == 0); + result_not_used = (callprojs.resproj == nullptr || callprojs.resproj->outcnt() == 0); } if (result_not_used) { @@ -754,7 +754,7 @@ void CallGenerator::do_late_inline_helper() { // Setup default node notes to be picked up by the inlining Node_Notes* old_nn = C->node_notes_at(call->_idx); - if (old_nn != NULL) { + if (old_nn != nullptr) { Node_Notes* entry_nn = old_nn->clone(C); entry_nn->set_jvms(jvms); C->set_default_node_notes(entry_nn); @@ -762,7 +762,7 @@ void CallGenerator::do_late_inline_helper() { // Now perform the inlining using the synthesized JVMState JVMState* new_jvms = inline_cg()->generate(jvms); - if (new_jvms == NULL) return; // no change + if (new_jvms == nullptr) return; // no change if (C->failing()) return; // Capture any exceptional control flow @@ -775,6 +775,10 @@ void CallGenerator::do_late_inline_helper() { result = (result_size == 1) ? kit.pop() : kit.pop_pair(); } + if (call->is_CallStaticJava() && call->as_CallStaticJava()->is_boxing_method()) { + result = kit.must_be_not_null(result, false); + } + if (inline_cg()->is_inline()) { C->set_has_loops(C->has_loops() || inline_cg()->method()->has_loops()); C->env()->notice_inlined_method(inline_cg()->method()); @@ -933,7 +937,7 @@ JVMState* PredictedCallGenerator::generate(JVMState* jvms) { // We share a map with the caller, so his JVMS gets adjusted. Node* receiver = kit.argument(0); CompileLog* log = kit.C->log(); - if (log != NULL) { + if (log != nullptr) { log->elem("predicted_call bci='%d' exact='%d' klass='%d'", jvms->bci(), (_exact_check ? 1 : 0), log->identify(_predicted_receiver)); } @@ -948,7 +952,7 @@ JVMState* PredictedCallGenerator::generate(JVMState* jvms) { replaced_nodes.clone(); Node* casted_receiver = receiver; // will get updated in place... - Node* slow_ctl = NULL; + Node* slow_ctl = nullptr; if (_exact_check) { slow_ctl = kit.type_check_receiver(receiver, _predicted_receiver, _hit_prob, &casted_receiver); @@ -957,15 +961,15 @@ JVMState* PredictedCallGenerator::generate(JVMState* jvms) { &casted_receiver); } - SafePointNode* slow_map = NULL; - JVMState* slow_jvms = NULL; + SafePointNode* slow_map = nullptr; + JVMState* slow_jvms = nullptr; { PreserveJVMState pjvms(&kit); kit.set_control(slow_ctl); if (!kit.stopped()) { slow_jvms = _if_missed->generate(kit.sync_jvms()); if (kit.failing()) - return NULL; // might happen because of NodeCountInliningCutoff - assert(slow_jvms != NULL, "must be"); + return nullptr; // might happen because of NodeCountInliningCutoff + assert(slow_jvms != nullptr, "must be"); kit.add_exception_states_from(slow_jvms); kit.set_map(slow_jvms->map()); if (!kit.stopped()) @@ -984,7 +988,7 @@ JVMState* PredictedCallGenerator::generate(JVMState* jvms) { // Make the hot call: JVMState* new_jvms = _if_hit->generate(kit.sync_jvms()); - if (new_jvms == NULL) { + if (new_jvms == nullptr) { // Inline failed, so make a direct call. assert(_if_hit->is_inline(), "must have been a failed inline"); CallGenerator* cg = CallGenerator::for_direct_call(_if_hit->method()); @@ -994,7 +998,7 @@ JVMState* PredictedCallGenerator::generate(JVMState* jvms) { kit.set_jvms(new_jvms); // Need to merge slow and fast? - if (slow_map == NULL) { + if (slow_map == nullptr) { // The fast path is the only path remaining. return kit.transfer_exceptions_into_jvms(); } @@ -1054,7 +1058,7 @@ CallGenerator* CallGenerator::for_method_handle_call(JVMState* jvms, ciMethod* c bool input_not_const; CallGenerator* cg = CallGenerator::for_method_handle_inline(jvms, caller, callee, allow_inline, input_not_const); Compile* C = Compile::current(); - if (cg != NULL) { + if (cg != nullptr) { if (AlwaysIncrementalInline) { return CallGenerator::for_late_inline(callee, cg); } else { @@ -1116,14 +1120,14 @@ CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod* if (receiver->Opcode() == Op_ConP) { input_not_const = false; const TypeOopPtr* recv_toop = receiver->bottom_type()->isa_oopptr(); - if (recv_toop != NULL) { + if (recv_toop != nullptr) { ciMethod* target = recv_toop->const_oop()->as_method_handle()->get_vmtarget(); const int vtable_index = Method::invalid_vtable_index; if (!ciMethod::is_consistent_info(callee, target)) { print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(), "signatures mismatch"); - return NULL; + return nullptr; } CallGenerator *cg = C->call_generator(target, vtable_index, @@ -1160,7 +1164,7 @@ CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod* if (!ciMethod::is_consistent_info(callee, target)) { print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(), "signatures mismatch"); - return NULL; + return nullptr; } // In lambda forms we erase signature types to avoid resolving issues @@ -1174,7 +1178,7 @@ CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod* Node* arg = kit.argument(0); const TypeOopPtr* arg_type = arg->bottom_type()->isa_oopptr(); const Type* sig_type = TypeOopPtr::make_from_klass(signature->accessing_klass()); - if (arg_type != NULL && !arg_type->higher_equal(sig_type)) { + if (arg_type != nullptr && !arg_type->higher_equal(sig_type)) { const Type* recv_type = arg_type->filter_speculative(sig_type); // keep speculative part Node* cast_obj = gvn.transform(new CheckCastPPNode(kit.control(), arg, recv_type)); kit.set_argument(0, cast_obj); @@ -1187,7 +1191,7 @@ CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod* Node* arg = kit.argument(receiver_skip + j); const TypeOopPtr* arg_type = arg->bottom_type()->isa_oopptr(); const Type* sig_type = TypeOopPtr::make_from_klass(t->as_klass()); - if (arg_type != NULL && !arg_type->higher_equal(sig_type)) { + if (arg_type != nullptr && !arg_type->higher_equal(sig_type)) { const Type* narrowed_arg_type = arg_type->filter_speculative(sig_type); // keep speculative part Node* cast_obj = gvn.transform(new CheckCastPPNode(kit.control(), arg, narrowed_arg_type)); kit.set_argument(receiver_skip + j, cast_obj); @@ -1202,7 +1206,7 @@ CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod* int vtable_index = Method::invalid_vtable_index; bool call_does_dispatch = false; - ciKlass* speculative_receiver_type = NULL; + ciKlass* speculative_receiver_type = nullptr; if (is_virtual_or_interface) { ciInstanceKlass* klass = target->holder(); Node* receiver_node = kit.argument(0); @@ -1217,7 +1221,7 @@ CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod* false /* check_access */); // We lack profiling at this call but type speculation may // provide us with a type - speculative_receiver_type = (receiver_type != NULL) ? receiver_type->speculative_type() : NULL; + speculative_receiver_type = (receiver_type != nullptr) ? receiver_type->speculative_type() : nullptr; } CallGenerator* cg = C->call_generator(target, vtable_index, call_does_dispatch, jvms, allow_inline, @@ -1254,7 +1258,7 @@ CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod* fatal("unexpected intrinsic %d: %s", vmIntrinsics::as_int(iid), vmIntrinsics::name_at(iid)); break; } - return NULL; + return nullptr; } @@ -1289,7 +1293,7 @@ CallGenerator* CallGenerator::for_predicated_intrinsic(CallGenerator* intrinsic, JVMState* PredicatedIntrinsicGenerator::generate(JVMState* jvms) { // The code we want to generate here is: - // if (receiver == NULL) + // if (receiver == nullptr) // uncommon_Trap // if (predicate(0)) // do_intrinsic(0) @@ -1304,7 +1308,7 @@ JVMState* PredicatedIntrinsicGenerator::generate(JVMState* jvms) { PhaseGVN& gvn = kit.gvn(); CompileLog* log = kit.C->log(); - if (log != NULL) { + if (log != nullptr) { log->elem("predicated_intrinsic bci='%d' method='%d'", jvms->bci(), log->identify(method())); } @@ -1348,7 +1352,7 @@ JVMState* PredicatedIntrinsicGenerator::generate(JVMState* jvms) { PreserveJVMState pjvms(&kit); // Generate intrinsic code: JVMState* new_jvms = _intrinsic->generate(kit.sync_jvms()); - if (new_jvms == NULL) { + if (new_jvms == nullptr) { // Intrinsic failed, use normal compilation path for this predicate. slow_region->add_req(kit.control()); } else { @@ -1359,7 +1363,7 @@ JVMState* PredicatedIntrinsicGenerator::generate(JVMState* jvms) { } } } - if (else_ctrl == NULL) { + if (else_ctrl == nullptr) { else_ctrl = kit.C->top(); } kit.set_control(else_ctrl); @@ -1374,8 +1378,8 @@ JVMState* PredicatedIntrinsicGenerator::generate(JVMState* jvms) { kit.set_control(gvn.transform(slow_region)); JVMState* new_jvms = _cg->generate(kit.sync_jvms()); if (kit.failing()) - return NULL; // might happen because of NodeCountInliningCutoff - assert(new_jvms != NULL, "must be"); + return nullptr; // might happen because of NodeCountInliningCutoff + assert(new_jvms != nullptr, "must be"); kit.add_exception_states_from(new_jvms); kit.set_jvms(new_jvms); if (!kit.stopped()) { @@ -1438,7 +1442,7 @@ JVMState* PredicatedIntrinsicGenerator::generate(JVMState* jvms) { for (int j = 1; j < results; j++) { JVMState* jvms = result_jvms[j]; Node* jmap = jvms->map(); - Node* m = NULL; + Node* m = nullptr; if (jmap->req() > i) { m = jmap->in(i); if (m != n) { @@ -1508,7 +1512,7 @@ JVMState* UncommonTrapCallGenerator::generate(JVMState* jvms) { // of a class cast failure for a monomorphic call as it will never let us convert // the call to either bi-morphic or megamorphic and can lead to unc-trap loops bool keep_exact_action = true; - kit.uncommon_trap(_reason, _action, NULL, "monomorphic vcall checkcast", false, keep_exact_action); + kit.uncommon_trap(_reason, _action, nullptr, "monomorphic vcall checkcast", false, keep_exact_action); } else { kit.uncommon_trap(_reason, _action); } diff --git a/src/hotspot/share/opto/callGenerator.hpp b/src/hotspot/share/opto/callGenerator.hpp index 1b711764f3282..abb84f244fcb7 100644 --- a/src/hotspot/share/opto/callGenerator.hpp +++ b/src/hotspot/share/opto/callGenerator.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -44,9 +44,9 @@ class CallGenerator : public ResourceObj { void do_late_inline_helper(); - virtual bool do_late_inline_check(Compile* C, JVMState* jvms) { ShouldNotReachHere(); return false; } - virtual CallGenerator* inline_cg() const { ShouldNotReachHere(); return NULL; } - virtual bool is_pure_call() const { ShouldNotReachHere(); return false; } + virtual bool do_late_inline_check(Compile* C, JVMState* jvms) { ShouldNotReachHere(); return false; } + virtual CallGenerator* inline_cg() const { ShouldNotReachHere(); return nullptr;} + virtual bool is_pure_call() const { ShouldNotReachHere(); return false; } public: // Accessors @@ -81,7 +81,7 @@ class CallGenerator : public ResourceObj { // Replace the call with an inline version of the code virtual void do_late_inline() { ShouldNotReachHere(); } - virtual CallNode* call_node() const { return NULL; } + virtual CallNode* call_node() const { return nullptr; } virtual CallGenerator* with_call_node(CallNode* call) { return this; } virtual void set_unique_id(jlong id) { fatal("unique id only for late inlines"); }; @@ -120,7 +120,7 @@ class CallGenerator : public ResourceObj { // If the call traps, the returned map must have a control edge of top. // If the call can throw, the returned map must report has_exceptions(). // - // If the result is NULL, it means that this CallGenerator was unable + // If the result is null, it means that this CallGenerator was unable // to handle the given call, and another CallGenerator should be consulted. virtual JVMState* generate(JVMState* jvms) = 0; @@ -170,7 +170,7 @@ class CallGenerator : public ResourceObj { static void register_intrinsic(ciMethod* m, CallGenerator* cg); static CallGenerator* for_predicated_intrinsic(CallGenerator* intrinsic, CallGenerator* cg); - virtual Node* generate_predicate(JVMState* jvms, int predicate) { return NULL; }; + virtual Node* generate_predicate(JVMState* jvms, int predicate) { return nullptr; }; virtual void print_inlining_late(const char* msg) { ShouldNotReachHere(); } diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index fc0d741945ca2..fa793454e83e3 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -63,7 +63,7 @@ void StartNode::dump_compact_spec(outputStream *st) const { /* empty */ } //------------------------------Ideal------------------------------------------ Node *StartNode::Ideal(PhaseGVN *phase, bool can_reshape){ - return remove_dead_region(phase, can_reshape) ? this : NULL; + return remove_dead_region(phase, can_reshape) ? this : nullptr; } //------------------------------calling_convention----------------------------- @@ -99,7 +99,7 @@ Node *StartNode::match( const ProjNode *proj, const Matcher *match ) { return new MachProjNode(this,proj->_con,rm,ideal_reg); } } - return NULL; + return nullptr; } //------------------------------StartOSRNode---------------------------------- @@ -176,7 +176,7 @@ ReturnNode::ReturnNode(uint edges, Node *cntrl, Node *i_o, Node *memory, Node *f } Node *ReturnNode::Ideal(PhaseGVN *phase, bool can_reshape){ - return remove_dead_region(phase, can_reshape) ? this : NULL; + return remove_dead_region(phase, can_reshape) ? this : nullptr; } const Type* ReturnNode::Value(PhaseGVN* phase) const { @@ -221,7 +221,7 @@ RethrowNode::RethrowNode( } Node *RethrowNode::Ideal(PhaseGVN *phase, bool can_reshape){ - return remove_dead_region(phase, can_reshape) ? this : NULL; + return remove_dead_region(phase, can_reshape) ? this : nullptr; } const Type* RethrowNode::Value(PhaseGVN* phase) const { @@ -261,13 +261,13 @@ uint TailJumpNode::match_edge(uint idx) const { //============================================================================= JVMState::JVMState(ciMethod* method, JVMState* caller) : _method(method) { - assert(method != NULL, "must be valid call site"); + assert(method != nullptr, "must be valid call site"); _bci = InvocationEntryBci; _reexecute = Reexecute_Undefined; debug_only(_bci = -99); // random garbage value debug_only(_map = (SafePointNode*)-1); _caller = caller; - _depth = 1 + (caller == NULL ? 0 : caller->depth()); + _depth = 1 + (caller == nullptr ? 0 : caller->depth()); _locoff = TypeFunc::Parms; _stkoff = _locoff + _method->max_locals(); _monoff = _stkoff + _method->max_stack(); @@ -276,11 +276,11 @@ JVMState::JVMState(ciMethod* method, JVMState* caller) : _sp = 0; } JVMState::JVMState(int stack_size) : - _method(NULL) { + _method(nullptr) { _bci = InvocationEntryBci; _reexecute = Reexecute_Undefined; debug_only(_map = (SafePointNode*)-1); - _caller = NULL; + _caller = nullptr; _depth = 1; _locoff = TypeFunc::Parms; _stkoff = _locoff; @@ -309,13 +309,13 @@ bool JVMState::same_calls_as(const JVMState* that) const { const JVMState* q = that; for (;;) { if (p->_method != q->_method) return false; - if (p->_method == NULL) return true; // bci is irrelevant + if (p->_method == nullptr) return true; // bci is irrelevant if (p->_bci != q->_bci) return false; if (p->_reexecute != q->_reexecute) return false; p = p->caller(); q = q->caller(); if (p == q) return true; - assert(p != NULL && q != NULL, "depth check ensures we don't run off end"); + assert(p != nullptr && q != nullptr, "depth check ensures we don't run off end"); } } @@ -336,7 +336,7 @@ uint JVMState::debug_end() const { //------------------------------debug_depth------------------------------------ uint JVMState::debug_depth() const { uint total = 0; - for (const JVMState* jvmp = this; jvmp != NULL; jvmp = jvmp->caller()) { + for (const JVMState* jvmp = this; jvmp != nullptr; jvmp = jvmp->caller()) { total += jvmp->debug_size(); } return total; @@ -348,7 +348,7 @@ uint JVMState::debug_depth() const { // Given an allocation (a Chaitin object) and a Node decide if the Node carries // any defined value or not. If it does, print out the register or constant. static void format_helper( PhaseRegAlloc *regalloc, outputStream* st, Node *n, const char *msg, uint i, GrowableArray *scobjs ) { - if (n == NULL) { st->print(" NULL"); return; } + if (n == nullptr) { st->print(" null"); return; } if (n->is_SafePointScalarObject()) { // Scalar replacement. SafePointScalarObjectNode* spobj = n->as_SafePointScalarObject(); @@ -371,7 +371,7 @@ static void format_helper( PhaseRegAlloc *regalloc, outputStream* st, Node *n, c break; case Type::AnyPtr: assert( t == TypePtr::NULL_PTR || n->in_dump(), "" ); - st->print(" %s%d]=#NULL",msg,i); + st->print(" %s%d]=#null",msg,i); break; case Type::AryPtr: case Type::InstPtr: @@ -472,7 +472,7 @@ void JVMState::format(PhaseRegAlloc *regalloc, const Node *n, outputStream* st) ciKlass* cik = spobj->bottom_type()->is_oopptr()->klass(); assert(cik->is_instance_klass() || cik->is_array_klass(), "Not supported allocation."); - ciInstanceKlass *iklass = NULL; + ciInstanceKlass *iklass = nullptr; if (cik->is_instance_klass()) { cik->print_name_on(st); iklass = cik->as_instance_klass(); @@ -500,7 +500,7 @@ void JVMState::format(PhaseRegAlloc *regalloc, const Node *n, outputStream* st) uint first_ind = spobj->first_index(mcall->jvms()); Node* fld_node = mcall->in(first_ind); ciField* cifield; - if (iklass != NULL) { + if (iklass != nullptr) { st->print(" ["); cifield = iklass->nonstatic_field_at(0); cifield->print_name_on(st); @@ -510,7 +510,7 @@ void JVMState::format(PhaseRegAlloc *regalloc, const Node *n, outputStream* st) } for (uint j = 1; j < nf; j++) { fld_node = mcall->in(first_ind+j); - if (iklass != NULL) { + if (iklass != nullptr) { st->print(", ["); cifield = iklass->nonstatic_field_at(j); cifield->print_name_on(st); @@ -524,12 +524,12 @@ void JVMState::format(PhaseRegAlloc *regalloc, const Node *n, outputStream* st) } } st->cr(); - if (caller() != NULL) caller()->format(regalloc, n, st); + if (caller() != nullptr) caller()->format(regalloc, n, st); } void JVMState::dump_spec(outputStream *st) const { - if (_method != NULL) { + if (_method != nullptr) { bool printed = false; if (!Verbose) { // The JVMS dumps make really, really long lines. @@ -541,8 +541,8 @@ void JVMState::dump_spec(outputStream *st) const { const char* name = namest.base(); if (name[0] == ' ') ++name; const char* endcn = strchr(name, ':'); // end of class name - if (endcn == NULL) endcn = strchr(name, '('); - if (endcn == NULL) endcn = name + strlen(name); + if (endcn == nullptr) endcn = strchr(name, '('); + if (endcn == nullptr) endcn = name + strlen(name); while (endcn > name && endcn[-1] != '.' && endcn[-1] != '/') --endcn; st->print(" %s", endcn); @@ -555,30 +555,30 @@ void JVMState::dump_spec(outputStream *st) const { } else { st->print(" runtime stub"); } - if (caller() != NULL) caller()->dump_spec(st); + if (caller() != nullptr) caller()->dump_spec(st); } void JVMState::dump_on(outputStream* st) const { bool print_map = _map && !((uintptr_t)_map & 1) && - ((caller() == NULL) || (caller()->map() != _map)); + ((caller() == nullptr) || (caller()->map() != _map)); if (print_map) { if (_map->len() > _map->req()) { // _map->has_exceptions() Node* ex = _map->in(_map->req()); // _map->next_exception() // skip the first one; it's already being printed - while (ex != NULL && ex->len() > ex->req()) { + while (ex != nullptr && ex->len() > ex->req()) { ex = ex->in(ex->req()); // ex->next_exception() ex->dump(1); } } _map->dump(Verbose ? 2 : 1); } - if (caller() != NULL) { + if (caller() != nullptr) { caller()->dump_on(st); } st->print("JVMS depth=%d loc=%d stk=%d arg=%d mon=%d scalar=%d end=%d mondepth=%d sp=%d bci=%d reexecute=%s method=", depth(), locoff(), stkoff(), argoff(), monoff(), scloff(), endoff(), monitor_depth(), sp(), bci(), should_reexecute()?"true":"false"); - if (_method == NULL) { + if (_method == nullptr) { st->print_cr("(none)"); } else { _method->print_name(st); @@ -615,7 +615,7 @@ JVMState* JVMState::clone_shallow(Compile* C) const { //---------------------------clone_deep---------------------------------------- JVMState* JVMState::clone_deep(Compile* C) const { JVMState* n = clone_shallow(C); - for (JVMState* p = n; p->_caller != NULL; p = p->_caller) { + for (JVMState* p = n; p->_caller != nullptr; p = p->_caller) { p->_caller = p->_caller->clone_shallow(C); } assert(n->depth() == depth(), "sanity"); @@ -627,7 +627,7 @@ JVMState* JVMState::clone_deep(Compile* C) const { * Reset map for all callers */ void JVMState::set_map_deep(SafePointNode* map) { - for (JVMState* p = this; p != NULL; p = p->_caller) { + for (JVMState* p = this; p != nullptr; p = p->_caller) { p->set_map(map); } } @@ -641,7 +641,7 @@ void JVMState::bind_map(SafePointNode* map) { // Adapt offsets in in-array after adding or removing an edge. // Prerequisite is that the JVMState is used by only one node. void JVMState::adapt_position(int delta) { - for (JVMState* jvms = this; jvms != NULL; jvms = jvms->caller()) { + for (JVMState* jvms = this; jvms != nullptr; jvms = jvms->caller()) { jvms->set_locoff(jvms->locoff() + delta); jvms->set_stkoff(jvms->stkoff() + delta); jvms->set_monoff(jvms->monoff() + delta); @@ -660,7 +660,7 @@ int JVMState::interpreter_frame_size() const { int callee_locals = 0; int extra_args = method()->max_stack() - stk_size(); - while (jvms != NULL) { + while (jvms != nullptr) { int locks = jvms->nof_monitors(); int temps = jvms->stk_size(); bool is_top_frame = (jvms == this); @@ -700,9 +700,9 @@ void CallNode::dump_req(outputStream *st) const { void CallNode::dump_spec(outputStream *st) const { st->print(" "); - if (tf() != NULL) tf()->dump_on(st); + if (tf() != nullptr) tf()->dump_on(st); if (_cnt != COUNT_UNKNOWN) st->print(" C=%f",_cnt); - if (jvms() != NULL) jvms()->dump_spec(st); + if (jvms() != nullptr) jvms()->dump_spec(st); } #endif @@ -764,7 +764,7 @@ Node *CallNode::match( const ProjNode *proj, const Matcher *match ) { default: ShouldNotReachHere(); } - return NULL; + return nullptr; } // Do we Match on this edge index or not? Match no edges @@ -777,10 +777,10 @@ uint CallNode::match_edge(uint idx) const { // instance at the specified offset. // bool CallNode::may_modify(const TypeOopPtr *t_oop, PhaseTransform *phase) { - assert((t_oop != NULL), "sanity"); + assert((t_oop != nullptr), "sanity"); if (is_call_to_arraycopystub() && strcmp(_name, "unsafe_arraycopy") != 0) { const TypeTuple* args = _tf->domain(); - Node* dest = NULL; + Node* dest = nullptr; // Stubs that can be called once an ArrayCopyNode is expanded have // different signatures. Look for the second pointer argument, // that is the destination of the copy. @@ -793,7 +793,7 @@ bool CallNode::may_modify(const TypeOopPtr *t_oop, PhaseTransform *phase) { } } } - guarantee(dest != NULL, "Call had only one ptr in, broken IR!"); + guarantee(dest != nullptr, "Call had only one ptr in, broken IR!"); if (!dest->is_top() && may_modify_arraycopy_helper(phase->type(dest)->is_oopptr(), t_oop, phase)) { return true; } @@ -809,21 +809,21 @@ bool CallNode::may_modify(const TypeOopPtr *t_oop, PhaseTransform *phase) { if (is_CallStaticJava() && as_CallStaticJava()->is_boxing_method()) { // Skip unrelated boxing methods. Node* proj = proj_out_or_null(TypeFunc::Parms); - if ((proj == NULL) || (phase->type(proj)->is_instptr()->klass() != boxing_klass)) { + if ((proj == nullptr) || (phase->type(proj)->is_instptr()->klass() != boxing_klass)) { return false; } } - if (is_CallJava() && as_CallJava()->method() != NULL) { + if (is_CallJava() && as_CallJava()->method() != nullptr) { ciMethod* meth = as_CallJava()->method(); if (meth->is_getter()) { return false; } // May modify (by reflection) if an boxing object is passed // as argument or returned. - Node* proj = returns_pointer() ? proj_out_or_null(TypeFunc::Parms) : NULL; - if (proj != NULL) { + Node* proj = returns_pointer() ? proj_out_or_null(TypeFunc::Parms) : nullptr; + if (proj != nullptr) { const TypeInstPtr* inst_t = phase->type(proj)->isa_instptr(); - if ((inst_t != NULL) && (!inst_t->klass_is_exact() || + if ((inst_t != nullptr) && (!inst_t->klass_is_exact() || (inst_t->klass() == boxing_klass))) { return true; } @@ -831,7 +831,7 @@ bool CallNode::may_modify(const TypeOopPtr *t_oop, PhaseTransform *phase) { const TypeTuple* d = tf()->domain(); for (uint i = TypeFunc::Parms; i < d->cnt(); i++) { const TypeInstPtr* inst_t = d->field_at(i)->isa_instptr(); - if ((inst_t != NULL) && (!inst_t->klass_is_exact() || + if ((inst_t != nullptr) && (!inst_t->klass_is_exact() || (inst_t->klass() == boxing_klass))) { return true; } @@ -856,18 +856,18 @@ bool CallNode::has_non_debug_use(Node *n) { // Returns the unique CheckCastPP of a call // or 'this' if there are several CheckCastPP or unexpected uses -// or returns NULL if there is no one. +// or returns null if there is no one. Node *CallNode::result_cast() { - Node *cast = NULL; + Node *cast = nullptr; Node *p = proj_out_or_null(TypeFunc::Parms); - if (p == NULL) - return NULL; + if (p == nullptr) + return nullptr; for (DUIterator_Fast imax, i = p->fast_outs(imax); i < imax; i++) { Node *use = p->fast_out(i); if (use->is_CheckCastPP()) { - if (cast != NULL) { + if (cast != nullptr) { return this; // more than 1 CheckCastPP } cast = use; @@ -886,15 +886,15 @@ Node *CallNode::result_cast() { void CallNode::extract_projections(CallProjections* projs, bool separate_io_proj, bool do_asserts) { - projs->fallthrough_proj = NULL; - projs->fallthrough_catchproj = NULL; - projs->fallthrough_ioproj = NULL; - projs->catchall_ioproj = NULL; - projs->catchall_catchproj = NULL; - projs->fallthrough_memproj = NULL; - projs->catchall_memproj = NULL; - projs->resproj = NULL; - projs->exobj = NULL; + projs->fallthrough_proj = nullptr; + projs->fallthrough_catchproj = nullptr; + projs->fallthrough_ioproj = nullptr; + projs->catchall_ioproj = nullptr; + projs->catchall_catchproj = nullptr; + projs->fallthrough_memproj = nullptr; + projs->catchall_memproj = nullptr; + projs->resproj = nullptr; + projs->exobj = nullptr; for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) { ProjNode *pn = fast_out(i)->as_Proj(); @@ -905,8 +905,8 @@ void CallNode::extract_projections(CallProjections* projs, bool separate_io_proj // For Control (fallthrough) and I_O (catch_all_index) we have CatchProj -> Catch -> Proj projs->fallthrough_proj = pn; const Node *cn = pn->unique_ctrl_out(); - if (cn != NULL && cn->is_Catch()) { - ProjNode *cpn = NULL; + if (cn != nullptr && cn->is_Catch()) { + ProjNode *cpn = nullptr; for (DUIterator_Fast kmax, k = cn->fast_outs(kmax); k < kmax; k++) { cpn = cn->fast_out(k)->as_Proj(); assert(cpn->is_CatchProj(), "must be a CatchProjNode"); @@ -928,7 +928,7 @@ void CallNode::extract_projections(CallProjections* projs, bool separate_io_proj for (DUIterator j = pn->outs(); pn->has_out(j); j++) { Node* e = pn->out(j); if (e->Opcode() == Op_CreateEx && e->in(0)->is_CatchProj() && e->outcnt() > 0) { - assert(projs->exobj == NULL, "only one"); + assert(projs->exobj == nullptr, "only one"); projs->exobj = e; } } @@ -950,15 +950,15 @@ void CallNode::extract_projections(CallProjections* projs, bool separate_io_proj // The resproj may not exist because the result could be ignored // and the exception object may not exist if an exception handler // swallows the exception but all the other must exist and be found. - assert(projs->fallthrough_proj != NULL, "must be found"); + assert(projs->fallthrough_proj != nullptr, "must be found"); do_asserts = do_asserts && !Compile::current()->inlining_incrementally(); - assert(!do_asserts || projs->fallthrough_catchproj != NULL, "must be found"); - assert(!do_asserts || projs->fallthrough_memproj != NULL, "must be found"); - assert(!do_asserts || projs->fallthrough_ioproj != NULL, "must be found"); - assert(!do_asserts || projs->catchall_catchproj != NULL, "must be found"); + assert(!do_asserts || projs->fallthrough_catchproj != nullptr, "must be found"); + assert(!do_asserts || projs->fallthrough_memproj != nullptr, "must be found"); + assert(!do_asserts || projs->fallthrough_ioproj != nullptr, "must be found"); + assert(!do_asserts || projs->catchall_catchproj != nullptr, "must be found"); if (separate_io_proj) { - assert(!do_asserts || projs->catchall_memproj != NULL, "must be found"); - assert(!do_asserts || projs->catchall_ioproj != NULL, "must be found"); + assert(!do_asserts || projs->catchall_memproj != nullptr, "must be found"); + assert(!do_asserts || projs->catchall_ioproj != nullptr, "must be found"); } } @@ -966,7 +966,7 @@ Node* CallNode::Ideal(PhaseGVN* phase, bool can_reshape) { #ifdef ASSERT // Validate attached generator CallGenerator* cg = generator(); - if (cg != NULL) { + if (cg != nullptr) { assert(is_CallStaticJava() && cg->is_mh_late_inline() || is_CallDynamicJava() && cg->is_virtual_late_inline(), "mismatch"); } @@ -975,7 +975,7 @@ Node* CallNode::Ideal(PhaseGVN* phase, bool can_reshape) { } bool CallNode::is_call_to_arraycopystub() const { - if (_name != NULL && strstr(_name, "arraycopy") != 0) { + if (_name != nullptr && strstr(_name, "arraycopy") != 0) { return true; } return false; @@ -1003,7 +1003,7 @@ void CallJavaNode::copy_call_debug_info(PhaseIterGVN* phase, SafePointNode* sfpt for (uint i = old_dbg_start; i < sfpt->req(); i++) { Node* old_in = sfpt->in(i); // Clone old SafePointScalarObjectNodes, adjusting their field contents. - if (old_in != NULL && old_in->is_SafePointScalarObject()) { + if (old_in != nullptr && old_in->is_SafePointScalarObject()) { SafePointScalarObjectNode* old_sosn = old_in->as_SafePointScalarObject(); bool new_node; Node* new_in = old_sosn->clone(sosn_map, new_node); @@ -1017,8 +1017,8 @@ void CallJavaNode::copy_call_debug_info(PhaseIterGVN* phase, SafePointNode* sfpt } // JVMS may be shared so clone it before we modify it - set_jvms(sfpt->jvms() != NULL ? sfpt->jvms()->clone_deep(C) : NULL); - for (JVMState *jvms = this->jvms(); jvms != NULL; jvms = jvms->caller()) { + set_jvms(sfpt->jvms() != nullptr ? sfpt->jvms()->clone_deep(C) : nullptr); + for (JVMState *jvms = this->jvms(); jvms != nullptr; jvms = jvms->caller()) { jvms->set_map(this); jvms->set_locoff(jvms->locoff()+jvms_adj); jvms->set_stkoff(jvms->stkoff()+jvms_adj); @@ -1030,7 +1030,7 @@ void CallJavaNode::copy_call_debug_info(PhaseIterGVN* phase, SafePointNode* sfpt #ifdef ASSERT bool CallJavaNode::validate_symbolic_info() const { - if (method() == NULL) { + if (method() == nullptr) { return true; // call into runtime or uncommon trap } ciMethod* symbolic_info = jvms()->method()->get_method_at_bci(jvms()->bci()); @@ -1067,7 +1067,7 @@ bool CallStaticJavaNode::cmp( const Node &n ) const { Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { CallGenerator* cg = generator(); - if (can_reshape && cg != NULL) { + if (can_reshape && cg != nullptr) { assert(IncrementalInlineMH, "required"); assert(cg->call_node() == this, "mismatch"); assert(cg->is_mh_late_inline(), "not virtual"); @@ -1078,7 +1078,7 @@ Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { if (iid == vmIntrinsics::_invokeBasic) { if (in(TypeFunc::Parms)->Opcode() == Op_ConP) { phase->C->prepend_late_inline(cg); - set_generator(NULL); + set_generator(nullptr); } } else if (iid == vmIntrinsics::_linkToNative) { if (in(TypeFunc::Parms + callee->arg_size() - 1)->Opcode() == Op_ConP /* NEP */ @@ -1090,7 +1090,7 @@ Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { assert(callee->has_member_arg(), "wrong type of call?"); if (in(TypeFunc::Parms + callee->arg_size() - 1)->Opcode() == Op_ConP) { phase->C->prepend_late_inline(cg); - set_generator(NULL); + set_generator(nullptr); } } } @@ -1100,7 +1100,7 @@ Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { //----------------------------uncommon_trap_request---------------------------- // If this is an uncommon trap, return the request code, else zero. int CallStaticJavaNode::uncommon_trap_request() const { - if (_name != NULL && !strcmp(_name, "uncommon_trap")) { + if (_name != nullptr && !strcmp(_name, "uncommon_trap")) { return extract_uncommon_trap_request(this); } return 0; @@ -1108,7 +1108,7 @@ int CallStaticJavaNode::uncommon_trap_request() const { int CallStaticJavaNode::extract_uncommon_trap_request(const Node* call) { #ifndef PRODUCT if (!(call->req() > TypeFunc::Parms && - call->in(TypeFunc::Parms) != NULL && + call->in(TypeFunc::Parms) != nullptr && call->in(TypeFunc::Parms)->is_Con() && call->in(TypeFunc::Parms)->bottom_type()->isa_int())) { assert(in_dump() != 0, "OK if dumping"); @@ -1122,7 +1122,7 @@ int CallStaticJavaNode::extract_uncommon_trap_request(const Node* call) { #ifndef PRODUCT void CallStaticJavaNode::dump_spec(outputStream *st) const { st->print("# Static "); - if (_name != NULL) { + if (_name != nullptr) { st->print("%s", _name); int trap_req = uncommon_trap_request(); if (trap_req != 0) { @@ -1156,7 +1156,7 @@ bool CallDynamicJavaNode::cmp( const Node &n ) const { Node* CallDynamicJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { CallGenerator* cg = generator(); - if (can_reshape && cg != NULL) { + if (can_reshape && cg != nullptr) { assert(IncrementalInlineVirtual, "required"); assert(cg->call_node() == this, "mismatch"); assert(cg->is_virtual_late_inline(), "not virtual"); @@ -1189,7 +1189,7 @@ Node* CallDynamicJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { // Register for late inlining. cg->set_callee_method(callee); phase->C->prepend_late_inline(cg); // MH late inlining prepends to the list, so do the same - set_generator(NULL); + set_generator(nullptr); } } return CallNode::Ideal(phase, can_reshape); @@ -1261,7 +1261,7 @@ Node* CallNativeNode::match(const ProjNode *proj, const Matcher *matcher) { default: ShouldNotReachHere(); } - return NULL; + return nullptr; } #ifndef PRODUCT void CallNativeNode::print_regs(const GrowableArray& regs, outputStream* st) { @@ -1378,9 +1378,9 @@ bool SafePointNode::cmp( const Node &n ) const { //-------------------------set_next_exception---------------------------------- void SafePointNode::set_next_exception(SafePointNode* n) { - assert(n == NULL || n->Opcode() == Op_SafePoint, "correct value for next_exception"); + assert(n == nullptr || n->Opcode() == Op_SafePoint, "correct value for next_exception"); if (len() == req()) { - if (n != NULL) add_prec(n); + if (n != nullptr) add_prec(n); } else { set_prec(req(), n); } @@ -1390,10 +1390,10 @@ void SafePointNode::set_next_exception(SafePointNode* n) { //----------------------------next_exception----------------------------------- SafePointNode* SafePointNode::next_exception() const { if (len() == req()) { - return NULL; + return nullptr; } else { Node* n = in(req()); - assert(n == NULL || n->Opcode() == Op_SafePoint, "no other uses of prec edges"); + assert(n == nullptr || n->Opcode() == Op_SafePoint, "no other uses of prec edges"); return (SafePointNode*) n; } } @@ -1402,8 +1402,8 @@ SafePointNode* SafePointNode::next_exception() const { //------------------------------Ideal------------------------------------------ // Skip over any collapsed Regions Node *SafePointNode::Ideal(PhaseGVN *phase, bool can_reshape) { - assert(_jvms == NULL || ((uintptr_t)_jvms->map() & 1) || _jvms->map() == this, "inconsistent JVMState"); - return remove_dead_region(phase, can_reshape) ? this : NULL; + assert(_jvms == nullptr || ((uintptr_t)_jvms->map() & 1) || _jvms->map() == this, "inconsistent JVMState"); + return remove_dead_region(phase, can_reshape) ? this : nullptr; } //------------------------------Identity--------------------------------------- @@ -1415,7 +1415,7 @@ Node* SafePointNode::Identity(PhaseGVN* phase) { Node* out_c = unique_ctrl_out(); // This can be the safepoint of an outer strip mined loop if the inner loop's backedge was removed. Replacing the // outer loop's safepoint could confuse removal of the outer loop. - if (out_c != NULL && !out_c->is_OuterStripMinedLoopEnd()) { + if (out_c != nullptr && !out_c->is_OuterStripMinedLoopEnd()) { return in(TypeFunc::Control); } } @@ -1611,7 +1611,7 @@ uint SafePointScalarObjectNode::match_edge(uint idx) const { SafePointScalarObjectNode* SafePointScalarObjectNode::clone(Dict* sosn_map, bool& new_node) const { void* cached = (*sosn_map)[(void*)this]; - if (cached != NULL) { + if (cached != nullptr) { new_node = false; return (SafePointScalarObjectNode*)cached; } @@ -1636,7 +1636,7 @@ uint AllocateNode::size_of() const { return sizeof(*this); } AllocateNode::AllocateNode(Compile* C, const TypeFunc *atype, Node *ctrl, Node *mem, Node *abio, Node *size, Node *klass_node, Node *initial_test) - : CallNode(atype, NULL, TypeRawPtr::BOTTOM) + : CallNode(atype, nullptr, TypeRawPtr::BOTTOM) { init_class_id(Class_Allocate); init_flags(Flag_is_macro); @@ -1654,17 +1654,18 @@ AllocateNode::AllocateNode(Compile* C, const TypeFunc *atype, init_req( KlassNode , klass_node); init_req( InitialTest , initial_test); init_req( ALength , topnode); + init_req( ValidLengthTest , topnode); C->add_macro_node(this); } void AllocateNode::compute_MemBar_redundancy(ciMethod* initializer) { - assert(initializer != NULL && + assert(initializer != nullptr && initializer->is_initializer() && !initializer->is_static(), "unexpected initializer method"); BCEscapeAnalyzer* analyzer = initializer->get_bcea(); - if (analyzer == NULL) { + if (analyzer == nullptr) { return; } @@ -1674,7 +1675,7 @@ void AllocateNode::compute_MemBar_redundancy(ciMethod* initializer) } } Node *AllocateNode::make_ideal_mark(PhaseGVN *phase, Node* obj, Node* control, Node* mem) { - Node* mark_node = NULL; + Node* mark_node = nullptr; // For now only enable fast locking for non-array types if (UseBiasedLocking && Opcode() == Op_Allocate) { Node* klass_node = in(AllocateNode::KlassNode); @@ -1686,65 +1687,17 @@ Node *AllocateNode::make_ideal_mark(PhaseGVN *phase, Node* obj, Node* control, N return mark_node; } -//============================================================================= -Node* AllocateArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) { - if (remove_dead_region(phase, can_reshape)) return this; - // Don't bother trying to transform a dead node - if (in(0) && in(0)->is_top()) return NULL; - - const Type* type = phase->type(Ideal_length()); - if (type->isa_int() && type->is_int()->_hi < 0) { - if (can_reshape) { - PhaseIterGVN *igvn = phase->is_IterGVN(); - // Unreachable fall through path (negative array length), - // the allocation can only throw so disconnect it. - Node* proj = proj_out_or_null(TypeFunc::Control); - Node* catchproj = NULL; - if (proj != NULL) { - for (DUIterator_Fast imax, i = proj->fast_outs(imax); i < imax; i++) { - Node *cn = proj->fast_out(i); - if (cn->is_Catch()) { - catchproj = cn->as_Multi()->proj_out_or_null(CatchProjNode::fall_through_index); - break; - } - } - } - if (catchproj != NULL && catchproj->outcnt() > 0 && - (catchproj->outcnt() > 1 || - catchproj->unique_out()->Opcode() != Op_Halt)) { - assert(catchproj->is_CatchProj(), "must be a CatchProjNode"); - Node* nproj = catchproj->clone(); - igvn->register_new_node_with_optimizer(nproj); - - Node *frame = new ParmNode( phase->C->start(), TypeFunc::FramePtr ); - frame = phase->transform(frame); - // Halt & Catch Fire - Node* halt = new HaltNode(nproj, frame, "unexpected negative array length"); - phase->C->root()->add_req(halt); - phase->transform(halt); - - igvn->replace_node(catchproj, phase->C->top()); - return this; - } - } else { - // Can't correct it during regular GVN so register for IGVN - phase->C->record_for_igvn(this); - } - } - return NULL; -} - // Retrieve the length from the AllocateArrayNode. Narrow the type with a // CastII, if appropriate. If we are not allowed to create new nodes, and -// a CastII is appropriate, return NULL. +// a CastII is appropriate, return null. Node *AllocateArrayNode::make_ideal_length(const TypeOopPtr* oop_type, PhaseTransform *phase, bool allow_new_nodes) { Node *length = in(AllocateNode::ALength); - assert(length != NULL, "length is not null"); + assert(length != nullptr, "length is not null"); const TypeInt* length_type = phase->find_int_type(length); const TypeAryPtr* ary_type = oop_type->isa_aryptr(); - if (ary_type != NULL && length_type != NULL) { + if (ary_type != nullptr && length_type != nullptr) { const TypeInt* narrow_length_type = ary_type->narrow_size_type(length_type); if (narrow_length_type != length_type) { // Assert one of: @@ -1757,14 +1710,14 @@ Node *AllocateArrayNode::make_ideal_length(const TypeOopPtr* oop_type, PhaseTran narrow_length_type->_lo >= length_type->_lo), "narrow type must be narrower than length type"); - // Return NULL if new nodes are not allowed + // Return null if new nodes are not allowed if (!allow_new_nodes) { - return NULL; + return nullptr; } // Create a cast which is control dependent on the initialization to // propagate the fact that the array length must be positive. InitializeNode* init = initialization(); - if (init != NULL) { + if (init != nullptr) { length = new CastIINode(length, narrow_length_type); length->set_req(TypeFunc::Control, init->proj_out_or_null(TypeFunc::Control)); } @@ -1905,13 +1858,13 @@ uint LockNode::size_of() const { return sizeof(*this); } // - eliminated locking nodes // static Node *next_control(Node *ctrl) { - if (ctrl == NULL) - return NULL; + if (ctrl == nullptr) + return nullptr; while (1) { if (ctrl->is_Region()) { RegionNode *r = ctrl->as_Region(); Node *n = r->is_copy(); - if (n == NULL) + if (n == nullptr) break; // hit a region, return it else ctrl = n; @@ -1934,10 +1887,10 @@ static Node *next_control(Node *ctrl) { // bool AbstractLockNode::find_matching_unlock(const Node* ctrl, LockNode* lock, GrowableArray &lock_ops) { - ProjNode *ctrl_proj = (ctrl->is_Proj()) ? ctrl->as_Proj() : NULL; - if (ctrl_proj != NULL && ctrl_proj->_con == TypeFunc::Control) { + ProjNode *ctrl_proj = (ctrl->is_Proj()) ? ctrl->as_Proj() : nullptr; + if (ctrl_proj != nullptr && ctrl_proj->_con == TypeFunc::Control) { Node *n = ctrl_proj->in(0); - if (n != NULL && n->is_Unlock()) { + if (n != nullptr && n->is_Unlock()) { UnlockNode *unlock = n->as_Unlock(); BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); Node* lock_obj = bs->step_over_gc_barrier(lock->obj_node()); @@ -1957,11 +1910,11 @@ bool AbstractLockNode::find_matching_unlock(const Node* ctrl, LockNode* lock, // Find the lock matching an unlock. Returns null if a safepoint // or complicated control is encountered first. LockNode *AbstractLockNode::find_matching_lock(UnlockNode* unlock) { - LockNode *lock_result = NULL; + LockNode *lock_result = nullptr; // find the matching lock, or an intervening safepoint Node *ctrl = next_control(unlock->in(0)); while (1) { - assert(ctrl != NULL, "invalid control graph"); + assert(ctrl != nullptr, "invalid control graph"); assert(!ctrl->is_Start(), "missing lock for unlock"); if (ctrl->is_top()) break; // dead control path if (ctrl->is_Proj()) ctrl = ctrl->in(0); @@ -1969,7 +1922,7 @@ LockNode *AbstractLockNode::find_matching_lock(UnlockNode* unlock) { break; // found a safepoint (may be the lock we are searching for) } else if (ctrl->is_Region()) { // Check for a simple diamond pattern. Punt on anything more complicated - if (ctrl->req() == 3 && ctrl->in(1) != NULL && ctrl->in(2) != NULL) { + if (ctrl->req() == 3 && ctrl->in(1) != nullptr && ctrl->in(2) != nullptr) { Node *in1 = next_control(ctrl->in(1)); Node *in2 = next_control(ctrl->in(2)); if (((in1->is_IfTrue() && in2->is_IfFalse()) || @@ -2008,7 +1961,7 @@ bool AbstractLockNode::find_lock_and_unlock_through_if(Node* node, LockNode* loc if (if_node->is_If() && if_node->outcnt() == 2 && (if_true || node->is_IfFalse())) { Node *lock_ctrl = next_control(if_node->in(0)); if (find_matching_unlock(lock_ctrl, lock, lock_ops)) { - Node* lock1_node = NULL; + Node* lock1_node = nullptr; ProjNode* proj = if_node->as_If()->proj_out(!if_true); if (if_true) { if (proj->is_IfFalse() && proj->outcnt() == 1) { @@ -2019,7 +1972,7 @@ bool AbstractLockNode::find_lock_and_unlock_through_if(Node* node, LockNode* loc lock1_node = proj->unique_out(); } } - if (lock1_node != NULL && lock1_node->is_Lock()) { + if (lock1_node != nullptr && lock1_node->is_Lock()) { LockNode *lock1 = lock1_node->as_Lock(); BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); Node* lock_obj = bs->step_over_gc_barrier(lock->obj_node()); @@ -2044,7 +1997,7 @@ bool AbstractLockNode::find_unlocks_for_region(const RegionNode* region, LockNod // in(0) should be self edge so skip it. for (int i = 1; i < (int)region->req(); i++) { Node *in_node = next_control(region->in(i)); - if (in_node != NULL) { + if (in_node != nullptr) { if (find_matching_unlock(in_node, lock, lock_ops)) { // found a match so keep on checking. continue; @@ -2109,11 +2062,11 @@ void AbstractLockNode::related(GrowableArray *in_rel, GrowableArrayis_top()) return NULL; + if (in(0) && in(0)->is_top()) return nullptr; // Now see if we can optimize away this lock. We don't actually // remove the locking here, we simply set the _eliminate flag which @@ -2125,7 +2078,7 @@ Node *LockNode::Ideal(PhaseGVN *phase, bool can_reshape) { // If we are locking an non-escaped object, the lock/unlock is unnecessary // ConnectionGraph *cgr = phase->C->congraph(); - if (cgr != NULL && cgr->not_global_escape(obj_node())) { + if (cgr != nullptr && cgr->not_global_escape(obj_node())) { assert(!is_eliminated() || is_coarsened(), "sanity"); // The lock could be marked eliminated by lock coarsening // code during first IGVN before EA. Replace coarsened flag @@ -2144,7 +2097,7 @@ Node *LockNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Try lock coarsening // PhaseIterGVN* iter = phase->is_IterGVN(); - if (iter != NULL && !is_eliminated()) { + if (iter != nullptr && !is_eliminated()) { GrowableArray lock_ops; @@ -2224,10 +2177,10 @@ Node *LockNode::Ideal(PhaseGVN *phase, bool can_reshape) { //============================================================================= bool LockNode::is_nested_lock_region() { - return is_nested_lock_region(NULL); + return is_nested_lock_region(nullptr); } -// p is used for access to compilation log; no logging if NULL +// p is used for access to compilation log; no logging if null bool LockNode::is_nested_lock_region(Compile * c) { BoxLockNode* box = box_node()->as_BoxLock(); int stk_slot = box->stack_slot(); @@ -2240,8 +2193,8 @@ bool LockNode::is_nested_lock_region(Compile * c) { // Ignore complex cases: merged locks or multiple locks. Node* obj = obj_node(); - LockNode* unique_lock = NULL; - Node* bad_lock = NULL; + LockNode* unique_lock = nullptr; + Node* bad_lock = nullptr; if (!box->is_simple_lock_region(&unique_lock, obj, &bad_lock)) { #ifdef ASSERT this->log_lock_optimization(c, "eliminate_lock_INLR_2a", bad_lock); @@ -2250,7 +2203,7 @@ bool LockNode::is_nested_lock_region(Compile * c) { } if (unique_lock != this) { #ifdef ASSERT - this->log_lock_optimization(c, "eliminate_lock_INLR_2b", (unique_lock != NULL ? unique_lock : bad_lock)); + this->log_lock_optimization(c, "eliminate_lock_INLR_2b", (unique_lock != nullptr ? unique_lock : bad_lock)); if (PrintEliminateLocks && Verbose) { tty->print_cr("=============== unique_lock != this ============"); tty->print(" this: "); @@ -2259,11 +2212,11 @@ bool LockNode::is_nested_lock_region(Compile * c) { box->dump(); tty->print(" obj: "); obj->dump(); - if (unique_lock != NULL) { + if (unique_lock != nullptr) { tty->print(" unique_lock: "); unique_lock->dump(); } - if (bad_lock != NULL) { + if (bad_lock != nullptr) { tty->print(" bad_lock: "); bad_lock->dump(); } @@ -2304,11 +2257,11 @@ uint UnlockNode::size_of() const { return sizeof(*this); } //============================================================================= Node *UnlockNode::Ideal(PhaseGVN *phase, bool can_reshape) { - // perform any generic optimizations first (returns 'this' or NULL) + // perform any generic optimizations first (returns 'this' or null) Node *result = SafePointNode::Ideal(phase, can_reshape); - if (result != NULL) return result; + if (result != nullptr) return result; // Don't bother trying to transform a dead node - if (in(0) && in(0)->is_top()) return NULL; + if (in(0) && in(0)->is_top()) return nullptr; // Now see if we can optimize away this unlock. We don't actually // remove the unlocking here, we simply set the _eliminate flag which @@ -2321,7 +2274,7 @@ Node *UnlockNode::Ideal(PhaseGVN *phase, bool can_reshape) { // If we are unlocking an non-escaped object, the lock/unlock is unnecessary. // ConnectionGraph *cgr = phase->C->congraph(); - if (cgr != NULL && cgr->not_global_escape(obj_node())) { + if (cgr != nullptr && cgr->not_global_escape(obj_node())) { assert(!is_eliminated() || is_coarsened(), "sanity"); // The lock could be marked eliminated by lock coarsening // code during first IGVN before EA. Replace coarsened flag @@ -2336,24 +2289,24 @@ Node *UnlockNode::Ideal(PhaseGVN *phase, bool can_reshape) { } void AbstractLockNode::log_lock_optimization(Compile *C, const char * tag, Node* bad_lock) const { - if (C == NULL) { + if (C == nullptr) { return; } CompileLog* log = C->log(); - if (log != NULL) { + if (log != nullptr) { Node* box = box_node(); Node* obj = obj_node(); - int box_id = box != NULL ? box->_idx : -1; - int obj_id = obj != NULL ? obj->_idx : -1; + int box_id = box != nullptr ? box->_idx : -1; + int obj_id = obj != nullptr ? obj->_idx : -1; log->begin_head("%s compile_id='%d' lock_id='%d' class='%s' kind='%s' box_id='%d' obj_id='%d' bad_id='%d'", tag, C->compile_id(), this->_idx, is_Unlock() ? "unlock" : is_Lock() ? "lock" : "?", - kind_as_string(), box_id, obj_id, (bad_lock != NULL ? bad_lock->_idx : -1)); + kind_as_string(), box_id, obj_id, (bad_lock != nullptr ? bad_lock->_idx : -1)); log->stamp(); log->end_head(); JVMState* p = is_Unlock() ? (as_Unlock()->dbg_jvms()) : jvms(); - while (p != NULL) { + while (p != nullptr) { log->elem("jvms bci='%d' method='%d'", p->bci(), log->identify(p->method())); p = p->caller(); } diff --git a/src/hotspot/share/opto/callnode.hpp b/src/hotspot/share/opto/callnode.hpp index 7c8baefbca914..8c6fa659c9b17 100644 --- a/src/hotspot/share/opto/callnode.hpp +++ b/src/hotspot/share/opto/callnode.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -249,7 +249,7 @@ class JVMState : public ResourceObj { int bci() const { return _bci; } bool should_reexecute() const { return _reexecute==Reexecute_True; } bool is_reexecute_undefined() const { return _reexecute==Reexecute_Undefined; } - bool has_method() const { return _method != NULL; } + bool has_method() const { return _method != nullptr; } ciMethod* method() const { assert(has_method(), ""); return _method; } JVMState* caller() const { return _caller; } SafePointNode* map() const { return _map; } @@ -337,13 +337,13 @@ class SafePointNode : public MultiNode { bool _has_ea_local_in_scope; // NoEscape or ArgEscape objects in JVM States void set_jvms(JVMState* s) { - assert(s != nullptr, "assign NULL value to _jvms"); + assert(s != nullptr, "assign null value to _jvms"); *(JVMState**)&_jvms = s; // override const attribute in the accessor } public: SafePointNode(uint edges, JVMState* jvms, - // A plain safepoint advertises no memory effects (NULL): - const TypePtr* adr_type = NULL) + // A plain safepoint advertises no memory effects (null): + const TypePtr* adr_type = nullptr) : MultiNode( edges ), _jvms(jvms), _adr_type(adr_type), @@ -355,7 +355,7 @@ class SafePointNode : public MultiNode { JVMState* jvms() const { return _jvms; } virtual bool needs_deep_clone_jvms(Compile* C) { return false; } void clone_jvms(Compile* C) { - if (jvms() != NULL) { + if (jvms() != nullptr) { if (needs_deep_clone_jvms(C)) { set_jvms(jvms()->clone_deep(C)); jvms()->set_map_deep(this); @@ -434,7 +434,7 @@ class SafePointNode : public MultiNode { } // The parser marks useless maps as dead when it's done with them: - bool is_killed() { return in(TypeFunc::Control) == NULL; } + bool is_killed() { return in(TypeFunc::Control) == nullptr; } // Exception states bubbling out of subgraphs such as inlined calls // are recorded here. (There might be more than one, hence the "next".) @@ -442,7 +442,7 @@ class SafePointNode : public MultiNode { // for JVM states during parsing, intrinsic expansion, etc. SafePointNode* next_exception() const; void set_next_exception(SafePointNode* n); - bool has_exceptions() const { return next_exception() != NULL; } + bool has_exceptions() const { return next_exception() != nullptr; } // Helper methods to operate on replaced nodes ReplacedNodes replaced_nodes() const { @@ -533,7 +533,7 @@ class SafePointScalarObjectNode: public TypeNode { virtual uint match_edge(uint idx) const; uint first_index(JVMState* jvms) const { - assert(jvms != NULL, "missed JVMS"); + assert(jvms != nullptr, "missed JVMS"); return jvms->scloff() + _first_index; } uint n_fields() const { return _n_fields; } @@ -591,15 +591,15 @@ class CallNode : public SafePointNode { address _entry_point; // Address of method being called float _cnt; // Estimate of number of times called CallGenerator* _generator; // corresponding CallGenerator for some late inline calls - const char* _name; // Printable name, if _method is NULL + const char* _name; // Printable name, if _method is null CallNode(const TypeFunc* tf, address addr, const TypePtr* adr_type, JVMState* jvms = nullptr) : SafePointNode(tf->domain()->cnt(), jvms, adr_type), _tf(tf), _entry_point(addr), _cnt(COUNT_UNKNOWN), - _generator(NULL), - _name(NULL) + _generator(nullptr), + _name(nullptr) { init_class_id(Class_Call); } @@ -637,7 +637,7 @@ class CallNode : public SafePointNode { bool has_non_debug_use(Node* n); // Returns the unique CheckCastPP of a call // or result projection is there are several CheckCastPP - // or returns NULL if there is no one. + // or returns null if there is no one. Node* result_cast(); // Does this node returns pointer? bool returns_pointer() const { @@ -723,13 +723,13 @@ class CallStaticJavaNode : public CallJavaNode { CallStaticJavaNode(Compile* C, const TypeFunc* tf, address addr, ciMethod* method) : CallJavaNode(tf, addr, method) { init_class_id(Class_CallStaticJava); - if (C->eliminate_boxing() && (method != NULL) && method->is_boxing_method()) { + if (C->eliminate_boxing() && (method != nullptr) && method->is_boxing_method()) { init_flags(Flag_is_macro); C->add_macro_node(this); } } CallStaticJavaNode(const TypeFunc* tf, address addr, const char* name, const TypePtr* adr_type) - : CallJavaNode(tf, addr, NULL) { + : CallJavaNode(tf, addr, nullptr) { init_class_id(Class_CallStaticJava); // This node calls a runtime stub, which often has narrow memory effects. _adr_type = adr_type; @@ -741,7 +741,7 @@ class CallStaticJavaNode : public CallJavaNode { static int extract_uncommon_trap_request(const Node* call); bool is_boxing_method() const { - return is_macro() && (method() != NULL) && method()->is_boxing_method(); + return is_macro() && (method() != nullptr) && method()->is_boxing_method(); } // Late inlining modifies the JVMState, so we need to deep clone it // when the call node is cloned (because it is macro node). @@ -913,6 +913,7 @@ class AllocateNode : public CallNode { KlassNode, // type (maybe dynamic) of the obj. InitialTest, // slow-path test (may be constant) ALength, // array length (or TOP if none) + ValidLengthTest, ParmLimit }; @@ -922,6 +923,7 @@ class AllocateNode : public CallNode { fields[KlassNode] = TypeInstPtr::NOTNULL; fields[InitialTest] = TypeInt::BOOL; fields[ALength] = t; // length (can be a bad length) + fields[ValidLengthTest] = TypeInt::BOOL; const TypeTuple *domain = TypeTuple::make(ParmLimit, fields); @@ -970,7 +972,7 @@ class AllocateNode : public CallNode { // Dig the klass operand out of a (possible) allocation site. static Node* Ideal_klass(Node* ptr, PhaseTransform* phase) { AllocateNode* allo = Ideal_allocation(ptr, phase); - return (allo == NULL) ? NULL : allo->in(KlassNode); + return (allo == nullptr) ? nullptr : allo->in(KlassNode); } // Conservatively small estimate of offset of first non-header byte. @@ -991,13 +993,13 @@ class AllocateNode : public CallNode { // Return true if allocation doesn't escape thread, its escape state // needs be noEscape or ArgEscape. InitializeNode._does_not_escape // is true when its allocation's escape state is noEscape or - // ArgEscape. In case allocation's InitializeNode is NULL, check + // ArgEscape. In case allocation's InitializeNode is null, check // AlllocateNode._is_non_escaping flag. // AlllocateNode._is_non_escaping is true when its escape state is // noEscape. bool does_not_escape_thread() { - InitializeNode* init = NULL; - return _is_non_escaping || (((init = initialization()) != NULL) && init->does_not_escape()); + InitializeNode* init = nullptr; + return _is_non_escaping || (((init = initialization()) != nullptr) && init->does_not_escape()); } // If object doesn't escape in <.init> method and there is memory barrier @@ -1016,18 +1018,16 @@ class AllocateNode : public CallNode { // class AllocateArrayNode : public AllocateNode { public: - AllocateArrayNode(Compile* C, const TypeFunc *atype, Node *ctrl, Node *mem, Node *abio, - Node* size, Node* klass_node, Node* initial_test, - Node* count_val - ) + AllocateArrayNode(Compile* C, const TypeFunc* atype, Node* ctrl, Node* mem, Node* abio, Node* size, Node* klass_node, + Node* initial_test, Node* count_val, Node* valid_length_test) : AllocateNode(C, atype, ctrl, mem, abio, size, klass_node, initial_test) { init_class_id(Class_AllocateArray); set_req(AllocateNode::ALength, count_val); + set_req(AllocateNode::ValidLengthTest, valid_length_test); } virtual int Opcode() const; - virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); // Dig the length operand out of a array allocation site. Node* Ideal_length() { @@ -1042,8 +1042,8 @@ class AllocateArrayNode : public AllocateNode { // Return null if no allocation is recognized. static AllocateArrayNode* Ideal_array_allocation(Node* ptr, PhaseTransform* phase) { AllocateNode* allo = Ideal_allocation(ptr, phase); - return (allo == NULL || !allo->is_AllocateArray()) - ? NULL : allo->as_AllocateArray(); + return (allo == nullptr || !allo->is_AllocateArray()) + ? nullptr : allo->as_AllocateArray(); } }; @@ -1080,11 +1080,11 @@ class AbstractLockNode: public CallNode { public: AbstractLockNode(const TypeFunc *tf) - : CallNode(tf, NULL, TypeRawPtr::BOTTOM), + : CallNode(tf, nullptr, TypeRawPtr::BOTTOM), _kind(Regular) { #ifndef PRODUCT - _counter = NULL; + _counter = nullptr; #endif } virtual int Opcode() const = 0; @@ -1103,7 +1103,7 @@ class AbstractLockNode: public CallNode { bool is_nested() const { return (_kind == Nested); } const char * kind_as_string() const; - void log_lock_optimization(Compile* c, const char * tag, Node* bad_lock = NULL) const; + void log_lock_optimization(Compile* c, const char * tag, Node* bad_lock = nullptr) const; void set_non_esc_obj() { _kind = NonEscObj; set_eliminated_lock_counter(); } void set_coarsened() { _kind = Coarsened; set_eliminated_lock_counter(); } @@ -1178,7 +1178,7 @@ class UnlockNode : public AbstractLockNode { virtual uint size_of() const; // Size is bigger UnlockNode(Compile* C, const TypeFunc *tf) : AbstractLockNode( tf ) #ifdef ASSERT - , _dbg_jvms(NULL) + , _dbg_jvms(nullptr) #endif { init_class_id(Class_Unlock); @@ -1194,7 +1194,7 @@ class UnlockNode : public AbstractLockNode { } JVMState* dbg_jvms() const { return _dbg_jvms; } #else - JVMState* dbg_jvms() const { return NULL; } + JVMState* dbg_jvms() const { return nullptr; } #endif }; #endif // SHARE_OPTO_CALLNODE_HPP diff --git a/src/hotspot/share/opto/castnode.cpp b/src/hotspot/share/opto/castnode.cpp index 44799635db094..5e79f32cc4cc9 100644 --- a/src/hotspot/share/opto/castnode.cpp +++ b/src/hotspot/share/opto/castnode.cpp @@ -36,7 +36,7 @@ // If input is already higher or equal to cast type, then this is an identity. Node* ConstraintCastNode::Identity(PhaseGVN* phase) { Node* dom = dominating_cast(phase, phase); - if (dom != NULL) { + if (dom != nullptr) { return dom; } if (_dependency != RegularDependency) { @@ -78,7 +78,7 @@ const Type* ConstraintCastNode::Value(PhaseGVN* phase) const { // Return a node which is more "ideal" than the current node. Strip out // control copies Node *ConstraintCastNode::Ideal(PhaseGVN *phase, bool can_reshape) { - return (in(0) && remove_dead_region(phase, can_reshape)) ? this : NULL; + return (in(0) && remove_dead_region(phase, can_reshape)) ? this : nullptr; } bool ConstraintCastNode::cmp(const Node &n) const { @@ -125,7 +125,7 @@ Node* ConstraintCastNode::make_cast(int opcode, Node* c, Node *n, const Type *t, default: fatal("Bad opcode %d", opcode); } - return NULL; + return nullptr; } Node* ConstraintCastNode::make(Node* c, Node *n, const Type *t, BasicType bt) { @@ -139,34 +139,34 @@ Node* ConstraintCastNode::make(Node* c, Node *n, const Type *t, BasicType bt) { default: fatal("Bad basic type %s", type2name(bt)); } - return NULL; + return nullptr; } TypeNode* ConstraintCastNode::dominating_cast(PhaseGVN* gvn, PhaseTransform* pt) const { if (_dependency == UnconditionalDependency) { - return NULL; + return nullptr; } Node* val = in(1); Node* ctl = in(0); int opc = Opcode(); - if (ctl == NULL) { - return NULL; + if (ctl == nullptr) { + return nullptr; } // Range check CastIIs may all end up under a single range check and // in that case only the narrower CastII would be kept by the code // below which would be incorrect. if (is_CastII() && as_CastII()->has_range_check()) { - return NULL; + return nullptr; } - if (type()->isa_rawptr() && (gvn->type_or_null(val) == NULL || gvn->type(val)->isa_oopptr())) { - return NULL; + if (type()->isa_rawptr() && (gvn->type_or_null(val) == nullptr || gvn->type(val)->isa_oopptr())) { + return nullptr; } for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { Node* u = val->fast_out(i); if (u != this && u->outcnt() > 0 && u->Opcode() == opc && - u->in(0) != NULL && + u->in(0) != nullptr && u->bottom_type()->higher_equal(type())) { if (pt->is_dominator(u->in(0), ctl)) { return u->as_Type(); @@ -180,7 +180,7 @@ TypeNode* ConstraintCastNode::dominating_cast(PhaseGVN* gvn, PhaseTransform* pt) } } } - return NULL; + return nullptr; } #ifndef PRODUCT @@ -198,7 +198,7 @@ const Type* CastIINode::Value(PhaseGVN* phase) const { // Try to improve the type of the CastII if we recognize a CmpI/If // pattern. if (_dependency != RegularDependency) { - if (in(0) != NULL && in(0)->in(0) != NULL && in(0)->in(0)->is_If()) { + if (in(0) != nullptr && in(0)->in(0) != nullptr && in(0)->in(0)->is_If()) { assert(in(0)->is_IfFalse() || in(0)->is_IfTrue(), "should be If proj"); Node* proj = in(0); if (proj->in(0)->in(1)->is_Bool()) { @@ -262,7 +262,7 @@ static Node* find_or_make_CastII(PhaseIterGVN* igvn, Node* parent, Node* control Node* n = new CastIINode(parent, type, dependency); n->set_req(0, control); Node* existing = igvn->hash_find_insert(n); - if (existing != NULL) { + if (existing != nullptr) { n->destruct(igvn); return existing; } @@ -271,22 +271,22 @@ static Node* find_or_make_CastII(PhaseIterGVN* igvn, Node* parent, Node* control Node *CastIINode::Ideal(PhaseGVN *phase, bool can_reshape) { Node* progress = ConstraintCastNode::Ideal(phase, can_reshape); - if (progress != NULL) { + if (progress != nullptr) { return progress; } PhaseIterGVN *igvn = phase->is_IterGVN(); const TypeInt* this_type = this->type()->is_int(); Node* z = in(1); - const TypeInteger* rx = NULL; - const TypeInteger* ry = NULL; + const TypeInteger* rx = nullptr; + const TypeInteger* ry = nullptr; // Similar to ConvI2LNode::Ideal() for the same reasons if (!_range_check_dependency && Compile::push_thru_add(phase, z, this_type, rx, ry, T_INT)) { - if (igvn == NULL) { + if (igvn == nullptr) { // Postpone this optimization to iterative GVN, where we can handle deep // AddI chains without an exponential number of recursive Ideal() calls. phase->record_for_igvn(this); - return NULL; + return nullptr; } int op = z->Opcode(); Node* x = z->in(1); @@ -309,7 +309,7 @@ Node *CastIINode::Ideal(PhaseGVN *phase, bool can_reshape) { if (phase->C->post_loop_opts_phase()) { const TypeInt* this_type = this->type()->is_int(); const TypeInt* in_type = phase->type(in(1))->isa_int(); - if (in_type != NULL && this_type != NULL && + if (in_type != nullptr && this_type != nullptr && (in_type->_lo != this_type->_lo || in_type->_hi != this_type->_hi)) { jint lo1 = this_type->_lo; @@ -337,7 +337,7 @@ Node *CastIINode::Ideal(PhaseGVN *phase, bool can_reshape) { phase->C->record_for_post_loop_opts_igvn(this); } } - return NULL; + return nullptr; } Node* CastIINode::Identity(PhaseGVN* phase) { @@ -377,7 +377,7 @@ void CastIINode::dump_spec(outputStream* st) const { // If input is already higher or equal to cast type, then this is an identity. Node* CheckCastPPNode::Identity(PhaseGVN* phase) { Node* dom = dominating_cast(phase, phase); - if (dom != NULL) { + if (dom != nullptr) { return dom; } if (_dependency != RegularDependency) { @@ -407,7 +407,7 @@ const Type* CheckCastPPNode::Value(PhaseGVN* phase) const { const TypePtr *in_type = inn->isa_ptr(); const TypePtr *my_type = _type->isa_ptr(); const Type *result = _type; - if( in_type != NULL && my_type != NULL ) { + if( in_type != nullptr && my_type != nullptr ) { TypePtr::PTR in_ptr = in_type->ptr(); if (in_ptr == TypePtr::Null) { result = in_type; @@ -447,8 +447,8 @@ const Type* CheckCastPPNode::Value(PhaseGVN* phase) const { // const TypeInstPtr *in_oop = in->isa_instptr(); // const TypeInstPtr *my_oop = _type->isa_instptr(); // // If either input is an 'interface', return destination type - // assert (in_oop == NULL || in_oop->klass() != NULL, ""); - // assert (my_oop == NULL || my_oop->klass() != NULL, ""); + // assert (in_oop == nullptr || in_oop->klass() != nullptr, ""); + // assert (my_oop == nullptr || my_oop->klass() != nullptr, ""); // if( (in_oop && in_oop->klass()->is_interface()) // ||(my_oop && my_oop->klass()->is_interface()) ) { // TypePtr::PTR in_ptr = in->isa_ptr() ? in->is_ptr()->_ptr : TypePtr::BotPTR; @@ -545,7 +545,7 @@ Node *CastX2PNode::Ideal(PhaseGVN *phase, bool can_reshape) { } break; } - return NULL; + return nullptr; } //------------------------------Identity--------------------------------------- @@ -567,7 +567,7 @@ const Type* CastP2XNode::Value(PhaseGVN* phase) const { } Node *CastP2XNode::Ideal(PhaseGVN *phase, bool can_reshape) { - return (in(0) && remove_dead_region(phase, can_reshape)) ? this : NULL; + return (in(0) && remove_dead_region(phase, can_reshape)) ? this : nullptr; } //------------------------------Identity--------------------------------------- @@ -577,7 +577,7 @@ Node* CastP2XNode::Identity(PhaseGVN* phase) { } Node* ConstraintCastNode::make_cast_for_type(Node* c, Node* in, const Type* type, DependencyType dependency) { - Node* cast= NULL; + Node* cast= nullptr; if (type->isa_int()) { cast = make_cast(Op_CastII, c, in, type, dependency); } else if (type->isa_long()) { diff --git a/src/hotspot/share/opto/castnode.hpp b/src/hotspot/share/opto/castnode.hpp index 2aa318c0e24c0..e7b55aa8fd7e8 100644 --- a/src/hotspot/share/opto/castnode.hpp +++ b/src/hotspot/share/opto/castnode.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -197,7 +197,7 @@ class CheckCastPPNode: public ConstraintCastNode { // convert a machine-pointer-sized integer to a raw pointer class CastX2PNode : public Node { public: - CastX2PNode( Node *n ) : Node(NULL, n) {} + CastX2PNode( Node *n ) : Node(nullptr, n) {} virtual int Opcode() const; virtual const Type* Value(PhaseGVN* phase) const; virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); diff --git a/src/hotspot/share/opto/cfgnode.cpp b/src/hotspot/share/opto/cfgnode.cpp index 50de630681ce8..734eba0000e96 100644 --- a/src/hotspot/share/opto/cfgnode.cpp +++ b/src/hotspot/share/opto/cfgnode.cpp @@ -39,6 +39,7 @@ #include "opto/narrowptrnode.hpp" #include "opto/mulnode.hpp" #include "opto/phaseX.hpp" +#include "opto/regalloc.hpp" #include "opto/regmask.hpp" #include "opto/runtime.hpp" #include "opto/subnode.hpp" @@ -76,8 +77,8 @@ Node* RegionNode::Identity(PhaseGVN* phase) { // hard to do if there is stuff that has to happen static Node *merge_region(RegionNode *region, PhaseGVN *phase) { if( region->Opcode() != Op_Region ) // Do not do to LoopNodes - return NULL; - Node *progress = NULL; // Progress flag + return nullptr; + Node *progress = nullptr; // Progress flag PhaseIterGVN *igvn = phase->is_IterGVN(); uint rreq = region->req(); @@ -90,7 +91,7 @@ static Node *merge_region(RegionNode *region, PhaseGVN *phase) { assert(!r->as_Region()->has_phi(), "no phi users"); if( !progress ) { // No progress if (region->has_phi()) { - return NULL; // Only flatten if no Phi users + return nullptr; // Only flatten if no Phi users // igvn->hash_delete( phi ); } igvn->hash_delete( region ); @@ -125,7 +126,7 @@ static Node *merge_region(RegionNode *region, PhaseGVN *phase) { //--------------------------------has_phi-------------------------------------- -// Helper function: Return any PhiNode that uses this region or NULL +// Helper function: Return any PhiNode that uses this region or null PhiNode* RegionNode::has_phi() const { for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) { Node* phi = fast_out(i); @@ -135,23 +136,23 @@ PhiNode* RegionNode::has_phi() const { } } - return NULL; + return nullptr; } //-----------------------------has_unique_phi---------------------------------- -// Helper function: Return the only PhiNode that uses this region or NULL +// Helper function: Return the only PhiNode that uses this region or null PhiNode* RegionNode::has_unique_phi() const { // Check that only one use is a Phi - PhiNode* only_phi = NULL; + PhiNode* only_phi = nullptr; for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) { Node* phi = fast_out(i); if (phi->is_Phi()) { // Check for Phi users assert(phi->in(0) == (Node*)this, "phi uses region only via in(0)"); - if (only_phi == NULL) { + if (only_phi == nullptr) { only_phi = phi->as_Phi(); } else { - return NULL; // multiple phis + return nullptr; // multiple phis } } } @@ -164,9 +165,9 @@ PhiNode* RegionNode::has_unique_phi() const { // Helper function for RegionNode's identification of FP clipping // Check inputs to the Phi static bool check_phi_clipping( PhiNode *phi, ConNode * &min, uint &min_idx, ConNode * &max, uint &max_idx, Node * &val, uint &val_idx ) { - min = NULL; - max = NULL; - val = NULL; + min = nullptr; + max = nullptr; + val = nullptr; min_idx = 0; max_idx = 0; val_idx = 0; @@ -178,11 +179,11 @@ static bool check_phi_clipping( PhiNode *phi, ConNode * &min, uint &min_idx, Con switch( opcode ) { case Op_ConI: { - if( min == NULL ) { - min = n->Opcode() == Op_ConI ? (ConNode*)n : NULL; + if( min == nullptr ) { + min = n->Opcode() == Op_ConI ? (ConNode*)n : nullptr; min_idx = j; } else { - max = n->Opcode() == Op_ConI ? (ConNode*)n : NULL; + max = n->Opcode() == Op_ConI ? (ConNode*)n : nullptr; max_idx = j; if( min->get_int() > max->get_int() ) { // Swap min and max @@ -219,8 +220,8 @@ static bool check_phi_clipping( PhiNode *phi, ConNode * &min, uint &min_idx, Con // RegionNode_inputs // static bool check_if_clipping( const RegionNode *region, IfNode * &bot_if, IfNode * &top_if ) { - top_if = NULL; - bot_if = NULL; + top_if = nullptr; + bot_if = nullptr; // Check control structure above RegionNode for (if ( if ) ) Node *in1 = region->in(1); @@ -232,14 +233,14 @@ static bool check_if_clipping( const RegionNode *region, IfNode * &bot_if, IfNod Node *in20 = in2->in(0); Node *in30 = in3->in(0); // Check that #1 and #2 are ifTrue and ifFalse from same If - if( in10 != NULL && in10->is_If() && - in20 != NULL && in20->is_If() && - in30 != NULL && in30->is_If() && in10 == in20 && + if( in10 != nullptr && in10->is_If() && + in20 != nullptr && in20->is_If() && + in30 != nullptr && in30->is_If() && in10 == in20 && (in1->Opcode() != in2->Opcode()) ) { Node *in100 = in10->in(0); - Node *in1000 = (in100 != NULL && in100->is_Proj()) ? in100->in(0) : NULL; + Node *in1000 = (in100 != nullptr && in100->is_Proj()) ? in100->in(0) : nullptr; // Check that control for in10 comes from other branch of IF from in3 - if( in1000 != NULL && in1000->is_If() && + if( in1000 != nullptr && in1000->is_If() && in30 == in1000 && (in3->Opcode() != in100->Opcode()) ) { // Control pattern checks top_if = (IfNode*)in1000; @@ -248,7 +249,7 @@ static bool check_if_clipping( const RegionNode *region, IfNode * &bot_if, IfNod } } - return (top_if != NULL); + return (top_if != nullptr); } @@ -256,7 +257,7 @@ static bool check_if_clipping( const RegionNode *region, IfNode * &bot_if, IfNod // Helper function for RegionNode's identification of FP clipping // Verify that the value input to the phi comes from "ConvF2I; LShift; RShift" static bool check_convf2i_clipping( PhiNode *phi, uint idx, ConvF2INode * &convf2i, Node *min, Node *max) { - convf2i = NULL; + convf2i = nullptr; // Check for the RShiftNode Node *rshift = phi->in(idx); @@ -317,7 +318,7 @@ static bool check_compare_clipping( bool less_than, IfNode *iff, ConNode *limit, // Check if the RegionNode is part of an unsafe loop and unreachable from root. bool RegionNode::is_unreachable_region(const PhaseGVN* phase) { Node* top = phase->C->top(); - assert(req() == 2 || (req() == 3 && in(1) != NULL && in(2) == top), "sanity check arguments"); + assert(req() == 2 || (req() == 3 && in(1) != nullptr && in(2) == top), "sanity check arguments"); if (_is_unreachable_region) { // Return cached result from previous evaluation which should still be valid assert(is_unreachable_from_root(phase), "walk the graph again and check if its indeed unreachable"); @@ -341,7 +342,7 @@ bool RegionNode::is_possible_unsafe_loop(const PhaseGVN* phase) const { uint i; for (i = 0; i < max; i++) { Node* n = raw_out(i); - if (n != NULL && n->is_Phi()) { + if (n != nullptr && n->is_Phi()) { PhiNode* phi = n->as_Phi(); assert(phi->in(0) == this, "sanity check phi"); if (phi->outcnt() == 0) { @@ -351,7 +352,7 @@ bool RegionNode::is_possible_unsafe_loop(const PhaseGVN* phase) const { Node* u = phi->raw_out(0); // Skip if only one use is an other Phi or Call or Uncommon trap. // It is safe to consider this case as fallthrough. - if (u != NULL && (u->is_Phi() || u->is_CFG())) { + if (u != nullptr && (u->is_Phi() || u->is_CFG())) { continue; } } @@ -381,7 +382,7 @@ bool RegionNode::is_unreachable_from_root(const PhaseGVN* phase) const { uint max = n->outcnt(); for (uint i = 0; i < max; i++) { Node* m = n->raw_out(i); - if (m != NULL && m->is_CFG()) { + if (m != nullptr && m->is_CFG()) { if (m == this) { return false; // We reached the Region node - it is not dead. } @@ -393,6 +394,47 @@ bool RegionNode::is_unreachable_from_root(const PhaseGVN* phase) const { return true; // The Region node is unreachable - it is dead. } +#ifdef ASSERT +// Is this region in an infinite subgraph? +// (no path to root except through false NeverBranch exit) +bool RegionNode::is_in_infinite_subgraph() { + ResourceMark rm; + Unique_Node_List worklist; + worklist.push(this); + return RegionNode::are_all_nodes_in_infinite_subgraph(worklist); +} + +// Are all nodes in worklist in infinite subgraph? +// (no path to root except through false NeverBranch exit) +// worklist is directly used for the traversal +bool RegionNode::are_all_nodes_in_infinite_subgraph(Unique_Node_List& worklist) { + // BFS traversal down the CFG, except through NeverBranch exits + for (uint i = 0; i < worklist.size(); ++i) { + Node* n = worklist.at(i); + assert(n->is_CFG(), "only traverse CFG"); + if (n->is_Root()) { + // Found root -> there was an exit! + return false; + } else if (n->is_NeverBranch()) { + // Only follow the loop-internal projection, not the NeverBranch exit + ProjNode* proj = n->as_NeverBranch()->proj_out_or_null(0); + assert(proj != nullptr, "must find loop-internal projection of NeverBranch"); + worklist.push(proj); + } else { + // Traverse all CFG outputs + for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { + Node* use = n->fast_out(i); + if (use->is_CFG()) { + worklist.push(use); + } + } + } + } + // No exit found for any loop -> all are infinite + return true; +} +#endif //ASSERT + bool RegionNode::try_clean_mem_phi(PhaseGVN *phase) { // Incremental inlining + PhaseStringOpts sometimes produce: // @@ -415,7 +457,7 @@ bool RegionNode::try_clean_mem_phi(PhaseGVN *phase) { PhiNode* phi = has_unique_phi(); if (phi && phi->type() == Type::MEMORY && req() == 3 && phi->is_diamond_phi(true)) { - MergeMemNode* m = NULL; + MergeMemNode* m = nullptr; assert(phi->req() == 3, "same as region"); for (uint i = 1; i < 3; ++i) { Node *mem = phi->in(i); @@ -440,14 +482,14 @@ bool RegionNode::try_clean_mem_phi(PhaseGVN *phase) { // Return a node which is more "ideal" than the current node. Must preserve // the CFG, but we can still strip out dead paths. Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) { - if( !can_reshape && !in(0) ) return NULL; // Already degraded to a Copy + if( !can_reshape && !in(0) ) return nullptr; // Already degraded to a Copy assert(!in(0) || !in(0)->is_Root(), "not a specially hidden merge"); // Check for RegionNode with no Phi users and both inputs come from either // arm of the same IF. If found, then the control-flow split is useless. bool has_phis = false; if (can_reshape) { // Need DU info to check for Phi users - has_phis = (has_phi() != NULL); // Cache result + has_phis = (has_phi() != nullptr); // Cache result if (has_phis && try_clean_mem_phi(phase)) { has_phis = false; } @@ -465,7 +507,7 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) { // will be eliminated if dead. phase->is_IterGVN()->add_users_to_worklist(iff); set_req(i, iff->in(0));// Skip around the useless IF diamond - set_req(j, NULL); + set_req(j, nullptr); return this; // Record progress } } @@ -473,7 +515,7 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) { } } - // Remove TOP or NULL input paths. If only 1 input path remains, this Region + // Remove TOP or null input paths. If only 1 input path remains, this Region // degrades to a copy. bool add_to_worklist = true; bool modified = false; @@ -483,7 +525,7 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) { // For all inputs... for( uint i=1; iis_Region() && n->as_Region()->is_copy() ) { set_req(i, n->nonnull_req()); @@ -501,7 +543,7 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) { } } if( phase->type(n) == Type::TOP ) { - set_req_X(i, NULL, phase); // Ignore TOP inputs + set_req_X(i, nullptr, phase); // Ignore TOP inputs modified = true; i--; continue; @@ -522,7 +564,7 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) { if( n->req() != req() && n->is_Phi() ) { assert( n->in(0) == this, "" ); igvn->hash_delete(n); // Yank from hash before hacking edges - n->set_req_X(i,NULL,igvn);// Correct DU info + n->set_req_X(i,nullptr,igvn);// Correct DU info n->del_req(i); // Yank path from Phis if( max != outcnt() ) { progress = true; @@ -595,10 +637,10 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) { } if( cnt <= 1 ) { // Only 1 path in? - set_req(0, NULL); // Null control input for region copy + set_req(0, nullptr); // Null control input for region copy if( cnt == 0 && !can_reshape) { // Parse phase - leave the node as it is. - // No inputs or all inputs are NULL. - return NULL; + // No inputs or all inputs are null. + return nullptr; } else if (can_reshape) { // Optimization phase - remove the node PhaseIterGVN *igvn = phase->is_IterGVN(); // Strip mined (inner) loop is going away, remove outer loop. @@ -606,7 +648,7 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) { as_Loop()->is_strip_mined()) { Node* outer_sfpt = as_CountedLoop()->outer_safepoint(); Node* outer_out = as_CountedLoop()->outer_loop_exit(); - if (outer_sfpt != NULL && outer_out != NULL) { + if (outer_sfpt != nullptr && outer_out != nullptr) { Node* in = outer_sfpt->in(0); igvn->replace_node(outer_out, in); LoopNode* outer = as_CountedLoop()->outer_loop(); @@ -615,7 +657,7 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) { } if (is_CountedLoop()) { Node* opaq = as_CountedLoop()->is_canonical_loop_entry(); - if (opaq != NULL) { + if (opaq != nullptr) { // This is not a loop anymore. No need to keep the Opaque1 node on the test that guards the loop as it won't be // subject to further loop opts. assert(opaq->Opcode() == Op_Opaque1, ""); @@ -633,7 +675,7 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) { } else { // The fallthrough case since we already checked dead loops above. parent_ctrl = in(1); - assert(parent_ctrl != NULL, "Region is a copy of some non-null control"); + assert(parent_ctrl != nullptr, "Region is a copy of some non-null control"); assert(parent_ctrl != this, "Close dead loop"); } if (add_to_worklist) { @@ -656,7 +698,7 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) { assert( n->req() == 1, "No data inputs expected" ); in = parent_ctrl; // replaced by top } else { - assert( n->req() == 2 && n->in(1) != NULL, "Only one data input expected" ); + assert( n->req() == 2 && n->in(1) != nullptr, "Only one data input expected" ); in = n->in(1); // replaced by unique input if( n->as_Phi()->is_unsafe_data_reference(in) ) in = phase->C->top(); // replaced by top @@ -682,7 +724,7 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) { } // Remove the RegionNode itself from DefUse info igvn->remove_dead_node(this); - return NULL; + return nullptr; } return this; // Record progress } @@ -691,14 +733,14 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) { // If a Region flows into a Region, merge into one big happy merge. if (can_reshape) { Node *m = merge_region(this, phase); - if (m != NULL) return m; + if (m != nullptr) return m; } // Check if this region is the root of a clipping idiom on floats if( ConvertFloat2IntClipping && can_reshape && req() == 4 ) { // Check that only one use is a Phi and that it simplifies to two constants + PhiNode* phi = has_unique_phi(); - if (phi != NULL) { // One Phi user + if (phi != nullptr) { // One Phi user // Check inputs to the Phi ConNode *min; ConNode *max; @@ -711,13 +753,13 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) { IfNode *bot_if; if( check_if_clipping( this, bot_if, top_if ) ) { // Control pattern checks, now verify compares - Node *top_in = NULL; // value being compared against - Node *bot_in = NULL; + Node *top_in = nullptr; // value being compared against + Node *bot_in = nullptr; if( check_compare_clipping( true, bot_if, min, bot_in ) && check_compare_clipping( false, top_if, max, top_in ) ) { if( bot_in == top_in ) { PhaseIterGVN *gvn = phase->is_IterGVN(); - assert( gvn != NULL, "Only had DefUse info in IterGVN"); + assert( gvn != nullptr, "Only had DefUse info in IterGVN"); // Only remaining check is that bot_in == top_in == (Phi's val + mods) // Check for the ConvF2INode @@ -759,7 +801,7 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) { modified |= optimize_trichotomy(phase->is_IterGVN()); } - return modified ? this : NULL; + return modified ? this : nullptr; } //------------------------------optimize_trichotomy-------------------------- @@ -794,19 +836,19 @@ Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) { // The method returns true if 'this' is modified and false otherwise. bool RegionNode::optimize_trichotomy(PhaseIterGVN* igvn) { int idx1 = 1, idx2 = 2; - Node* region = NULL; - if (req() == 3 && in(1) != NULL && in(2) != NULL) { + Node* region = nullptr; + if (req() == 3 && in(1) != nullptr && in(2) != nullptr) { // Shape 1: Check if one of the inputs is a region that merges two control // inputs and has no other users (especially no Phi users). region = in(1)->isa_Region() ? in(1) : in(2)->isa_Region(); - if (region == NULL || region->outcnt() != 2 || region->req() != 3) { + if (region == nullptr || region->outcnt() != 2 || region->req() != 3) { return false; // No suitable region input found } } else if (req() == 4) { // Shape 2: Check if two control inputs map to the same value of the unique phi // user and treat these as if they would come from another region (shape (1)). PhiNode* phi = has_unique_phi(); - if (phi == NULL) { + if (phi == nullptr) { return false; // No unique phi user } if (phi->in(idx1) != phi->in(idx2)) { @@ -821,22 +863,22 @@ bool RegionNode::optimize_trichotomy(PhaseIterGVN* igvn) { assert(phi->in(idx1) == phi->in(idx2), "must be"); // Region is merging same value region = this; } - if (region == NULL || region->in(idx1) == NULL || region->in(idx2) == NULL) { + if (region == nullptr || region->in(idx1) == nullptr || region->in(idx2) == nullptr) { return false; // Region does not merge two control inputs } // At this point we know that region->in(idx1) and region->(idx2) map to the same // value and control flow. Now search for ifs that feed into these region inputs. ProjNode* proj1 = region->in(idx1)->isa_Proj(); ProjNode* proj2 = region->in(idx2)->isa_Proj(); - if (proj1 == NULL || proj1->outcnt() != 1 || - proj2 == NULL || proj2->outcnt() != 1) { + if (proj1 == nullptr || proj1->outcnt() != 1 || + proj2 == nullptr || proj2->outcnt() != 1) { return false; // No projection inputs with region as unique user found } assert(proj1 != proj2, "should be different projections"); IfNode* iff1 = proj1->in(0)->isa_If(); IfNode* iff2 = proj2->in(0)->isa_If(); - if (iff1 == NULL || iff1->outcnt() != 2 || - iff2 == NULL || iff2->outcnt() != 2) { + if (iff1 == nullptr || iff1->outcnt() != 2 || + iff2 == nullptr || iff2->outcnt() != 2) { return false; // No ifs found } if (iff1 == iff2) { @@ -847,7 +889,7 @@ bool RegionNode::optimize_trichotomy(PhaseIterGVN* igvn) { } BoolNode* bol1 = iff1->in(1)->isa_Bool(); BoolNode* bol2 = iff2->in(1)->isa_Bool(); - if (bol1 == NULL || bol2 == NULL) { + if (bol1 == nullptr || bol2 == nullptr) { return false; // No bool inputs found } Node* cmp1 = bol1->in(1); @@ -917,7 +959,7 @@ Node *Node::nonnull_req() const { if( in(i) ) return in(i); ShouldNotReachHere(); - return NULL; + return nullptr; } @@ -932,7 +974,7 @@ bool PhiNode::cmp( const Node &n ) const { } static inline const TypePtr* flatten_phi_adr_type(const TypePtr* at) { - if (at == NULL || at == TypePtr::BOTTOM) return at; + if (at == nullptr || at == TypePtr::BOTTOM) return at; return Compile::current()->alias_type(at)->adr_type(); } @@ -944,20 +986,20 @@ PhiNode* PhiNode::make(Node* r, Node* x, const Type *t, const TypePtr* at) { PhiNode* p = new PhiNode(r, t, at); for (uint j = 1; j < preds; j++) { // Fill in all inputs, except those which the region does not yet have - if (r->in(j) != NULL) + if (r->in(j) != nullptr) p->init_req(j, x); } return p; } PhiNode* PhiNode::make(Node* r, Node* x) { const Type* t = x->bottom_type(); - const TypePtr* at = NULL; + const TypePtr* at = nullptr; if (t == Type::MEMORY) at = flatten_phi_adr_type(x->adr_type()); return make(r, x, t, at); } PhiNode* PhiNode::make_blank(Node* r, Node* x) { const Type* t = x->bottom_type(); - const TypePtr* at = NULL; + const TypePtr* at = nullptr; if (t == Type::MEMORY) at = flatten_phi_adr_type(x->adr_type()); return new PhiNode(r, t, at); } @@ -980,7 +1022,7 @@ PhiNode* PhiNode::slice_memory(const TypePtr* adr_type) const { // Split out an instance type from a bottom phi. PhiNode* PhiNode::split_out_instance(const TypePtr* at, PhaseIterGVN *igvn) const { const TypeOopPtr *t_oop = at->isa_oopptr(); - assert(t_oop != NULL && t_oop->is_known_instance(), "expecting instance oopptr"); + assert(t_oop != nullptr && t_oop->is_known_instance(), "expecting instance oopptr"); const TypePtr *t = adr_type(); assert(type() == Type::MEMORY && (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM || @@ -1017,13 +1059,13 @@ PhiNode* PhiNode::split_out_instance(const TypePtr* at, PhaseIterGVN *igvn) cons nphi = node_map[ophi->_idx]->as_Phi(); for (; i < ophi->req(); i++) { Node *in = ophi->in(i); - if (in == NULL || igvn->type(in) == Type::TOP) + if (in == nullptr || igvn->type(in) == Type::TOP) continue; - Node *opt = MemNode::optimize_simple_memory_chain(in, t_oop, NULL, igvn); - PhiNode *optphi = opt->is_Phi() ? opt->as_Phi() : NULL; - if (optphi != NULL && optphi->adr_type() == TypePtr::BOTTOM) { + Node *opt = MemNode::optimize_simple_memory_chain(in, t_oop, nullptr, igvn); + PhiNode *optphi = opt->is_Phi() ? opt->as_Phi() : nullptr; + if (optphi != nullptr && optphi->adr_type() == TypePtr::BOTTOM) { opt = node_map[optphi->_idx]; - if (opt == NULL) { + if (opt == nullptr) { stack.push(ophi, i); nphi = optphi->slice_memory(at); igvn->register_new_node_with_optimizer( nphi ); @@ -1054,7 +1096,7 @@ void PhiNode::verify_adr_type(VectorSet& visited, const TypePtr* at) const { // walk around for (uint i = 1; i < req(); i++) { Node* n = in(i); - if (n == NULL) continue; + if (n == nullptr) continue; const Node* np = in(i); if (np->is_Phi()) { np->as_Phi()->verify_adr_type(visited, at); @@ -1064,7 +1106,7 @@ void PhiNode::verify_adr_type(VectorSet& visited, const TypePtr* at) const { } else { const TypePtr* nat = flatten_phi_adr_type(n->adr_type()); // recheck phi/non-phi consistency at leaves: - assert((nat != NULL) == (at != NULL), ""); + assert((nat != nullptr) == (at != nullptr), ""); assert(nat == at || nat == TypePtr::BOTTOM, "adr_type must be consistent at leaves of phi nest"); } @@ -1076,7 +1118,7 @@ void PhiNode::verify_adr_type(bool recursive) const { if (VMError::is_error_reported()) return; // muzzle asserts when debugging an error if (Node::in_dump()) return; // muzzle asserts when printing - assert((_type == Type::MEMORY) == (_adr_type != NULL), "adr_type for memory phis only"); + assert((_type == Type::MEMORY) == (_adr_type != nullptr), "adr_type for memory phis only"); if (!VerifyAliases) return; // verify thoroughly only if requested @@ -1104,18 +1146,18 @@ const Type* PhiNode::Value(PhaseGVN* phase) const { return Type::TOP; // Check for trip-counted loop. If so, be smarter. - BaseCountedLoopNode* l = r->is_BaseCountedLoop() ? r->as_BaseCountedLoop() : NULL; + BaseCountedLoopNode* l = r->is_BaseCountedLoop() ? r->as_BaseCountedLoop() : nullptr; if (l && ((const Node*)l->phi() == this)) { // Trip counted loop! - // protect against init_trip() or limit() returning NULL + // protect against init_trip() or limit() returning null if (l->can_be_counted_loop(phase)) { const Node* init = l->init_trip(); const Node* limit = l->limit(); const Node* stride = l->stride(); - if (init != NULL && limit != NULL && stride != NULL) { + if (init != nullptr && limit != nullptr && stride != nullptr) { const TypeInteger* lo = phase->type(init)->isa_integer(l->bt()); const TypeInteger* hi = phase->type(limit)->isa_integer(l->bt()); const TypeInteger* stride_t = phase->type(stride)->isa_integer(l->bt()); - if (lo != NULL && hi != NULL && stride_t != NULL) { // Dying loops might have TOP here + if (lo != nullptr && hi != nullptr && stride_t != nullptr) { // Dying loops might have TOP here assert(stride_t->hi_as_long() >= stride_t->lo_as_long(), "bad stride type"); BoolTest::mask bt = l->loopexit()->test_trip(); // If the loop exit condition is "not equal", the condition @@ -1132,8 +1174,8 @@ const Type* PhiNode::Value(PhaseGVN* phase) const { } } } - } else if (l->in(LoopNode::LoopBackControl) != NULL && - in(LoopNode::EntryControl) != NULL && + } else if (l->in(LoopNode::LoopBackControl) != nullptr && + in(LoopNode::EntryControl) != nullptr && phase->type(l->in(LoopNode::LoopBackControl)) == Type::TOP) { // During CCP, if we saturate the type of a counted loop's Phi // before the special code for counted loop above has a chance @@ -1341,19 +1383,19 @@ Node* PhiNode::is_cmove_id(PhaseTransform* phase, int true_path) { Node* tval = in(true_path); Node* fval = in(3-true_path); Node* id = CMoveNode::is_cmove_id(phase, cmp, tval, fval, b); - if (id == NULL) - return NULL; + if (id == nullptr) + return nullptr; // Either value might be a cast that depends on a branch of 'iff'. // Since the 'id' value will float free of the diamond, either // decast or return failure. Node* ctl = id->in(0); - if (ctl != NULL && ctl->in(0) == iff) { + if (ctl != nullptr && ctl->in(0) == iff) { if (id->is_ConstraintCast()) { return id->in(1); } else { // Don't know how to disentangle this value. - return NULL; + return nullptr; } } @@ -1369,7 +1411,7 @@ Node* PhiNode::Identity(PhaseGVN* phase) { // trivially, perhaps with a single cast. The unique_input method // does all this and more, by reducing such tributaries to 'this'.) Node* uin = unique_input(phase, false); - if (uin != NULL) { + if (uin != nullptr) { return uin; } @@ -1377,7 +1419,7 @@ Node* PhiNode::Identity(PhaseGVN* phase) { // Delay CMove'ing identity if Ideal has not had the chance to handle unsafe cases, yet. if (true_path != 0 && !(phase->is_IterGVN() && wait_for_region_igvn(phase))) { Node* id = is_cmove_id(phase, true_path); - if (id != NULL) { + if (id != nullptr) { return id; } } @@ -1395,11 +1437,11 @@ Node* PhiNode::Identity(PhaseGVN* phase) { u->req() == phi_len) { for (uint j = 1; j < phi_len; j++) { if (in(j) != u->in(j)) { - u = NULL; + u = nullptr; break; } } - if (u != NULL) { + if (u != nullptr) { return u; } } @@ -1425,21 +1467,21 @@ Node* PhiNode::unique_input(PhaseTransform* phase, bool uncast) { // phi / -- Node* r = in(0); // RegionNode - Node* input = NULL; // The unique direct input (maybe uncasted = ConstraintCasts removed) + Node* input = nullptr; // The unique direct input (maybe uncasted = ConstraintCasts removed) for (uint i = 1, cnt = req(); i < cnt; ++i) { Node* rc = r->in(i); - if (rc == NULL || phase->type(rc) == Type::TOP) + if (rc == nullptr || phase->type(rc) == Type::TOP) continue; // ignore unreachable control path Node* n = in(i); - if (n == NULL) + if (n == nullptr) continue; Node* un = n; if (uncast) { #ifdef ASSERT Node* m = un->uncast(); #endif - while (un != NULL && un->req() == 2 && un->is_ConstraintCast()) { + while (un != nullptr && un->req() == 2 && un->is_ConstraintCast()) { Node* next = un->in(1); if (phase->type(next)->isa_rawptr() && phase->type(un)->isa_oopptr()) { // risk exposing raw ptr at safepoint @@ -1449,17 +1491,17 @@ Node* PhiNode::unique_input(PhaseTransform* phase, bool uncast) { } assert(m == un || un->in(1) == m, "Only expected at CheckCastPP from allocation"); } - if (un == NULL || un == this || phase->type(un) == Type::TOP) { + if (un == nullptr || un == this || phase->type(un) == Type::TOP) { continue; // ignore if top, or in(i) and "this" are in a data cycle } // Check for a unique input (maybe uncasted) - if (input == NULL) { + if (input == nullptr) { input = un; } else if (input != un) { input = NodeSentinel; // no unique input } } - if (input == NULL) { + if (input == nullptr) { return phase->C->top(); // no inputs } @@ -1468,7 +1510,7 @@ Node* PhiNode::unique_input(PhaseTransform* phase, bool uncast) { } // Nothing. - return NULL; + return nullptr; } //------------------------------is_x2logic------------------------------------- @@ -1498,25 +1540,25 @@ static Node *is_x2logic( PhaseGVN *phase, PhiNode *phi, int true_path ) { if( tcmp != TypeInt::ZERO && tcmp != TypePtr::NULL_PTR ) { // Allow cmp-vs-1 if the other input is bounded by 0-1 if( !(tcmp == TypeInt::ONE && phase->type(cmp->in(1)) == TypeInt::BOOL) ) - return NULL; + return nullptr; flipped = 1-flipped; // Test is vs 1 instead of 0! } // Check for setting zero/one opposite expected if( tzero == TypeInt::ZERO ) { if( tone == TypeInt::ONE ) { - } else return NULL; + } else return nullptr; } else if( tzero == TypeInt::ONE ) { if( tone == TypeInt::ZERO ) { flipped = 1-flipped; - } else return NULL; - } else return NULL; + } else return nullptr; + } else return nullptr; // Check for boolean test backwards if( b->_test._test == BoolTest::ne ) { } else if( b->_test._test == BoolTest::eq ) { flipped = 1-flipped; - } else return NULL; + } else return nullptr; // Build int->bool conversion Node *n = new Conv2BNode(cmp->in(1)); @@ -1545,16 +1587,16 @@ static Node* is_cond_add(PhaseGVN *phase, PhiNode *phi, int true_path) { const CmpNode *cmp = (CmpNode*)b->in(1); // Make sure only merging this one phi here - if (region->has_unique_phi() != phi) return NULL; + if (region->has_unique_phi() != phi) return nullptr; // Make sure each arm of the diamond has exactly one output, which we assume // is the region. Otherwise, the control flow won't disappear. - if (region->in(1)->outcnt() != 1) return NULL; - if (region->in(2)->outcnt() != 1) return NULL; + if (region->in(1)->outcnt() != 1) return nullptr; + if (region->in(2)->outcnt() != 1) return nullptr; // Check for "(P < Q)" of type signed int - if (b->_test._test != BoolTest::lt) return NULL; - if (cmp->Opcode() != Op_CmpI) return NULL; + if (b->_test._test != BoolTest::lt) return nullptr; + if (cmp->Opcode() != Op_CmpI) return nullptr; Node *p = cmp->in(1); Node *q = cmp->in(2); @@ -1567,19 +1609,19 @@ static Node* is_cond_add(PhaseGVN *phase, PhiNode *phi, int true_path) { op != Op_AddP && op != Op_XorI && op != Op_OrI*/ ) - return NULL; + return nullptr; Node *x = n2; - Node *y = NULL; + Node *y = nullptr; if( x == n1->in(1) ) { y = n1->in(2); } else if( x == n1->in(2) ) { y = n1->in(1); - } else return NULL; + } else return nullptr; // Not so profitable if compare and add are constants if( q->is_Con() && phase->type(q) != TypeInt::ZERO && y->is_Con() ) - return NULL; + return nullptr; Node *cmplt = phase->transform( new CmpLTMaskNode(p,q) ); Node *j_and = phase->transform( new AndINode(cmplt,y) ); @@ -1610,7 +1652,7 @@ static Node* is_absolute( PhaseGVN *phase, PhiNode *phi_root, int true_path) { case BoolTest::le: cmp_zero_idx = 2; phi_x_idx = false_path; break; case BoolTest::gt: cmp_zero_idx = 2; phi_x_idx = true_path; break; case BoolTest::ge: cmp_zero_idx = 1; phi_x_idx = false_path; break; - default: return NULL; break; + default: return nullptr; break; } } else if (cmp->Opcode() == Op_CmpI || cmp->Opcode() == Op_CmpL) { switch (bol->_test._test) { @@ -1618,22 +1660,22 @@ static Node* is_absolute( PhaseGVN *phase, PhiNode *phi_root, int true_path) { case BoolTest::le: cmp_zero_idx = 2; phi_x_idx = false_path; break; case BoolTest::gt: case BoolTest::ge: cmp_zero_idx = 2; phi_x_idx = true_path; break; - default: return NULL; break; + default: return nullptr; break; } } // Test is next - const Type *tzero = NULL; + const Type *tzero = nullptr; switch (cmp->Opcode()) { case Op_CmpI: tzero = TypeInt::ZERO; break; // Integer ABS case Op_CmpL: tzero = TypeLong::ZERO; break; // Long ABS case Op_CmpF: tzero = TypeF::ZERO; break; // Float ABS case Op_CmpD: tzero = TypeD::ZERO; break; // Double ABS - default: return NULL; + default: return nullptr; } // Find zero input of compare; the other input is being abs'd - Node *x = NULL; + Node *x = nullptr; bool flip = false; if( phase->type(cmp->in(cmp_zero_idx)) == tzero ) { x = cmp->in(3 - cmp_zero_idx); @@ -1642,12 +1684,12 @@ static Node* is_absolute( PhaseGVN *phase, PhiNode *phi_root, int true_path) { x = cmp->in(cmp_zero_idx); flip = true; } else { - return NULL; + return nullptr; } // Next get the 2 pieces being selected, one is the original value // and the other is the negated value. - if( phi_root->in(phi_x_idx) != x ) return NULL; + if( phi_root->in(phi_x_idx) != x ) return nullptr; // Check other phi input for subtract node Node *sub = phi_root->in(3 - phi_x_idx); @@ -1656,7 +1698,7 @@ static Node* is_absolute( PhaseGVN *phase, PhiNode *phi_root, int true_path) { sub->Opcode() == Op_SubI || sub->Opcode() == Op_SubL; // Allow only Sub(0,X) and fail out for all others; Neg is not OK - if (!is_sub || phase->type(sub->in(1)) != tzero || sub->in(2) != x) return NULL; + if (!is_sub || phase->type(sub->in(1)) != tzero || sub->in(2) != x) return nullptr; if (tzero == TypeF::ZERO) { x = new AbsFNode(x); @@ -1678,7 +1720,7 @@ static Node* is_absolute( PhaseGVN *phase, PhiNode *phi_root, int true_path) { if (flip) { x = new SubLNode(sub->in(1), phase->transform(x)); } - } else return NULL; + } else return nullptr; return x; } @@ -1712,21 +1754,21 @@ static void split_once(PhaseIterGVN *igvn, Node *phi, Node *val, Node *n, Node * static Node* split_flow_path(PhaseGVN *phase, PhiNode *phi) { BasicType bt = phi->type()->basic_type(); if( bt == T_ILLEGAL || type2size[bt] <= 0 ) - return NULL; // Bail out on funny non-value stuff + return nullptr; // Bail out on funny non-value stuff if( phi->req() <= 3 ) // Need at least 2 matched inputs and a - return NULL; // third unequal input to be worth doing + return nullptr; // third unequal input to be worth doing // Scan for a constant uint i; for( i = 1; i < phi->req()-1; i++ ) { Node *n = phi->in(i); - if( !n ) return NULL; - if( phase->type(n) == Type::TOP ) return NULL; + if( !n ) return nullptr; + if( phase->type(n) == Type::TOP ) return nullptr; if( n->Opcode() == Op_ConP || n->Opcode() == Op_ConN || n->Opcode() == Op_ConNKlass ) break; } if( i >= phi->req() ) // Only split for constants - return NULL; + return nullptr; Node *val = phi->in(i); // Constant to split for uint hit = 0; // Number of times it occurs @@ -1734,19 +1776,19 @@ static Node* split_flow_path(PhaseGVN *phase, PhiNode *phi) { for( ; i < phi->req(); i++ ){ // Count occurrences of constant Node *n = phi->in(i); - if( !n ) return NULL; - if( phase->type(n) == Type::TOP ) return NULL; + if( !n ) return nullptr; + if( phase->type(n) == Type::TOP ) return nullptr; if( phi->in(i) == val ) { hit++; - if (PhaseIdealLoop::find_predicate(r->in(i)) != NULL) { - return NULL; // don't split loop entry path + if (PhaseIdealLoop::find_predicate(r->in(i)) != nullptr) { + return nullptr; // don't split loop entry path } } } if( hit <= 1 || // Make sure we find 2 or more hit == phi->req()-1 ) // and not ALL the same value - return NULL; + return nullptr; // Now start splitting out the flow paths that merge the same value. // Split first the RegionNode. @@ -1799,7 +1841,7 @@ PhiNode::LoopSafety PhiNode::simple_data_loop_check(Node *in) const { // Unsafe loop if the phi node references itself through an unsafe data node. // Exclude cases with null inputs or data nodes which could reference // itself (safe for dead loops). - if (in != NULL && !in->is_dead_loop_safe()) { + if (in != nullptr && !in->is_dead_loop_safe()) { // Check inputs of phi's inputs also. // It is much less expensive then full graph walk. uint cnt = in->req(); @@ -1808,13 +1850,13 @@ PhiNode::LoopSafety PhiNode::simple_data_loop_check(Node *in) const { Node* m = in->in(i); if (m == (Node*)this) return UnsafeLoop; // Unsafe loop - if (m != NULL && !m->is_dead_loop_safe()) { + if (m != nullptr && !m->is_dead_loop_safe()) { // Check the most common case (about 30% of all cases): // phi->Load/Store->AddP->(ConP ConP Con)/(Parm Parm Con). - Node *m1 = (m->is_AddP() && m->req() > 3) ? m->in(1) : NULL; + Node *m1 = (m->is_AddP() && m->req() > 3) ? m->in(1) : nullptr; if (m1 == (Node*)this) return UnsafeLoop; // Unsafe loop - if (m1 != NULL && m1 == m->in(2) && + if (m1 != nullptr && m1 == m->in(2) && m1->is_dead_loop_safe() && m->in(3)->is_Con()) { continue; // Safe case } @@ -1857,7 +1899,7 @@ bool PhiNode::is_unsafe_data_reference(Node *in) const { if (m == (Node*)this) { return true; // Data loop } - if (m != NULL && !m->is_dead_loop_safe()) { // Only look for unsafe cases. + if (m != nullptr && !m->is_dead_loop_safe()) { // Only look for unsafe cases. if (!visited.test_set(m->_idx)) nstack.push(m); } @@ -1876,19 +1918,19 @@ bool PhiNode::wait_for_region_igvn(PhaseGVN* phase) { for (uint j = 1; j < req(); j++) { Node* rc = r->in(j); Node* n = in(j); - if (rc != NULL && + if (rc != nullptr && rc->is_Proj()) { if (worklist.member(rc)) { delay = true; - } else if (rc->in(0) != NULL && + } else if (rc->in(0) != nullptr && rc->in(0)->is_If()) { if (worklist.member(rc->in(0))) { delay = true; - } else if (rc->in(0)->in(1) != NULL && + } else if (rc->in(0)->in(1) != nullptr && rc->in(0)->in(1)->is_Bool()) { if (worklist.member(rc->in(0)->in(1))) { delay = true; - } else if (rc->in(0)->in(1)->in(1) != NULL && + } else if (rc->in(0)->in(1)->in(1) != nullptr && rc->in(0)->in(1)->in(1)->is_Cmp()) { if (worklist.member(rc->in(0)->in(1)->in(1))) { delay = true; @@ -1909,34 +1951,34 @@ bool PhiNode::wait_for_region_igvn(PhaseGVN* phase) { // the CFG, but we can still strip out dead paths. Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { Node *r = in(0); // RegionNode - assert(r != NULL && r->is_Region(), "this phi must have a region"); - assert(r->in(0) == NULL || !r->in(0)->is_Root(), "not a specially hidden merge"); + assert(r != nullptr && r->is_Region(), "this phi must have a region"); + assert(r->in(0) == nullptr || !r->in(0)->is_Root(), "not a specially hidden merge"); // Note: During parsing, phis are often transformed before their regions. // This means we have to use type_or_null to defend against untyped regions. if( phase->type_or_null(r) == Type::TOP ) // Dead code? - return NULL; // No change + return nullptr; // No change Node *top = phase->C->top(); bool new_phi = (outcnt() == 0); // transforming new Phi // No change for igvn if new phi is not hooked if (new_phi && can_reshape) - return NULL; + return nullptr; // The are 2 situations when only one valid phi's input is left // (in addition to Region input). // One: region is not loop - replace phi with this input. // Two: region is loop - replace phi with top since this data path is dead // and we need to break the dead data loop. - Node* progress = NULL; // Record if any progress made + Node* progress = nullptr; // Record if any progress made for( uint j = 1; j < req(); ++j ){ // For all paths in // Check unreachable control paths Node* rc = r->in(j); Node* n = in(j); // Get the input - if (rc == NULL || phase->type(rc) == Type::TOP) { + if (rc == nullptr || phase->type(rc) == Type::TOP) { if (n != top) { // Not already top? PhaseIterGVN *igvn = phase->is_IterGVN(); - if (can_reshape && igvn != NULL) { + if (can_reshape && igvn != nullptr) { igvn->_worklist.push(r); } // Nuke it down @@ -1954,7 +1996,7 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { bool uncasted = false; Node* uin = unique_input(phase, false); - if (uin == NULL && can_reshape && + if (uin == nullptr && can_reshape && // If there is a chance that the region can be optimized out do // not add a cast node that we can't remove yet. !wait_for_region_igvn(phase)) { @@ -1965,12 +2007,12 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (can_reshape) // IGVN transformation return top; else - return NULL; // Identity will return TOP - } else if (uin != NULL) { - // Only one not-NULL unique input path is left. + return nullptr; // Identity will return TOP + } else if (uin != nullptr) { + // Only one not-null unique input path is left. // Determine if this input is backedge of a loop. // (Skip new phis which have no uses and dead regions). - if (outcnt() > 0 && r->in(0) != NULL) { + if (outcnt() > 0 && r->in(0) != nullptr) { if (is_data_loop(r->as_Region(), uin, phase)) { // Break this data loop to avoid creation of a dead loop. if (can_reshape) { @@ -1979,7 +2021,7 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { // We can't return top if we are in Parse phase - cut inputs only // let Identity to handle the case. replace_edge(uin, top, phase); - return NULL; + return nullptr; } } } @@ -1991,7 +2033,7 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { const Type* phi_type = bottom_type(); // Add casts to carry the control dependency of the Phi that is // going away - Node* cast = NULL; + Node* cast = nullptr; if (phi_type->isa_ptr()) { const Type* uin_type = phase->type(uin); if (!phi_type->isa_oopptr() && !uin_type->isa_oopptr()) { @@ -2013,20 +2055,20 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { // that of phi if (phi_type->join_speculative(TypePtr::NOTNULL) != uin_type->join_speculative(TypePtr::NOTNULL)) { Node* n = uin; - if (cast != NULL) { + if (cast != nullptr) { cast = phase->transform(cast); n = cast; } cast = ConstraintCastNode::make_cast(Op_CheckCastPP, r, n, phi_type, ConstraintCastNode::StrongDependency); } - if (cast == NULL) { + if (cast == nullptr) { cast = ConstraintCastNode::make_cast(Op_CastPP, r, uin, phi_type, ConstraintCastNode::StrongDependency); } } } else { cast = ConstraintCastNode::make_cast_for_type(r, uin, phi_type, ConstraintCastNode::StrongDependency); } - assert(cast != NULL, "cast should be set"); + assert(cast != nullptr, "cast should be set"); cast = phase->transform(cast); // set all inputs to the new cast(s) so the Phi is removed by Identity PhaseIterGVN* igvn = phase->is_IterGVN(); @@ -2049,10 +2091,10 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { } #endif assert(ident == uin || ident->is_top(), "Identity must clean this up"); - return NULL; + return nullptr; } - Node* opt = NULL; + Node* opt = nullptr; int true_path = is_diamond_phi(); if (true_path != 0 && // If one of the diamond's branch is in the process of dying then, the Phi's input for that branch might transform @@ -2062,24 +2104,24 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Check for CMove'ing identity. If it would be unsafe, // handle it here. In the safe case, let Identity handle it. Node* unsafe_id = is_cmove_id(phase, true_path); - if( unsafe_id != NULL && is_unsafe_data_reference(unsafe_id) ) + if( unsafe_id != nullptr && is_unsafe_data_reference(unsafe_id) ) opt = unsafe_id; // Check for simple convert-to-boolean pattern - if( opt == NULL ) + if( opt == nullptr ) opt = is_x2logic(phase, this, true_path); // Check for absolute value - if( opt == NULL ) + if( opt == nullptr ) opt = is_absolute(phase, this, true_path); // Check for conditional add - if( opt == NULL && can_reshape ) + if( opt == nullptr && can_reshape ) opt = is_cond_add(phase, this, true_path); // These 4 optimizations could subsume the phi: // have to check for a dead data loop creation. - if( opt != NULL ) { + if( opt != nullptr ) { if( opt == unsafe_id || is_unsafe_data_reference(opt) ) { // Found dead loop. if( can_reshape ) @@ -2089,7 +2131,7 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { assert(req() == 3, "only diamond merge phi here"); set_req(1, top); set_req(2, top); - return NULL; + return nullptr; } else { return opt; } @@ -2100,11 +2142,11 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (can_reshape) { opt = split_flow_path(phase, this); // This optimization only modifies phi - don't need to check for dead loop. - assert(opt == NULL || opt == this, "do not elide phi"); - if (opt != NULL) return opt; + assert(opt == nullptr || opt == this, "do not elide phi"); + if (opt != nullptr) return opt; } - if (in(1) != NULL && in(1)->Opcode() == Op_AddP && can_reshape) { + if (in(1) != nullptr && in(1)->Opcode() == Op_AddP && can_reshape) { // Try to undo Phi of AddP: // (Phi (AddP base address offset) (AddP base2 address2 offset2)) // becomes: @@ -2120,7 +2162,7 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { Node* base = addp->in(AddPNode::Base); Node* address = addp->in(AddPNode::Address); Node* offset = addp->in(AddPNode::Offset); - if (base != NULL && address != NULL && offset != NULL && + if (base != nullptr && address != nullptr && offset != nullptr && !base->is_top() && !address->is_top() && !offset->is_top()) { const Type* base_type = base->bottom_type(); const Type* address_type = address->bottom_type(); @@ -2128,11 +2170,11 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { // i.e. AddP with base == address and same offset as first AddP bool doit = true; for (uint i = 2; i < req(); i++) { - if (in(i) == NULL || + if (in(i) == nullptr || in(i)->Opcode() != Op_AddP || - in(i)->in(AddPNode::Base) == NULL || - in(i)->in(AddPNode::Address) == NULL || - in(i)->in(AddPNode::Offset) == NULL || + in(i)->in(AddPNode::Base) == nullptr || + in(i)->in(AddPNode::Address) == nullptr || + in(i)->in(AddPNode::Offset) == nullptr || in(i)->in(AddPNode::Base)->is_top() || in(i)->in(AddPNode::Address)->is_top() || in(i)->in(AddPNode::Offset)->is_top()) { @@ -2140,27 +2182,27 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { break; } if (in(i)->in(AddPNode::Base) != base) { - base = NULL; + base = nullptr; } if (in(i)->in(AddPNode::Offset) != offset) { - offset = NULL; + offset = nullptr; } if (in(i)->in(AddPNode::Address) != address) { - address = NULL; + address = nullptr; } // Accumulate type for resulting Phi base_type = base_type->meet_speculative(in(i)->in(AddPNode::Base)->bottom_type()); address_type = address_type->meet_speculative(in(i)->in(AddPNode::Address)->bottom_type()); } - if (doit && base == NULL) { + if (doit && base == nullptr) { // Check for neighboring AddP nodes in a tree. // If they have a base, use that it. for (DUIterator_Fast kmax, k = this->fast_outs(kmax); k < kmax; k++) { Node* u = this->fast_out(k); if (u->is_AddP()) { Node* base2 = u->in(AddPNode::Base); - if (base2 != NULL && !base2->is_top()) { - if (base == NULL) + if (base2 != nullptr && !base2->is_top()) { + if (base == nullptr) base = base2; else if (base != base2) { doit = false; break; } @@ -2169,22 +2211,22 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { } } if (doit) { - if (base == NULL) { - base = new PhiNode(in(0), base_type, NULL); + if (base == nullptr) { + base = new PhiNode(in(0), base_type, nullptr); for (uint i = 1; i < req(); i++) { base->init_req(i, in(i)->in(AddPNode::Base)); } phase->is_IterGVN()->register_new_node_with_optimizer(base); } - if (address == NULL) { - address = new PhiNode(in(0), address_type, NULL); + if (address == nullptr) { + address = new PhiNode(in(0), address_type, nullptr); for (uint i = 1; i < req(); i++) { address->init_req(i, in(i)->in(AddPNode::Address)); } phase->is_IterGVN()->register_new_node_with_optimizer(address); } - if (offset == NULL) { - offset = new PhiNode(in(0), TypeX_X, NULL); + if (offset == nullptr) { + offset = new PhiNode(in(0), TypeX_X, nullptr); for (uint i = 1; i < req(); i++) { offset->init_req(i, in(i)->in(AddPNode::Offset)); } @@ -2201,7 +2243,7 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { // (Do not attempt this optimization unless parsing is complete. // It would make the parser's memory-merge logic sick.) // (MergeMemNode is not dead_loop_safe - need to check for dead loop.) - if (progress == NULL && can_reshape && type() == Type::MEMORY) { + if (progress == nullptr && can_reshape && type() == Type::MEMORY) { // see if this phi should be sliced uint merge_width = 0; bool saw_self = false; @@ -2211,7 +2253,7 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Phi references itself through all other inputs then splitting the // Phi through memory merges would create dead loop at later stage. if (ii == top) { - return NULL; // Delay optimization until graph is cleaned. + return nullptr; // Delay optimization until graph is cleaned. } if (ii->is_MergeMem()) { MergeMemNode* n = ii->as_MergeMem(); @@ -2280,7 +2322,7 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Phi(...MergeMem(m0, m1:AT1, m2:AT2)...) into // MergeMem(Phi(...m0...), Phi:AT1(...m1...), Phi:AT2(...m2...)) PhaseIterGVN* igvn = phase->is_IterGVN(); - assert(igvn != NULL, "sanity check"); + assert(igvn != nullptr, "sanity check"); Node* hook = new Node(1); PhiNode* new_base = (PhiNode*) clone(); // Must eagerly register phis, since they participate in loops. @@ -2340,7 +2382,7 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { const TypePtr* at = adr_type(); for( uint i=1; iis_DecodeNarrowPtr()) { assert(ii->bottom_type() == bottom_type(), "sanity"); new_ii = ii->in(1); @@ -2411,7 +2453,7 @@ Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) { #endif // Phi (VB ... VB) => VB (Phi ...) (Phi ...) - if (EnableVectorReboxing && can_reshape && progress == NULL) { + if (EnableVectorReboxing && can_reshape && progress == nullptr) { PhaseIterGVN* igvn = phase->is_IterGVN(); bool all_inputs_are_equiv_vboxes = true; @@ -2472,7 +2514,7 @@ bool PhiNode::is_data_loop(RegionNode* r, Node* uin, const PhaseGVN* phase) { //------------------------------is_tripcount----------------------------------- bool PhiNode::is_tripcount(BasicType bt) const { - return (in(0) != NULL && in(0)->is_BaseCountedLoop() && + return (in(0) != nullptr && in(0)->is_BaseCountedLoop() && in(0)->as_BaseCountedLoop()->operates_on(bt, true) && in(0)->as_BaseCountedLoop()->phi() == this); } @@ -2587,7 +2629,7 @@ const Type* PCTableNode::Value(PhaseGVN* phase) const { // Return a node which is more "ideal" than the current node. Strip out // control copies Node *PCTableNode::Ideal(PhaseGVN *phase, bool can_reshape) { - return remove_dead_region(phase, can_reshape) ? this : NULL; + return remove_dead_region(phase, can_reshape) ? this : nullptr; } //============================================================================= @@ -2630,7 +2672,7 @@ const Type* CatchNode::Value(PhaseGVN* phase) const { for( uint i = 0; i < _size; i++ ) f[i] = Type::CONTROL; // Identify cases that will always throw an exception // () rethrow call - // () virtual or interface call with NULL receiver + // () virtual or interface call with null receiver // () call is a check cast with incompatible arguments if( in(1)->is_Proj() ) { Node *i10 = in(1)->in(0); @@ -2639,6 +2681,17 @@ const Type* CatchNode::Value(PhaseGVN* phase) const { // Rethrows always throw exceptions, never return if (call->entry_point() == OptoRuntime::rethrow_stub()) { f[CatchProjNode::fall_through_index] = Type::TOP; + } else if (call->is_AllocateArray()) { + Node* klass_node = call->in(AllocateNode::KlassNode); + Node* length = call->in(AllocateNode::ALength); + const Type* length_type = phase->type(length); + const Type* klass_type = phase->type(klass_node); + Node* valid_length_test = call->in(AllocateNode::ValidLengthTest); + const Type* valid_length_test_t = phase->type(valid_length_test); + if (length_type == Type::TOP || klass_type == Type::TOP || valid_length_test_t == Type::TOP || + valid_length_test_t->is_int()->is_con(0)) { + f[CatchProjNode::fall_through_index] = Type::TOP; + } } else if( call->req() > TypeFunc::Parms ) { const Type *arg0 = phase->type( call->in(TypeFunc::Parms) ); // Check for null receiver to virtual or interface calls @@ -2677,7 +2730,7 @@ Node* CatchProjNode::Identity(PhaseGVN* phase) { // an exception) or for "rethrow", because a further optimization will // yank the rethrow (happens when we inline a function that can throw an // exception and the caller has no handler). Not legal, e.g., for passing - // a NULL receiver to a v-call, or passing bad types to a slow-check-cast. + // a null receiver to a v-call, or passing bad types to a slow-check-cast. // These cases MUST throw an exception via the runtime system, so the VM // will be looking for a table entry. Node *proj = in(0)->in(1); // Expect a proj feeding CatchNode @@ -2736,12 +2789,12 @@ Node *NeverBranchNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Dead code elimination can sometimes delete this projection so // if it's not there, there's nothing to do. Node* fallthru = proj_out_or_null(0); - if (fallthru != NULL) { + if (fallthru != nullptr) { phase->is_IterGVN()->replace_node(fallthru, in(0)); } return phase->C->top(); } - return NULL; + return nullptr; } #ifndef PRODUCT @@ -2749,3 +2802,25 @@ void NeverBranchNode::format( PhaseRegAlloc *ra_, outputStream *st) const { st->print("%s", Name()); } #endif + +#ifndef PRODUCT +void BlackholeNode::format(PhaseRegAlloc* ra, outputStream* st) const { + st->print("blackhole "); + bool first = true; + for (uint i = 0; i < req(); i++) { + Node* n = in(i); + if (n != nullptr && OptoReg::is_valid(ra->get_reg_first(n))) { + if (first) { + first = false; + } else { + st->print(", "); + } + char buf[128]; + ra->dump_register(n, buf); + st->print("%s", buf); + } + } + st->cr(); +} +#endif + diff --git a/src/hotspot/share/opto/cfgnode.hpp b/src/hotspot/share/opto/cfgnode.hpp index 0f754aee45393..96b4305accaff 100644 --- a/src/hotspot/share/opto/cfgnode.hpp +++ b/src/hotspot/share/opto/cfgnode.hpp @@ -47,6 +47,7 @@ class PCTableNode; class JumpNode; class CatchNode; class NeverBranchNode; +class BlackholeNode; class ProjNode; class CProjNode; class IfTrueNode; @@ -82,14 +83,18 @@ class RegionNode : public Node { Node* is_copy() const { const Node* r = _in[Region]; - if (r == NULL) + if (r == nullptr) return nonnull_req(); - return NULL; // not a copy! + return nullptr; // not a copy! } - PhiNode* has_phi() const; // returns an arbitrary phi user, or NULL - PhiNode* has_unique_phi() const; // returns the unique phi user, or NULL + PhiNode* has_phi() const; // returns an arbitrary phi user, or null + PhiNode* has_unique_phi() const; // returns the unique phi user, or null // Is this region node unreachable from root? bool is_unreachable_region(const PhaseGVN* phase); +#ifdef ASSERT + bool is_in_infinite_subgraph(); + static bool are_all_nodes_in_infinite_subgraph(Unique_Node_List& worklist); +#endif //ASSERT virtual int Opcode() const; virtual uint size_of() const { return sizeof(*this); } virtual bool pinned() const { return (const Node*)in(0) == this; } @@ -149,7 +154,7 @@ class PhiNode : public TypeNode { Input // Input values are [1..len) }; - PhiNode( Node *r, const Type *t, const TypePtr* at = NULL, + PhiNode( Node *r, const Type *t, const TypePtr* at = nullptr, const int imid = -1, const int iid = TypeOopPtr::InstanceTop, const int iidx = Compile::AliasIdxTop, @@ -168,7 +173,7 @@ class PhiNode : public TypeNode { // create a new phi with in edges matching r and set (initially) to x static PhiNode* make( Node* r, Node* x ); // extra type arguments override the new phi's bottom_type and adr_type - static PhiNode* make( Node* r, Node* x, const Type *t, const TypePtr* at = NULL ); + static PhiNode* make( Node* r, Node* x, const Type *t, const TypePtr* at = nullptr ); // create a new phi with narrowed memory type PhiNode* slice_memory(const TypePtr* adr_type) const; PhiNode* split_out_instance(const TypePtr* at, PhaseIterGVN *igvn) const; @@ -181,11 +186,11 @@ class PhiNode : public TypeNode { bool is_tripcount(BasicType bt) const; // Determine a unique non-trivial input, if any. - // Ignore casts if it helps. Return NULL on failure. + // Ignore casts if it helps. Return null on failure. Node* unique_input(PhaseTransform *phase, bool uncast); Node* unique_input(PhaseTransform *phase) { Node* uin = unique_input(phase, false); - if (uin == NULL) { + if (uin == nullptr) { uin = unique_input(phase, true); } return uin; @@ -398,7 +403,7 @@ class IfNode : public MultiBranchNode { // Takes the type of val and filters it through the test represented // by if_proj and returns a more refined type if one is produced. - // Returns NULL is it couldn't improve the type. + // Returns null is it couldn't improve the type. static const TypeInt* filtered_int_type(PhaseGVN* phase, Node* val, Node* if_proj); #ifndef PRODUCT @@ -619,4 +624,28 @@ class NeverBranchNode : public MultiBranchNode { #endif }; +//------------------------------BlackholeNode---------------------------- +// Blackhole all arguments. This node would survive through the compiler +// the effects on its arguments, and would be finally matched to nothing. +class BlackholeNode : public MultiNode { +public: + BlackholeNode(Node* ctrl) : MultiNode(1) { + init_req(TypeFunc::Control, ctrl); + } + virtual int Opcode() const; + virtual uint ideal_reg() const { return 0; } // not matched in the AD file + virtual const Type* bottom_type() const { return TypeTuple::MEMBAR; } + + const RegMask &in_RegMask(uint idx) const { + // Fake the incoming arguments mask for blackholes: accept all registers + // and all stack slots. This would avoid any redundant register moves + // for blackhole inputs. + return RegMask::All; + } +#ifndef PRODUCT + virtual void format(PhaseRegAlloc* ra, outputStream* st) const; +#endif +}; + + #endif // SHARE_OPTO_CFGNODE_HPP diff --git a/src/hotspot/share/opto/chaitin.cpp b/src/hotspot/share/opto/chaitin.cpp index 0a1f8e22021cc..d0d09ce619004 100644 --- a/src/hotspot/share/opto/chaitin.cpp +++ b/src/hotspot/share/opto/chaitin.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -61,7 +61,7 @@ void LRG::dump() const { if( is_multidef() ) { tty->print("MultiDef "); - if (_defs != NULL) { + if (_defs != nullptr) { tty->print("("); for (int i = 0; i < _defs->length(); i++) { tty->print("N%d ", _defs->at(i)->_idx); @@ -199,7 +199,7 @@ PhaseChaitin::PhaseChaitin(uint unique, PhaseCFG &cfg, Matcher &matcher, bool sc #ifndef PRODUCT print_chaitin_statistics #else - NULL + nullptr #endif ) , _live(0) @@ -233,7 +233,7 @@ PhaseChaitin::PhaseChaitin(uint unique, PhaseCFG &cfg, Matcher &matcher, bool sc cutoff *= 0.001; buckval[i] = cutoff; for (uint j = 0; j < _cfg.number_of_blocks(); j++) { - buckets[i][j] = NULL; + buckets[i][j] = nullptr; } } // Sort blocks into buckets @@ -378,7 +378,7 @@ void PhaseChaitin::Register_Allocate() { { Compile::TracePhase tp("computeLive", &timers[_t_computeLive]); - _live = NULL; // Mark live as being not available + _live = nullptr; // Mark live as being not available rm.reset_to_mark(); // Reclaim working storage IndexSet::reset_memory(C, &live_arena); ifg.init(_lrg_map.max_lrg_id()); // Empty IFG @@ -396,7 +396,7 @@ void PhaseChaitin::Register_Allocate() { if (stretch_base_pointer_live_ranges(&live_arena)) { Compile::TracePhase tp("computeLive (sbplr)", &timers[_t_computeLive]); // Since some live range stretched, I need to recompute live - _live = NULL; + _live = nullptr; rm.reset_to_mark(); // Reclaim working storage IndexSet::reset_memory(C, &live_arena); ifg.init(_lrg_map.max_lrg_id()); @@ -435,7 +435,7 @@ void PhaseChaitin::Register_Allocate() { // To color, we need the IFG and for that we need LIVE. { Compile::TracePhase tp("computeLive", &timers[_t_computeLive]); - _live = NULL; + _live = nullptr; rm.reset_to_mark(); // Reclaim working storage IndexSet::reset_memory(C, &live_arena); ifg.init(_lrg_map.max_lrg_id()); @@ -473,7 +473,7 @@ void PhaseChaitin::Register_Allocate() { { Compile::TracePhase tp("computeLive", &timers[_t_computeLive]); - _live = NULL; + _live = nullptr; rm.reset_to_mark(); // Reclaim working storage IndexSet::reset_memory(C, &live_arena); ifg.init(_lrg_map.max_lrg_id()); // Build a new interference graph @@ -543,7 +543,7 @@ void PhaseChaitin::Register_Allocate() { // Nuke the live-ness and interference graph and LiveRanGe info { Compile::TracePhase tp("computeLive", &timers[_t_computeLive]); - _live = NULL; + _live = nullptr; rm.reset_to_mark(); // Reclaim working storage IndexSet::reset_memory(C, &live_arena); ifg.init(_lrg_map.max_lrg_id()); @@ -621,7 +621,7 @@ void PhaseChaitin::Register_Allocate() { // Log regalloc results CompileLog* log = Compile::current()->log(); - if (log != NULL) { + if (log != nullptr) { log->elem("regalloc attempts='%d' success='%d'", _trip_cnt, !C->failing()); } @@ -676,9 +676,9 @@ void PhaseChaitin::Register_Allocate() { } // Done! - _live = NULL; - _ifg = NULL; - C->set_indexSet_arena(NULL); // ResourceArea is at end of scope + _live = nullptr; + _ifg = nullptr; + C->set_indexSet_arena(nullptr); // ResourceArea is at end of scope } void PhaseChaitin::de_ssa() { @@ -785,10 +785,10 @@ void PhaseChaitin::gather_lrg_masks( bool after_aggressive ) { copy_src._has_copy = 1; } - if (trace_spilling() && lrg._def != NULL) { + if (trace_spilling() && lrg._def != nullptr) { // collect defs for MultiDef printing - if (lrg._defs == NULL) { - lrg._defs = new (_ifg->_arena) GrowableArray(_ifg->_arena, 2, 0, NULL); + if (lrg._defs == nullptr) { + lrg._defs = new (_ifg->_arena) GrowableArray(_ifg->_arena, 2, 0, nullptr); lrg._defs->append(lrg._def); } lrg._defs->append(n); @@ -796,7 +796,7 @@ void PhaseChaitin::gather_lrg_masks( bool after_aggressive ) { #endif // Check for a single def LRG; these can spill nicely - // via rematerialization. Flag as NULL for no def found + // via rematerialization. Flag as null for no def found // yet, or 'n' for single def or -1 for many defs. lrg._def = lrg._def ? NodeSentinel : n; @@ -824,7 +824,7 @@ void PhaseChaitin::gather_lrg_masks( bool after_aggressive ) { lrg.set_scalable_reg_slots(Matcher::scalable_vector_reg_size(T_FLOAT)); } } - assert(n_type->isa_vect() == NULL || lrg._is_vector || + assert(n_type->isa_vect() == nullptr || lrg._is_vector || ireg == Op_RegD || ireg == Op_RegL || ireg == Op_RegVectMask, "vector must be in vector registers"); @@ -1041,7 +1041,7 @@ void PhaseChaitin::gather_lrg_masks( bool after_aggressive ) { const RegMask &lrgmask = lrg.mask(); uint kreg = n->in(k)->ideal_reg(); bool is_vect = RegMask::is_vector(kreg); - assert(n->in(k)->bottom_type()->isa_vect() == NULL || is_vect || + assert(n->in(k)->bottom_type()->isa_vect() == nullptr || is_vect || kreg == Op_RegD || kreg == Op_RegL || kreg == Op_RegVectMask, "vector must be in vector registers"); if (lrgmask.is_bound(kreg)) @@ -1070,7 +1070,7 @@ void PhaseChaitin::gather_lrg_masks( bool after_aggressive ) { // if the LRG is an unaligned pair, we will have to spill // so clear the LRG's register mask if it is not already spilled if (!is_vect && !n->is_SpillCopy() && - (lrg._def == NULL || lrg.is_multidef() || !lrg._def->is_SpillCopy()) && + (lrg._def == nullptr || lrg.is_multidef() || !lrg._def->is_SpillCopy()) && lrgmask.is_misaligned_pair()) { lrg.Clear(); } @@ -1750,22 +1750,22 @@ Node *PhaseChaitin::find_base_for_derived( Node **derived_base_map, Node *derive // See if this happens to be a base. // NOTE: we use TypePtr instead of TypeOopPtr because we can have - // pointers derived from NULL! These are always along paths that + // pointers derived from null! These are always along paths that // can't happen at run-time but the optimizer cannot deduce it so // we have to handle it gracefully. assert(!derived->bottom_type()->isa_narrowoop() || derived->bottom_type()->make_ptr()->is_ptr()->_offset == 0, "sanity"); const TypePtr *tj = derived->bottom_type()->isa_ptr(); // If its an OOP with a non-zero offset, then it is derived. - if( tj == NULL || tj->_offset == 0 ) { + if( tj == nullptr || tj->_offset == 0 ) { derived_base_map[derived->_idx] = derived; return derived; } - // Derived is NULL+offset? Base is NULL! + // Derived is null+offset? Base is null! if( derived->is_Con() ) { Node *base = _matcher.mach_null(); - assert(base != NULL, "sanity"); - if (base->in(0) == NULL) { + assert(base != nullptr, "sanity"); + if (base->in(0) == nullptr) { // Initialize it once and make it shared: // set control to _root and place it into Start block // (where top() node is placed). @@ -1790,7 +1790,7 @@ Node *PhaseChaitin::find_base_for_derived( Node **derived_base_map, Node *derive if (_lrg_map.live_range_id(base) == 0) { new_lrg(base, maxlrg++); } - assert(base->in(0) == _cfg.get_root_node() && _cfg.get_block_for_node(base) == _cfg.get_block_for_node(C->top()), "base NULL should be shared"); + assert(base->in(0) == _cfg.get_root_node() && _cfg.get_block_for_node(base) == _cfg.get_block_for_node(C->top()), "base null should be shared"); derived_base_map[derived->_idx] = base; return base; } @@ -1839,7 +1839,7 @@ Node *PhaseChaitin::find_base_for_derived( Node **derived_base_map, Node *derive uint j; for( j = 1; j < base->req(); j++ ) if( phi->in(j) != base->in(j) && - !(phi->in(j)->is_Con() && base->in(j)->is_Con()) ) // allow different NULLs + !(phi->in(j)->is_Con() && base->in(j)->is_Con()) ) // allow different nulls break; if( j == base->req() ) { // All inputs match? base = phi; // Then use existing 'phi' and drop 'base' @@ -2401,7 +2401,7 @@ void PhaseChaitin::verify_base_ptrs(ResourceArea* a) const { if (n->is_MachSafePoint()) { MachSafePointNode* sfpt = n->as_MachSafePoint(); JVMState* jvms = sfpt->jvms(); - if (jvms != NULL) { + if (jvms != nullptr) { // Now scan for a live derived pointer if (jvms->oopoff() < sfpt->req()) { // Check each derived/base pair @@ -2425,11 +2425,11 @@ void PhaseChaitin::verify_base_ptrs(ResourceArea* a) const { } } else if (check->is_Con()) { if (is_derived && check->bottom_type()->is_ptr()->_offset != 0) { - // Derived is NULL+non-zero offset, base must be NULL. + // Derived is null+non-zero offset, base must be null. assert(check->bottom_type()->is_ptr()->ptr() == TypePtr::Null, "Bad derived pointer"); } else { assert(check->bottom_type()->is_ptr()->_offset == 0, "Bad base pointer"); - // Base either ConP(NULL) or loadConP + // Base either ConP(nullptr) or loadConP if (check->is_Mach()) { assert(check->as_Mach()->ideal_Opcode() == Op_ConP, "Bad base pointer"); } else { diff --git a/src/hotspot/share/opto/chaitin.hpp b/src/hotspot/share/opto/chaitin.hpp index 646ba100f2086..650190571d911 100644 --- a/src/hotspot/share/opto/chaitin.hpp +++ b/src/hotspot/share/opto/chaitin.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -216,7 +216,7 @@ class LRG : public ResourceObj { // Alive if non-zero, dead if zero - bool alive() const { return _def != NULL; } + bool alive() const { return _def != nullptr; } bool is_multidef() const { return _def == NodeSentinel; } bool is_singledef() const { return _def != NodeSentinel; } @@ -746,7 +746,7 @@ class PhaseChaitin : public PhaseRegAlloc { Node* _def; Node* _first_use; public: - RegDefUse() : _def(NULL), _first_use(NULL) { } + RegDefUse() : _def(nullptr), _first_use(nullptr) { } Node* def() const { return _def; } Node* first_use() const { return _first_use; } @@ -757,8 +757,8 @@ class PhaseChaitin : public PhaseRegAlloc { } } void clear() { - _def = NULL; - _first_use = NULL; + _def = nullptr; + _first_use = nullptr; } }; typedef GrowableArray RegToDefUseMap; diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp index 582ec51fbde80..614d0b4e11227 100644 --- a/src/hotspot/share/opto/classes.hpp +++ b/src/hotspot/share/opto/classes.hpp @@ -219,6 +219,7 @@ macro(MemBarAcquireLock) macro(MemBarCPUOrder) macro(MemBarRelease) macro(StoreFence) +macro(StoreStoreFence) macro(MemBarReleaseLock) macro(MemBarVolatile) macro(MemBarStoreStore) diff --git a/src/hotspot/share/opto/coalesce.cpp b/src/hotspot/share/opto/coalesce.cpp index 3fa627d42cb71..0220bdfb2722c 100644 --- a/src/hotspot/share/opto/coalesce.cpp +++ b/src/hotspot/share/opto/coalesce.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -260,7 +260,7 @@ void PhaseAggressiveCoalesce::insert_copies( Matcher &matcher ) { Node *def = n->in(cidx); if (_phc._lrg_map.find(n) == _phc._lrg_map.find(def)) { n->replace_by(def); - n->set_req(cidx,NULL); + n->set_req(cidx,nullptr); b->remove_node(l); l--; continue; @@ -503,7 +503,7 @@ void PhaseConservativeCoalesce::union_helper( Node *lr1_node, Node *lr2_node, ui lrgs(lr1)._def = (lrgs(lr1).is_multidef() || lrgs(lr2).is_multidef() ) ? NodeSentinel : src_def; - lrgs(lr2)._def = NULL; // No def for lrg 2 + lrgs(lr2)._def = nullptr; // No def for lrg 2 lrgs(lr2).Clear(); // Force empty mask for LRG 2 //lrgs(lr2)._size = 0; // Live-range 2 goes dead lrgs(lr1)._is_oop |= lrgs(lr2)._is_oop; @@ -520,7 +520,7 @@ void PhaseConservativeCoalesce::union_helper( Node *lr1_node, Node *lr2_node, ui // _phc.free_spillcopy(b->_nodes[bindex]); assert( b->get_node(bindex) == dst_copy, "" ); dst_copy->replace_by( dst_copy->in(didx) ); - dst_copy->set_req( didx, NULL); + dst_copy->set_req( didx, nullptr); b->remove_node(bindex); if( bindex < b->_ihrp_index ) b->_ihrp_index--; if( bindex < b->_fhrp_index ) b->_fhrp_index--; diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index e3de3affbc18d..4b22284f977db 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -85,7 +85,7 @@ // -------------------- Compile::mach_constant_base_node ----------------------- // Constant table base node singleton. MachConstantBaseNode* Compile::mach_constant_base_node() { - if (_mach_constant_base_node == NULL) { + if (_mach_constant_base_node == nullptr) { _mach_constant_base_node = new MachConstantBaseNode(); _mach_constant_base_node->add_req(C->root()); } @@ -153,7 +153,7 @@ CallGenerator* Compile::find_intrinsic(ciMethod* m, bool is_virtual) { if (m->intrinsic_id() != vmIntrinsics::_none && m->intrinsic_id() <= vmIntrinsics::LAST_COMPILER_INLINE) { CallGenerator* cg = make_vm_intrinsic(m, is_virtual); - if (cg != NULL) { + if (cg != nullptr) { // Save it for next time: register_intrinsic(cg); return cg; @@ -161,7 +161,7 @@ CallGenerator* Compile::find_intrinsic(ciMethod* m, bool is_virtual) { gather_intrinsic_statistics(m->intrinsic_id(), is_virtual, _intrinsic_disabled); } } - return NULL; + return nullptr; } // Compile::make_vm_intrinsic is defined in library_call.cpp. @@ -228,7 +228,7 @@ static char* format_flags(int flags, char* buf) { void Compile::print_intrinsic_statistics() { char flagsbuf[100]; ttyLocker ttyl; - if (xtty != NULL) xtty->head("statistics type='intrinsic'"); + if (xtty != nullptr) xtty->head("statistics type='intrinsic'"); tty->print_cr("Compiler intrinsic usage:"); juint total = _intrinsic_hist_count[as_int(vmIntrinsics::_none)]; if (total == 0) total = 1; // avoid div0 in case of no successes @@ -242,19 +242,19 @@ void Compile::print_intrinsic_statistics() { } } PRINT_STAT_LINE("total", total, format_flags(_intrinsic_hist_flags[as_int(vmIntrinsics::_none)], flagsbuf)); - if (xtty != NULL) xtty->tail("statistics"); + if (xtty != nullptr) xtty->tail("statistics"); } void Compile::print_statistics() { { ttyLocker ttyl; - if (xtty != NULL) xtty->head("statistics type='opto'"); + if (xtty != nullptr) xtty->head("statistics type='opto'"); Parse::print_statistics(); PhaseCCP::print_statistics(); PhaseRegAlloc::print_statistics(); PhaseOutput::print_statistics(); PhasePeephole::print_statistics(); PhaseIdealLoop::print_statistics(); - if (xtty != NULL) xtty->tail("statistics"); + if (xtty != nullptr) xtty->tail("statistics"); } if (_intrinsic_hist_flags[as_int(vmIntrinsics::_none)] != 0) { // put this under its own element. @@ -292,12 +292,12 @@ void Compile::gvn_replace_by(Node* n, Node* nn) { // recursive traversal is slower. void Compile::identify_useful_nodes(Unique_Node_List &useful) { int estimated_worklist_size = live_nodes(); - useful.map( estimated_worklist_size, NULL ); // preallocate space + useful.map( estimated_worklist_size, nullptr ); // preallocate space // Initialize worklist - if (root() != NULL) { useful.push(root()); } + if (root() != nullptr) { useful.push(root()); } // If 'top' is cached, declare it useful to preserve cached node - if( cached_top_node() ) { useful.push(cached_top_node()); } + if (cached_top_node()) { useful.push(cached_top_node()); } // Push all useful nodes onto the list, breadthfirst for( uint next = 0; next < useful.size(); ++next ) { @@ -345,7 +345,7 @@ void Compile::remove_useless_late_inlines(GrowableArray* inlines } void Compile::remove_useless_late_inlines(GrowableArray* inlines, Node* dead) { - assert(dead != NULL && dead->is_Call(), "sanity"); + assert(dead != nullptr && dead->is_Call(), "sanity"); int found = 0; for (int i = 0; i < inlines->length(); i++) { if (inlines->at(i)->call_node() == dead) { @@ -434,7 +434,7 @@ void Compile::disconnect_useless_nodes(Unique_Node_List &useful, Unique_Node_Lis remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass remove_useless_coarsened_locks(useful); // remove useless coarsened locks nodes #ifdef ASSERT - if (_modified_nodes != NULL) { + if (_modified_nodes != nullptr) { _modified_nodes->remove_useless_nodes(useful.member_set()); } #endif @@ -463,17 +463,17 @@ CompileWrapper::CompileWrapper(Compile* compile) : _compile(compile) { // the Compile* pointer is stored in the current ciEnv: ciEnv* env = compile->env(); assert(env == ciEnv::current(), "must already be a ciEnv active"); - assert(env->compiler_data() == NULL, "compile already active?"); + assert(env->compiler_data() == nullptr, "compile already active?"); env->set_compiler_data(compile); assert(compile == Compile::current(), "sanity"); - compile->set_type_dict(NULL); + compile->set_type_dict(nullptr); compile->set_clone_map(new Dict(cmpkey, hashkey, _compile->comp_arena())); compile->clone_map().set_clone_idx(0); compile->set_type_last_size(0); - compile->set_last_tf(NULL, NULL); - compile->set_indexSet_arena(NULL); - compile->set_indexSet_free_block_list(NULL); + compile->set_last_tf(nullptr, nullptr); + compile->set_indexSet_arena(nullptr); + compile->set_indexSet_free_block_list(nullptr); compile->init_type_arena(); Type::Initialize(compile); _compile->begin_method(); @@ -481,7 +481,7 @@ CompileWrapper::CompileWrapper(Compile* compile) : _compile(compile) { } CompileWrapper::~CompileWrapper() { _compile->end_method(); - _compile->env()->set_compiler_data(NULL); + _compile->env()->set_compiler_data(nullptr); } @@ -551,9 +551,9 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, _do_locks_coarsening(do_locks_coarsening), _method(target), _entry_bci(osr_bci), - _stub_function(NULL), - _stub_name(NULL), - _stub_entry_point(NULL), + _stub_function(nullptr), + _stub_name(nullptr), + _stub_entry_point(nullptr), _max_node_limit(MaxNodeLimit), _post_loop_opts_phase(false), _inlining_progress(false), @@ -573,36 +573,36 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, _env(ci_env), _directive(directive), _log(ci_env->log()), - _failure_reason(NULL), - _intrinsics (comp_arena(), 0, 0, NULL), - _macro_nodes (comp_arena(), 8, 0, NULL), - _predicate_opaqs (comp_arena(), 8, 0, NULL), - _skeleton_predicate_opaqs (comp_arena(), 8, 0, NULL), - _expensive_nodes (comp_arena(), 8, 0, NULL), - _for_post_loop_igvn(comp_arena(), 8, 0, NULL), - _coarsened_locks (comp_arena(), 8, 0, NULL), - _congraph(NULL), - NOT_PRODUCT(_printer(NULL) COMMA) + _failure_reason(nullptr), + _intrinsics (comp_arena(), 0, 0, nullptr), + _macro_nodes (comp_arena(), 8, 0, nullptr), + _predicate_opaqs (comp_arena(), 8, 0, nullptr), + _skeleton_predicate_opaqs (comp_arena(), 8, 0, nullptr), + _expensive_nodes (comp_arena(), 8, 0, nullptr), + _for_post_loop_igvn(comp_arena(), 8, 0, nullptr), + _coarsened_locks (comp_arena(), 8, 0, nullptr), + _congraph(nullptr), + NOT_PRODUCT(_printer(nullptr) COMMA) _dead_node_list(comp_arena()), _dead_node_count(0), _node_arena(mtCompiler), _old_arena(mtCompiler), - _mach_constant_base_node(NULL), + _mach_constant_base_node(nullptr), _Compile_types(mtCompiler), - _initial_gvn(NULL), - _for_igvn(NULL), - _late_inlines(comp_arena(), 2, 0, NULL), - _string_late_inlines(comp_arena(), 2, 0, NULL), - _boxing_late_inlines(comp_arena(), 2, 0, NULL), - _vector_reboxing_late_inlines(comp_arena(), 2, 0, NULL), + _initial_gvn(nullptr), + _for_igvn(nullptr), + _late_inlines(comp_arena(), 2, 0, nullptr), + _string_late_inlines(comp_arena(), 2, 0, nullptr), + _boxing_late_inlines(comp_arena(), 2, 0, nullptr), + _vector_reboxing_late_inlines(comp_arena(), 2, 0, nullptr), _late_inlines_pos(0), _number_of_mh_late_inlines(0), _native_invokers(comp_arena(), 1, 0, NULL), _print_inlining_stream(NULL), - _print_inlining_list(NULL), + _print_inlining_list(nullptr), _print_inlining_idx(0), - _print_inlining_output(NULL), - _replay_inline_data(NULL), + _print_inlining_output(nullptr), + _replay_inline_data(nullptr), _java_calls(0), _inner_loops(0), _interpreter_frame_size(0) @@ -621,7 +621,7 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, tty->print(" "); } TraceTime t1("Total compilation time", &_t_totalCompilation, CITime, CITimeVerbose); - TraceTime t2(NULL, &_t_methodCompilation, CITime, false); + TraceTime t2(nullptr, &_t_methodCompilation, CITime, false); #if defined(SUPPORT_ASSEMBLY) || defined(SUPPORT_ABSTRACT_ASSEMBLY) bool print_opto_assembly = directive->PrintOptoAssemblyOption; @@ -682,7 +682,7 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, initial_gvn()->transform_no_reclaim(top()); // Set up tf(), start(), and find a CallGenerator. - CallGenerator* cg = NULL; + CallGenerator* cg = nullptr; if (is_osr_compilation()) { const TypeTuple *domain = StartOSRNode::osr_domain(); const TypeTuple *range = TypeTuple::make_range(method()->signature()); @@ -705,19 +705,19 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, // the pre-barrier code. cg = find_intrinsic(method(), false); } - if (cg == NULL) { + if (cg == nullptr) { float past_uses = method()->interpreter_invocation_count(); float expected_uses = past_uses; cg = CallGenerator::for_inline(method(), expected_uses); } } if (failing()) return; - if (cg == NULL) { + if (cg == nullptr) { record_method_not_compilable("cannot parse method"); return; } JVMState* jvms = build_start_state(start(), tf()); - if ((jvms = cg->generate(jvms)) == NULL) { + if ((jvms = cg->generate(jvms)) == nullptr) { if (!failure_reason_is(C2Compiler::retry_class_loading_during_parsing())) { record_method_not_compilable("method parse failed"); } @@ -761,7 +761,7 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, // After parsing, node notes are no longer automagic. // They must be propagated by register_new_node_with_optimizer(), // clone(), or the like. - set_default_node_notes(NULL); + set_default_node_notes(nullptr); #ifndef PRODUCT if (should_print(1)) { @@ -781,7 +781,7 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, } else { _stress_seed = StressSeed; } - if (_log != NULL) { + if (_log != nullptr) { _log->elem("stress_test seed='%u'", _stress_seed); } } @@ -797,13 +797,13 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, // This output goes directly to the tty, not the compiler log. // To enable tools to match it up with the compilation activity, // be sure to tag this tty output with the compile ID. - if (xtty != NULL) { + if (xtty != nullptr) { xtty->head("ideal compile_id='%d'%s", compile_id(), is_osr_compilation() ? " compile_kind='osr'" : ""); } root()->dump(9999); - if (xtty != NULL) { + if (xtty != nullptr) { xtty->tail("ideal"); } } @@ -818,7 +818,7 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, if (directive->DumpReplayOption) { env()->dump_replay_data(_compile_id); } - if (directive->DumpInlineOption && (ilt() != NULL)) { + if (directive->DumpInlineOption && (ilt() != nullptr)) { env()->dump_inline_data(_compile_id); } @@ -852,11 +852,11 @@ Compile::Compile( ciEnv* ci_env, _install_code(true), _eliminate_boxing(false), _do_locks_coarsening(false), - _method(NULL), + _method(nullptr), _entry_bci(InvocationEntryBci), _stub_function(stub_function), _stub_name(stub_name), - _stub_entry_point(NULL), + _stub_entry_point(nullptr), _max_node_limit(MaxNodeLimit), _post_loop_opts_phase(false), _inlining_progress(false), @@ -875,24 +875,24 @@ Compile::Compile( ciEnv* ci_env, _env(ci_env), _directive(directive), _log(ci_env->log()), - _failure_reason(NULL), - _congraph(NULL), - NOT_PRODUCT(_printer(NULL) COMMA) + _failure_reason(nullptr), + _congraph(nullptr), + NOT_PRODUCT(_printer(nullptr) COMMA) _dead_node_list(comp_arena()), _dead_node_count(0), _node_arena(mtCompiler), _old_arena(mtCompiler), - _mach_constant_base_node(NULL), + _mach_constant_base_node(nullptr), _Compile_types(mtCompiler), - _initial_gvn(NULL), - _for_igvn(NULL), + _initial_gvn(nullptr), + _for_igvn(nullptr), _number_of_mh_late_inlines(0), _native_invokers(), _print_inlining_stream(NULL), - _print_inlining_list(NULL), + _print_inlining_list(nullptr), _print_inlining_idx(0), - _print_inlining_output(NULL), - _replay_inline_data(NULL), + _print_inlining_output(nullptr), + _replay_inline_data(nullptr), _java_calls(0), _inner_loops(0), _interpreter_frame_size(0), @@ -902,8 +902,8 @@ Compile::Compile( ciEnv* ci_env, _allowed_reasons(0) { C = this; - TraceTime t1(NULL, &_t_totalCompilation, CITime, false); - TraceTime t2(NULL, &_t_stubCompilation, CITime, false); + TraceTime t1(nullptr, &_t_totalCompilation, CITime, false); + TraceTime t2(nullptr, &_t_stubCompilation, CITime, false); #ifndef PRODUCT set_print_assembly(PrintFrameConverterAssembly); @@ -938,28 +938,34 @@ Compile::Compile( ciEnv* ci_env, // Prepare for a single compilation void Compile::Init(int aliaslevel) { _unique = 0; - _regalloc = NULL; + _regalloc = nullptr; - _tf = NULL; // filled in later - _top = NULL; // cached later - _matcher = NULL; // filled in later - _cfg = NULL; // filled in later + _tf = nullptr; // filled in later + _top = nullptr; // cached later + _matcher = nullptr; // filled in later + _cfg = nullptr; // filled in later IA32_ONLY( set_24_bit_selection_and_mode(true, false); ) - _node_note_array = NULL; - _default_node_notes = NULL; - DEBUG_ONLY( _modified_nodes = NULL; ) // Used in Optimize() + _node_note_array = nullptr; + _default_node_notes = nullptr; + DEBUG_ONLY( _modified_nodes = nullptr; ) // Used in Optimize() - _immutable_memory = NULL; // filled in at first inquiry + _immutable_memory = nullptr; // filled in at first inquiry + +#ifdef ASSERT + _type_verify_symmetry = true; + _phase_optimize_finished = false; + _exception_backedge = false; +#endif // Globally visible Nodes - // First set TOP to NULL to give safe behavior during creation of RootNode - set_cached_top_node(NULL); + // First set TOP to null to give safe behavior during creation of RootNode + set_cached_top_node(nullptr); set_root(new RootNode()); // Now that you have a Root to point to, create the real TOP set_cached_top_node( new ConNode(Type::TOP) ); - set_recent_alloc(NULL, NULL); + set_recent_alloc(nullptr, nullptr); // Create Debug Information Recorder to record scopes, oopmaps, etc. env()->set_oop_recorder(new OopRecorder(env()->arena())); @@ -1005,7 +1011,7 @@ void Compile::Init(int aliaslevel) { _max_node_limit = _directive->MaxNodeLimitOption; #if INCLUDE_RTM_OPT - if (UseRTMLocking && has_method() && (method()->method_data_or_null() != NULL)) { + if (UseRTMLocking && has_method() && (method()->method_data_or_null() != nullptr)) { int rtm_state = method()->method_data()->rtm_state(); if (method_has_option(CompileCommand::NoRTMLockEliding) || ((rtm_state & NoRTM) != 0)) { // Don't generate RTM lock eliding code. @@ -1025,7 +1031,7 @@ void Compile::Init(int aliaslevel) { } if (debug_info()->recording_non_safepoints()) { set_node_note_array(new(comp_arena()) GrowableArray - (comp_arena(), 8, 0, NULL)); + (comp_arena(), 8, 0, nullptr)); set_default_node_notes(Node_Notes::make(this)); } @@ -1047,20 +1053,14 @@ void Compile::Init(int aliaslevel) { for (int i = 0; i < grow_ats; i++) _alias_types[i] = &ats[i]; } // Initialize the first few types. - _alias_types[AliasIdxTop]->Init(AliasIdxTop, NULL); + _alias_types[AliasIdxTop]->Init(AliasIdxTop, nullptr); _alias_types[AliasIdxBot]->Init(AliasIdxBot, TypePtr::BOTTOM); _alias_types[AliasIdxRaw]->Init(AliasIdxRaw, TypeRawPtr::BOTTOM); _num_alias_types = AliasIdxRaw+1; // Zero out the alias type cache. Copy::zero_to_bytes(_alias_cache, sizeof(_alias_cache)); - // A NULL adr_type hits in the cache right away. Preload the right answer. - probe_alias_cache(NULL)->_index = AliasIdxTop; - -#ifdef ASSERT - _type_verify_symmetry = true; - _phase_optimize_finished = false; - _exception_backedge = false; -#endif + // A null adr_type hits in the cache right away. Preload the right answer. + probe_alias_cache(nullptr)->_index = AliasIdxTop; } //---------------------------init_start---------------------------------------- @@ -1085,13 +1085,13 @@ StartNode* Compile::start() const { } } fatal("Did not find Start node!"); - return NULL; + return nullptr; } //-------------------------------immutable_memory------------------------------------- // Access immutable memory Node* Compile::immutable_memory() { - if (_immutable_memory != NULL) { + if (_immutable_memory != nullptr) { return _immutable_memory; } StartNode* s = start(); @@ -1103,20 +1103,20 @@ Node* Compile::immutable_memory() { } } ShouldNotReachHere(); - return NULL; + return nullptr; } //----------------------set_cached_top_node------------------------------------ // Install the cached top node, and make sure Node::is_top works correctly. void Compile::set_cached_top_node(Node* tn) { - if (tn != NULL) verify_top(tn); + if (tn != nullptr) verify_top(tn); Node* old_top = _top; _top = tn; // Calling Node::setup_is_top allows the nodes the chance to adjust // their _out arrays. - if (_top != NULL) _top->setup_is_top(); - if (old_top != NULL) old_top->setup_is_top(); - assert(_top == NULL || top()->is_top(), ""); + if (_top != nullptr) _top->setup_is_top(); + if (old_top != nullptr) old_top->setup_is_top(); + assert(_top == nullptr || top()->is_top(), ""); } #ifdef ASSERT @@ -1129,8 +1129,8 @@ uint Compile::count_live_nodes_by_graph_walk() { void Compile::print_missing_nodes() { - // Return if CompileLog is NULL and PrintIdealNodeCount is false. - if ((_log == NULL) && (! PrintIdealNodeCount)) { + // Return if CompileLog is null and PrintIdealNodeCount is false. + if ((_log == nullptr) && (! PrintIdealNodeCount)) { return; } @@ -1148,7 +1148,7 @@ void Compile::print_missing_nodes() { uint l_nodes_by_walk = useful.size(); if (l_nodes != l_nodes_by_walk) { - if (_log != NULL) { + if (_log != nullptr) { _log->begin_head("mismatched_nodes count='%d'", abs((int) (l_nodes - l_nodes_by_walk))); _log->stamp(); _log->end_head(); @@ -1158,7 +1158,7 @@ void Compile::print_missing_nodes() { for (int i = 0; i < last_idx; i++) { if (useful_member_set.test(i)) { if (_dead_node_list.test(i)) { - if (_log != NULL) { + if (_log != nullptr) { _log->elem("mismatched_node_info node_idx='%d' type='both live and dead'", i); } if (PrintIdealNodeCount) { @@ -1169,7 +1169,7 @@ void Compile::print_missing_nodes() { } } else if (! _dead_node_list.test(i)) { - if (_log != NULL) { + if (_log != nullptr) { _log->elem("mismatched_node_info node_idx='%d' type='neither live nor dead'", i); } if (PrintIdealNodeCount) { @@ -1178,19 +1178,19 @@ void Compile::print_missing_nodes() { } } } - if (_log != NULL) { + if (_log != nullptr) { _log->tail("mismatched_nodes"); } } } void Compile::record_modified_node(Node* n) { - if (_modified_nodes != NULL && !_inlining_incrementally && !n->is_Con()) { + if (_modified_nodes != nullptr && !_inlining_incrementally && !n->is_Con()) { _modified_nodes->push(n); } } void Compile::remove_modified_node(Node* n) { - if (_modified_nodes != NULL) { + if (_modified_nodes != nullptr) { _modified_nodes->remove(n); } } @@ -1198,10 +1198,10 @@ void Compile::remove_modified_node(Node* n) { #ifndef PRODUCT void Compile::verify_top(Node* tn) const { - if (tn != NULL) { + if (tn != nullptr) { assert(tn->is_Con(), "top node must be a constant"); assert(((ConNode*)tn)->type() == Type::TOP, "top node must have correct type"); - assert(tn->in(0) != NULL, "must have live top node"); + assert(tn->in(0) != nullptr, "must have live top node"); } } #endif @@ -1210,7 +1210,7 @@ void Compile::verify_top(Node* tn) const { ///-------------------Managing Per-Node Debug & Profile Info------------------- void Compile::grow_node_notes(GrowableArray* arr, int grow_by) { - guarantee(arr != NULL, ""); + guarantee(arr != nullptr, ""); int num_blocks = arr->length(); if (grow_by < num_blocks) grow_by = num_blocks; int num_notes = grow_by * _node_notes_block_size; @@ -1225,27 +1225,27 @@ void Compile::grow_node_notes(GrowableArray* arr, int grow_by) { } bool Compile::copy_node_notes_to(Node* dest, Node* source) { - if (source == NULL || dest == NULL) return false; + if (source == nullptr || dest == nullptr) return false; if (dest->is_Con()) return false; // Do not push debug info onto constants. #ifdef ASSERT // Leave a bread crumb trail pointing to the original node: - if (dest != NULL && dest != source && dest->debug_orig() == NULL) { + if (dest != nullptr && dest != source && dest->debug_orig() == nullptr) { dest->set_debug_orig(source); } #endif - if (node_note_array() == NULL) + if (node_note_array() == nullptr) return false; // Not collecting any notes now. // This is a copy onto a pre-existing node, which may already have notes. // If both nodes have notes, do not overwrite any pre-existing notes. Node_Notes* source_notes = node_notes_at(source->_idx); - if (source_notes == NULL || source_notes->is_clear()) return false; + if (source_notes == nullptr || source_notes->is_clear()) return false; Node_Notes* dest_notes = node_notes_at(dest->_idx); - if (dest_notes == NULL || dest_notes->is_clear()) { + if (dest_notes == nullptr || dest_notes->is_clear()) { return set_node_notes_at(dest->_idx, source_notes); } @@ -1278,7 +1278,7 @@ const TypePtr *Compile::flatten_alias_type( const TypePtr *tj ) const { TypePtr::PTR ptr = tj->ptr(); // Known instance (scalarizable allocation) alias only with itself. - bool is_known_inst = tj->isa_oopptr() != NULL && + bool is_known_inst = tj->isa_oopptr() != nullptr && tj->is_oopptr()->is_known_instance(); // Process weird unsafe references. @@ -1334,11 +1334,11 @@ const TypePtr *Compile::flatten_alias_type( const TypePtr *tj ) const { // Arrays of known objects become arrays of unknown objects. if (ta->elem()->isa_narrowoop() && ta->elem() != TypeNarrowOop::BOTTOM) { const TypeAry *tary = TypeAry::make(TypeNarrowOop::BOTTOM, ta->size()); - tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,NULL,false,offset); + tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,offset); } if (ta->elem()->isa_oopptr() && ta->elem() != TypeInstPtr::BOTTOM) { const TypeAry *tary = TypeAry::make(TypeInstPtr::BOTTOM, ta->size()); - tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,NULL,false,offset); + tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,offset); } // Arrays of bytes and of booleans both use 'bastore' and 'baload' so // cannot be distinguished by bytecode alone. @@ -1350,7 +1350,7 @@ const TypePtr *Compile::flatten_alias_type( const TypePtr *tj ) const { // During the 2nd round of IterGVN, NotNull castings are removed. // Make sure the Bottom and NotNull variants alias the same. // Also, make sure exact and non-exact variants alias the same. - if (ptr == TypePtr::NotNull || ta->klass_is_exact() || ta->speculative() != NULL) { + if (ptr == TypePtr::NotNull || ta->klass_is_exact() || ta->speculative() != nullptr) { tj = ta = TypeAryPtr::make(TypePtr::BotPTR,ta->ary(),ta->klass(),false,offset); } } @@ -1375,7 +1375,7 @@ const TypePtr *Compile::flatten_alias_type( const TypePtr *tj ) const { // Also, make sure exact and non-exact variants alias the same. tj = to = TypeInstPtr::make(TypePtr::BotPTR,to->klass(),false,0,offset); } - if (to->speculative() != NULL) { + if (to->speculative() != nullptr) { tj = to = TypeInstPtr::make(to->ptr(),to->klass(),to->klass_is_exact(),to->const_oop(),to->offset(), to->instance_id()); } // Canonicalize the holder of this field @@ -1383,13 +1383,13 @@ const TypePtr *Compile::flatten_alias_type( const TypePtr *tj ) const { // First handle header references such as a LoadKlassNode, even if the // object's klass is unloaded at compile time (4965979). if (!is_known_inst) { // Do it only for non-instance types - tj = to = TypeInstPtr::make(TypePtr::BotPTR, env()->Object_klass(), false, NULL, offset); + tj = to = TypeInstPtr::make(TypePtr::BotPTR, env()->Object_klass(), false, nullptr, offset); } } else if (offset < 0 || offset >= k->layout_helper_size_in_bytes()) { // Static fields are in the space above the normal instance // fields in the java.lang.Class instance. if (to->klass() != ciEnv::current()->Class_klass()) { - to = NULL; + to = nullptr; tj = TypeOopPtr::BOTTOM; offset = tj->offset(); } @@ -1398,9 +1398,9 @@ const TypePtr *Compile::flatten_alias_type( const TypePtr *tj ) const { assert(offset < canonical_holder->layout_helper_size_in_bytes(), ""); if (!k->equals(canonical_holder) || tj->offset() != offset) { if( is_known_inst ) { - tj = to = TypeInstPtr::make(to->ptr(), canonical_holder, true, NULL, offset, to->instance_id()); + tj = to = TypeInstPtr::make(to->ptr(), canonical_holder, true, nullptr, offset, to->instance_id()); } else { - tj = to = TypeInstPtr::make(to->ptr(), canonical_holder, false, NULL, offset); + tj = to = TypeInstPtr::make(to->ptr(), canonical_holder, false, nullptr, offset); } } } @@ -1500,11 +1500,11 @@ void Compile::AliasType::Init(int i, const TypePtr* at) { assert(AliasIdxTop <= i && i < Compile::current()->_max_alias_types, "Invalid alias index"); _index = i; _adr_type = at; - _field = NULL; - _element = NULL; + _field = nullptr; + _element = nullptr; _is_rewritable = true; // default - const TypeOopPtr *atoop = (at != NULL) ? at->isa_oopptr() : NULL; - if (atoop != NULL && atoop->is_known_instance()) { + const TypeOopPtr *atoop = (at != nullptr) ? at->isa_oopptr() : nullptr; + if (atoop != nullptr && atoop->is_known_instance()) { const TypeOopPtr *gt = atoop->cast_to_instance_id(TypeOopPtr::InstanceBot); _general_index = Compile::current()->get_alias_index(gt); } else { @@ -1513,10 +1513,10 @@ void Compile::AliasType::Init(int i, const TypePtr* at) { } BasicType Compile::AliasType::basic_type() const { - if (element() != NULL) { + if (element() != nullptr) { const Type* element = adr_type()->is_aryptr()->elem(); return element->isa_narrowoop() ? T_OBJECT : element->array_element_basic_type(); - } if (field() != NULL) { + } if (field() != nullptr) { return field()->layout_type(); } else { return T_ILLEGAL; // unknown @@ -1537,7 +1537,7 @@ void Compile::AliasType::print_on(outputStream* st) { st->print(" in "); adr_type()->dump_on(st); const TypeOopPtr* tjp = adr_type()->isa_oopptr(); - if (field() != NULL && tjp) { + if (field() != nullptr && tjp) { if (tjp->klass() != field()->holder() || tjp->offset() != field()->offset_in_bytes()) { st->print(" != "); @@ -1590,7 +1590,7 @@ Compile::AliasType* Compile::find_alias_type(const TypePtr* adr_type, bool no_cr } // Handle special cases. - if (adr_type == NULL) return alias_type(AliasIdxTop); + if (adr_type == nullptr) return alias_type(AliasIdxTop); if (adr_type == TypePtr::BOTTOM) return alias_type(AliasIdxBot); // Do it the slow way. @@ -1623,7 +1623,7 @@ Compile::AliasType* Compile::find_alias_type(const TypePtr* adr_type, bool no_cr } if (idx == AliasIdxTop) { - if (no_create) return NULL; + if (no_create) return nullptr; // Grow the array if necessary. if (_num_alias_types == _max_alias_types) grow_alias_types(); // Add a new alias type. @@ -1666,7 +1666,7 @@ Compile::AliasType* Compile::find_alias_type(const TypePtr* adr_type, bool no_cr const TypeInstPtr* tinst = flat->isa_instptr(); if (tinst && tinst->offset() >= instanceOopDesc::base_offset_in_bytes()) { ciField* field; - if (tinst->const_oop() != NULL && + if (tinst->const_oop() != nullptr && tinst->klass() == ciEnv::current()->Class_klass() && tinst->offset() >= (tinst->klass()->as_instance_klass()->layout_helper_size_in_bytes())) { // static field @@ -1676,13 +1676,13 @@ Compile::AliasType* Compile::find_alias_type(const TypePtr* adr_type, bool no_cr ciInstanceKlass *k = tinst->klass()->as_instance_klass(); field = k->get_field_by_offset(tinst->offset(), false); } - assert(field == NULL || - original_field == NULL || + assert(field == nullptr || + original_field == nullptr || (field->holder() == original_field->holder() && field->offset() == original_field->offset() && field->is_static() == original_field->is_static()), "wrong field?"); // Set field() and is_rewritable() attributes. - if (field != NULL) alias_type(idx)->set_field(field); + if (field != nullptr) alias_type(idx)->set_field(field); } } @@ -1693,7 +1693,7 @@ Compile::AliasType* Compile::find_alias_type(const TypePtr* adr_type, bool no_cr // Might as well try to fill the cache for the flattened version, too. AliasCacheEntry* face = probe_alias_cache(flat); - if (face->_adr_type == NULL) { + if (face->_adr_type == nullptr) { face->_adr_type = flat; face->_index = idx; assert(alias_type(flat) == alias_type(idx), "flat type must work too"); @@ -1723,17 +1723,17 @@ bool Compile::have_alias_type(const TypePtr* adr_type) { } // Handle special cases. - if (adr_type == NULL) return true; + if (adr_type == nullptr) return true; if (adr_type == TypePtr::BOTTOM) return true; - return find_alias_type(adr_type, true, NULL) != NULL; + return find_alias_type(adr_type, true, nullptr) != nullptr; } //-----------------------------must_alias-------------------------------------- // True if all values of the given address type are in the given alias category. bool Compile::must_alias(const TypePtr* adr_type, int alias_idx) { if (alias_idx == AliasIdxBot) return true; // the universal category - if (adr_type == NULL) return true; // NULL serves as TypePtr::TOP + if (adr_type == nullptr) return true; // null serves as TypePtr::TOP if (alias_idx == AliasIdxTop) return false; // the empty category if (adr_type->base() == Type::AnyPtr) return false; // TypePtr::BOTTOM or its twins @@ -1751,7 +1751,7 @@ bool Compile::must_alias(const TypePtr* adr_type, int alias_idx) { // True if any values of the given address type are in the given alias category. bool Compile::can_alias(const TypePtr* adr_type, int alias_idx) { if (alias_idx == AliasIdxTop) return false; // the empty category - if (adr_type == NULL) return false; // NULL serves as TypePtr::TOP + if (adr_type == nullptr) return false; // null serves as TypePtr::TOP // Known instance doesn't alias with bottom memory if (alias_idx == AliasIdxBot) return !adr_type->is_known_instance(); // the universal category if (adr_type->base() == Type::AnyPtr) return !C->get_adr_type(alias_idx)->is_known_instance(); // TypePtr::BOTTOM or its twins @@ -1946,7 +1946,7 @@ void Compile::inline_incrementally(PhaseIterGVN& igvn) { if (live_nodes() > (uint)LiveNodeCountInliningCutoff) { bool do_print_inlining = print_inlining() || print_intrinsics(); - if (do_print_inlining || log() != NULL) { + if (do_print_inlining || log() != nullptr) { // Print inlining message for candidates that we couldn't inline for lack of space. for (int i = 0; i < _late_inlines.length(); i++) { CallGenerator* cg = _late_inlines.at(i); @@ -1999,10 +1999,10 @@ void Compile::inline_incrementally(PhaseIterGVN& igvn) { void Compile::process_late_inline_calls_no_inline(PhaseIterGVN& igvn) { // "inlining_incrementally() == false" is used to signal that no inlining is allowed // (see LateInlineVirtualCallGenerator::do_late_inline_check() for details). - // Tracking and verification of modified nodes is disabled by setting "_modified_nodes == NULL" + // Tracking and verification of modified nodes is disabled by setting "_modified_nodes == nullptr" // as if "inlining_incrementally() == true" were set. assert(inlining_incrementally() == false, "not allowed"); - assert(_modified_nodes == NULL, "not allowed"); + assert(_modified_nodes == nullptr, "not allowed"); assert(_late_inlines.length() > 0, "sanity"); while (_late_inlines.length() > 0) { @@ -2037,10 +2037,10 @@ bool Compile::optimize_loops(PhaseIterGVN& igvn, LoopOptsMode mode) { // useful. void Compile::remove_root_to_sfpts_edges(PhaseIterGVN& igvn) { Node *r = root(); - if (r != NULL) { + if (r != nullptr) { for (uint i = r->req(); i < r->len(); ++i) { Node *n = r->in(i); - if (n != NULL && n->is_SafePoint()) { + if (n != nullptr && n->is_SafePoint()) { r->rm_prec(i); if (n->outcnt() == 0) { igvn.remove_dead_node(n); @@ -2175,7 +2175,7 @@ void Compile::Optimize() { if (failing()) return; - if (congraph() != NULL && macro_count() > 0) { + if (congraph() != nullptr && macro_count() > 0) { TracePhase tp("macroEliminate", &timers[_t_macroEliminate]); PhaseMacroExpand mexp(igvn); mexp.eliminate_macro_nodes(); @@ -2283,7 +2283,7 @@ void Compile::Optimize() { igvn.optimize(); } - DEBUG_ONLY( _modified_nodes = NULL; ) + DEBUG_ONLY( _modified_nodes = nullptr; ) assert(igvn._worklist.size() == 0, "not empty"); @@ -2508,7 +2508,7 @@ uint Compile::eval_macro_logic_op(uint func, uint in1 , uint in2, uint in3) { } static uint eval_operand(Node* n, ResourceHashtable& eval_map) { - assert(n != NULL, ""); + assert(n != nullptr, ""); assert(eval_map.contains(n), "absent"); return *(eval_map.get(n)); } @@ -2850,7 +2850,7 @@ void Compile::eliminate_redundant_card_marks(Node* n) { // Already converted to precedence edge for (uint i = mem->req(); i < mem->len(); i++) { // Accumulate any precedence edges - if (mem->in(i) != NULL) { + if (mem->in(i) != nullptr) { n->add_prec(mem->in(i)); } } @@ -2871,7 +2871,7 @@ void Compile::eliminate_redundant_card_marks(Node* n) { //------------------------------final_graph_reshaping_impl---------------------- // Implement items 1-5 from final_graph_reshaping below. -void Compile::final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc) { +void Compile::final_graph_reshaping_impl(Node *n, Final_Reshape_Counts& frc, Unique_Node_List& dead_nodes) { if ( n->outcnt() == 0 ) return; // dead node uint nop = n->Opcode(); @@ -2903,7 +2903,7 @@ void Compile::final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc) { #ifdef ASSERT if( n->is_Mem() ) { int alias_idx = get_alias_index(n->as_Mem()->adr_type()); - assert( n->in(0) != NULL || alias_idx != Compile::AliasIdxRaw || + assert( n->in(0) != nullptr || alias_idx != Compile::AliasIdxRaw || // oop will be recorded in oop map if load crosses safepoint n->is_Load() && (n->as_Load()->bottom_type()->isa_oopptr() || LoadNode::is_immutable_value(n->in(MemNode::Address))), @@ -2922,9 +2922,9 @@ void Compile::final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc) { } #endif // Count FPU ops and common calls, implements item (3) - bool gc_handled = BarrierSet::barrier_set()->barrier_set_c2()->final_graph_reshaping(this, n, nop); + bool gc_handled = BarrierSet::barrier_set()->barrier_set_c2()->final_graph_reshaping(this, n, nop, dead_nodes); if (!gc_handled) { - final_graph_reshaping_main_switch(n, frc, nop); + final_graph_reshaping_main_switch(n, frc, nop, dead_nodes); } // Collect CFG split points @@ -2933,7 +2933,7 @@ void Compile::final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc) { } } -void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& frc, uint nop) { +void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& frc, uint nop, Unique_Node_List& dead_nodes) { switch( nop ) { // Count all float operations that may use FPU case Op_AddF: @@ -3099,12 +3099,12 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f // Some platforms can use the constant pool to load ConP. // Do this transformation here since IGVN will convert ConN back to ConP. const Type* t = addp->bottom_type(); - bool is_oop = t->isa_oopptr() != NULL; - bool is_klass = t->isa_klassptr() != NULL; + bool is_oop = t->isa_oopptr() != nullptr; + bool is_klass = t->isa_klassptr() != nullptr; if ((is_oop && Matcher::const_oop_prefer_decode() ) || (is_klass && Matcher::const_klass_prefer_decode())) { - Node* nn = NULL; + Node* nn = nullptr; int op = is_oop ? Op_ConN : Op_ConNKlass; @@ -3113,13 +3113,13 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f uint cnt = r->outcnt(); for (uint i = 0; i < cnt; i++) { Node* m = r->raw_out(i); - if (m!= NULL && m->Opcode() == op && + if (m!= nullptr && m->Opcode() == op && m->bottom_type()->make_ptr() == t) { nn = m; break; } } - if (nn != NULL) { + if (nn != nullptr) { // Decode a narrow oop to match address // [R12 + narrow_oop_reg<<3 + offset] if (is_oop) { @@ -3136,7 +3136,7 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f #ifdef ASSERT for (uint j = 0; j < out_i->outcnt(); ++j) { Node *out_j = out_i->raw_out(j); - assert(out_j == NULL || !out_j->is_AddP() || out_j->in(AddPNode::Base) != addp, + assert(out_j == nullptr || !out_j->is_AddP() || out_j->in(AddPNode::Base) != addp, "more than 2 AddP nodes in a chain (out_j %u)", out_j->_idx); } #endif @@ -3160,7 +3160,7 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f // (if control is set already) on memory operations. Some CastPP // nodes don't have a control (don't carry a dependency): skip // those. - if (n->in(0) != NULL) { + if (n->in(0) != nullptr) { ResourceMark rm; Unique_Node_List wq; wq.push(n); @@ -3195,20 +3195,20 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f // // x86, ARM and friends can handle 2 adds in addressing mode // and Matcher can fold a DecodeN node into address by using - // a narrow oop directly and do implicit NULL check in address: + // a narrow oop directly and do implicit null check in address: // // [R12 + narrow_oop_reg<<3 + offset] // NullCheck narrow_oop_reg // // On other platforms (Sparc) we have to keep new DecodeN node and - // use it to do implicit NULL check in address: + // use it to do implicit null check in address: // // decode_not_null narrow_oop_reg, base_reg // [base_reg + offset] // NullCheck base_reg // // Pin the new DecodeN node to non-null path on these platform (Sparc) - // to keep the information to which NULL check the new DecodeN node + // to keep the information to which null check the new DecodeN node // corresponds to use it as value in implicit_null_check(). // new_in1->set_req(0, n->in(0)); @@ -3239,7 +3239,7 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f } assert(in1->is_DecodeNarrowPtr(), "sanity"); - Node* new_in2 = NULL; + Node* new_in2 = nullptr; if (in2->is_DecodeNarrowPtr()) { assert(in2->Opcode() == in1->Opcode(), "must be same node type"); new_in2 = in2->in(1); @@ -3254,24 +3254,24 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f new_in2 = ConNode::make(TypeNarrowOop::NULL_PTR); // // This transformation together with CastPP transformation above - // will generated code for implicit NULL checks for compressed oops. + // will generated code for implicit null checks for compressed oops. // // The original code after Optimize() // // LoadN memory, narrow_oop_reg // decode narrow_oop_reg, base_reg - // CmpP base_reg, NULL + // CmpP base_reg, nullptr // CastPP base_reg // NotNull // Load [base_reg + offset], val_reg // // after these transformations will be // // LoadN memory, narrow_oop_reg - // CmpN narrow_oop_reg, NULL + // CmpN narrow_oop_reg, nullptr // decode_not_null narrow_oop_reg, base_reg // Load [base_reg + offset], val_reg // - // and the uncommon path (== NULL) will use narrow_oop_reg directly + // and the uncommon path (== nullptr) will use narrow_oop_reg directly // since narrow oops can be used in debug info now (see the code in // final_graph_reshaping_walk()). // @@ -3295,7 +3295,7 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f new_in2 = ConNode::make(t->make_narrowklass()); } } - if (new_in2 != NULL) { + if (new_in2 != nullptr) { Node* cmpN = new CmpNNode(in1->in(1), new_in2); n->subsume_by(cmpN, this); if (in1->outcnt() == 0) { @@ -3313,7 +3313,7 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f assert(!n->in(1)->is_EncodeNarrowPtr(), "should be optimized out"); // DecodeN could be pinned when it can't be fold into // an address expression, see the code for Op_CastPP above. - assert(n->in(0) == NULL || (UseCompressedOops && !Matcher::narrow_oop_use_complex_address()), "no control"); + assert(n->in(0) == nullptr || (UseCompressedOops && !Matcher::narrow_oop_use_complex_address()), "no control"); break; case Op_EncodeP: @@ -3348,7 +3348,7 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f // then they will hang around and should just be replaced with // the original one. Merge them. Node* non_io_proj = proj->in(0)->as_Multi()->proj_out_or_null(proj->_con, false /*is_io_use*/); - if (non_io_proj != NULL) { + if (non_io_proj != nullptr) { proj->subsume_by(non_io_proj , this); } } @@ -3361,15 +3361,15 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f // The EncodeP optimization may create Phi with the same edges // for all paths. It is not handled well by Register Allocator. Node* unique_in = n->in(1); - assert(unique_in != NULL, ""); + assert(unique_in != nullptr, ""); uint cnt = n->req(); for (uint i = 2; i < cnt; i++) { Node* m = n->in(i); - assert(m != NULL, ""); + assert(m != nullptr, ""); if (unique_in != m) - unique_in = NULL; + unique_in = nullptr; } - if (unique_in != NULL) { + if (unique_in != nullptr) { n->subsume_by(unique_in, this); } } @@ -3487,13 +3487,13 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f Node* in2 = n->in(2); juint mask = (n->bottom_type() == TypeInt::INT) ? (BitsPerInt - 1) : (BitsPerLong - 1); const TypeInt* t = in2->find_int_type(); - if (t != NULL && t->is_con()) { + if (t != nullptr && t->is_con()) { juint shift = t->get_con(); if (shift > mask) { // Unsigned cmp n->set_req(2, ConNode::make(TypeInt::make(shift & mask))); } } else { - if (t == NULL || t->_lo < 0 || t->_hi > (int)mask) { + if (t == nullptr || t->_lo < 0 || t->_hi > (int)mask) { Node* shift = new AndINode(in2, ConNode::make(TypeInt::make(mask))); n->set_req(2, shift); } @@ -3518,22 +3518,8 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f // that input may be a chain of Phis. If those phis have no // other use, then the MemBarAcquire keeps them alive and // register allocation can be confused. - ResourceMark rm; - Unique_Node_List wq; - wq.push(n->in(MemBarNode::Precedent)); + dead_nodes.push(n->in(MemBarNode::Precedent)); n->set_req(MemBarNode::Precedent, top()); - while (wq.size() > 0) { - Node* m = wq.pop(); - if (m->outcnt() == 0 && m != top()) { - for (uint j = 0; j < m->req(); j++) { - Node* in = m->in(j); - if (in != NULL) { - wq.push(in); - } - } - m->disconnect_inputs(this); - } - } } break; } @@ -3561,7 +3547,7 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f for(;;) { // Loop over all nodes with identical inputs edges as m Node* k = m->find_similar(m->Opcode()); - if (k == NULL) { + if (k == nullptr) { break; } // Push their uses so we get a chance to remove node made @@ -3606,7 +3592,7 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f //------------------------------final_graph_reshaping_walk--------------------- // Replacing Opaque nodes with their input in final_graph_reshaping_impl(), // requires that the walk visits a node's inputs before visiting the node. -void Compile::final_graph_reshaping_walk( Node_Stack &nstack, Node *root, Final_Reshape_Counts &frc ) { +void Compile::final_graph_reshaping_walk(Node_Stack& nstack, Node* root, Final_Reshape_Counts& frc, Unique_Node_List& dead_nodes) { Unique_Node_List sfpt; frc._visited.set(root->_idx); // first, mark node as visited @@ -3618,8 +3604,8 @@ void Compile::final_graph_reshaping_walk( Node_Stack &nstack, Node *root, Final_ // Place all non-visited non-null inputs onto stack Node* m = n->in(i); ++i; - if (m != NULL && !frc._visited.test_set(m->_idx)) { - if (m->is_SafePoint() && m->as_SafePoint()->jvms() != NULL) { + if (m != nullptr && !frc._visited.test_set(m->_idx)) { + if (m->is_SafePoint() && m->as_SafePoint()->jvms() != nullptr) { // compute worst case interpreter size in case of a deoptimization update_interpreter_frame_size(m->as_SafePoint()->jvms()->interpreter_frame_size()); @@ -3632,7 +3618,7 @@ void Compile::final_graph_reshaping_walk( Node_Stack &nstack, Node *root, Final_ } } else { // Now do post-visit work - final_graph_reshaping_impl( n, frc ); + final_graph_reshaping_impl(n, frc, dead_nodes); if (nstack.is_empty()) break; // finished n = nstack.node(); // Get node from stack @@ -3653,7 +3639,7 @@ void Compile::final_graph_reshaping_walk( Node_Stack &nstack, Node *root, Final_ while (sfpt.size() > 0) { n = sfpt.pop(); JVMState *jvms = n->as_SafePoint()->jvms(); - assert(jvms != NULL, "sanity"); + assert(jvms != nullptr, "sanity"); int start = jvms->debug_start(); int end = n->req(); bool is_uncommon = (n->is_CallStaticJava() && @@ -3724,7 +3710,7 @@ bool Compile::final_graph_reshaping() { // be freely moved to the least frequent code path by gcm. assert(OptimizeExpensiveOps || expensive_count() == 0, "optimization off but list non empty?"); for (int i = 0; i < expensive_count(); i++) { - _expensive_nodes.at(i)->set_req(0, NULL); + _expensive_nodes.at(i)->set_req(0, nullptr); } Final_Reshape_Counts frc; @@ -3732,7 +3718,8 @@ bool Compile::final_graph_reshaping() { // Visit everybody reachable! // Allocate stack of size C->live_nodes()/2 to avoid frequent realloc Node_Stack nstack(live_nodes() >> 1); - final_graph_reshaping_walk(nstack, root(), frc); + Unique_Node_List dead_nodes; + final_graph_reshaping_walk(nstack, root(), frc, dead_nodes); // Check for unreachable (from below) code (i.e., infinite loops). for( uint i = 0; i < frc._tests.size(); i++ ) { @@ -3745,7 +3732,7 @@ bool Compile::final_graph_reshaping() { // 'fall-thru' path, so expected kids is 1 less. if (n->is_PCTable() && n->in(0) && n->in(0)->in(0)) { if (n->in(0)->in(0)->is_Call()) { - CallNode *call = n->in(0)->in(0)->as_Call(); + CallNode* call = n->in(0)->in(0)->as_Call(); if (call->entry_point() == OptoRuntime::rethrow_stub()) { required_outcnt--; // Rethrow always has 1 less kid } else if (call->req() > TypeFunc::Parms && @@ -3754,22 +3741,27 @@ bool Compile::final_graph_reshaping() { // detected that the virtual call will always result in a null // pointer exception. The fall-through projection of this CatchNode // will not be populated. - Node *arg0 = call->in(TypeFunc::Parms); + Node* arg0 = call->in(TypeFunc::Parms); if (arg0->is_Type() && arg0->as_Type()->type()->higher_equal(TypePtr::NULL_PTR)) { required_outcnt--; } - } else if (call->entry_point() == OptoRuntime::new_array_Java() && - call->req() > TypeFunc::Parms+1 && - call->is_CallStaticJava()) { - // Check for negative array length. In such case, the optimizer has + } else if (call->entry_point() == OptoRuntime::new_array_Java() || + call->entry_point() == OptoRuntime::new_array_nozero_Java()) { + // Check for illegal array length. In such case, the optimizer has // detected that the allocation attempt will always result in an // exception. There is no fall-through projection of this CatchNode . - Node *arg1 = call->in(TypeFunc::Parms+1); - if (arg1->is_Type() && - arg1->as_Type()->type()->join(TypeInt::POS)->empty()) { + assert(call->is_CallStaticJava(), "static call expected"); + assert(call->req() == call->jvms()->endoff() + 1, "missing extra input"); + uint valid_length_test_input = call->req() - 1; + Node* valid_length_test = call->in(valid_length_test_input); + call->del_req(valid_length_test_input); + if (valid_length_test->find_int_con(1) == 0) { required_outcnt--; } + dead_nodes.push(valid_length_test); + assert(n->outcnt() == required_outcnt, "malformed control flow"); + continue; } } } @@ -3778,6 +3770,16 @@ bool Compile::final_graph_reshaping() { record_method_not_compilable("malformed control flow"); return true; // Not all targets reachable! } + } else if (n->is_PCTable() && n->in(0) && n->in(0)->in(0) && n->in(0)->in(0)->is_Call()) { + CallNode* call = n->in(0)->in(0)->as_Call(); + if (call->entry_point() == OptoRuntime::new_array_Java() || + call->entry_point() == OptoRuntime::new_array_nozero_Java()) { + assert(call->is_CallStaticJava(), "static call expected"); + assert(call->req() == call->jvms()->endoff() + 1, "missing extra input"); + uint valid_length_test_input = call->req() - 1; + dead_nodes.push(call->in(valid_length_test_input)); + call->del_req(valid_length_test_input); // valid length test useless now + } } // Check that I actually visited all kids. Unreached kids // must be infinite loops. @@ -3796,6 +3798,19 @@ bool Compile::final_graph_reshaping() { } } + while (dead_nodes.size() > 0) { + Node* m = dead_nodes.pop(); + if (m->outcnt() == 0 && m != top()) { + for (uint j = 0; j < m->req(); j++) { + Node* in = m->in(j); + if (in != nullptr) { + dead_nodes.push(in); + } + } + m->disconnect_inputs(this); + } + } + #ifdef IA32 // If original bytecodes contained a mixture of floats and doubles // check if the optimizer has made it homogenous, item (3). @@ -3826,7 +3841,7 @@ bool Compile::too_many_traps(ciMethod* method, // because of a transient condition during start-up in the interpreter. return false; } - ciMethod* m = Deoptimization::reason_is_speculate(reason) ? this->method() : NULL; + ciMethod* m = Deoptimization::reason_is_speculate(reason) ? this->method() : nullptr; if (md->has_trap_at(bci, m, reason) != 0) { // Assume PerBytecodeTrapLimit==0, for a more conservative heuristic. // Also, if there are multiple reasons, or if there is no per-BCI record, @@ -3849,7 +3864,7 @@ bool Compile::too_many_traps(Deoptimization::DeoptReason reason, // Too many traps globally. // Note that we use cumulative trap_count, not just md->trap_count. if (log()) { - int mcount = (logmd == NULL)? -1: (int)logmd->trap_count(reason); + int mcount = (logmd == nullptr)? -1: (int)logmd->trap_count(reason); log()->elem("observe trap='%s' count='0' mcount='%d' ccount='%d'", Deoptimization::trap_reason_name(reason), mcount, trap_count(reason)); @@ -3880,7 +3895,7 @@ bool Compile::too_many_recompiles(ciMethod* method, uint m_cutoff = (uint) PerMethodRecompilationCutoff / 2 + 1; // not zero Deoptimization::DeoptReason per_bc_reason = Deoptimization::reason_recorded_per_bytecode_if_any(reason); - ciMethod* m = Deoptimization::reason_is_speculate(reason) ? this->method() : NULL; + ciMethod* m = Deoptimization::reason_is_speculate(reason) ? this->method() : nullptr; if ((per_bc_reason == Deoptimization::Reason_none || md->has_trap_at(bci, m, reason) != 0) // The trap frequency measure we care about is the recompile count: @@ -4008,10 +4023,10 @@ void Compile::verify_graph_edges(bool no_dead_code) { // behavior, the Compile's failure reason is quietly copied up to the ciEnv // by the logic in C2Compiler. void Compile::record_failure(const char* reason) { - if (log() != NULL) { + if (log() != nullptr) { log()->elem("failure reason='%s' phase='compile'", reason); } - if (_failure_reason == NULL) { + if (_failure_reason == nullptr) { // Record the first failure reason. _failure_reason = reason; } @@ -4019,7 +4034,7 @@ void Compile::record_failure(const char* reason) { if (!C->failure_reason_is(C2Compiler::retry_no_subsuming_loads())) { C->print_method(PHASE_FAILURE); } - _root = NULL; // flush the graph, too + _root = nullptr; // flush the graph, too } Compile::TracePhase::TracePhase(const char* name, elapsedTimer* accumulator) @@ -4030,10 +4045,10 @@ Compile::TracePhase::TracePhase(const char* name, elapsedTimer* accumulator) C = Compile::current(); _log = C->log(); } else { - C = NULL; - _log = NULL; + C = nullptr; + _log = nullptr; } - if (_log != NULL) { + if (_log != nullptr) { _log->begin_head("phase name='%s' nodes='%d' live='%d'", _phase_name, C->unique(), C->live_nodes()); _log->stamp(); _log->end_head(); @@ -4046,7 +4061,7 @@ Compile::TracePhase::~TracePhase() { if (_dolog) { _log = C->log(); } else { - _log = NULL; + _log = nullptr; } #ifdef ASSERT @@ -4060,7 +4075,7 @@ Compile::TracePhase::~TracePhase() { } #endif - if (_log != NULL) { + if (_log != nullptr) { _log->done("phase name='%s' nodes='%d' live='%d'", _phase_name, C->unique(), C->live_nodes()); } } @@ -4134,7 +4149,7 @@ Node* Compile::conv_I2X_index(PhaseGVN* phase, Node* idx, const TypeInt* sizetyp // number. (The prior range check has ensured this.) // This assertion is used by ConvI2LNode::Ideal. int index_max = max_jint - 1; // array size is max_jint, index is one less - if (sizetype != NULL) index_max = sizetype->_hi - 1; + if (sizetype != nullptr) index_max = sizetype->_hi - 1; const TypeInt* iidxtype = TypeInt::make(0, index_max, Type::WidenMax); idx = constrained_convI2L(phase, idx, iidxtype, ctrl); #endif @@ -4143,7 +4158,7 @@ Node* Compile::conv_I2X_index(PhaseGVN* phase, Node* idx, const TypeInt* sizetyp // Convert integer value to a narrowed long type dependent on ctrl (for example, a range check) Node* Compile::constrained_convI2L(PhaseGVN* phase, Node* value, const TypeInt* itype, Node* ctrl, bool carry_dependency) { - if (ctrl != NULL) { + if (ctrl != nullptr) { // Express control dependency by a CastII node with a narrow type. value = new CastIINode(value, itype, carry_dependency ? ConstraintCastNode::StrongDependency : ConstraintCastNode::RegularDependency, true /* range check dependency */); // Make the CastII node dependent on the control input to prevent the narrowed ConvI2L @@ -4215,14 +4230,14 @@ void Compile::print_inlining_update(CallGenerator* cg) { if (print_inlining() || print_intrinsics()) { if (cg->is_late_inline()) { if (print_inlining_current()->cg() != cg && - (print_inlining_current()->cg() != NULL || + (print_inlining_current()->cg() != nullptr || print_inlining_current()->ss()->size() != 0)) { print_inlining_push(); } print_inlining_commit(); print_inlining_current()->set_cg(cg); } else { - if (print_inlining_current()->cg() != NULL) { + if (print_inlining_current()->cg() != nullptr) { print_inlining_push(); } print_inlining_commit(); @@ -4264,16 +4279,16 @@ void Compile::process_print_inlining() { if (print_inlining() || print_intrinsics()) { ResourceMark rm; stringStream ss; - assert(_print_inlining_list != NULL, "process_print_inlining should be called only once."); + assert(_print_inlining_list != nullptr, "process_print_inlining should be called only once."); for (int i = 0; i < _print_inlining_list->length(); i++) { PrintInliningBuffer* pib = _print_inlining_list->at(i); ss.print("%s", pib->ss()->as_string()); delete pib; - DEBUG_ONLY(_print_inlining_list->at_put(i, NULL)); + DEBUG_ONLY(_print_inlining_list->at_put(i, nullptr)); } // Reset _print_inlining_list, it only contains destructed objects. // It is on the arena, so it will be freed when the arena is reset. - _print_inlining_list = NULL; + _print_inlining_list = nullptr; // _print_inlining_stream won't be used anymore, either. print_inlining_stream_free(); size_t end = ss.size(); @@ -4284,17 +4299,17 @@ void Compile::process_print_inlining() { } void Compile::dump_print_inlining() { - if (_print_inlining_output != NULL) { + if (_print_inlining_output != nullptr) { tty->print_raw(_print_inlining_output); } } void Compile::log_late_inline(CallGenerator* cg) { - if (log() != NULL) { + if (log() != nullptr) { log()->head("late_inline method='%d' inline_id='" JLONG_FORMAT "'", log()->identify(cg->method()), cg->unique_id()); JVMState* p = cg->call_node()->jvms(); - while (p != NULL) { + while (p != nullptr) { log()->elem("jvms bci='%d' method='%d'", p->bci(), log()->identify(p->method())); p = p->caller(); } @@ -4304,13 +4319,13 @@ void Compile::log_late_inline(CallGenerator* cg) { void Compile::log_late_inline_failure(CallGenerator* cg, const char* msg) { log_late_inline(cg); - if (log() != NULL) { + if (log() != nullptr) { log()->inline_fail(msg); } } void Compile::log_inline_id(CallGenerator* cg) { - if (log() != NULL) { + if (log() != nullptr) { // The LogCompilation tool needs a unique way to identify late // inline call sites. This id must be unique for this call site in // this compilation. Try to have it unique across compilations as @@ -4325,7 +4340,7 @@ void Compile::log_inline_id(CallGenerator* cg) { } void Compile::log_inline_failure(const char* msg) { - if (C->log() != NULL) { + if (C->log() != nullptr) { C->log()->inline_fail(msg); } } @@ -4335,7 +4350,7 @@ void Compile::log_inline_failure(const char* msg) { // Don't change thread state and acquire any locks. void Compile::dump_inline_data(outputStream* out) { InlineTree* inl_tree = ilt(); - if (inl_tree != NULL) { + if (inl_tree != nullptr) { out->print(" inline %d", inl_tree->count()); inl_tree->dump_replay_data(out); } @@ -4436,7 +4451,7 @@ void Compile::cleanup_expensive_nodes(PhaseIterGVN &igvn) { identical = 0; } else { Node* n = _expensive_nodes.at(i); - igvn.replace_input_of(n, 0, NULL); + igvn.replace_input_of(n, 0, nullptr); igvn.hash_insert(n); modified = true; } @@ -4445,7 +4460,7 @@ void Compile::cleanup_expensive_nodes(PhaseIterGVN &igvn) { _expensive_nodes.at_put(j++, _expensive_nodes.at(i)); } else if (_expensive_nodes.length() >= 1) { Node* n = _expensive_nodes.at(i); - igvn.replace_input_of(n, 0, NULL); + igvn.replace_input_of(n, 0, nullptr); igvn.hash_insert(n); modified = true; } @@ -4464,7 +4479,7 @@ void Compile::add_expensive_node(Node * n) { } else { // Clear control input and let IGVN optimize expensive nodes if // OptimizeExpensiveOps is off. - n->set_req(0, NULL); + n->set_req(0, nullptr); } } @@ -4612,7 +4627,7 @@ void Compile::remove_speculative_types(PhaseIterGVN &igvn) { for (uint next = 0; next < worklist.size(); ++next) { Node *n = worklist.at(next); const Type* t = igvn.type_or_null(n); - assert((t == NULL) || (t == t->remove_speculative()), "no more speculative types"); + assert((t == nullptr) || (t == t->remove_speculative()), "no more speculative types"); if (n->is_Type()) { t = n->as_Type()->type(); assert(t == t->remove_speculative(), "no more speculative types"); @@ -4776,10 +4791,10 @@ void Compile::print_method(CompilerPhaseType cpt, Node* n, int level) { ResourceMark rm; stringStream ss; ss.print_raw(CompilerPhaseTypeHelper::to_string(cpt)); - if (n != NULL) { + if (n != nullptr) { ss.print(": %d %s ", n->_idx, NodeClassNames[n->Opcode()]); } else { - ss.print_raw(": NULL"); + ss.print_raw(": nullptr"); } C->print_method(cpt, ss.as_string(), level); } @@ -4791,7 +4806,7 @@ void Compile::end_method(int level) { } #ifndef PRODUCT - if (_method != NULL && should_print(level)) { + if (_method != nullptr && should_print(level)) { _printer->end_method(); } #endif @@ -4799,8 +4814,8 @@ void Compile::end_method(int level) { #ifndef PRODUCT -IdealGraphPrinter* Compile::_debug_file_printer = NULL; -IdealGraphPrinter* Compile::_debug_network_printer = NULL; +IdealGraphPrinter* Compile::_debug_file_printer = nullptr; +IdealGraphPrinter* Compile::_debug_network_printer = nullptr; // Called from debugger. Prints method to the default file with the default phase name. // This works regardless of any Ideal Graph Visualizer flags set or not. @@ -4852,7 +4867,7 @@ void igv_append(const char* phase_name) { void Compile::igv_print_method_to_file(const char* phase_name, bool append) { const char* file_name = "custom_debug.xml"; - if (_debug_file_printer == NULL) { + if (_debug_file_printer == nullptr) { _debug_file_printer = new IdealGraphPrinter(C, file_name, append); } else { _debug_file_printer->update_compiled_method(C->method()); @@ -4862,7 +4877,7 @@ void Compile::igv_print_method_to_file(const char* phase_name, bool append) { } void Compile::igv_print_method_to_network(const char* phase_name) { - if (_debug_network_printer == NULL) { + if (_debug_network_printer == nullptr) { _debug_network_printer = new IdealGraphPrinter(C); } else { _debug_network_printer->update_compiled_method(C->method()); @@ -4877,10 +4892,10 @@ void Compile::add_native_invoker(RuntimeStub* stub) { } Node* Compile::narrow_value(BasicType bt, Node* value, const Type* type, PhaseGVN* phase, bool transform_res) { - if (type != NULL && phase->type(value)->higher_equal(type)) { + if (type != nullptr && phase->type(value)->higher_equal(type)) { return value; } - Node* result = NULL; + Node* result = nullptr; if (bt == T_BYTE) { result = phase->transform(new LShiftINode(value, phase->intcon(24))); result = new RShiftINode(result, phase->intcon(24)); diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index 6e5f2ed2305d5..cae1cb27c27db 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -140,7 +140,7 @@ class CloneMap { void* _2p(node_idx_t key) const { return (void*)(intptr_t)key; } // 2 conversion functions to make gcc happy node_idx_t _2_node_idx_t(const void* k) const { return (node_idx_t)(intptr_t)k; } Dict* dict() const { return _dict; } - void insert(node_idx_t key, uint64_t val) { assert(_dict->operator[](_2p(key)) == NULL, "key existed"); _dict->Insert(_2p(key), (void*)val); } + void insert(node_idx_t key, uint64_t val) { assert(_dict->operator[](_2p(key)) == nullptr, "key existed"); _dict->Insert(_2p(key), (void*)val); } void insert(node_idx_t key, NodeCloneInfo& ci) { insert(key, ci.get()); } void remove(node_idx_t key) { _dict->Delete(_2p(key)); } uint64_t value(node_idx_t key) const { return (uint64_t)_dict->operator[](_2p(key)); } @@ -176,7 +176,7 @@ class Compile : public Phase { AliasIdxRaw = 3 // hard-wired index for TypeRawPtr::BOTTOM }; - // Variant of TraceTime(NULL, &_t_accumulator, CITime); + // Variant of TraceTime(nullptr, &_t_accumulator, CITime); // Integrated with logging. If logging is turned on, and CITimeVerbose is true, // then brackets are put into the log, with time stamps and node counts. // (The time collection itself is always conditionalized on CITime.) @@ -225,7 +225,7 @@ class Compile : public Phase { } } void set_element(const Type* e) { - assert(_element == NULL, ""); + assert(_element == nullptr, ""); _element = e; } @@ -255,9 +255,9 @@ class Compile : public Phase { int _entry_bci; // entry bci for osr methods. const TypeFunc* _tf; // My kind of signature InlineTree* _ilt; // Ditto (temporary). - address _stub_function; // VM entry for stub being compiled, or NULL - const char* _stub_name; // Name of stub or adapter being compiled, or NULL - address _stub_entry_point; // Compile code entry for generated stub, or NULL + address _stub_function; // VM entry for stub being compiled, or null + const char* _stub_name; // Name of stub or adapter being compiled, or null + address _stub_entry_point; // Compile code entry for generated stub, or null // Control of this compilation. int _max_inline_size; // Max inline size for this compilation @@ -340,7 +340,7 @@ class Compile : public Phase { debug_only(static int _debug_idx;) // Monotonic counter (not reset), use -XX:BreakAtNode= Arena _node_arena; // Arena for new-space Nodes Arena _old_arena; // Arena for old-space Nodes, lifetime during xform - RootNode* _root; // Unique root of compilation, or NULL after bail-out. + RootNode* _root; // Unique root of compilation, or null after bail-out. Node* _top; // Unique top node. (Reset by various phases.) Node* _immutable_memory; // Initial memory state @@ -403,7 +403,7 @@ class Compile : public Phase { public: PrintInliningBuffer() - : _cg(NULL), _ss(default_stream_buffer_size) {} + : _cg(nullptr), _ss(default_stream_buffer_size) {} stringStream* ss() { return &_ss; } CallGenerator* cg() { return _cg; } @@ -451,7 +451,7 @@ class Compile : public Phase { void print_inlining_assert_ready(); void print_inlining_reset(); - void print_inlining(ciMethod* method, int inline_level, int bci, const char* msg = NULL) { + void print_inlining(ciMethod* method, int inline_level, int bci, const char* msg = nullptr) { stringStream ss; CompileTask::print_inlining_inner(&ss, method, inline_level, bci, msg); print_inlining_stream()->print("%s", ss.as_string()); @@ -519,9 +519,9 @@ class Compile : public Phase { ciMethod* method() const { return _method; } int entry_bci() const { return _entry_bci; } bool is_osr_compilation() const { return _entry_bci != InvocationEntryBci; } - bool is_method_compilation() const { return (_method != NULL && !_method->flags().is_native()); } - const TypeFunc* tf() const { assert(_tf!=NULL, ""); return _tf; } - void init_tf(const TypeFunc* tf) { assert(_tf==NULL, ""); _tf = tf; } + bool is_method_compilation() const { return (_method != nullptr && !_method->flags().is_native()); } + const TypeFunc* tf() const { assert(_tf!=nullptr, ""); return _tf; } + void init_tf(const TypeFunc* tf) { assert(_tf==nullptr, ""); _tf = tf; } InlineTree* ilt() const { return _ilt; } address stub_function() const { return _stub_function; } const char* stub_name() const { return _stub_name; } @@ -598,7 +598,7 @@ class Compile : public Phase { // check the CompilerOracle for special behaviours for this compile bool method_has_option(enum CompileCommand option) { - return method() != NULL && method()->has_option(option); + return method() != nullptr && method()->has_option(option); } #ifndef PRODUCT @@ -739,11 +739,11 @@ class Compile : public Phase { Arena* comp_arena() { return &_comp_arena; } ciEnv* env() const { return _env; } CompileLog* log() const { return _log; } - bool failing() const { return _env->failing() || _failure_reason != NULL; } + bool failing() const { return _env->failing() || _failure_reason != nullptr; } const char* failure_reason() const { return (_env->failing()) ? _env->failure_reason() : _failure_reason; } bool failure_reason_is(const char* r) const { - return (r == _failure_reason) || (r != NULL && _failure_reason != NULL && strcmp(r, _failure_reason) == 0); + return (r == _failure_reason) || (r != nullptr && _failure_reason != nullptr && strcmp(r, _failure_reason) == 0); } void record_failure(const char* reason); @@ -805,7 +805,7 @@ class Compile : public Phase { DEBUG_ONLY( Unique_Node_List* modified_nodes() const { return _modified_nodes; } ) MachConstantBaseNode* mach_constant_base_node(); - bool has_mach_constant_base_node() const { return _mach_constant_base_node != NULL; } + bool has_mach_constant_base_node() const { return _mach_constant_base_node != nullptr; } // Generated by adlc, true if CallNode requires MachConstantBase. bool needs_deep_clone_jvms(); @@ -849,16 +849,16 @@ class Compile : public Phase { void set_type_last_size(size_t sz) { _type_last_size = sz; } const TypeFunc* last_tf(ciMethod* m) { - return (m == _last_tf_m) ? _last_tf : NULL; + return (m == _last_tf_m) ? _last_tf : nullptr; } void set_last_tf(ciMethod* m, const TypeFunc* tf) { - assert(m != NULL || tf == NULL, ""); + assert(m != nullptr || tf == nullptr, ""); _last_tf_m = m; _last_tf = tf; } AliasType* alias_type(int idx) { assert(idx < num_alias_types(), "oob"); return _alias_types[idx]; } - AliasType* alias_type(const TypePtr* adr_type, ciField* field = NULL) { return find_alias_type(adr_type, false, field); } + AliasType* alias_type(const TypePtr* adr_type, ciField* field = nullptr) { return find_alias_type(adr_type, false, field); } bool have_alias_type(const TypePtr* adr_type); AliasType* alias_type(ciField* field); @@ -874,7 +874,7 @@ class Compile : public Phase { // Decide how to build a call. // The profile factor is a discount to apply to this site's interp. profile. CallGenerator* call_generator(ciMethod* call_method, int vtable_index, bool call_does_dispatch, - JVMState* jvms, bool allow_inline, float profile_factor, ciKlass* speculative_receiver_type = NULL, + JVMState* jvms, bool allow_inline, float profile_factor, ciKlass* speculative_receiver_type = nullptr, bool allow_intrinsics = true); bool should_delay_inlining(ciMethod* call_method, JVMState* jvms) { return should_delay_string_inlining(call_method, jvms) || @@ -904,7 +904,7 @@ class Compile : public Phase { // PerMethodTrapLimit was exceeded for all inlined methods seen so far. bool too_many_traps(Deoptimization::DeoptReason reason, // Privately used parameter for logging: - ciMethodData* logmd = NULL); + ciMethodData* logmd = nullptr); // Report if there were too many recompiles at a method and bci. bool too_many_recompiles(ciMethod* method, int bci, Deoptimization::DeoptReason reason); // Report if there were too many traps or recompiles at a method and bci. @@ -921,7 +921,8 @@ class Compile : public Phase { // Parsing, optimization PhaseGVN* initial_gvn() { return _initial_gvn; } Unique_Node_List* for_igvn() { return _for_igvn; } - inline void record_for_igvn(Node* n); // Body is after class Unique_Node_List. + inline void record_for_igvn(Node* n); // Body is after class Unique_Node_List in node.hpp. + inline void remove_for_igvn(Node* n); // Body is after class Unique_Node_List in node.hpp. void set_initial_gvn(PhaseGVN *gvn) { _initial_gvn = gvn; } void set_for_igvn(Unique_Node_List *for_igvn) { _for_igvn = for_igvn; } @@ -1047,7 +1048,7 @@ class Compile : public Phase { bool return_pc, DirectiveSet* directive); // Are we compiling a method? - bool has_method() { return method() != NULL; } + bool has_method() { return method() != nullptr; } // Maybe print some information about this compile. void print_compile_messages(); @@ -1113,9 +1114,9 @@ class Compile : public Phase { #endif // Function calls made by the public function final_graph_reshaping. // No need to be made public as they are not called elsewhere. - void final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc); - void final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& frc, uint nop); - void final_graph_reshaping_walk( Node_Stack &nstack, Node *root, Final_Reshape_Counts &frc ); + void final_graph_reshaping_impl(Node *n, Final_Reshape_Counts& frc, Unique_Node_List& dead_nodes); + void final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& frc, uint nop, Unique_Node_List& dead_nodes); + void final_graph_reshaping_walk(Node_Stack& nstack, Node* root, Final_Reshape_Counts& frc, Unique_Node_List& dead_nodes); void eliminate_redundant_card_marks(Node* n); // Logic cone optimization. @@ -1165,7 +1166,7 @@ class Compile : public Phase { static Node* conv_I2X_index(PhaseGVN* phase, Node* offset, const TypeInt* sizetype, // Optional control dependency (for example, on range check) - Node* ctrl = NULL); + Node* ctrl = nullptr); // Convert integer value to a narrowed long type dependent on ctrl (for example, a range check) static Node* constrained_convI2L(PhaseGVN* phase, Node* value, const TypeInt* itype, Node* ctrl, bool carry_dependency = false); diff --git a/src/hotspot/share/opto/connode.cpp b/src/hotspot/share/opto/connode.cpp index 1bf7aaaf796df..00049c6f19e36 100644 --- a/src/hotspot/share/opto/connode.cpp +++ b/src/hotspot/share/opto/connode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -62,6 +62,6 @@ ConNode *ConNode::make(const Type *t) { // or else TypeOopPtr::NULL_PTR. Then set Type::_basic_type[AnyPtr] = T_ILLEGAL default: ShouldNotReachHere(); - return NULL; + return nullptr; } } diff --git a/src/hotspot/share/opto/connode.hpp b/src/hotspot/share/opto/connode.hpp index ef0318b22750e..565c42033300f 100644 --- a/src/hotspot/share/opto/connode.hpp +++ b/src/hotspot/share/opto/connode.hpp @@ -39,6 +39,7 @@ class ConNode : public TypeNode { ConNode( const Type *t ) : TypeNode(t->remove_speculative(),1) { init_req(0, (Node*)Compile::current()->root()); init_flags(Flag_is_Con); + init_class_id(Class_Con); } virtual int Opcode() const; virtual uint hash() const; @@ -53,7 +54,9 @@ class ConNode : public TypeNode { // Simple integer constants class ConINode : public ConNode { public: - ConINode( const TypeInt *t ) : ConNode(t) {} + ConINode(const TypeInt *t) : ConNode(t) { + init_class_id(Class_ConI); + } virtual int Opcode() const; // Factory method: @@ -67,15 +70,16 @@ class ConINode : public ConNode { // Simple pointer constants class ConPNode : public ConNode { public: - ConPNode( const TypePtr *t ) : ConNode(t) {} + ConPNode(const TypePtr *t) : ConNode(t) {} virtual int Opcode() const; // Factory methods: static ConPNode* make(address con) { - if (con == NULL) - return new ConPNode( TypePtr::NULL_PTR ) ; - else - return new ConPNode( TypeRawPtr::make(con) ); + if (con == nullptr) { + return new ConPNode(TypePtr::NULL_PTR); + } else { + return new ConPNode(TypeRawPtr::make(con)); + } } }; diff --git a/src/hotspot/share/opto/constantTable.cpp b/src/hotspot/share/opto/constantTable.cpp index 285c273bb22cc..37277fa916b80 100644 --- a/src/hotspot/share/opto/constantTable.cpp +++ b/src/hotspot/share/opto/constantTable.cpp @@ -118,7 +118,7 @@ bool ConstantTable::emit(CodeBuffer& cb) const { MacroAssembler _masm(&cb); for (int i = 0; i < _constants.length(); i++) { Constant con = _constants.at(i); - address constant_addr = NULL; + address constant_addr = nullptr; switch (con.type()) { case T_INT: constant_addr = _masm.int_constant( con.get_jint() ); break; case T_LONG: constant_addr = _masm.long_constant( con.get_jlong() ); break; @@ -143,17 +143,17 @@ bool ConstantTable::emit(CodeBuffer& cb) const { // filled in later in fill_jump_table. address dummy = (address) n; constant_addr = _masm.address_constant(dummy); - if (constant_addr == NULL) { + if (constant_addr == nullptr) { return false; } assert((constant_addr - _masm.code()->consts()->start()) == con.offset(), "must be: %d == %d", (int)(constant_addr - _masm.code()->consts()->start()), (int)(con.offset())); // Expand jump-table - address last_addr = NULL; + address last_addr = nullptr; for (uint j = 1; j < n->outcnt(); j++) { last_addr = _masm.address_constant(dummy + j); - if (last_addr == NULL) { + if (last_addr == nullptr) { return false; } } @@ -177,7 +177,7 @@ bool ConstantTable::emit(CodeBuffer& cb) const { default: ShouldNotReachHere(); } - if (constant_addr == NULL) { + if (constant_addr == nullptr) { return false; } assert((constant_addr - _masm.code()->consts()->start()) == con.offset(), diff --git a/src/hotspot/share/opto/convertnode.cpp b/src/hotspot/share/opto/convertnode.cpp index 501cb08c45209..961e1f4a5f201 100644 --- a/src/hotspot/share/opto/convertnode.cpp +++ b/src/hotspot/share/opto/convertnode.cpp @@ -49,7 +49,7 @@ const Type* Conv2BNode::Value(PhaseGVN* phase) const { if( t == TypeInt::ZERO ) return TypeInt::ZERO; if( t == TypePtr::NULL_PTR ) return TypeInt::ZERO; const TypePtr *tp = t->isa_ptr(); - if( tp != NULL ) { + if(tp != nullptr) { if( tp->ptr() == TypePtr::AnyNull ) return Type::TOP; if( tp->ptr() == TypePtr::Constant) return TypeInt::ONE; if (tp->ptr() == TypePtr::NotNull) return TypeInt::ONE; @@ -85,7 +85,7 @@ Node *ConvD2FNode::Ideal(PhaseGVN *phase, bool can_reshape) { } } } - return NULL; + return nullptr; } //------------------------------Identity--------------------------------------- @@ -112,7 +112,7 @@ Node *ConvD2INode::Ideal(PhaseGVN *phase, bool can_reshape) { set_req(1, in(1)->in(1)); return this; } - return NULL; + return nullptr; } //------------------------------Identity--------------------------------------- @@ -148,7 +148,7 @@ Node *ConvD2LNode::Ideal(PhaseGVN *phase, bool can_reshape) { set_req(1, in(1)->in(1)); return this; } - return NULL; + return nullptr; } //============================================================================= @@ -187,7 +187,7 @@ Node *ConvF2INode::Ideal(PhaseGVN *phase, bool can_reshape) { set_req(1, in(1)->in(1)); return this; } - return NULL; + return nullptr; } //============================================================================= @@ -216,7 +216,7 @@ Node *ConvF2LNode::Ideal(PhaseGVN *phase, bool can_reshape) { set_req(1, in(1)->in(1)); return this; } - return NULL; + return nullptr; } //============================================================================= @@ -275,7 +275,7 @@ static Node* find_or_make_convI2L(PhaseIterGVN* igvn, Node* parent, const TypeLong* type) { Node* n = new ConvI2LNode(parent, type); Node* existing = igvn->hash_find_insert(n); - if (existing != NULL) { + if (existing != nullptr) { n->destruct(igvn); return existing; } @@ -430,10 +430,10 @@ Node *ConvI2LNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Addressing arithmetic will not absorb it as part of a 64-bit AddL. Node* z = in(1); - const TypeInteger* rx = NULL; - const TypeInteger* ry = NULL; + const TypeInteger* rx = nullptr; + const TypeInteger* ry = nullptr; if (Compile::push_thru_add(phase, z, this_type, rx, ry, T_LONG)) { - if (igvn == NULL) { + if (igvn == nullptr) { // Postpone this optimization to iterative GVN, where we can handle deep // AddI chains without an exponential number of recursive Ideal() calls. phase->record_for_igvn(this); @@ -518,13 +518,13 @@ Node *ConvL2INode::Ideal(PhaseGVN *phase, bool can_reshape) { if( andl_op == Op_AddL ) { // Don't do this for nodes which have more than one user since // we'll end up computing the long add anyway. - if (andl->outcnt() > 1) return NULL; + if (andl->outcnt() > 1) return nullptr; Node* x = andl->in(1); Node* y = andl->in(2); assert( x != andl && y != andl, "dead loop in ConvL2INode::Ideal" ); - if (phase->type(x) == Type::TOP) return NULL; - if (phase->type(y) == Type::TOP) return NULL; + if (phase->type(x) == Type::TOP) return nullptr; + if (phase->type(y) == Type::TOP) return nullptr; Node *add1 = phase->transform(new ConvL2INode(x)); Node *add2 = phase->transform(new ConvL2INode(y)); return new AddINode(add1,add2); @@ -533,7 +533,7 @@ Node *ConvL2INode::Ideal(PhaseGVN *phase, bool can_reshape) { // Disable optimization: LoadL->ConvL2I ==> LoadI. // It causes problems (sizes of Load and Store nodes do not match) // in objects initialization code and Escape Analysis. - return NULL; + return nullptr; } diff --git a/src/hotspot/share/opto/divnode.cpp b/src/hotspot/share/opto/divnode.cpp index 26cf1c941238e..3335638d95430 100644 --- a/src/hotspot/share/opto/divnode.cpp +++ b/src/hotspot/share/opto/divnode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -89,7 +89,7 @@ static bool magic_int_divide_constants(jint d, jint &M, jint &s) { //--------------------------transform_int_divide------------------------------- // Convert a division by constant divisor into an alternate Ideal graph. -// Return NULL if no transformation occurs. +// Return null if no transformation occurs. static Node *transform_int_divide( PhaseGVN *phase, Node *dividend, jint divisor ) { // Check for invalid divisors @@ -101,7 +101,7 @@ static Node *transform_int_divide( PhaseGVN *phase, Node *dividend, jint divisor const int N = 32; // Result - Node *q = NULL; + Node *q = nullptr; if (d == 1) { // division by +/- 1 @@ -334,7 +334,7 @@ static Node* long_by_long_mulhi(PhaseGVN* phase, Node* dividend, jlong magic_con //--------------------------transform_long_divide------------------------------ // Convert a division by constant divisor into an alternate Ideal graph. -// Return NULL if no transformation occurs. +// Return null if no transformation occurs. static Node *transform_long_divide( PhaseGVN *phase, Node *dividend, jlong divisor ) { // Check for invalid divisors assert( divisor != 0L && divisor != min_jlong, @@ -345,7 +345,7 @@ static Node *transform_long_divide( PhaseGVN *phase, Node *dividend, jlong divis const int N = 64; // Result - Node *q = NULL; + Node *q = nullptr; if (d == 1) { // division by +/- 1 @@ -460,29 +460,29 @@ Node* DivINode::Identity(PhaseGVN* phase) { Node *DivINode::Ideal(PhaseGVN *phase, bool can_reshape) { if (in(0) && remove_dead_region(phase, can_reshape)) return this; // Don't bother trying to transform a dead node - if( in(0) && in(0)->is_top() ) return NULL; + if( in(0) && in(0)->is_top() ) return nullptr; const Type *t = phase->type( in(2) ); - if( t == TypeInt::ONE ) // Identity? - return NULL; // Skip it + if( t == TypeInt::ONE ) // Identity? + return nullptr; // Skip it const TypeInt *ti = t->isa_int(); - if( !ti ) return NULL; + if( !ti ) return nullptr; // Check for useless control input // Check for excluding div-zero case if (in(0) && (ti->_hi < 0 || ti->_lo > 0)) { - set_req(0, NULL); // Yank control input + set_req(0, nullptr); // Yank control input return this; } - if( !ti->is_con() ) return NULL; + if( !ti->is_con() ) return nullptr; jint i = ti->get_con(); // Get divisor - if (i == 0) return NULL; // Dividing by zero constant does not idealize + if (i == 0) return nullptr; // Dividing by zero constant does not idealize // Dividing by MININT does not optimize as a power-of-2 shift. - if( i == min_jint ) return NULL; + if( i == min_jint ) return nullptr; return transform_int_divide( phase, in(1), i ); } @@ -566,29 +566,29 @@ Node* DivLNode::Identity(PhaseGVN* phase) { Node *DivLNode::Ideal( PhaseGVN *phase, bool can_reshape) { if (in(0) && remove_dead_region(phase, can_reshape)) return this; // Don't bother trying to transform a dead node - if( in(0) && in(0)->is_top() ) return NULL; + if( in(0) && in(0)->is_top() ) return nullptr; const Type *t = phase->type( in(2) ); if( t == TypeLong::ONE ) // Identity? - return NULL; // Skip it + return nullptr; // Skip it const TypeLong *tl = t->isa_long(); - if( !tl ) return NULL; + if( !tl ) return nullptr; // Check for useless control input // Check for excluding div-zero case if (in(0) && (tl->_hi < 0 || tl->_lo > 0)) { - set_req(0, NULL); // Yank control input + set_req(0, nullptr); // Yank control input return this; } - if( !tl->is_con() ) return NULL; + if( !tl->is_con() ) return nullptr; jlong l = tl->get_con(); // Get divisor - if (l == 0) return NULL; // Dividing by zero constant does not idealize + if (l == 0) return nullptr; // Dividing by zero constant does not idealize // Dividing by MINLONG does not optimize as a power-of-2 shift. - if( l == min_jlong ) return NULL; + if( l == min_jlong ) return nullptr; return transform_long_divide( phase, in(1), l ); } @@ -717,28 +717,28 @@ Node* DivFNode::Identity(PhaseGVN* phase) { Node *DivFNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (in(0) && remove_dead_region(phase, can_reshape)) return this; // Don't bother trying to transform a dead node - if( in(0) && in(0)->is_top() ) return NULL; + if( in(0) && in(0)->is_top() ) return nullptr; const Type *t2 = phase->type( in(2) ); if( t2 == TypeF::ONE ) // Identity? - return NULL; // Skip it + return nullptr; // Skip it const TypeF *tf = t2->isa_float_constant(); - if( !tf ) return NULL; - if( tf->base() != Type::FloatCon ) return NULL; + if( !tf ) return nullptr; + if( tf->base() != Type::FloatCon ) return nullptr; // Check for out of range values - if( tf->is_nan() || !tf->is_finite() ) return NULL; + if( tf->is_nan() || !tf->is_finite() ) return nullptr; // Get the value float f = tf->getf(); int exp; // Only for special case of dividing by a power of 2 - if( frexp((double)f, &exp) != 0.5 ) return NULL; + if( frexp((double)f, &exp) != 0.5 ) return nullptr; // Limit the range of acceptable exponents - if( exp < -126 || exp > 126 ) return NULL; + if( exp < -126 || exp > 126 ) return nullptr; // Compute the reciprocal float reciprocal = ((float)1.0) / f; @@ -809,28 +809,28 @@ Node* DivDNode::Identity(PhaseGVN* phase) { Node *DivDNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (in(0) && remove_dead_region(phase, can_reshape)) return this; // Don't bother trying to transform a dead node - if( in(0) && in(0)->is_top() ) return NULL; + if( in(0) && in(0)->is_top() ) return nullptr; const Type *t2 = phase->type( in(2) ); if( t2 == TypeD::ONE ) // Identity? - return NULL; // Skip it + return nullptr; // Skip it const TypeD *td = t2->isa_double_constant(); - if( !td ) return NULL; - if( td->base() != Type::DoubleCon ) return NULL; + if( !td ) return nullptr; + if( td->base() != Type::DoubleCon ) return nullptr; // Check for out of range values - if( td->is_nan() || !td->is_finite() ) return NULL; + if( td->is_nan() || !td->is_finite() ) return nullptr; // Get the value double d = td->getd(); int exp; // Only for special case of dividing by a power of 2 - if( frexp(d, &exp) != 0.5 ) return NULL; + if( frexp(d, &exp) != 0.5 ) return nullptr; // Limit the range of acceptable exponents - if( exp < -1021 || exp > 1022 ) return NULL; + if( exp < -1021 || exp > 1022 ) return nullptr; // Compute the reciprocal double reciprocal = 1.0 / d; @@ -847,22 +847,22 @@ Node *ModINode::Ideal(PhaseGVN *phase, bool can_reshape) { // Check for dead control input if( in(0) && remove_dead_region(phase, can_reshape) ) return this; // Don't bother trying to transform a dead node - if( in(0) && in(0)->is_top() ) return NULL; + if( in(0) && in(0)->is_top() ) return nullptr; // Get the modulus const Type *t = phase->type( in(2) ); - if( t == Type::TOP ) return NULL; + if( t == Type::TOP ) return nullptr; const TypeInt *ti = t->is_int(); // Check for useless control input // Check for excluding mod-zero case if (in(0) && (ti->_hi < 0 || ti->_lo > 0)) { - set_req(0, NULL); // Yank control input + set_req(0, nullptr); // Yank control input return this; } // See if we are MOD'ing by 2^k or 2^k-1. - if( !ti->is_con() ) return NULL; + if( !ti->is_con() ) return nullptr; jint con = ti->get_con(); Node *hook = new Node(1); @@ -915,7 +915,7 @@ Node *ModINode::Ideal(PhaseGVN *phase, bool can_reshape) { // into a long multiply/int multiply/subtract case // Cannot handle mod 0, and min_jint isn't handled by the transform - if( con == 0 || con == min_jint ) return NULL; + if( con == 0 || con == min_jint ) return nullptr; // Get the absolute value of the constant; at this point, we can use this jint pos_con = (con >= 0) ? con : -con; @@ -942,11 +942,11 @@ Node *ModINode::Ideal(PhaseGVN *phase, bool can_reshape) { // Divide using the transform from DivI to MulL Node *result = transform_int_divide( phase, in(1), pos_con ); - if (result != NULL) { + if (result != nullptr) { Node *divide = phase->transform(result); // Re-multiply, using a shift if this is a power of two - Node *mult = NULL; + Node *mult = nullptr; if( log2_con >= 0 ) mult = phase->transform( new LShiftINode( divide, phase->intcon( log2_con ) ) ); @@ -1012,22 +1012,22 @@ Node *ModLNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Check for dead control input if( in(0) && remove_dead_region(phase, can_reshape) ) return this; // Don't bother trying to transform a dead node - if( in(0) && in(0)->is_top() ) return NULL; + if( in(0) && in(0)->is_top() ) return nullptr; // Get the modulus const Type *t = phase->type( in(2) ); - if( t == Type::TOP ) return NULL; + if( t == Type::TOP ) return nullptr; const TypeLong *tl = t->is_long(); // Check for useless control input // Check for excluding mod-zero case if (in(0) && (tl->_hi < 0 || tl->_lo > 0)) { - set_req(0, NULL); // Yank control input + set_req(0, nullptr); // Yank control input return this; } // See if we are MOD'ing by 2^k or 2^k-1. - if( !tl->is_con() ) return NULL; + if( !tl->is_con() ) return nullptr; jlong con = tl->get_con(); Node *hook = new Node(1); @@ -1082,7 +1082,7 @@ Node *ModLNode::Ideal(PhaseGVN *phase, bool can_reshape) { // into a long multiply/int multiply/subtract case // Cannot handle mod 0, and min_jlong isn't handled by the transform - if( con == 0 || con == min_jlong ) return NULL; + if( con == 0 || con == min_jlong ) return nullptr; // Get the absolute value of the constant; at this point, we can use this jlong pos_con = (con >= 0) ? con : -con; @@ -1109,11 +1109,11 @@ Node *ModLNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Divide using the transform from DivL to MulL Node *result = transform_long_divide( phase, in(1), pos_con ); - if (result != NULL) { + if (result != nullptr) { Node *divide = phase->transform(result); // Re-multiply, using a shift if this is a power of two - Node *mult = NULL; + Node *mult = nullptr; if( log2_con >= 0 ) mult = phase->transform( new LShiftLNode( divide, phase->intcon( log2_con ) ) ); diff --git a/src/hotspot/share/opto/divnode.hpp b/src/hotspot/share/opto/divnode.hpp index 7af3432a2d3ec..2347f2d5d4937 100644 --- a/src/hotspot/share/opto/divnode.hpp +++ b/src/hotspot/share/opto/divnode.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -148,7 +148,7 @@ class DivModNode : public MultiNode { }; virtual int Opcode() const; virtual Node* Identity(PhaseGVN* phase) { return this; } - virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { return NULL; } + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { return nullptr; } virtual const Type* Value(PhaseGVN* phase) const { return bottom_type(); } virtual uint hash() const { return Node::hash(); } virtual bool is_CFG() const { return false; } diff --git a/src/hotspot/share/opto/doCall.cpp b/src/hotspot/share/opto/doCall.cpp index d343075e8aaa1..d42f9367dac42 100644 --- a/src/hotspot/share/opto/doCall.cpp +++ b/src/hotspot/share/opto/doCall.cpp @@ -67,7 +67,7 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool JVMState* jvms, bool allow_inline, float prof_factor, ciKlass* speculative_receiver_type, bool allow_intrinsics) { - assert(callee != NULL, "failed method resolution"); + assert(callee != nullptr, "failed method resolution"); ciMethod* caller = jvms->method(); int bci = jvms->bci(); @@ -99,7 +99,7 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool } CompileLog* log = this->log(); - if (log != NULL) { + if (log != nullptr) { int rid = (receiver_count >= 0)? log->identify(profile.receiver(0)): -1; int r2id = (rid != -1 && profile.has_receiver(1))? log->identify(profile.receiver(1)):-1; log->begin_elem("call method='%d' count='%d' prof_factor='%f'", @@ -121,15 +121,15 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool // Special case the handling of certain common, profitable library // methods. If these methods are replaced with specialized code, // then we return it as the inlined version of the call. - CallGenerator* cg_intrinsic = NULL; + CallGenerator* cg_intrinsic = nullptr; if (allow_inline && allow_intrinsics) { CallGenerator* cg = find_intrinsic(callee, call_does_dispatch); - if (cg != NULL) { + if (cg != nullptr) { if (cg->is_predicated()) { // Code without intrinsic but, hopefully, inlined. CallGenerator* inline_cg = this->call_generator(callee, vtable_index, call_does_dispatch, jvms, allow_inline, prof_factor, speculative_receiver_type, false); - if (inline_cg != NULL) { + if (inline_cg != nullptr) { cg = CallGenerator::for_predicated_intrinsic(cg, inline_cg); } } @@ -139,7 +139,7 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool // We will retry the intrinsic if nothing had claimed it afterwards. if (cg->does_virtual_dispatch()) { cg_intrinsic = cg; - cg = NULL; + cg = nullptr; } else if (IncrementalInline && should_delay_vector_inlining(callee, jvms)) { return CallGenerator::for_late_inline(callee, cg); } else { @@ -177,13 +177,13 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool // sometimes has a broader type. Similar scenario is possible with // default methods when type system loses information about implemented // interfaces. - if (cg != NULL && is_virtual_or_interface && !callee->is_static()) { + if (cg != nullptr && is_virtual_or_interface && !callee->is_static()) { CallGenerator* trap_cg = CallGenerator::for_uncommon_trap(callee, Deoptimization::Reason_receiver_constraint, Deoptimization::Action_none); cg = CallGenerator::for_guarded_call(callee->holder(), trap_cg, cg); } - if (cg != NULL) { + if (cg != nullptr) { // Delay the inlining of this method to give us the // opportunity to perform some high level optimizations // first. @@ -206,10 +206,10 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool if (call_does_dispatch && site_count > 0 && UseTypeProfile) { // The major receiver's count >= TypeProfileMajorReceiverPercent of site_count. bool have_major_receiver = profile.has_receiver(0) && (100.*profile.receiver_prob(0) >= (float)TypeProfileMajorReceiverPercent); - ciMethod* receiver_method = NULL; + ciMethod* receiver_method = nullptr; int morphism = profile.morphism(); - if (speculative_receiver_type != NULL) { + if (speculative_receiver_type != nullptr) { if (!too_many_traps_or_recompiles(caller, bci, Deoptimization::Reason_speculate_class_check)) { // We have a speculative type, we should be able to resolve // the call. We do that before looking at the profiling at @@ -217,18 +217,18 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool // a speculative type should help us avoid. receiver_method = callee->resolve_invoke(jvms->method()->holder(), speculative_receiver_type); - if (receiver_method == NULL) { - speculative_receiver_type = NULL; + if (receiver_method == nullptr) { + speculative_receiver_type = nullptr; } else { morphism = 1; } } else { // speculation failed before. Use profiling at the call // (could allow bimorphic inlining for instance). - speculative_receiver_type = NULL; + speculative_receiver_type = nullptr; } } - if (receiver_method == NULL && + if (receiver_method == nullptr && (have_major_receiver || morphism == 1 || (morphism == 2 && UseBimorphicInlining))) { // receiver_method = profile.method(); @@ -236,33 +236,33 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool receiver_method = callee->resolve_invoke(jvms->method()->holder(), profile.receiver(0)); } - if (receiver_method != NULL) { + if (receiver_method != nullptr) { // The single majority receiver sufficiently outweighs the minority. CallGenerator* hit_cg = this->call_generator(receiver_method, vtable_index, !call_does_dispatch, jvms, allow_inline, prof_factor); - if (hit_cg != NULL) { + if (hit_cg != nullptr) { // Look up second receiver. - CallGenerator* next_hit_cg = NULL; - ciMethod* next_receiver_method = NULL; + CallGenerator* next_hit_cg = nullptr; + ciMethod* next_receiver_method = nullptr; if (morphism == 2 && UseBimorphicInlining) { next_receiver_method = callee->resolve_invoke(jvms->method()->holder(), profile.receiver(1)); - if (next_receiver_method != NULL) { + if (next_receiver_method != nullptr) { next_hit_cg = this->call_generator(next_receiver_method, vtable_index, !call_does_dispatch, jvms, allow_inline, prof_factor); - if (next_hit_cg != NULL && !next_hit_cg->is_inline() && + if (next_hit_cg != nullptr && !next_hit_cg->is_inline() && have_major_receiver && UseOnlyInlinedBimorphic) { // Skip if we can't inline second receiver's method - next_hit_cg = NULL; + next_hit_cg = nullptr; } } } CallGenerator* miss_cg; Deoptimization::DeoptReason reason = (morphism == 2 ? Deoptimization::Reason_bimorphic - : Deoptimization::reason_class_check(speculative_receiver_type != NULL)); - if ((morphism == 1 || (morphism == 2 && next_hit_cg != NULL)) && + : Deoptimization::reason_class_check(speculative_receiver_type != nullptr)); + if ((morphism == 1 || (morphism == 2 && next_hit_cg != nullptr)) && !too_many_traps_or_recompiles(caller, bci, reason) ) { // Generate uncommon trap for class check failure path @@ -275,20 +275,20 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool miss_cg = (IncrementalInlineVirtual ? CallGenerator::for_late_inline_virtual(callee, vtable_index, prof_factor) : CallGenerator::for_virtual_call(callee, vtable_index)); } - if (miss_cg != NULL) { - if (next_hit_cg != NULL) { - assert(speculative_receiver_type == NULL, "shouldn't end up here if we used speculation"); + if (miss_cg != nullptr) { + if (next_hit_cg != nullptr) { + assert(speculative_receiver_type == nullptr, "shouldn't end up here if we used speculation"); trace_type_profile(C, jvms->method(), jvms->depth() - 1, jvms->bci(), next_receiver_method, profile.receiver(1), site_count, profile.receiver_count(1)); // We don't need to record dependency on a receiver here and below. // Whenever we inline, the dependency is added by Parse::Parse(). miss_cg = CallGenerator::for_predicted_call(profile.receiver(1), miss_cg, next_hit_cg, PROB_MAX); } - if (miss_cg != NULL) { - ciKlass* k = speculative_receiver_type != NULL ? speculative_receiver_type : profile.receiver(0); + if (miss_cg != nullptr) { + ciKlass* k = speculative_receiver_type != nullptr ? speculative_receiver_type : profile.receiver(0); trace_type_profile(C, jvms->method(), jvms->depth() - 1, jvms->bci(), receiver_method, k, site_count, receiver_count); - float hit_prob = speculative_receiver_type != NULL ? 1.0 : profile.receiver_prob(0); + float hit_prob = speculative_receiver_type != nullptr ? 1.0 : profile.receiver_prob(0); CallGenerator* cg = CallGenerator::for_predicted_call(k, miss_cg, hit_cg, hit_prob); - if (cg != NULL) return cg; + if (cg != nullptr) return cg; } } } @@ -314,13 +314,13 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool caller->get_declared_method_holder_at_bci(bci)->as_instance_klass(); ciInstanceKlass* singleton = declared_interface->unique_implementor(); - if (singleton != NULL) { + if (singleton != nullptr) { assert(singleton != declared_interface, "not a unique implementor"); ciMethod* cha_monomorphic_target = callee->find_monomorphic_target(caller->holder(), declared_interface, singleton); - if (cha_monomorphic_target != NULL && + if (cha_monomorphic_target != nullptr && cha_monomorphic_target->holder() != env()->Object_klass()) { // subtype check against Object is useless ciKlass* holder = cha_monomorphic_target->holder(); @@ -334,7 +334,7 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool ciKlass* constraint = (holder->is_subclass_of(singleton) ? holder : singleton); // avoid upcasts CallGenerator* cg = CallGenerator::for_guarded_call(constraint, miss_cg, hit_cg); - if (hit_cg != NULL && cg != NULL) { + if (hit_cg != nullptr && cg != nullptr) { dependencies()->assert_unique_implementor(declared_interface, singleton); dependencies()->assert_unique_concrete_method(declared_interface, cha_monomorphic_target, declared_interface, callee); return cg; @@ -345,7 +345,7 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool // Nothing claimed the intrinsic, we go with straight-forward inlining // for already discovered intrinsic. - if (allow_intrinsics && cg_intrinsic != NULL) { + if (allow_intrinsics && cg_intrinsic != nullptr) { assert(cg_intrinsic->does_virtual_dispatch(), "sanity"); return cg_intrinsic; } @@ -369,7 +369,7 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool CallGenerator* cg = CallGenerator::for_direct_call(callee, should_delay_inlining(callee, jvms)); // For optimized virtual calls assert at runtime that receiver object // is a subtype of the method holder. - if (cg != NULL && is_virtual_or_interface && !callee->is_static()) { + if (cg != nullptr && is_virtual_or_interface && !callee->is_static()) { CallGenerator* trap_cg = CallGenerator::for_uncommon_trap(callee, Deoptimization::Reason_receiver_constraint, Deoptimization::Action_none); cg = CallGenerator::for_guarded_call(callee->holder(), trap_cg, cg); @@ -415,7 +415,7 @@ bool Compile::should_delay_string_inlining(ciMethod* call_method, JVMState* jvms if (receiver->is_Proj() && receiver->in(0)->is_CallStaticJava()) { CallStaticJavaNode* csj = receiver->in(0)->as_CallStaticJava(); ciMethod* m = csj->method(); - if (m != NULL && + if (m != nullptr && (m->intrinsic_id() == vmIntrinsics::_StringBuffer_toString || m->intrinsic_id() == vmIntrinsics::_StringBuilder_toString)) // Delay String.(new SB()) @@ -505,12 +505,12 @@ void Parse::do_call() { // Find target being called bool will_link; - ciSignature* declared_signature = NULL; + ciSignature* declared_signature = nullptr; ciMethod* orig_callee = iter().get_method(will_link, &declared_signature); // callee in the bytecode ciInstanceKlass* holder_klass = orig_callee->holder(); ciKlass* holder = iter().get_declared_method_holder(); ciInstanceKlass* klass = ciEnv::get_instance_klass_for_declared_method_holder(holder); - assert(declared_signature != NULL, "cannot be null"); + assert(declared_signature != nullptr, "cannot be null"); // Bump max node limit for JSR292 users if (bc() == Bytecodes::_invokedynamic || orig_callee->is_method_handle_intrinsic()) { @@ -530,7 +530,7 @@ void Parse::do_call() { //assert((bc_callee->is_static() || is_invokedynamic) == !has_receiver , "must match bc"); // XXX invokehandle (cur_bc_raw) // Note: this takes into account invokeinterface of methods declared in java/lang/Object, // which should be invokevirtuals but according to the VM spec may be invokeinterfaces - assert(holder_klass->is_interface() || holder_klass->super() == NULL || (bc() != Bytecodes::_invokeinterface), "must match bc"); + assert(holder_klass->is_interface() || holder_klass->super() == nullptr || (bc() != Bytecodes::_invokeinterface), "must match bc"); // Note: In the absence of miranda methods, an abstract class K can perform // an invokevirtual directly on an interface method I.m if K implements I. @@ -560,7 +560,7 @@ void Parse::do_call() { bool call_does_dispatch = false; // Speculative type of the receiver if any - ciKlass* speculative_receiver_type = NULL; + ciKlass* speculative_receiver_type = nullptr; if (is_virtual_or_interface) { Node* receiver_node = stack(sp() - nargs); const TypeOopPtr* receiver_type = _gvn.type(receiver_node)->isa_oopptr(); @@ -574,11 +574,11 @@ void Parse::do_call() { callee = C->optimize_virtual_call(method(), klass, holder, orig_callee, receiver_type, is_virtual, call_does_dispatch, vtable_index); // out-parameters - speculative_receiver_type = receiver_type != NULL ? receiver_type->speculative_type() : NULL; + speculative_receiver_type = receiver_type != nullptr ? receiver_type->speculative_type() : nullptr; } // Additional receiver subtype checks for interface calls via invokespecial or invokeinterface. - ciKlass* receiver_constraint = NULL; + ciKlass* receiver_constraint = nullptr; if (iter().cur_bc_raw() == Bytecodes::_invokespecial && !orig_callee->is_object_initializer()) { ciInstanceKlass* calling_klass = method()->holder(); ciInstanceKlass* sender_klass = calling_klass; @@ -590,12 +590,12 @@ void Parse::do_call() { receiver_constraint = holder; } - if (receiver_constraint != NULL) { + if (receiver_constraint != nullptr) { Node* receiver_node = stack(sp() - nargs); Node* cls_node = makecon(TypeKlassPtr::make(receiver_constraint)); - Node* bad_type_ctrl = NULL; + Node* bad_type_ctrl = nullptr; Node* casted_receiver = gen_checkcast(receiver_node, cls_node, &bad_type_ctrl); - if (bad_type_ctrl != NULL) { + if (bad_type_ctrl != nullptr) { PreserveJVMState pjvms(this); set_control(bad_type_ctrl); uncommon_trap(Deoptimization::Reason_class_check, @@ -623,7 +623,7 @@ void Parse::do_call() { CallGenerator* cg = C->call_generator(callee, vtable_index, call_does_dispatch, jvms, try_inline, prof_factor(), speculative_receiver_type); // NOTE: Don't use orig_callee and callee after this point! Use cg->method() instead. - orig_callee = callee = NULL; + orig_callee = callee = nullptr; // --------------------- // Round double arguments before call @@ -645,17 +645,17 @@ void Parse::do_call() { assert(jvms_in_sync(), "jvms must carry full info into CG"); // save across call, for a subsequent cast_not_null. - Node* receiver = has_receiver ? argument(0) : NULL; + Node* receiver = has_receiver ? argument(0) : nullptr; // The extra CheckCastPPs for speculative types mess with PhaseStringOpts - if (receiver != NULL && !call_does_dispatch && !cg->is_string_late_inline()) { + if (receiver != nullptr && !call_does_dispatch && !cg->is_string_late_inline()) { // Feed profiling data for a single receiver to the type system so // it can propagate it as a speculative type receiver = record_profiled_receiver_for_speculation(receiver); } JVMState* new_jvms = cg->generate(jvms); - if (new_jvms == NULL) { + if (new_jvms == nullptr) { // When inlining attempt fails (e.g., too many arguments), // it may contaminate the current compile state, making it // impossible to pull back and try again. Once we call @@ -669,7 +669,7 @@ void Parse::do_call() { // get a normal java call that may inline in that case cg = C->call_generator(cg->method(), vtable_index, call_does_dispatch, jvms, try_inline, prof_factor(), speculative_receiver_type, /* allow_intrinsics= */ false); new_jvms = cg->generate(jvms); - if (new_jvms == NULL) { + if (new_jvms == nullptr) { guarantee(failing(), "call failed to generate: calls should work"); return; } @@ -695,7 +695,7 @@ void Parse::do_call() { if (!stopped()) { // This was some sort of virtual call, which did a null check for us. // Now we can assert receiver-not-null, on the normal return path. - if (receiver != NULL && cg->is_virtual()) { + if (receiver != nullptr && cg->is_virtual()) { Node* cast = cast_not_null(receiver); // %%% assert(receiver == cast, "should already have cast the receiver"); } @@ -721,7 +721,7 @@ void Parse::do_call() { if (ctype->is_loaded()) { const TypeOopPtr* arg_type = TypeOopPtr::make_from_klass(rtype->as_klass()); const Type* sig_type = TypeOopPtr::make_from_klass(ctype->as_klass()); - if (arg_type != NULL && !arg_type->higher_equal(sig_type)) { + if (arg_type != nullptr && !arg_type->higher_equal(sig_type)) { Node* retnode = pop(); Node* cast_obj = _gvn.transform(new CheckCastPPNode(control(), retnode, sig_type)); push(cast_obj); @@ -754,7 +754,7 @@ void Parse::do_call() { method()->print_name(); tty->print_cr(" asserting nullness of result at bci: %d", bci()); cg->method()->print_name(); tty->cr(); } - if (C->log() != NULL) { + if (C->log() != nullptr) { C->log()->elem("assert_null reason='return' klass='%d'", C->log()->identify(rtype)); } @@ -786,7 +786,7 @@ void Parse::catch_call_exceptions(ciExceptionHandlerStream& handlers) { // Add a CatchNode. GrowableArray* bcis = new (C->node_arena()) GrowableArray(C->node_arena(), 8, 0, -1); - GrowableArray* extypes = new (C->node_arena()) GrowableArray(C->node_arena(), 8, 0, NULL); + GrowableArray* extypes = new (C->node_arena()) GrowableArray(C->node_arena(), 8, 0, nullptr); GrowableArray* saw_unloaded = new (C->node_arena()) GrowableArray(C->node_arena(), 8, 0, 0); bool default_handler = false; @@ -819,7 +819,9 @@ void Parse::catch_call_exceptions(ciExceptionHandlerStream& handlers) { if (!default_handler) { bcis->append(-1); - extypes->append(TypeOopPtr::make_from_klass(env()->Throwable_klass())->is_instptr()); + const Type* extype = TypeOopPtr::make_from_klass(env()->Throwable_klass())->is_instptr(); + extype = extype->join(TypeInstPtr::NOTNULL); + extypes->append(extype); } int len = bcis->length(); @@ -896,7 +898,7 @@ void Parse::catch_call_exceptions(ciExceptionHandlerStream& handlers) { // So we insert a RethrowCall and all the logic that goes with it. void Parse::catch_inline_exceptions(SafePointNode* ex_map) { // Caller is responsible for saving away the map for normal control flow! - assert(stopped(), "call set_map(NULL) first"); + assert(stopped(), "call set_map(nullptr) first"); assert(method()->has_exception_handlers(), "don't come here w/o work to do"); Node* ex_node = saved_ex_oop(ex_map); @@ -905,8 +907,8 @@ void Parse::catch_inline_exceptions(SafePointNode* ex_map) { return; } const TypeInstPtr* ex_type = _gvn.type(ex_node)->isa_instptr(); - NOT_PRODUCT(if (ex_type==NULL) tty->print_cr("*** Exception not InstPtr")); - if (ex_type == NULL) + NOT_PRODUCT(if (ex_type==nullptr) tty->print_cr("*** Exception not InstPtr")); + if (ex_type == nullptr) ex_type = TypeOopPtr::make_from_klass(env()->Throwable_klass())->is_instptr(); // determine potential exception handlers @@ -919,10 +921,10 @@ void Parse::catch_inline_exceptions(SafePointNode* ex_map) { ex_node = use_exception_state(ex_map); // Get the exception oop klass from its header - Node* ex_klass_node = NULL; + Node* ex_klass_node = nullptr; if (has_ex_handler() && !ex_type->klass_is_exact()) { Node* p = basic_plus_adr( ex_node, ex_node, oopDesc::klass_offset_in_bytes()); - ex_klass_node = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, TypeInstPtr::KLASS, TypeKlassPtr::OBJECT)); + ex_klass_node = _gvn.transform(LoadKlassNode::make(_gvn, nullptr, immutable_memory(), p, TypeInstPtr::KLASS, TypeKlassPtr::OBJECT)); // Compute the exception klass a little more cleverly. // Obvious solution is to simple do a LoadKlass from the 'ex_node'. @@ -934,13 +936,13 @@ void Parse::catch_inline_exceptions(SafePointNode* ex_map) { ex_klass_node = new PhiNode(ex_node->in(0), TypeKlassPtr::OBJECT); for (uint i = 1; i < ex_node->req(); i++) { Node* ex_in = ex_node->in(i); - if (ex_in == top() || ex_in == NULL) { + if (ex_in == top() || ex_in == nullptr) { // This path was not taken. ex_klass_node->init_req(i, top()); continue; } Node* p = basic_plus_adr(ex_in, ex_in, oopDesc::klass_offset_in_bytes()); - Node* k = _gvn.transform( LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, TypeInstPtr::KLASS, TypeKlassPtr::OBJECT)); + Node* k = _gvn.transform( LoadKlassNode::make(_gvn, nullptr, immutable_memory(), p, TypeInstPtr::KLASS, TypeKlassPtr::OBJECT)); ex_klass_node->init_req( i, k ); } _gvn.set_type(ex_klass_node, TypeKlassPtr::OBJECT); @@ -1026,7 +1028,7 @@ void Parse::catch_inline_exceptions(SafePointNode* ex_map) { make_runtime_call(RC_NO_LEAF | RC_MUST_THROW, OptoRuntime::rethrow_Type(), OptoRuntime::rethrow_stub(), - NULL, NULL, + nullptr, nullptr, ex_node); // Rethrow is a pure call, no side effects, only a result. @@ -1089,7 +1091,7 @@ ciMethod* Compile::optimize_virtual_call(ciMethod* caller, ciInstanceKlass* klas receiver_type, check_access); // Have the call been sufficiently improved such that it is no longer a virtual? - if (optimized_virtual_method != NULL) { + if (optimized_virtual_method != nullptr) { callee = optimized_virtual_method; call_does_dispatch = false; } else if (!UseInlineCaches && is_virtual && callee->is_loaded()) { @@ -1113,8 +1115,8 @@ ciMethod* Compile::optimize_inlining(ciMethod* caller, ciInstanceKlass* klass, c return callee; } - if (receiver_type == NULL) { - return NULL; // no receiver type info + if (receiver_type == nullptr) { + return nullptr; // no receiver type info } // Attempt to improve the receiver @@ -1130,7 +1132,7 @@ ciMethod* Compile::optimize_inlining(ciMethod* caller, ciInstanceKlass* klass, c // All other interesting cases are instance klasses. if (!receiver_type->isa_instptr()) { - return NULL; + return nullptr; } ciInstanceKlass* receiver_klass = receiver_type->klass()->as_instance_klass(); @@ -1146,7 +1148,7 @@ ciMethod* Compile::optimize_inlining(ciMethod* caller, ciInstanceKlass* klass, c ciInstanceKlass* calling_klass = caller->holder(); ciMethod* cha_monomorphic_target = callee->find_monomorphic_target(calling_klass, klass, actual_receiver, check_access); - if (cha_monomorphic_target != NULL) { + if (cha_monomorphic_target != nullptr) { // Hardwiring a virtual. assert(!callee->can_be_statically_bound(), "should have been handled earlier"); assert(!cha_monomorphic_target->is_abstract(), ""); @@ -1167,10 +1169,10 @@ ciMethod* Compile::optimize_inlining(ciMethod* caller, ciInstanceKlass* klass, c // In case of evolution, there is a dependence on every inlined method, since each // such method can be changed when its class is redefined. ciMethod* exact_method = callee->resolve_invoke(calling_klass, actual_receiver); - if (exact_method != NULL) { + if (exact_method != nullptr) { return exact_method; } } - return NULL; + return nullptr; } diff --git a/src/hotspot/share/opto/domgraph.cpp b/src/hotspot/share/opto/domgraph.cpp index 03b0c52ab5557..d896abbbab89b 100644 --- a/src/hotspot/share/opto/domgraph.cpp +++ b/src/hotspot/share/opto/domgraph.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -129,12 +129,12 @@ void PhaseCFG::build_dominator_tree() { Tarjan *w = &tarjan[i]; if( w->_dom != &tarjan[w->_semi] ) w->_dom = w->_dom->_dom; - w->_dom_next = w->_dom_child = NULL; // Initialize for building tree later + w->_dom_next = w->_dom_child = nullptr; // Initialize for building tree later } // No immediate dominator for the root Tarjan *w = &tarjan[get_root_block()->_pre_order]; - w->_dom = NULL; - w->_dom_next = w->_dom_child = NULL; // Initialize for building tree later + w->_dom = nullptr; + w->_dom_next = w->_dom_child = nullptr; // Initialize for building tree later // Convert the dominator tree array into my kind of graph for(uint i = 1; i <= number_of_blocks(); i++){ // For all Tarjan vertices @@ -145,7 +145,7 @@ void PhaseCFG::build_dominator_tree() { t->_dom_next = tdom->_dom_child; // Make me a sibling of parent's child tdom->_dom_child = t; // Make me a child of my parent } else - t->_block->_idom = NULL; // Root + t->_block->_idom = nullptr; // Root } w->setdepth(number_of_blocks() + 1); // Set depth in dominator tree @@ -175,12 +175,12 @@ class Block_Stack { t->_block = b; // Save actual block t->_semi = pre_order; // Block to DFS map t->_label = t; // DFS to vertex map - t->_ancestor = NULL; // Fast LINK & EVAL setup + t->_ancestor = nullptr; // Fast LINK & EVAL setup t->_child = &_tarjan[0]; // Sentenial t->_size = 1; - t->_bucket = NULL; + t->_bucket = nullptr; if (pre_order == 1) - t->_parent = NULL; // first block doesn't have parent + t->_parent = nullptr; // first block doesn't have parent else { // Save parent (current top block on stack) in DFS t->_parent = &_tarjan[_stack_top->block->_pre_order]; @@ -341,11 +341,11 @@ void Tarjan::setdepth( uint stack_size ) { t->_block->_dom_depth = depth; // Set depth in dominator tree Tarjan *dom_child = t->_dom_child; t = t->_dom_next; // next tarjan - if (dom_child != NULL) { + if (dom_child != nullptr) { *top = dom_child; // save child on stack ++top; } - } while (t != NULL); + } while (t != nullptr); } while (next < last); } while (last < top); } @@ -395,7 +395,7 @@ void PhaseIdealLoop::Dominators() { // Initialize _control field for fast reference int i; for( i= C->unique()-1; i>=0; i-- ) - ntarjan[i]._control = NULL; + ntarjan[i]._control = nullptr; // Store the DFS order for the main loop const uint fill_value = max_juint; @@ -413,12 +413,12 @@ void PhaseIdealLoop::Dominators() { for( i = dfsnum-1; i>1; i-- ) { // For all nodes in reverse DFS order NTarjan *w = &ntarjan[i]; // Get Node from DFS - assert(w->_control != NULL,"bad DFS walk"); + assert(w->_control != nullptr,"bad DFS walk"); // Step 2: Node *whead = w->_control; for( uint j=0; j < whead->req(); j++ ) { // For each predecessor - if( whead->in(j) == NULL || !whead->in(j)->is_CFG() ) + if( whead->in(j) == nullptr || !whead->in(j)->is_CFG() ) continue; // Only process control nodes uint b = dfsorder[whead->in(j)->_idx]; if(b == fill_value) continue; @@ -468,28 +468,28 @@ void PhaseIdealLoop::Dominators() { // Step 4: for( i=2; i < dfsnum; i++ ) { // DFS order NTarjan *w = &ntarjan[i]; - assert(w->_control != NULL,"Bad DFS walk"); + assert(w->_control != nullptr,"Bad DFS walk"); if( w->_dom != &ntarjan[w->_semi] ) w->_dom = w->_dom->_dom; - w->_dom_next = w->_dom_child = NULL; // Initialize for building tree later + w->_dom_next = w->_dom_child = nullptr; // Initialize for building tree later } // No immediate dominator for the root NTarjan *w = &ntarjan[dfsorder[C->root()->_idx]]; - w->_dom = NULL; - w->_parent = NULL; - w->_dom_next = w->_dom_child = NULL; // Initialize for building tree later + w->_dom = nullptr; + w->_parent = nullptr; + w->_dom_next = w->_dom_child = nullptr; // Initialize for building tree later // Convert the dominator tree array into my kind of graph for( i=1; i_control != NULL,"Bad DFS walk"); + assert(t->_control != nullptr,"Bad DFS walk"); NTarjan *tdom = t->_dom; // Handy access to immediate dominator if( tdom ) { // Root has no immediate dominator _idom[t->_control->_idx] = tdom->_control; // Set immediate dominator t->_dom_next = tdom->_dom_child; // Make me a sibling of parent's child tdom->_dom_child = t; // Make me a child of my parent } else - _idom[C->root()->_idx] = NULL; // Root + _idom[C->root()->_idx] = nullptr; // Root } w->setdepth( C->unique()+1, _dom_depth ); // Set depth in dominator tree // Pick up the 'top' node as well @@ -525,10 +525,10 @@ int NTarjan::DFS( NTarjan *ntarjan, VectorSet &visited, PhaseIdealLoop *pil, uin dfsorder[b->_idx] = dfsnum; // Save DFS order info w->_semi = dfsnum; // Node to DFS map w->_label = w; // DFS to vertex map - w->_ancestor = NULL; // Fast LINK & EVAL setup + w->_ancestor = nullptr; // Fast LINK & EVAL setup w->_child = &ntarjan[0]; // Sentinal w->_size = 1; - w->_bucket = NULL; + w->_bucket = nullptr; // Need DEF-USE info for this pass for ( int i = b->outcnt(); i-- > 0; ) { // Put on stack backwards @@ -604,11 +604,11 @@ void NTarjan::setdepth( uint stack_size, uint *dom_depth ) { dom_depth[t->_control->_idx] = depth; // Set depth in dominator tree NTarjan *dom_child = t->_dom_child; t = t->_dom_next; // next tarjan - if (dom_child != NULL) { + if (dom_child != nullptr) { *top = dom_child; // save child on stack ++top; } - } while (t != NULL); + } while (t != nullptr); } while (next < last); } while (last < top); } @@ -628,13 +628,13 @@ void NTarjan::dump(int offset) const { for(i = offset; i >0; i--) // Use indenting for tree structure tty->print(" "); tty->print("DFS Parent: "); - if(_parent != NULL) + if(_parent != nullptr) _parent->_control->dump(); // Parent in DFS tty->print("\n"); for(i = offset; i >0; i--) // Use indenting for tree structure tty->print(" "); tty->print("Dom Parent: "); - if(_dom != NULL) + if(_dom != nullptr) _dom->_control->dump(); // Parent in Dominator Tree tty->print("\n"); diff --git a/src/hotspot/share/opto/escape.cpp b/src/hotspot/share/opto/escape.cpp index fd7302d4ec98e..0c84d8bd97e0b 100644 --- a/src/hotspot/share/opto/escape.cpp +++ b/src/hotspot/share/opto/escape.cpp @@ -42,7 +42,7 @@ #include "utilities/macros.hpp" ConnectionGraph::ConnectionGraph(Compile * C, PhaseIterGVN *igvn) : - _nodes(C->comp_arena(), C->unique(), C->unique(), NULL), + _nodes(C->comp_arena(), C->unique(), C->unique(), nullptr), _in_worklist(C->comp_arena()), _next_pidx(0), _collecting(true), @@ -53,7 +53,7 @@ ConnectionGraph::ConnectionGraph(Compile * C, PhaseIterGVN *igvn) : // Add unknown java object. add_java_object(C->top(), PointsToNode::GlobalEscape); phantom_obj = ptnode_adr(C->top()->_idx)->as_JavaObject(); - // Add ConP(#NULL) and ConN(#NULL) nodes. + // Add ConP and ConN null oop nodes Node* oop_null = igvn->zerocon(T_OBJECT); assert(oop_null->_idx < nodes_size(), "should be created already"); add_java_object(oop_null, PointsToNode::NoEscape); @@ -92,7 +92,7 @@ void ConnectionGraph::do_analysis(Compile *C, PhaseIterGVN *igvn) { Compile::TracePhase tp("escapeAnalysis", &Phase::timers[Phase::_t_escapeAnalysis]); ResourceMark rm; - // Add ConP#NULL and ConN#NULL nodes before ConnectionGraph construction + // Add ConP and ConN null oop nodes before ConnectionGraph construction // to create space for them in ConnectionGraph::_nodes[]. Node* oop_null = igvn->zerocon(T_OBJECT); Node* noop_null = igvn->zerocon(T_NARROWOOP); @@ -131,9 +131,9 @@ bool ConnectionGraph::compute_escape() { { Compile::TracePhase tp("connectionGraph", &Phase::timers[Phase::_t_connectionGraph]); // 1. Populate Connection Graph (CG) with PointsTo nodes. - ideal_nodes.map(C->live_nodes(), NULL); // preallocate space + ideal_nodes.map(C->live_nodes(), nullptr); // preallocate space // Initialize worklist - if (C->root() != NULL) { + if (C->root() != nullptr) { ideal_nodes.push(C->root()); } // Processed ideal nodes are unique on ideal_nodes list @@ -148,7 +148,7 @@ bool ConnectionGraph::compute_escape() { // only once per ideal node since ideal_nodes is Unique_Node list. add_node_to_connection_graph(n, &delayed_worklist); PointsToNode* ptn = ptnode_adr(n->_idx); - if (ptn != NULL && ptn != phantom_obj) { + if (ptn != nullptr && ptn != phantom_obj) { ptnodes_worklist.append(ptn); if (ptn->is_JavaObject()) { java_objects_worklist.append(ptn->as_JavaObject()); @@ -361,7 +361,7 @@ bool ConnectionGraph::compute_escape() { // Returns true if there is an object in the scope of sfn that does not escape globally. bool ConnectionGraph::has_ea_local_in_scope(SafePointNode* sfn) { Compile* C = _compile; - for (JVMState* jvms = sfn->jvms(); jvms != NULL; jvms = jvms->caller()) { + for (JVMState* jvms = sfn->jvms(); jvms != nullptr; jvms = jvms->caller()) { if (C->env()->should_retain_local_variables() || C->env()->jvmti_can_walk_any_space() || DeoptimizeObjectsALot) { // Jvmti agents can access locals. Must provide info about local objects at runtime. @@ -379,7 +379,7 @@ bool ConnectionGraph::has_ea_local_in_scope(SafePointNode* sfn) { int num_mon = jvms->nof_monitors(); for (int idx = 0; idx < num_mon; idx++) { Node* m = sfn->monitor_obj(jvms, idx); - if (m != NULL && not_global_escape(m)) { + if (m != nullptr && not_global_escape(m)) { return true; } } @@ -391,7 +391,7 @@ bool ConnectionGraph::has_ea_local_in_scope(SafePointNode* sfn) { // Returns true if at least one of the arguments to the call is an object // that does not escape globally. bool ConnectionGraph::has_arg_escape(CallJavaNode* call) { - if (call->method() != NULL) { + if (call->method() != nullptr) { uint max_idx = TypeFunc::Parms + call->method()->arg_size(); for (uint idx = TypeFunc::Parms; idx < max_idx; idx++) { Node* p = call->in(idx); @@ -401,14 +401,14 @@ bool ConnectionGraph::has_arg_escape(CallJavaNode* call) { } } else { const char* name = call->as_CallStaticJava()->_name; - assert(name != NULL, "no name"); + assert(name != nullptr, "no name"); // no arg escapes through uncommon traps if (strcmp(name, "uncommon_trap") != 0) { // process_call_arguments() assumes that all arguments escape globally const TypeTuple* d = call->tf()->domain(); for (uint i = TypeFunc::Parms; i < d->cnt(); i++) { const Type* at = d->field_at(i); - if (at->isa_oopptr() != NULL) { + if (at->isa_oopptr() != nullptr) { return true; } } @@ -424,13 +424,13 @@ void ConnectionGraph::add_objload_to_connection_graph(Node *n, Unique_Node_List // Using isa_ptr() instead of isa_oopptr() for LoadP and Phi because // ThreadLocal has RawPtr type. const Type* t = _igvn->type(n); - if (t->make_ptr() != NULL) { + if (t->make_ptr() != nullptr) { Node* adr = n->in(MemNode::Address); #ifdef ASSERT if (!adr->is_AddP()) { assert(_igvn->type(adr)->isa_rawptr(), "sanity"); } else { - assert((ptnode_adr(adr->_idx) == NULL || + assert((ptnode_adr(adr->_idx) == nullptr || ptnode_adr(adr->_idx)->as_Field()->is_oop()), "sanity"); } #endif @@ -446,7 +446,7 @@ void ConnectionGraph::add_node_to_connection_graph(Node *n, Unique_Node_List *de PhaseGVN* igvn = _igvn; uint n_idx = n->_idx; PointsToNode* n_ptn = ptnode_adr(n_idx); - if (n_ptn != NULL) { + if (n_ptn != nullptr) { return; // No need to redefine PointsTo node during first iteration. } int opcode = n->Opcode(); @@ -467,7 +467,7 @@ void ConnectionGraph::add_node_to_connection_graph(Node *n, Unique_Node_List *de } else { if (n->is_CallStaticJava()) { const char* name = n->as_CallStaticJava()->_name; - if (name != NULL && strcmp(name, "uncommon_trap") == 0) { + if (name != nullptr && strcmp(name, "uncommon_trap") == 0) { return; // Skip uncommon traps } } @@ -475,7 +475,7 @@ void ConnectionGraph::add_node_to_connection_graph(Node *n, Unique_Node_List *de delayed_worklist->push(n); // Check if a call returns an object. if ((n->as_Call()->returns_pointer() && - n->as_Call()->proj_out_or_null(TypeFunc::Parms) != NULL) || + n->as_Call()->proj_out_or_null(TypeFunc::Parms) != nullptr) || (n->is_CallStaticJava() && n->as_CallStaticJava()->is_boxing_method())) { add_call_node(n->as_Call()); @@ -498,7 +498,7 @@ void ConnectionGraph::add_node_to_connection_graph(Node *n, Unique_Node_List *de // Graph because such fields are not used for oop loads and stores. int offset = address_offset(n, igvn); add_field(n, PointsToNode::NoEscape, offset); - if (ptn_base == NULL) { + if (ptn_base == nullptr) { delayed_worklist->push(n); // Process it later. } else { n_ptn = ptnode_adr(n_idx); @@ -572,7 +572,7 @@ void ConnectionGraph::add_node_to_connection_graph(Node *n, Unique_Node_List *de // Using isa_ptr() instead of isa_oopptr() for LoadP and Phi because // ThreadLocal has RawPtr type. const Type* t = n->as_Phi()->type(); - if (t->make_ptr() != NULL) { + if (t->make_ptr() != nullptr) { add_local_var(n, PointsToNode::NoEscape); // Do not add edges during first iteration because some could be // not defined yet. @@ -682,7 +682,7 @@ void ConnectionGraph::add_final_edges(Node *n) { return; } assert(n->is_Store() || n->is_LoadStore() || - (n_ptn != NULL) && (n_ptn->ideal_node() != NULL), + (n_ptn != nullptr) && (n_ptn->ideal_node() != nullptr), "node should be registered already"); int opcode = n->Opcode(); bool gc_handled = BarrierSet::barrier_set()->barrier_set_c2()->escape_add_final_edges(this, _igvn, n, opcode); @@ -693,7 +693,7 @@ void ConnectionGraph::add_final_edges(Node *n) { case Op_AddP: { Node* base = get_addp_base(n); PointsToNode* ptn_base = ptnode_adr(base->_idx); - assert(ptn_base != NULL, "field's base should be registered"); + assert(ptn_base != nullptr, "field's base should be registered"); add_base(n_ptn->as_Field(), ptn_base); break; } @@ -704,21 +704,21 @@ void ConnectionGraph::add_final_edges(Node *n) { case Op_EncodePKlass: case Op_DecodeNKlass: { add_local_var_and_edge(n, PointsToNode::NoEscape, - n->in(1), NULL); + n->in(1), nullptr); break; } case Op_CMoveP: { for (uint i = CMoveNode::IfFalse; i < n->req(); i++) { Node* in = n->in(i); - if (in == NULL) { - continue; // ignore NULL + if (in == nullptr) { + continue; // ignore null } Node* uncast_in = in->uncast(); if (uncast_in->is_top() || uncast_in == n) { continue; // ignore top or inputs which go back this node } PointsToNode* ptn = ptnode_adr(in->_idx); - assert(ptn != NULL, "node should be registered"); + assert(ptn != nullptr, "node should be registered"); add_edge(n_ptn, ptn); } break; @@ -729,9 +729,9 @@ void ConnectionGraph::add_final_edges(Node *n) { // Using isa_ptr() instead of isa_oopptr() for LoadP and Phi because // ThreadLocal has RawPtr type. const Type* t = _igvn->type(n); - if (t->make_ptr() != NULL) { + if (t->make_ptr() != nullptr) { Node* adr = n->in(MemNode::Address); - add_local_var_and_edge(n, PointsToNode::NoEscape, adr, NULL); + add_local_var_and_edge(n, PointsToNode::NoEscape, adr, nullptr); break; } ELSE_FAIL("Op_LoadP"); @@ -740,18 +740,18 @@ void ConnectionGraph::add_final_edges(Node *n) { // Using isa_ptr() instead of isa_oopptr() for LoadP and Phi because // ThreadLocal has RawPtr type. const Type* t = n->as_Phi()->type(); - if (t->make_ptr() != NULL) { + if (t->make_ptr() != nullptr) { for (uint i = 1; i < n->req(); i++) { Node* in = n->in(i); - if (in == NULL) { - continue; // ignore NULL + if (in == nullptr) { + continue; // ignore null } Node* uncast_in = in->uncast(); if (uncast_in->is_top() || uncast_in == n) { continue; // ignore top or inputs which go back this node } PointsToNode* ptn = ptnode_adr(in->_idx); - assert(ptn != NULL, "node should be registered"); + assert(ptn != nullptr, "node should be registered"); add_edge(n_ptn, ptn); } break; @@ -762,7 +762,7 @@ void ConnectionGraph::add_final_edges(Node *n) { // we are only interested in the oop result projection from a call if (n->as_Proj()->_con == TypeFunc::Parms && n->in(0)->is_Call() && n->in(0)->as_Call()->returns_pointer()) { - add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), NULL); + add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), nullptr); break; } ELSE_FAIL("Op_Proj"); @@ -773,7 +773,7 @@ void ConnectionGraph::add_final_edges(Node *n) { _igvn->type(n->in(TypeFunc::Parms))->isa_oopptr()) { // Treat Return value as LocalVar with GlobalEscape escape state. add_local_var_and_edge(n, PointsToNode::GlobalEscape, - n->in(TypeFunc::Parms), NULL); + n->in(TypeFunc::Parms), nullptr); break; } ELSE_FAIL("Op_Return"); @@ -812,12 +812,12 @@ void ConnectionGraph::add_final_edges(Node *n) { const Type* at = _igvn->type(adr); if (!adr->is_top() && at->isa_ptr()) { assert(at == Type::TOP || at == TypePtr::NULL_PTR || - at->isa_ptr() != NULL, "expecting a pointer"); + at->isa_ptr() != nullptr, "expecting a pointer"); if (adr->is_AddP()) { adr = get_addp_base(adr); } PointsToNode* ptn = ptnode_adr(adr->_idx); - assert(ptn != NULL, "node should be registered"); + assert(ptn != nullptr, "node should be registered"); add_edge(n_ptn, ptn); } } @@ -859,7 +859,7 @@ void ConnectionGraph::add_to_congraph_unsafe_access(Node* n, uint opcode, Unique Node* adr = n->in(MemNode::Address); const Type* adr_type = _igvn->type(adr); adr_type = adr_type->make_ptr(); - if (adr_type == NULL) { + if (adr_type == nullptr) { return; // skip dead nodes } if (adr_type->isa_oopptr() @@ -897,9 +897,9 @@ bool ConnectionGraph::add_final_edges_unsafe_access(Node* n, uint opcode) { const Type *adr_type = _igvn->type(adr); adr_type = adr_type->make_ptr(); #ifdef ASSERT - if (adr_type == NULL) { + if (adr_type == nullptr) { n->dump(1); - assert(adr_type != NULL, "dead node should not be on list"); + assert(adr_type != nullptr, "dead node should not be on list"); return true; } #endif @@ -915,22 +915,22 @@ bool ConnectionGraph::add_final_edges_unsafe_access(Node* n, uint opcode) { && is_captured_store_address(adr))) { // Point Address to Value PointsToNode* adr_ptn = ptnode_adr(adr->_idx); - assert(adr_ptn != NULL && + assert(adr_ptn != nullptr && adr_ptn->as_Field()->is_oop(), "node should be registered"); Node* val = n->in(MemNode::ValueIn); PointsToNode* ptn = ptnode_adr(val->_idx); - assert(ptn != NULL, "node should be registered"); + assert(ptn != nullptr, "node should be registered"); add_edge(adr_ptn, ptn); return true; } else if ((opcode == Op_StoreP) && adr_type->isa_rawptr()) { // Stored value escapes in unsafe access. Node* val = n->in(MemNode::ValueIn); PointsToNode* ptn = ptnode_adr(val->_idx); - assert(ptn != NULL, "node should be registered"); + assert(ptn != nullptr, "node should be registered"); set_escape_state(ptn, PointsToNode::GlobalEscape); // Add edge to object for unsafe access with offset. PointsToNode* adr_ptn = ptnode_adr(adr->_idx); - assert(adr_ptn != NULL, "node should be registered"); + assert(adr_ptn != nullptr, "node should be registered"); if (adr_ptn->is_Field()) { assert(adr_ptn->as_Field()->is_oop(), "should be oop field"); add_edge(adr_ptn, ptn); @@ -946,7 +946,7 @@ void ConnectionGraph::add_call_node(CallNode* call) { if (call->is_Allocate()) { Node* k = call->in(AllocateNode::KlassNode); const TypeKlassPtr* kt = k->bottom_type()->isa_klassptr(); - assert(kt != NULL, "TypeKlassPtr required."); + assert(kt != nullptr, "TypeKlassPtr required."); ciKlass* cik = kt->klass(); PointsToNode::EscapeState es = PointsToNode::NoEscape; bool scalar_replaceable = true; @@ -1004,7 +1004,7 @@ void ConnectionGraph::add_call_node(CallNode* call) { // For a static call, we know exactly what method is being called. // Use bytecode estimator to record whether the call's return value escapes. ciMethod* meth = call->as_CallJava()->method(); - if (meth == NULL) { + if (meth == nullptr) { const char* name = call->as_CallStaticJava()->_name; assert(strncmp(name, "_multianewarray", 15) == 0, "TODO: add failed case check"); // Returns a newly allocated non-escaped object. @@ -1037,7 +1037,7 @@ void ConnectionGraph::add_call_node(CallNode* call) { const TypeTuple* d = call->tf()->domain(); bool ret_arg = false; for (uint i = TypeFunc::Parms; i < d->cnt(); i++) { - if (d->field_at(i)->isa_ptr() != NULL && + if (d->field_at(i)->isa_ptr() != nullptr && call_analyzer->is_arg_returned(i - TypeFunc::Parms)) { ret_arg = true; break; @@ -1087,7 +1087,7 @@ void ConnectionGraph::process_call_arguments(CallNode *call) { for (uint i = TypeFunc::Parms; i < d->cnt(); i++) { const Type* at = d->field_at(i); Node *arg = call->in(i); - if (arg == NULL) { + if (arg == nullptr) { continue; } const Type *aat = _igvn->type(arg); @@ -1106,13 +1106,13 @@ void ConnectionGraph::process_call_arguments(CallNode *call) { arg = get_addp_base(arg); } PointsToNode* arg_ptn = ptnode_adr(arg->_idx); - assert(arg_ptn != NULL, "should be registered"); + assert(arg_ptn != nullptr, "should be registered"); PointsToNode::EscapeState arg_esc = arg_ptn->escape_state(); if (is_arraycopy || arg_esc < PointsToNode::ArgEscape) { assert(aat == Type::TOP || aat == TypePtr::NULL_PTR || - aat->isa_ptr() != NULL, "expecting an Ptr"); + aat->isa_ptr() != nullptr, "expecting an Ptr"); bool arg_has_oops = aat->isa_oopptr() && - (aat->isa_oopptr()->klass() == NULL || aat->isa_instptr() || + (aat->isa_oopptr()->klass() == nullptr || aat->isa_instptr() || (aat->isa_aryptr() && aat->isa_aryptr()->klass()->is_obj_array_klass())); if (i == TypeFunc::Parms) { src_has_oops = arg_has_oops; @@ -1130,7 +1130,7 @@ void ConnectionGraph::process_call_arguments(CallNode *call) { #ifdef ASSERT if (!(is_arraycopy || BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(call) || - (call->as_CallLeaf()->_name != NULL && + (call->as_CallLeaf()->_name != nullptr && (strcmp(call->as_CallLeaf()->_name, "updateBytesCRC32") == 0 || strcmp(call->as_CallLeaf()->_name, "updateBytesCRC32C") == 0 || strcmp(call->as_CallLeaf()->_name, "updateBytesAdler32") == 0 || @@ -1192,7 +1192,7 @@ void ConnectionGraph::process_call_arguments(CallNode *call) { src = get_addp_base(src); } PointsToNode* src_ptn = ptnode_adr(src->_idx); - assert(src_ptn != NULL, "should be registered"); + assert(src_ptn != nullptr, "should be registered"); if (arg_ptn != src_ptn) { // Special arraycopy edge: // A destination object's field can't have the source object @@ -1211,15 +1211,15 @@ void ConnectionGraph::process_call_arguments(CallNode *call) { // Use bytecode estimator to record the call's escape affects #ifdef ASSERT const char* name = call->as_CallStaticJava()->_name; - assert((name == NULL || strcmp(name, "uncommon_trap") != 0), "normal calls only"); + assert((name == nullptr || strcmp(name, "uncommon_trap") != 0), "normal calls only"); #endif ciMethod* meth = call->as_CallJava()->method(); - if ((meth != NULL) && meth->is_boxing_method()) { + if ((meth != nullptr) && meth->is_boxing_method()) { break; // Boxing methods do not modify any oops. } - BCEscapeAnalyzer* call_analyzer = (meth !=NULL) ? meth->get_bcea() : NULL; + BCEscapeAnalyzer* call_analyzer = (meth !=nullptr) ? meth->get_bcea() : nullptr; // fall-through if not a Java method or no analyzer information - if (call_analyzer != NULL) { + if (call_analyzer != nullptr) { PointsToNode* call_ptn = ptnode_adr(call->_idx); const TypeTuple* d = call->tf()->domain(); for (uint i = TypeFunc::Parms; i < d->cnt(); i++) { @@ -1227,16 +1227,16 @@ void ConnectionGraph::process_call_arguments(CallNode *call) { int k = i - TypeFunc::Parms; Node* arg = call->in(i); PointsToNode* arg_ptn = ptnode_adr(arg->_idx); - if (at->isa_ptr() != NULL && + if (at->isa_ptr() != nullptr && call_analyzer->is_arg_returned(k)) { // The call returns arguments. - if (call_ptn != NULL) { // Is call's result used? + if (call_ptn != nullptr) { // Is call's result used? assert(call_ptn->is_LocalVar(), "node should be registered"); - assert(arg_ptn != NULL, "node should be registered"); + assert(arg_ptn != nullptr, "node should be registered"); add_edge(call_ptn, arg_ptn); } } - if (at->isa_oopptr() != NULL && + if (at->isa_oopptr() != nullptr && arg_ptn->escape_state() < PointsToNode::GlobalEscape) { if (!call_analyzer->is_arg_stack(k)) { // The argument global escapes @@ -1250,7 +1250,7 @@ void ConnectionGraph::process_call_arguments(CallNode *call) { } } } - if (call_ptn != NULL && call_ptn->is_LocalVar()) { + if (call_ptn != nullptr && call_ptn->is_LocalVar()) { // The call returns arguments. assert(call_ptn->edge_count() > 0, "sanity"); if (!call_analyzer->is_return_local()) { @@ -1268,12 +1268,12 @@ void ConnectionGraph::process_call_arguments(CallNode *call) { const TypeTuple* d = call->tf()->domain(); for (uint i = TypeFunc::Parms; i < d->cnt(); i++) { const Type* at = d->field_at(i); - if (at->isa_oopptr() != NULL) { + if (at->isa_oopptr() != nullptr) { Node* arg = call->in(i); if (arg->is_AddP()) { arg = get_addp_base(arg); } - assert(ptnode_adr(arg->_idx) != NULL, "should be defined already"); + assert(ptnode_adr(arg->_idx) != nullptr, "should be defined already"); set_escape_state(ptnode_adr(arg->_idx), PointsToNode::GlobalEscape); } } @@ -1385,7 +1385,7 @@ bool ConnectionGraph::complete_connection_graph( // Bailout if passed limits. if ((iterations >= GRAPH_BUILD_ITER_LIMIT) || timeout) { Compile* C = _compile; - if (C->log() != NULL) { + if (C->log() != nullptr) { C->log()->begin_elem("connectionGraph_bailout reason='reached "); C->log()->text("%s", timeout ? "time" : "iterations"); C->log()->end_elem(" limit'"); @@ -1405,7 +1405,7 @@ bool ConnectionGraph::complete_connection_graph( #undef GRAPH_BUILD_ITER_LIMIT - // Find fields initialized by NULL for non-escaping Allocations. + // Find fields initialized by null for non-escaping Allocations. int non_escaped_length = non_escaped_allocs_worklist.length(); for (int next = 0; next < non_escaped_length; next++) { JavaObjectNode* ptn = non_escaped_allocs_worklist.at(next); @@ -1413,8 +1413,8 @@ bool ConnectionGraph::complete_connection_graph( assert(es <= PointsToNode::ArgEscape, "sanity"); if (es == PointsToNode::NoEscape) { if (find_init_values_null(ptn, _igvn) > 0) { - // Adding references to NULL object does not change escape states - // since it does not escape. Also no fields are added to NULL object. + // Adding references to null object does not change escape states + // since it does not escape. Also no fields are added to null object. add_java_object_edges(null_obj, false); } } @@ -1424,7 +1424,7 @@ bool ConnectionGraph::complete_connection_graph( // seen by an other thread. Mark it so that when it is // expanded no MemBarStoreStore is added. InitializeNode* ini = n->as_Allocate()->initialization(); - if (ini != NULL) + if (ini != nullptr) ini->set_does_not_escape(); } } @@ -1540,7 +1540,7 @@ int ConnectionGraph::add_java_object_edges(JavaObjectNode* jobj, bool populate_w } assert(!use->is_JavaObject(), "sanity"); if (use->is_Arraycopy()) { - if (jobj == null_obj) { // NULL object does not have field edges + if (jobj == null_obj) { // null object does not have field edges continue; } // Added edge from Arraycopy node to arraycopy's source java object @@ -1561,7 +1561,7 @@ int ConnectionGraph::add_java_object_edges(JavaObjectNode* jobj, bool populate_w for (EdgeIterator i(use); i.has_next(); i.next()) { PointsToNode* e = i.get(); if (e->is_Arraycopy()) { - if (jobj == null_obj) { // NULL object does not have field edges + if (jobj == null_obj) { // null object does not have field edges continue; } // Add edge from arraycopy's destination java object to Arraycopy node. @@ -1636,7 +1636,7 @@ void ConnectionGraph::add_fields_to_worklist(FieldNode* field, PointsToNode* bas if (// Skip phantom_object since it is only used to indicate that // this field's content globally escapes. (base != phantom_obj) && - // NULL object node does not have fields. + // null object node does not have fields. (base != null_obj)) { for (EdgeIterator i(base); i.has_next(); i.next()) { PointsToNode* f = i.get(); @@ -1670,7 +1670,7 @@ int ConnectionGraph::find_field_value(FieldNode* field) { if (base->ideal_node()->is_Allocate()) { return 0; } - assert(base == null_obj, "only NULL ptr base expected here"); + assert(base == null_obj, "only null ptr base expected here"); } } if (add_edge(field, phantom_obj)) { @@ -1693,7 +1693,7 @@ int ConnectionGraph::find_init_values_phantom(JavaObjectNode* pta) { } assert(pta->arraycopy_dst() || alloc->as_CallStaticJava(), "sanity"); #ifdef ASSERT - if (!pta->arraycopy_dst() && alloc->as_CallStaticJava()->method() == NULL) { + if (!pta->arraycopy_dst() && alloc->as_CallStaticJava()->method() == nullptr) { const char* name = alloc->as_CallStaticJava()->_name; assert(strncmp(name, "_multianewarray", 15) == 0, "sanity"); } @@ -1727,8 +1727,8 @@ int ConnectionGraph::find_init_values_null(JavaObjectNode* pta, PhaseTransform* int new_edges = 0; // Check if an oop field's initializing value is recorded and add - // a corresponding NULL if field's value if it is not recorded. - // Connection Graph does not record a default initialization by NULL + // a corresponding null if field's value if it is not recorded. + // Connection Graph does not record a default initialization by null // captured by Initialize node. // for (EdgeIterator i(pta); i.has_next(); i.next()) { @@ -1740,7 +1740,7 @@ int ConnectionGraph::find_init_values_null(JavaObjectNode* pta, PhaseTransform* if (offset == Type::OffsetBot) { if (!visited_bottom_offset) { // OffsetBot is used to reference array's element, - // always add reference to NULL to all Field nodes since we don't + // always add reference to null to all Field nodes since we don't // known which element is referenced. if (add_edge(field, null_obj)) { // New edge was added @@ -1763,23 +1763,23 @@ int ConnectionGraph::find_init_values_null(JavaObjectNode* pta, PhaseTransform* } if (!offsets_worklist.contains(offset)) { offsets_worklist.append(offset); - Node* value = NULL; - if (ini != NULL) { + Node* value = nullptr; + if (ini != nullptr) { // StoreP::memory_type() == T_ADDRESS BasicType ft = UseCompressedOops ? T_NARROWOOP : T_ADDRESS; Node* store = ini->find_captured_store(offset, type2aelembytes(ft, true), phase); // Make sure initializing store has the same type as this AddP. // This AddP may reference non existing field because it is on a // dead branch of bimorphic call which is not eliminated yet. - if (store != NULL && store->is_Store() && + if (store != nullptr && store->is_Store() && store->as_Store()->memory_type() == ft) { value = store->in(MemNode::ValueIn); #ifdef ASSERT if (VerifyConnectionGraph) { // Verify that AddP already points to all objects the value points to. PointsToNode* val = ptnode_adr(value->_idx); - assert((val != NULL), "should be processed already"); - PointsToNode* missed_obj = NULL; + assert((val != nullptr), "should be processed already"); + PointsToNode* missed_obj = nullptr; if (val->is_JavaObject()) { if (!field->points_to(val->as_JavaObject())) { missed_obj = val; @@ -1801,7 +1801,7 @@ int ConnectionGraph::find_init_values_null(JavaObjectNode* pta, PhaseTransform* } } } - if (missed_obj != NULL) { + if (missed_obj != nullptr) { tty->print_cr("----------field---------------------------------"); field->dump(); tty->print_cr("----------missed referernce to object-----------"); @@ -1819,12 +1819,12 @@ int ConnectionGraph::find_init_values_null(JavaObjectNode* pta, PhaseTransform* // by Initialize node. // // Need to check for dependent loads to separate such stores from - // stores which follow loads. For now, add initial value NULL so + // stores which follow loads. For now, add initial value null so // that compare pointers optimization works correctly. } } - if (value == NULL) { - // A field's initializing value was not recorded. Add NULL. + if (value == nullptr) { + // A field's initializing value was not recorded. Add null. if (add_edge(field, null_obj)) { // New edge was added new_edges++; @@ -2138,7 +2138,7 @@ const TypeInt* ConnectionGraph::optimize_ptr_compare(Node* n) { assert(ptn2->is_JavaObject() || ptn2->is_LocalVar(), "sanity"); // Check simple cases first. - if (jobj1 != NULL) { + if (jobj1 != nullptr) { if (jobj1->escape_state() == PointsToNode::NoEscape) { if (jobj1 == jobj2) { // Comparing the same not escaping object. @@ -2152,7 +2152,7 @@ const TypeInt* ConnectionGraph::optimize_ptr_compare(Node* n) { } } } - if (jobj2 != NULL) { + if (jobj2 != nullptr) { if (jobj2->escape_state() == PointsToNode::NoEscape) { Node* obj = jobj2->ideal_node(); // Comparing not escaping allocation. @@ -2162,8 +2162,8 @@ const TypeInt* ConnectionGraph::optimize_ptr_compare(Node* n) { } } } - if (jobj1 != NULL && jobj1 != phantom_obj && - jobj2 != NULL && jobj2 != phantom_obj && + if (jobj1 != nullptr && jobj1 != phantom_obj && + jobj2 != nullptr && jobj2 != phantom_obj && jobj1->ideal_node()->is_Con() && jobj2->ideal_node()->is_Con()) { // Klass or String constants compare. Need to be careful with @@ -2212,7 +2212,7 @@ const TypeInt* ConnectionGraph::optimize_ptr_compare(Node* n) { void ConnectionGraph::add_local_var(Node *n, PointsToNode::EscapeState es) { PointsToNode* ptadr = _nodes.at(n->_idx); - if (ptadr != NULL) { + if (ptadr != nullptr) { assert(ptadr->is_LocalVar() && ptadr->ideal_node() == n, "sanity"); return; } @@ -2223,7 +2223,7 @@ void ConnectionGraph::add_local_var(Node *n, PointsToNode::EscapeState es) { void ConnectionGraph::add_java_object(Node *n, PointsToNode::EscapeState es) { PointsToNode* ptadr = _nodes.at(n->_idx); - if (ptadr != NULL) { + if (ptadr != nullptr) { assert(ptadr->is_JavaObject() && ptadr->ideal_node() == n, "sanity"); return; } @@ -2234,7 +2234,7 @@ void ConnectionGraph::add_java_object(Node *n, PointsToNode::EscapeState es) { void ConnectionGraph::add_field(Node *n, PointsToNode::EscapeState es, int offset) { PointsToNode* ptadr = _nodes.at(n->_idx); - if (ptadr != NULL) { + if (ptadr != nullptr) { assert(ptadr->is_Field() && ptadr->ideal_node() == n, "sanity"); return; } @@ -2251,9 +2251,9 @@ void ConnectionGraph::add_field(Node *n, PointsToNode::EscapeState es, int offse void ConnectionGraph::add_arraycopy(Node *n, PointsToNode::EscapeState es, PointsToNode* src, PointsToNode* dst) { assert(!src->is_Field() && !dst->is_Field(), "only for JavaObject and LocalVar"); - assert((src != null_obj) && (dst != null_obj), "not for ConP NULL"); + assert((src != null_obj) && (dst != null_obj), "not for ConP null"); PointsToNode* ptadr = _nodes.at(n->_idx); - if (ptadr != NULL) { + if (ptadr != nullptr) { assert(ptadr->is_Arraycopy() && ptadr->ideal_node() == n, "sanity"); return; } @@ -2274,17 +2274,17 @@ bool ConnectionGraph::is_oop_field(Node* n, int offset, bool* unsafe) { if (offset == Type::OffsetBot) { // Check only oop fields. if (!adr_type->isa_aryptr() || - (adr_type->isa_aryptr()->klass() == NULL) || + (adr_type->isa_aryptr()->klass() == nullptr) || adr_type->isa_aryptr()->klass()->is_obj_array_klass()) { // OffsetBot is used to reference array's element. Ignore first AddP. - if (find_second_addp(n, n->in(AddPNode::Base)) == NULL) { + if (find_second_addp(n, n->in(AddPNode::Base)) == nullptr) { bt = T_OBJECT; } } } else if (offset != oopDesc::klass_offset_in_bytes()) { if (adr_type->isa_instptr()) { ciField* field = _compile->alias_type(adr_type->isa_instptr())->field(); - if (field != NULL) { + if (field != nullptr) { bt = field->layout_type(); } else { // Check for unsafe oop field access @@ -2299,7 +2299,7 @@ bool ConnectionGraph::is_oop_field(Node* n, int offset, bool* unsafe) { } else if (adr_type->isa_aryptr()) { if (offset == arrayOopDesc::length_offset_in_bytes()) { // Ignore array length load. - } else if (find_second_addp(n, n->in(AddPNode::Base)) != NULL) { + } else if (find_second_addp(n, n->in(AddPNode::Base)) != nullptr) { // Ignore first AddP. } else { const Type* elemtype = adr_type->isa_aryptr()->elem(); @@ -2319,31 +2319,31 @@ bool ConnectionGraph::is_oop_field(Node* n, int offset, bool* unsafe) { return (is_reference_type(bt) || bt == T_NARROWOOP); } -// Returns unique pointed java object or NULL. +// Returns unique pointed java object or null. JavaObjectNode* ConnectionGraph::unique_java_object(Node *n) { assert(!_collecting, "should not call when constructed graph"); // If the node was created after the escape computation we can't answer. uint idx = n->_idx; if (idx >= nodes_size()) { - return NULL; + return nullptr; } PointsToNode* ptn = ptnode_adr(idx); - if (ptn == NULL) { - return NULL; + if (ptn == nullptr) { + return nullptr; } if (ptn->is_JavaObject()) { return ptn->as_JavaObject(); } assert(ptn->is_LocalVar(), "sanity"); // Check all java objects it points to. - JavaObjectNode* jobj = NULL; + JavaObjectNode* jobj = nullptr; for (EdgeIterator i(ptn); i.has_next(); i.next()) { PointsToNode* e = i.get(); if (e->is_JavaObject()) { - if (jobj == NULL) { + if (jobj == nullptr) { jobj = e->as_JavaObject(); } else if (jobj != e) { - return NULL; + return nullptr; } } } @@ -2384,7 +2384,7 @@ bool ConnectionGraph::not_global_escape(Node *n) { return false; } PointsToNode* ptn = ptnode_adr(idx); - if (ptn == NULL) { + if (ptn == nullptr) { return false; // not in congraph (e.g. ConI) } PointsToNode::EscapeState es = ptn->escape_state(); @@ -2458,7 +2458,7 @@ bool FieldNode::has_base(JavaObjectNode* jobj) const { bool ConnectionGraph::is_captured_store_address(Node* addp) { // Handle simple case first. - assert(_igvn->type(addp)->isa_oopptr() == NULL, "should be raw access"); + assert(_igvn->type(addp)->isa_oopptr() == nullptr, "should be raw access"); if (addp->in(AddPNode::Address)->is_Proj() && addp->in(AddPNode::Address)->in(0)->is_Allocate()) { return true; } else if (addp->in(AddPNode::Address)->is_Phi()) { @@ -2478,7 +2478,7 @@ bool ConnectionGraph::is_captured_store_address(Node* addp) { int ConnectionGraph::address_offset(Node* adr, PhaseTransform *phase) { const Type *adr_type = phase->type(adr); - if (adr->is_AddP() && adr_type->isa_oopptr() == NULL && is_captured_store_address(adr)) { + if (adr->is_AddP() && adr_type->isa_oopptr() == nullptr && is_captured_store_address(adr)) { // We are computing a raw address for a store captured by an Initialize // compute an appropriate address type. AddP cases #3 and #5 (see below). int offs = (int)phase->find_intptr_t_con(adr->in(AddPNode::Offset), Type::OffsetBot); @@ -2488,7 +2488,7 @@ int ConnectionGraph::address_offset(Node* adr, PhaseTransform *phase) { return offs; } const TypePtr *t_ptr = adr_type->isa_ptr(); - assert(t_ptr != NULL, "must be a pointer type"); + assert(t_ptr != nullptr, "must be a pointer type"); return t_ptr->offset(); } @@ -2580,7 +2580,7 @@ Node* ConnectionGraph::get_addp_base(Node *addp) { int opcode = uncast_base->Opcode(); assert(opcode == Op_ConP || opcode == Op_ThreadLocal || opcode == Op_CastX2P || uncast_base->is_DecodeNarrowPtr() || - (uncast_base->is_Mem() && (uncast_base->bottom_type()->isa_rawptr() != NULL)) || + (uncast_base->is_Mem() && (uncast_base->bottom_type()->isa_rawptr() != nullptr)) || is_captured_store_address(addp), "sanity"); } } @@ -2624,7 +2624,7 @@ Node* ConnectionGraph::find_second_addp(Node* addp, Node* n) { // return addp2; } - return NULL; + return nullptr; } // @@ -2634,9 +2634,9 @@ Node* ConnectionGraph::find_second_addp(Node* addp, Node* n) { bool ConnectionGraph::split_AddP(Node *addp, Node *base) { PhaseGVN* igvn = _igvn; const TypeOopPtr *base_t = igvn->type(base)->isa_oopptr(); - assert(base_t != NULL && base_t->is_known_instance(), "expecting instance oopptr"); + assert(base_t != nullptr && base_t->is_known_instance(), "expecting instance oopptr"); const TypeOopPtr *t = igvn->type(addp)->isa_oopptr(); - if (t == NULL) { + if (t == nullptr) { // We are computing a raw address for a store captured by an Initialize // compute an appropriate address type (cases #3 and #5). assert(igvn->type(addp) == TypeRawPtr::NOTNULL, "must be raw pointer"); @@ -2694,7 +2694,7 @@ bool ConnectionGraph::split_AddP(Node *addp, Node *base) { // AddP case #4 (adr is array's element offset AddP node) #ifdef ASSERT const TypeOopPtr *atype = igvn->type(adr)->isa_oopptr(); - assert(adr->is_AddP() && atype != NULL && + assert(adr->is_AddP() && atype != nullptr && atype->instance_id() == inst_id, "array's element offset should be processed first"); #endif } @@ -2722,12 +2722,12 @@ PhiNode *ConnectionGraph::create_split_phi(PhiNode *orig_phi, int alias_idx, Gro } // Have we recently created a Phi for this alias index? PhiNode *result = get_map_phi(orig_phi->_idx); - if (result != NULL && C->get_alias_index(result->adr_type()) == alias_idx) { + if (result != nullptr && C->get_alias_index(result->adr_type()) == alias_idx) { return result; } // Previous check may fail when the same wide memory Phi was split into Phis // for different memory slices. Search all Phis for this region. - if (result != NULL) { + if (result != nullptr) { Node* region = orig_phi->in(0); for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) { Node* phi = region->fast_out(i); @@ -2745,11 +2745,11 @@ PhiNode *ConnectionGraph::create_split_phi(PhiNode *orig_phi, int alias_idx, Gro // to the Compile object, and the C2Compiler will see it and retry. C->record_failure(C2Compiler::retry_no_escape_analysis()); } - return NULL; + return nullptr; } orig_phi_worklist.append_if_missing(orig_phi); const TypePtr *atype = C->get_adr_type(alias_idx); - result = PhiNode::make(orig_phi->in(0), NULL, Type::MEMORY, atype); + result = PhiNode::make(orig_phi->in(0), nullptr, Type::MEMORY, atype); C->copy_node_notes_to(result, orig_phi); igvn->set_type(result, result->bottom_type()); record_for_optimizer(result); @@ -2779,7 +2779,7 @@ PhiNode *ConnectionGraph::split_memory_phi(PhiNode *orig_phi, int alias_idx, Gro while(!finished) { while (idx < phi->req()) { Node *mem = find_inst_mem(phi->in(idx), alias_idx, orig_phi_worklist); - if (mem != NULL && mem->is_Phi()) { + if (mem != nullptr && mem->is_Phi()) { PhiNode *newphi = create_split_phi(mem->as_Phi(), alias_idx, orig_phi_worklist, new_phi_created); if (new_phi_created) { // found an phi for which we created a new split, push current one on worklist and begin @@ -2795,20 +2795,20 @@ PhiNode *ConnectionGraph::split_memory_phi(PhiNode *orig_phi, int alias_idx, Gro } } if (C->failing()) { - return NULL; + return nullptr; } result->set_req(idx++, mem); } #ifdef ASSERT // verify that the new Phi has an input for each input of the original assert( phi->req() == result->req(), "must have same number of inputs."); - assert( result->in(0) != NULL && result->in(0) == phi->in(0), "regions must match"); + assert( result->in(0) != nullptr && result->in(0) == phi->in(0), "regions must match"); #endif // Check if all new phi's inputs have specified alias index. // Otherwise use old phi. for (uint i = 1; i < phi->req(); i++) { Node* in = result->in(i); - assert((phi->in(i) == NULL) == (in == NULL), "inputs must correspond."); + assert((phi->in(i) == nullptr) == (in == nullptr), "inputs must correspond."); } // we have finished processing a Phi, see if there are any more to do finished = (phi_list.length() == 0 ); @@ -2848,7 +2848,7 @@ void ConnectionGraph::move_inst_mem(Node* n, GrowableArray &orig_phi Compile* C = _compile; PhaseGVN* igvn = _igvn; const TypePtr* tp = igvn->type(n->in(MemNode::Address))->isa_ptr(); - assert(tp != NULL, "ptr type"); + assert(tp != nullptr, "ptr type"); int alias_idx = C->get_alias_index(tp); int general_idx = C->get_general_index(alias_idx); @@ -2877,7 +2877,7 @@ void ConnectionGraph::move_inst_mem(Node* n, GrowableArray &orig_phi continue; } tp = use->as_MemBar()->adr_type()->isa_ptr(); - if ((tp != NULL && C->get_alias_index(tp) == alias_idx) || + if ((tp != nullptr && C->get_alias_index(tp) == alias_idx) || alias_idx == general_idx) { continue; // Nothing to do } @@ -2898,14 +2898,14 @@ void ConnectionGraph::move_inst_mem(Node* n, GrowableArray &orig_phi } // Memory nodes should have new memory input. tp = igvn->type(use->in(MemNode::Address))->isa_ptr(); - assert(tp != NULL, "ptr type"); + assert(tp != nullptr, "ptr type"); int idx = C->get_alias_index(tp); - assert(get_map(use->_idx) != NULL || idx == alias_idx, + assert(get_map(use->_idx) != nullptr || idx == alias_idx, "Following memory nodes should have new memory input or be on the same memory slice"); } else if (use->is_Phi()) { // Phi nodes should be split and moved already. tp = use->as_Phi()->adr_type()->isa_ptr(); - assert(tp != NULL, "ptr type"); + assert(tp != nullptr, "ptr type"); int idx = C->get_alias_index(tp); assert(idx == alias_idx, "Following Phi nodes should be on the same memory slice"); } else { @@ -2921,15 +2921,15 @@ void ConnectionGraph::move_inst_mem(Node* n, GrowableArray &orig_phi // is the specified alias index. // Node* ConnectionGraph::find_inst_mem(Node *orig_mem, int alias_idx, GrowableArray &orig_phis) { - if (orig_mem == NULL) { + if (orig_mem == nullptr) { return orig_mem; } Compile* C = _compile; PhaseGVN* igvn = _igvn; const TypeOopPtr *toop = C->get_adr_type(alias_idx)->isa_oopptr(); - bool is_instance = (toop != NULL) && toop->is_known_instance(); + bool is_instance = (toop != nullptr) && toop->is_known_instance(); Node *start_mem = C->start()->proj_out_or_null(TypeFunc::Memory); - Node *prev = NULL; + Node *prev = nullptr; Node *result = orig_mem; while (prev != result) { prev = result; @@ -2941,12 +2941,12 @@ Node* ConnectionGraph::find_inst_mem(Node *orig_mem, int alias_idx, GrowableArra if (at == Type::TOP) { break; // Dead } - assert (at->isa_ptr() != NULL, "pointer type required."); + assert (at->isa_ptr() != nullptr, "pointer type required."); int idx = C->get_alias_index(at->is_ptr()); if (idx == alias_idx) { break; // Found } - if (!is_instance && (at->isa_oopptr() == NULL || + if (!is_instance && (at->isa_oopptr() == nullptr || !at->is_oopptr()->is_known_instance())) { break; // Do not skip store to general memory slice. } @@ -2970,7 +2970,7 @@ Node* ConnectionGraph::find_inst_mem(Node *orig_mem, int alias_idx, GrowableArra AllocateNode* alloc = proj_in->as_Initialize()->allocation(); // Stop if this is the initialization for the object instance which // which contains this memory slice, otherwise skip over it. - if (alloc == NULL || alloc->_idx != (uint)toop->instance_id()) { + if (alloc == nullptr || alloc->_idx != (uint)toop->instance_id()) { result = proj_in->in(TypeFunc::Memory); } } else if (proj_in->is_MemBar()) { @@ -2996,14 +2996,14 @@ Node* ConnectionGraph::find_inst_mem(Node *orig_mem, int alias_idx, GrowableArra result = mmem->memory_at(C->get_general_index(alias_idx)); result = find_inst_mem(result, alias_idx, orig_phis); if (C->failing()) { - return NULL; + return nullptr; } mmem->set_memory_at(alias_idx, result); } } else if (result->is_Phi() && C->get_alias_index(result->as_Phi()->adr_type()) != alias_idx) { Node *un = result->as_Phi()->unique_input(igvn); - if (un != NULL) { + if (un != nullptr) { orig_phis.append_if_missing(result->as_Phi()); result = un; } else { @@ -3018,7 +3018,7 @@ Node* ConnectionGraph::find_inst_mem(Node *orig_mem, int alias_idx, GrowableArra // Otherwise skip it (the call updated 'result' value). } else if (result->Opcode() == Op_SCMemProj) { Node* mem = result->in(0); - Node* adr = NULL; + Node* adr = nullptr; if (mem->is_LoadStore()) { adr = mem->in(MemNode::Address); } else { @@ -3028,7 +3028,7 @@ Node* ConnectionGraph::find_inst_mem(Node *orig_mem, int alias_idx, GrowableArra } const Type *at = igvn->type(adr); if (at != Type::TOP) { - assert(at->isa_ptr() != NULL, "pointer type required."); + assert(at->isa_ptr() != nullptr, "pointer type required."); int idx = C->get_alias_index(at->is_ptr()); if (idx == alias_idx) { // Assert in debug mode @@ -3041,7 +3041,7 @@ Node* ConnectionGraph::find_inst_mem(Node *orig_mem, int alias_idx, GrowableArra Node* adr = result->in(3); // Memory edge corresponds to destination array const Type *at = igvn->type(adr); if (at != Type::TOP) { - assert(at->isa_ptr() != NULL, "pointer type required."); + assert(at->isa_ptr() != nullptr, "pointer type required."); int idx = C->get_alias_index(at->is_ptr()); if (idx == alias_idx) { // Assert in debug mode @@ -3189,7 +3189,7 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, } // Find CheckCastPP for the allocate or for the return value of a call n = alloc->result_cast(); - if (n == NULL) { // No uses except Initialize node + if (n == nullptr) { // No uses except Initialize node if (alloc->is_Allocate()) { // Set the scalar_replaceable flag for allocation // so it could be eliminated if it has no uses. @@ -3214,7 +3214,7 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, if (alloc->is_Allocate() && n->as_Type()->type() == TypeInstPtr::NOTNULL && (alloc->is_AllocateArray() || igvn->type(alloc->in(AllocateNode::KlassNode)) != TypeKlassPtr::OBJECT)) { - Node *cast2 = NULL; + Node *cast2 = nullptr; for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { Node *use = n->fast_out(i); if (use->is_CheckCastPP()) { @@ -3222,7 +3222,7 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, break; } } - if (cast2 != NULL) { + if (cast2 != nullptr) { n = cast2; } else { // Non-scalar replaceable if the allocation type is unknown statically @@ -3233,7 +3233,7 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, } const TypeOopPtr *t = igvn->type(n)->isa_oopptr(); - if (t == NULL) { + if (t == nullptr) { continue; // not a TypeOopPtr } if (!t->klass_is_exact()) { @@ -3277,7 +3277,7 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, "only AddP nodes are Field edges in CG"); if (use->outcnt() > 0) { // Don't process dead nodes Node* addp2 = find_second_addp(use, use->in(AddPNode::Base)); - if (addp2 != NULL) { + if (addp2 != nullptr) { assert(alloc->is_AllocateArray(),"array allocation was expected"); alloc_worklist.append_if_missing(addp2); } @@ -3289,12 +3289,12 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, // the users of the raw allocation result and push AddP users // on alloc_worklist. Node *raw_result = alloc->proj_out_or_null(TypeFunc::Parms); - assert (raw_result != NULL, "must have an allocation result"); + assert (raw_result != nullptr, "must have an allocation result"); for (DUIterator_Fast imax, i = raw_result->fast_outs(imax); i < imax; i++) { Node *use = raw_result->fast_out(i); if (use->is_AddP() && use->outcnt() > 0) { // Don't process dead nodes Node* addp2 = find_second_addp(use, raw_result); - if (addp2 != NULL) { + if (addp2 != nullptr) { assert(alloc->is_AllocateArray(),"array allocation was expected"); alloc_worklist.append_if_missing(addp2); } @@ -3306,11 +3306,11 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, } } else if (n->is_AddP()) { JavaObjectNode* jobj = unique_java_object(get_addp_base(n)); - if (jobj == NULL || jobj == phantom_obj) { + if (jobj == nullptr || jobj == phantom_obj) { #ifdef ASSERT ptnode_adr(get_addp_base(n)->_idx)->dump(); ptnode_adr(n->_idx)->dump(); - assert(jobj != NULL && jobj != phantom_obj, "escaped allocation"); + assert(jobj != nullptr && jobj != phantom_obj, "escaped allocation"); #endif _compile->record_failure(C2Compiler::retry_no_escape_analysis()); return; @@ -3327,10 +3327,10 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, continue; // already processed } JavaObjectNode* jobj = unique_java_object(n); - if (jobj == NULL || jobj == phantom_obj) { + if (jobj == nullptr || jobj == phantom_obj) { #ifdef ASSERT ptnode_adr(n->_idx)->dump(); - assert(jobj != NULL && jobj != phantom_obj, "escaped allocation"); + assert(jobj != nullptr && jobj != phantom_obj, "escaped allocation"); #endif _compile->record_failure(C2Compiler::retry_no_escape_analysis()); return; @@ -3338,7 +3338,7 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, Node *val = get_map(jobj->idx()); // CheckCastPP node TypeNode *tn = n->as_Type(); const TypeOopPtr* tinst = igvn->type(val)->isa_oopptr(); - assert(tinst != NULL && tinst->is_known_instance() && + assert(tinst != nullptr && tinst->is_known_instance() && tinst->instance_id() == jobj->idx() , "instance type expected."); const Type *tn_type = igvn->type(tn); @@ -3348,7 +3348,7 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, } else { tn_t = tn_type->isa_oopptr(); } - if (tn_t != NULL && tinst->klass()->is_subtype_of(tn_t->klass())) { + if (tn_t != nullptr && tinst->klass()->is_subtype_of(tn_t->klass())) { if (tn_type->isa_narrowoop()) { tn_type = tinst->make_narrowoop(); } else { @@ -3361,7 +3361,7 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, record_for_optimizer(n); } else { assert(tn_type == TypePtr::NULL_PTR || - tn_t != NULL && !tinst->klass()->is_subtype_of(tn_t->klass()), + tn_t != nullptr && !tinst->klass()->is_subtype_of(tn_t->klass()), "unexpected type"); continue; // Skip dead path with different type } @@ -3383,7 +3383,7 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, } } else if (use->is_AddP() && use->outcnt() > 0) { // No dead nodes Node* addp2 = find_second_addp(use, n); - if (addp2 != NULL) { + if (addp2 != nullptr) { alloc_worklist.append_if_missing(addp2); } alloc_worklist.append_if_missing(use); @@ -3443,9 +3443,9 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, dest = get_addp_base(dest); } JavaObjectNode* jobj = unique_java_object(dest); - if (jobj != NULL) { + if (jobj != nullptr) { Node *base = get_map(jobj->idx()); - if (base != NULL) { + if (base != nullptr) { const TypeOopPtr *base_t = _igvn->type(base)->isa_oopptr(); ac->_dest_type = base_t; } @@ -3455,9 +3455,9 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, src = get_addp_base(src); } jobj = unique_java_object(src); - if (jobj != NULL) { + if (jobj != nullptr) { Node* base = get_map(jobj->idx()); - if (base != NULL) { + if (base != nullptr) { const TypeOopPtr *base_t = _igvn->type(base)->isa_oopptr(); ac->_src_type = base_t; } @@ -3483,14 +3483,14 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, } else if (n->is_MemBar()) { // Initialize, MemBar nodes // we don't need to do anything, but the users must be pushed n = n->as_MemBar()->proj_out_or_null(TypeFunc::Memory); - if (n == NULL) { + if (n == nullptr) { continue; } } else if (n->Opcode() == Op_StrCompressedCopy || n->Opcode() == Op_EncodeISOArray) { // get the memory projection n = n->find_out_with(Op_SCMemProj); - assert(n != NULL && n->Opcode() == Op_SCMemProj, "memory projection required"); + assert(n != nullptr && n->Opcode() == Op_SCMemProj, "memory projection required"); } else { assert(n->is_Mem(), "memory node required."); Node *addr = n->in(MemNode::Address); @@ -3498,7 +3498,7 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, if (addr_t == Type::TOP) { continue; } - assert (addr_t->isa_ptr() != NULL, "pointer type required."); + assert (addr_t->isa_ptr() != nullptr, "pointer type required."); int alias_idx = _compile->get_alias_index(addr_t->is_ptr()); assert ((uint)alias_idx < new_index_end, "wrong alias index"); Node *mem = find_inst_mem(n->in(MemNode::Memory), alias_idx, orig_phis); @@ -3515,7 +3515,7 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, } else if (n->is_LoadStore()) { // get the memory projection n = n->find_out_with(Op_SCMemProj); - assert(n != NULL && n->Opcode() == Op_SCMemProj, "memory projection required"); + assert(n != nullptr && n->Opcode() == Op_SCMemProj, "memory projection required"); } } // push user on appropriate worklist @@ -3576,8 +3576,8 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, uint nslices = MIN2(nmm->req(), new_index_start); for (uint i = Compile::AliasIdxRaw+1; i < nslices; i++) { Node* mem = nmm->in(i); - Node* cur = NULL; - if (mem == NULL || mem->is_top()) { + Node* cur = nullptr; + if (mem == nullptr || mem->is_top()) { continue; } // First, update mergemem by moving memory nodes to corresponding slices @@ -3585,10 +3585,10 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, while (mem->is_Mem()) { const Type *at = igvn->type(mem->in(MemNode::Address)); if (at != Type::TOP) { - assert (at->isa_ptr() != NULL, "pointer type required."); + assert (at->isa_ptr() != nullptr, "pointer type required."); uint idx = (uint)_compile->get_alias_index(at->is_ptr()); if (idx == i) { - if (cur == NULL) { + if (cur == nullptr) { cur = mem; } } else { @@ -3599,7 +3599,7 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, } mem = mem->in(MemNode::Memory); } - nmm->set_memory_at(i, (cur != NULL) ? cur : mem); + nmm->set_memory_at(i, (cur != nullptr) ? cur : mem); // Find any instance of the current type if we haven't encountered // already a memory slice of the instance along the memory chain. for (uint ni = new_index_start; ni < new_index_end; ni++) { @@ -3669,7 +3669,7 @@ void ConnectionGraph::split_unique_types(GrowableArray &alloc_worklist, for (uint i = 0; i < ideal_nodes.size(); i++) { Node* n = ideal_nodes.at(i); Node* nmem = get_map(n->_idx); - assert(nmem != NULL, "sanity"); + assert(nmem != nullptr, "sanity"); if (n->is_Mem()) { #if 0 // ifdef ASSERT Node* old_mem = n->in(MemNode::Memory); @@ -3761,7 +3761,7 @@ void PointsToNode::dump(bool print_state) const { tty->print(" %d%s%s", u->idx(), is_base ? "b" : "", u->is_Arraycopy() ? "cp" : ""); } tty->print(" ]] "); - if (_node == NULL) { + if (_node == nullptr) { tty->print_cr(""); } else { _node->dump(); @@ -3773,7 +3773,7 @@ void ConnectionGraph::dump(GrowableArray& ptnodes_worklist) { int ptnodes_length = ptnodes_worklist.length(); for (int i = 0; i < ptnodes_length; i++) { PointsToNode *ptn = ptnodes_worklist.at(i); - if (ptn == NULL || !ptn->is_JavaObject()) { + if (ptn == nullptr || !ptn->is_JavaObject()) { continue; } PointsToNode::EscapeState es = ptn->escape_state(); diff --git a/src/hotspot/share/opto/escape.hpp b/src/hotspot/share/opto/escape.hpp index 2171c344c3187..16d58f5d93806 100644 --- a/src/hotspot/share/opto/escape.hpp +++ b/src/hotspot/share/opto/escape.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -294,7 +294,7 @@ class PointsToIterator: public StackObj { inline PointsToIterator(const PointsToNode* n, int cnt) : node(n), cnt(cnt), i(0) { } inline bool has_next() const { return i < cnt; } inline void next() { i++; } - PointsToNode* get() const { ShouldNotCallThis(); return NULL; } + PointsToNode* get() const { ShouldNotCallThis(); return nullptr; } }; class EdgeIterator: public PointsToIterator { @@ -429,7 +429,7 @@ class ConnectionGraph: public ResourceObj { // Set the escape state of an object and its fields. void set_escape_state(PointsToNode* ptn, PointsToNode::EscapeState esc) { - // Don't change non-escaping state of NULL pointer. + // Don't change non-escaping state of null pointer. if (ptn != null_obj) { if (ptn->escape_state() < esc) { ptn->set_escape_state(esc); @@ -440,7 +440,7 @@ class ConnectionGraph: public ResourceObj { } } void set_fields_escape_state(PointsToNode* ptn, PointsToNode::EscapeState esc) { - // Don't change non-escaping state of NULL pointer. + // Don't change non-escaping state of null pointer. if (ptn != null_obj) { if (ptn->fields_escape_state() < esc) { ptn->set_fields_escape_state(esc); @@ -465,7 +465,7 @@ class ConnectionGraph: public ResourceObj { // Optimize objects compare. const TypeInt* optimize_ptr_compare(Node* n); - // Returns unique corresponding java object or NULL. + // Returns unique corresponding java object or null. JavaObjectNode* unique_java_object(Node *n); // Add an edge of the specified type pointing to the specified target. @@ -503,7 +503,7 @@ class ConnectionGraph: public ResourceObj { if (is_new) { // New edge? assert(!_verify, "graph is incomplete"); if (to == null_obj) { - return is_new; // Don't add fields to NULL pointer. + return is_new; // Don't add fields to null pointer. } if (to->is_JavaObject()) { is_new = to->add_edge(from); @@ -558,7 +558,7 @@ class ConnectionGraph: public ResourceObj { PhiNode* get_map_phi(int idx) { Node* phi = _node_map[idx]; - return (phi == NULL) ? NULL : phi->as_Phi(); + return (phi == nullptr) ? nullptr : phi->as_Phi(); } // Returns true if there is an object in the scope of sfn that does not escape globally. @@ -598,21 +598,21 @@ class ConnectionGraph: public ResourceObj { void add_local_var_and_edge(Node* n, PointsToNode::EscapeState es, Node* to, Unique_Node_List *delayed_worklist) { PointsToNode* ptn = ptnode_adr(to->_idx); - if (delayed_worklist != NULL) { // First iteration of CG construction + if (delayed_worklist != nullptr) { // First iteration of CG construction add_local_var(n, es); - if (ptn == NULL) { + if (ptn == nullptr) { delayed_worklist->push(n); return; // Process it later. } } else { - assert(ptn != NULL, "node should be registered"); + assert(ptn != nullptr, "node should be registered"); } add_edge(ptnode_adr(n->_idx), ptn); } // Map ideal node to existing PointsTo node (usually phantom_object). void map_ideal_node(Node *n, PointsToNode* ptn) { - assert(ptn != NULL, "only existing PointsTo node"); + assert(ptn != nullptr, "only existing PointsTo node"); _nodes.at_put(n->_idx, ptn); } @@ -625,8 +625,8 @@ class ConnectionGraph: public ResourceObj { }; inline PointsToNode::PointsToNode(ConnectionGraph *CG, Node* n, EscapeState es, NodeType type): - _edges(CG->_compile->comp_arena(), 2, 0, NULL), - _uses (CG->_compile->comp_arena(), 2, 0, NULL), + _edges(CG->_compile->comp_arena(), 2, 0, nullptr), + _uses (CG->_compile->comp_arena(), 2, 0, nullptr), _type((u1)type), _flags(ScalarReplaceable), _escape((u1)es), @@ -634,12 +634,12 @@ inline PointsToNode::PointsToNode(ConnectionGraph *CG, Node* n, EscapeState es, _node(n), _idx(n->_idx), _pidx(CG->next_pidx()) { - assert(n != NULL && es != UnknownEscape, "sanity"); + assert(n != nullptr && es != UnknownEscape, "sanity"); } inline FieldNode::FieldNode(ConnectionGraph *CG, Node* n, EscapeState es, int offs, bool is_oop): PointsToNode(CG, n, es, Field), - _bases(CG->_compile->comp_arena(), 2, 0, NULL), + _bases(CG->_compile->comp_arena(), 2, 0, nullptr), _offset(offs), _is_oop(is_oop), _has_unknown_base(false) { } diff --git a/src/hotspot/share/opto/gcm.cpp b/src/hotspot/share/opto/gcm.cpp index 33d8058009074..1ad8fd8a53f96 100644 --- a/src/hotspot/share/opto/gcm.cpp +++ b/src/hotspot/share/opto/gcm.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -61,7 +61,7 @@ void PhaseCFG::schedule_node_into_block( Node *n, Block *b ) { if (use->is_Proj()) { Block* buse = get_block_for_node(use); if (buse != b) { // In wrong block? - if (buse != NULL) { + if (buse != nullptr) { buse->find_remove(use); // Remove from wrong block } map_node_to_block(use, b); @@ -77,9 +77,9 @@ void PhaseCFG::schedule_node_into_block( Node *n, Block *b ) { // the projection will be in a predecessor block. void PhaseCFG::replace_block_proj_ctrl( Node *n ) { const Node *in0 = n->in(0); - assert(in0 != NULL, "Only control-dependent"); + assert(in0 != nullptr, "Only control-dependent"); const Node *p = in0->is_block_proj(); - if (p != NULL && p != n) { // Control from a block projection? + if (p != nullptr && p != n) { // Control from a block projection? assert(!n->pinned() || n->is_MachConstantBase(), "only pinned MachConstantBase node is expected here"); // Find trailing Region Block *pb = get_block_for_node(in0); // Block-projection already has basic block @@ -109,7 +109,7 @@ bool PhaseCFG::is_dominator(Node* dom_node, Node* node) { } Block* d = find_block_for_node(dom_node); Block* n = find_block_for_node(node); - assert(n != NULL && d != NULL, "blocks must exist"); + assert(n != nullptr && d != nullptr, "blocks must exist"); if (d == n) { if (dom_node->is_block_start()) { @@ -212,15 +212,15 @@ void PhaseCFG::schedule_pinned_nodes(VectorSet &visited) { // removed in final_graph_reshaping), fix the control of the // node to cover the precedence edges and remove the // dependencies. - Node* n = NULL; + Node* n = nullptr; for (uint i = node->len()-1; i >= node->req(); i--) { Node* m = node->in(i); - if (m == NULL) continue; + if (m == nullptr) continue; // Only process precedence edges that are CFG nodes. Safepoints and control projections can be in the middle of a block if (is_CFG(m)) { node->rm_prec(i); - if (n == NULL) { + if (n == nullptr) { n = m; } else { assert(is_dominator(n, m) || is_dominator(m, n), "one must dominate the other"); @@ -231,7 +231,7 @@ void PhaseCFG::schedule_pinned_nodes(VectorSet &visited) { assert(node->as_Mach()->ideal_Opcode() == Op_StoreCM, "must be StoreCM node"); } } - if (n != NULL) { + if (n != nullptr) { assert(node->in(0), "control should have been set"); assert(is_dominator(n, node->in(0)) || is_dominator(node->in(0), n), "one must dominate the other"); if (!is_dominator(n, node->in(0))) { @@ -239,9 +239,9 @@ void PhaseCFG::schedule_pinned_nodes(VectorSet &visited) { } } - // process all inputs that are non NULL + // process all inputs that are non null for (int i = node->req()-1; i >= 0; --i) { - if (node->in(i) != NULL) { + if (node->in(i) != nullptr) { spstack.push(node->in(i)); } } @@ -254,10 +254,10 @@ void PhaseCFG::schedule_pinned_nodes(VectorSet &visited) { // Check this by by seeing that it is dominated by b1, the deepest // input observed until b2. static void assert_dom(Block* b1, Block* b2, Node* n, const PhaseCFG* cfg) { - if (b1 == NULL) return; + if (b1 == nullptr) return; assert(b1->_dom_depth < b2->_dom_depth, "sanity"); Block* tmp = b2; - while (tmp != b1 && tmp != NULL) { + while (tmp != b1 && tmp != nullptr) { tmp = tmp->_idom; } if (tmp != b1) { @@ -265,7 +265,7 @@ static void assert_dom(Block* b1, Block* b2, Node* n, const PhaseCFG* cfg) { tty->print_cr("!!! Unschedulable graph !!!"); for (uint j=0; jlen(); j++) { // For all inputs Node* inn = n->in(j); // Get input - if (inn == NULL) continue; // Ignore NULL, missing inputs + if (inn == nullptr) continue; // Ignore null, missing inputs Block* inb = cfg->get_block_for_node(inn); tty->print("B%d idom=B%d depth=%2d ",inb->_pre_order, inb->_idom ? inb->_idom->_pre_order : 0, inb->_dom_depth); @@ -280,13 +280,13 @@ static void assert_dom(Block* b1, Block* b2, Node* n, const PhaseCFG* cfg) { static Block* find_deepest_input(Node* n, const PhaseCFG* cfg) { // Find the last input dominated by all other inputs. - Block* deepb = NULL; // Deepest block so far + Block* deepb = nullptr; // Deepest block so far int deepb_dom_depth = 0; for (uint k = 0; k < n->len(); k++) { // For all inputs Node* inn = n->in(k); // Get input - if (inn == NULL) continue; // Ignore NULL, missing inputs + if (inn == nullptr) continue; // Ignore null, missing inputs Block* inb = cfg->get_block_for_node(inn); - assert(inb != NULL, "must already have scheduled this input"); + assert(inb != nullptr, "must already have scheduled this input"); if (deepb_dom_depth < (int) inb->_dom_depth) { // The new inb must be dominated by the previous deepb. // The various inputs must be linearly ordered in the dom @@ -296,7 +296,7 @@ static Block* find_deepest_input(Node* n, const PhaseCFG* cfg) { deepb_dom_depth = deepb->_dom_depth; } } - assert(deepb != NULL, "must be at least one input to n"); + assert(deepb != nullptr, "must be at least one input to n"); return deepb; } @@ -325,7 +325,7 @@ bool PhaseCFG::schedule_early(VectorSet &visited, Node_Stack &roots) { // to root and nodes that use is_block_proj() nodes should be attached // to the region that starts their block. const Node* control_input = parent_node->in(0); - if (control_input != NULL) { + if (control_input != nullptr) { replace_block_proj_ctrl(parent_node); } else { // Is a constant with NO inputs? @@ -345,7 +345,7 @@ bool PhaseCFG::schedule_early(VectorSet &visited, Node_Stack &roots) { while (input_index < parent_node->len()) { Node* in = parent_node->in(input_index++); - if (in == NULL) { + if (in == nullptr) { continue; } @@ -401,10 +401,10 @@ bool PhaseCFG::schedule_early(VectorSet &visited, Node_Stack &roots) { //------------------------------dom_lca---------------------------------------- // Find least common ancestor in dominator tree // LCA is a current notion of LCA, to be raised above 'this'. -// As a convenient boundary condition, return 'this' if LCA is NULL. +// As a convenient boundary condition, return 'this' if LCA is null. // Find the LCA of those two nodes. Block* Block::dom_lca(Block* LCA) { - if (LCA == NULL || LCA == this) return this; + if (LCA == nullptr || LCA == this) return this; Block* anc = this; while (anc->_dom_depth > LCA->_dom_depth) @@ -428,7 +428,7 @@ Block* Block::dom_lca(Block* LCA) { // the LCA only with the phi input paths which actually use this def. static Block* raise_LCA_above_use(Block* LCA, Node* use, Node* def, const PhaseCFG* cfg) { Block* buse = cfg->get_block_for_node(use); - if (buse == NULL) return LCA; // Unused killing Projs have no use block + if (buse == nullptr) return LCA; // Unused killing Projs have no use block if (!use->is_Phi()) return buse->dom_lca(LCA); uint pmax = use->req(); // Number of Phi inputs // Why does not this loop just break after finding the matching input to @@ -507,9 +507,9 @@ static Block* memory_early_block(Node* load, Block* early, const PhaseCFG* cfg) Node* mem_inputs[4]; int mem_inputs_length = 0; - if (base != NULL) mem_inputs[mem_inputs_length++] = base; - if (index != NULL) mem_inputs[mem_inputs_length++] = index; - if (store != NULL) mem_inputs[mem_inputs_length++] = store; + if (base != nullptr) mem_inputs[mem_inputs_length++] = base; + if (index != nullptr) mem_inputs[mem_inputs_length++] = index; + if (store != nullptr) mem_inputs[mem_inputs_length++] = store; // In the comparision below, add one to account for the control input, // which may be null, but always takes up a spot in the in array. @@ -519,9 +519,9 @@ static Block* memory_early_block(Node* load, Block* early, const PhaseCFG* cfg) // from the early block of only the address portion of the instruction, // and ignore other blocks that may have factored into the wider // schedule_early calculation. - if (load->in(0) != NULL) mem_inputs[mem_inputs_length++] = load->in(0); + if (load->in(0) != nullptr) mem_inputs[mem_inputs_length++] = load->in(0); - Block* deepb = NULL; // Deepest block so far + Block* deepb = nullptr; // Deepest block so far int deepb_dom_depth = 0; for (int i = 0; i < mem_inputs_length; i++) { Block* inb = cfg->get_block_for_node(mem_inputs[i]); @@ -554,9 +554,9 @@ bool PhaseCFG::unrelated_load_in_store_null_block(Node* store, Node* load) { Node* end = store_block->end(); if (end->is_MachNullCheck() && (end->in(1) == store) && store_block->dominates(load_block)) { Node* if_true = end->find_out_with(Op_IfTrue); - assert(if_true != NULL, "null check without null projection"); + assert(if_true != nullptr, "null check without null projection"); Node* null_block_region = if_true->find_out_with(Op_Region); - assert(null_block_region != NULL, "null check without null region"); + assert(null_block_region != nullptr, "null check without null region"); return get_block_for_node(null_block_region) == load_block; } return false; @@ -580,7 +580,7 @@ bool PhaseCFG::unrelated_load_in_store_null_block(Node* store, Node* load) { // above the LCA, if it is not the early block. Block* PhaseCFG::insert_anti_dependences(Block* LCA, Node* load, bool verify) { assert(load->needs_anti_dependence_check(), "must be a load of some sort"); - assert(LCA != NULL, ""); + assert(LCA != nullptr, ""); DEBUG_ONLY(Block* LCA_orig = LCA); // Compute the alias index. Loads and stores with different alias indices @@ -650,7 +650,7 @@ Block* PhaseCFG::insert_anti_dependences(Block* LCA, Node* load, bool verify) { Node* initial_mem = load->in(MemNode::Memory); worklist_store.push(initial_mem); worklist_visited.push(initial_mem); - worklist_mem.push(NULL); + worklist_mem.push(nullptr); while (worklist_store.size() > 0) { // Examine a nearby store to see if it might interfere with our load. Node* mem = worklist_mem.pop(); @@ -665,7 +665,7 @@ Block* PhaseCFG::insert_anti_dependences(Block* LCA, Node* load, bool verify) { ) { mem = store; // It's not a possibly interfering store. if (store == initial_mem) - initial_mem = NULL; // only process initial memory once + initial_mem = nullptr; // only process initial memory once for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) { store = mem->fast_out(i); @@ -708,7 +708,7 @@ Block* PhaseCFG::insert_anti_dependences(Block* LCA, Node* load, bool verify) { MachSafePointNode* ms = (MachSafePointNode*) mstore; assert(ms->is_MachCallJava(), ""); MachCallJavaNode* mcj = (MachCallJavaNode*) ms; - if (mcj->_method == NULL) { + if (mcj->_method == nullptr) { // These runtime calls do not write to Java visible memory // (other than Raw) and so do not require anti-dependence edges. continue; @@ -737,7 +737,7 @@ Block* PhaseCFG::insert_anti_dependences(Block* LCA, Node* load, bool verify) { // earliest legal block for 'load'. In the latter case, // immediately insert an anti-dependence edge. Block* store_block = get_block_for_node(store); - assert(store_block != NULL, "unused killing projections skipped above"); + assert(store_block != nullptr, "unused killing projections skipped above"); if (store->is_Phi()) { // Loop-phis need to raise load before input. (Other phis are treated @@ -887,9 +887,9 @@ Node_Backward_Iterator::Node_Backward_Iterator( Node *root, VectorSet &visited, // Iterator for the Node_Backward_Iterator Node *Node_Backward_Iterator::next() { - // If the _stack is empty, then just return NULL: finished. + // If the _stack is empty, then just return null: finished. if ( !_stack.size() ) - return NULL; + return nullptr; // I visit unvisited not-anti-dependence users first, then anti-dependent // children next. I iterate backwards to support removal of nodes. @@ -911,7 +911,7 @@ Node *Node_Backward_Iterator::next() { uint src_rpo = _cfg.get_block_for_node(src)->_rpo; // Schedule all nodes in a post-order visit - Node *unvisited = NULL; // Unvisited anti-dependent Node, if any + Node *unvisited = nullptr; // Unvisited anti-dependent Node, if any // Scan for unvisited nodes while (idx > 0) { @@ -1150,7 +1150,7 @@ Block* PhaseCFG::hoist_to_cheaper_block(Block* LCA, Block* early, Node* self) { // Do not hoist (to cover latency) instructions which target a // single register. Hoisting stretches the live range of the // single register and may force spilling. - MachNode* mach = self->is_Mach() ? self->as_Mach() : NULL; + MachNode* mach = self->is_Mach() ? self->as_Mach() : nullptr; if (mach && mach->out_RegMask().is_bound1() && mach->out_RegMask().is_NotEmpty()) in_latency = true; @@ -1175,10 +1175,10 @@ Block* PhaseCFG::hoist_to_cheaper_block(Block* LCA, Block* early, Node* self) { while (LCA != early) { LCA = LCA->_idom; // Follow up the dominator tree - if (LCA == NULL) { + if (LCA == nullptr) { // Bailout without retry assert(false, "graph should be schedulable"); - C->record_method_not_compilable("late schedule failed: LCA == NULL"); + C->record_method_not_compilable("late schedule failed: LCA is null"); return least; } @@ -1292,7 +1292,7 @@ void PhaseCFG::schedule_late(VectorSet &visited, Node_Stack &stack) { } #endif - MachNode* mach = self->is_Mach() ? self->as_Mach() : NULL; + MachNode* mach = self->is_Mach() ? self->as_Mach() : nullptr; if (mach) { switch (mach->ideal_Opcode()) { case Op_CreateEx: @@ -1304,7 +1304,7 @@ void PhaseCFG::schedule_late(VectorSet &visited, Node_Stack &stack) { // Don't move CheckCastPP nodes away from their input, if the input // is a rawptr (5071820). Node *def = self->in(1); - if (def != NULL && def->bottom_type()->base() == Type::RawPtr) { + if (def != nullptr && def->bottom_type()->base() == Type::RawPtr) { early->add_inst(self); #ifdef ASSERT _raw_oops.push(def); @@ -1362,20 +1362,20 @@ void PhaseCFG::schedule_late(VectorSet &visited, Node_Stack &stack) { } // Gather LCA of all uses - Block *LCA = NULL; + Block *LCA = nullptr; { for (DUIterator_Fast imax, i = self->fast_outs(imax); i < imax; i++) { // For all uses, find LCA Node* use = self->fast_out(i); LCA = raise_LCA_above_use(LCA, use, self, this); } - guarantee(LCA != NULL, "There must be a LCA"); + guarantee(LCA != nullptr, "There must be a LCA"); } // (Hide defs of imax, i from rest of block.) // Place temps in the block of their use. This isn't a // requirement for correctness but it reduces useless // interference between temps and other nodes. - if (mach != NULL && mach->is_MachTemp()) { + if (mach != nullptr && mach->is_MachTemp()) { map_node_to_block(self, LCA); LCA->add_inst(self); continue; @@ -1410,7 +1410,7 @@ void PhaseCFG::schedule_late(VectorSet &visited, Node_Stack &stack) { while (LCA->_loop->depth() > early->_loop->depth()) { LCA = LCA->_idom; } - assert(LCA != NULL, "a valid LCA must exist"); + assert(LCA != nullptr, "a valid LCA must exist"); verify_memory_writer_placement(LCA, self); } @@ -1423,10 +1423,10 @@ void PhaseCFG::schedule_late(VectorSet &visited, Node_Stack &stack) { // allocatable (hoisting can make a value live longer, leading to // anti and output dependency problems which are normally resolved // by the register allocator giving everyone a different register). - if (mach != NULL && must_clone[mach->ideal_Opcode()]) + if (mach != nullptr && must_clone[mach->ideal_Opcode()]) try_to_hoist = false; - Block* late = NULL; + Block* late = nullptr; if (try_to_hoist) { // Now find the block with the least execution frequency. // Start at the latest schedule and work up to the earliest schedule @@ -1506,8 +1506,8 @@ void PhaseCFG::global_code_motion() { } #endif - // Detect implicit-null-check opportunities. Basically, find NULL checks - // with suitable memory ops nearby. Use the memory op to do the NULL check. + // Detect implicit-null-check opportunities. Basically, find null checks + // with suitable memory ops nearby. Use the memory op to do the null check. // I can generate a memory op if there is not one nearby. if (C->is_method_compilation()) { // By reversing the loop direction we get a very minor gain on mpegaudio. @@ -1527,7 +1527,7 @@ void PhaseCFG::global_code_motion() { } bool block_size_threshold_ok = false; - intptr_t *recalc_pressure_nodes = NULL; + intptr_t *recalc_pressure_nodes = nullptr; if (OptoRegScheduling) { for (uint i = 0; i < number_of_blocks(); i++) { Block* block = get_block(i); @@ -1580,11 +1580,11 @@ void PhaseCFG::global_code_motion() { if (!C->failure_reason_is(C2Compiler::retry_no_subsuming_loads())) { C->record_method_not_compilable("local schedule failed"); } - _regalloc = NULL; + _regalloc = nullptr; return; } } - _regalloc = NULL; + _regalloc = nullptr; // If we inserted any instructions between a Call and his CatchNode, // clone the instructions on all paths below the Catch. @@ -1725,7 +1725,7 @@ CFGLoop* PhaseCFG::create_loop_tree() { for (uint i = 0; i < number_of_blocks(); i++) { Block* block = get_block(i); // Check that _loop field are clear...we could clear them if not. - assert(block->_loop == NULL, "clear _loop expected"); + assert(block->_loop == nullptr, "clear _loop expected"); // Sanity check that the RPO numbering is reflected in the _blocks array. // It doesn't have to be for the loop tree to be built, but if it is not, // then the blocks have been reordered since dom graph building...which @@ -1758,7 +1758,7 @@ CFGLoop* PhaseCFG::create_loop_tree() { assert(worklist.size() == 0, "nonempty worklist"); CFGLoop* nloop = new CFGLoop(idct++); - assert(loop_head->_loop == NULL, "just checking"); + assert(loop_head->_loop == nullptr, "just checking"); loop_head->_loop = nloop; // Add to nloop so push_pred() will skip over inner loops nloop->add_member(loop_head); @@ -1781,7 +1781,7 @@ CFGLoop* PhaseCFG::create_loop_tree() { for (uint i = 0; i < number_of_blocks(); i++) { Block* block = get_block(i); CFGLoop* lp = block->_loop; - if (lp == NULL) { + if (lp == nullptr) { // Not assigned to a loop. Add it to the method's pseudo loop. block->_loop = root_loop; lp = root_loop; @@ -1790,7 +1790,7 @@ CFGLoop* PhaseCFG::create_loop_tree() { lp->add_member(block); } if (lp != root_loop) { - if (lp->parent() == NULL) { + if (lp->parent() == nullptr) { // Not a nested loop. Make it a child of the method's pseudo loop. root_loop->add_nested_loop(lp); } @@ -1809,7 +1809,7 @@ void CFGLoop::push_pred(Block* blk, int i, Block_List& worklist, PhaseCFG* cfg) Node* pred_n = blk->pred(i); Block* pred = cfg->get_block_for_node(pred_n); CFGLoop *pred_loop = pred->_loop; - if (pred_loop == NULL) { + if (pred_loop == nullptr) { // Filter out blocks for non-single-entry loops. // For all reasonable loops, the head occurs before the tail in RPO. if (pred->_rpo > head()->_rpo) { @@ -1818,11 +1818,11 @@ void CFGLoop::push_pred(Block* blk, int i, Block_List& worklist, PhaseCFG* cfg) } } else if (pred_loop != this) { // Nested loop. - while (pred_loop->_parent != NULL && pred_loop->_parent != this) { + while (pred_loop->_parent != nullptr && pred_loop->_parent != this) { pred_loop = pred_loop->_parent; } // Make pred's loop be a child - if (pred_loop->_parent == NULL) { + if (pred_loop->_parent == nullptr) { add_nested_loop(pred_loop); // Continue with loop entry predecessor. Block* pred_head = pred_loop->head(); @@ -1830,7 +1830,7 @@ void CFGLoop::push_pred(Block* blk, int i, Block_List& worklist, PhaseCFG* cfg) assert(pred_head != head(), "loop head in only one loop"); push_pred(pred_head, LoopNode::EntryControl, worklist, cfg); } else { - assert(pred_loop->_parent == this && _parent == NULL, "just checking"); + assert(pred_loop->_parent == this && _parent == nullptr, "just checking"); } } } @@ -1838,14 +1838,14 @@ void CFGLoop::push_pred(Block* blk, int i, Block_List& worklist, PhaseCFG* cfg) //------------------------------add_nested_loop-------------------------------- // Make cl a child of the current loop in the loop tree. void CFGLoop::add_nested_loop(CFGLoop* cl) { - assert(_parent == NULL, "no parent yet"); + assert(_parent == nullptr, "no parent yet"); assert(cl != this, "not my own parent"); cl->_parent = this; CFGLoop* ch = _child; - if (ch == NULL) { + if (ch == nullptr) { _child = cl; } else { - while (ch->_sibling != NULL) { ch = ch->_sibling; } + while (ch->_sibling != nullptr) { ch = ch->_sibling; } ch->_sibling = cl; } } @@ -1856,7 +1856,7 @@ void CFGLoop::add_nested_loop(CFGLoop* cl) { void CFGLoop::compute_loop_depth(int depth) { _depth = depth; CFGLoop* ch = _child; - while (ch != NULL) { + while (ch != nullptr) { ch->compute_loop_depth(depth + 1); ch = ch->_sibling; } @@ -1875,7 +1875,7 @@ void CFGLoop::compute_freq() { // Nested loops first CFGLoop* ch = _child; - while (ch != NULL) { + while (ch != nullptr) { ch->compute_freq(); ch = ch->_sibling; } @@ -2205,7 +2205,7 @@ void CFGLoop::scale_freq() { s->_freq = block_freq; } CFGLoop* ch = _child; - while (ch != NULL) { + while (ch != nullptr) { ch->scale_freq(); ch = ch->_sibling; } @@ -2213,7 +2213,7 @@ void CFGLoop::scale_freq() { // Frequency of outer loop double CFGLoop::outer_loop_freq() const { - if (_child != NULL) { + if (_child != nullptr) { return _child->_freq; } return _freq; @@ -2223,8 +2223,8 @@ double CFGLoop::outer_loop_freq() const { //------------------------------dump_tree-------------------------------------- void CFGLoop::dump_tree() const { dump(); - if (_child != NULL) _child->dump_tree(); - if (_sibling != NULL) _sibling->dump_tree(); + if (_child != nullptr) _child->dump_tree(); + if (_sibling != nullptr) _sibling->dump_tree(); } //------------------------------dump------------------------------------------- diff --git a/src/hotspot/share/opto/generateOptoStub.cpp b/src/hotspot/share/opto/generateOptoStub.cpp index c5dfd1eeb408e..49a4d6998b55e 100644 --- a/src/hotspot/share/opto/generateOptoStub.cpp +++ b/src/hotspot/share/opto/generateOptoStub.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -96,7 +96,7 @@ void GraphKit::gen_stub(address C_function, // Drop in the last_Java_sp. last_Java_fp is not touched. // Always do this after the other "last_Java_frame" fields are set since - // as soon as last_Java_sp != NULL the has_last_Java_frame is true and + // as soon as last_Java_sp != nullptr the has_last_Java_frame is true and // users will look at the other fields. // Node *adr_sp = basic_plus_adr(top(), thread, in_bytes(JavaThread::last_Java_sp_offset())); @@ -234,7 +234,7 @@ void GraphKit::gen_stub(address C_function, // Runtime call returning oop in TLS? Fetch it out if( pass_tls ) { Node* adr = basic_plus_adr(top(), thread, in_bytes(JavaThread::vm_result_offset())); - Node* vm_result = make_load(NULL, adr, TypeOopPtr::BOTTOM, T_OBJECT, NoAlias, MemNode::unordered); + Node* vm_result = make_load(nullptr, adr, TypeOopPtr::BOTTOM, T_OBJECT, NoAlias, MemNode::unordered); map()->set_req(TypeFunc::Parms, vm_result); // vm_result passed as result // clear thread-local-storage(tls) store_to_memory(control(), adr, null(), T_ADDRESS, NoAlias, MemNode::unordered); @@ -243,7 +243,7 @@ void GraphKit::gen_stub(address C_function, //----------------------------- // check exception Node* adr = basic_plus_adr(top(), thread, in_bytes(Thread::pending_exception_offset())); - Node* pending = make_load(NULL, adr, TypeOopPtr::BOTTOM, T_OBJECT, NoAlias, MemNode::unordered); + Node* pending = make_load(nullptr, adr, TypeOopPtr::BOTTOM, T_OBJECT, NoAlias, MemNode::unordered); Node* exit_memory = reset_memory(); @@ -254,7 +254,7 @@ void GraphKit::gen_stub(address C_function, Node* if_null = _gvn.transform( new IfFalseNode(iff) ); Node* if_not_null = _gvn.transform( new IfTrueNode(iff) ); - assert (StubRoutines::forward_exception_entry() != NULL, "must be generated before"); + assert (StubRoutines::forward_exception_entry() != nullptr, "must be generated before"); Node *exc_target = makecon(TypeRawPtr::make( StubRoutines::forward_exception_entry() )); Node *to_exc = new TailCallNode(if_not_null, i_o(), @@ -267,7 +267,7 @@ void GraphKit::gen_stub(address C_function, //----------------------------- // If this is a normal subroutine return, issue the return and be done. - Node *ret = NULL; + Node *ret = nullptr; switch( is_fancy_jump ) { case 0: // Make a return instruction // Return to caller, free any space for return address diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp index a3df43c23818d..c7cfbf0ce9328 100644 --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -61,7 +61,7 @@ GraphKit::GraphKit(JVMState* jvms) _barrier_set(BarrierSet::barrier_set()->barrier_set_c2()) { _exceptions = jvms->map()->next_exception(); - if (_exceptions != NULL) jvms->map()->set_next_exception(NULL); + if (_exceptions != nullptr) jvms->map()->set_next_exception(nullptr); set_jvms(jvms); } @@ -72,8 +72,8 @@ GraphKit::GraphKit() _gvn(*C->initial_gvn()), _barrier_set(BarrierSet::barrier_set()->barrier_set_c2()) { - _exceptions = NULL; - set_map(NULL); + _exceptions = nullptr; + set_map(nullptr); debug_only(_sp = -99); debug_only(set_bci(-99)); } @@ -120,7 +120,7 @@ JVMState* GraphKit::sync_jvms_for_reexecute() { #ifdef ASSERT bool GraphKit::jvms_in_sync() const { Parse* parse = is_Parse(); - if (parse == NULL) { + if (parse == nullptr) { if (bci() != jvms()->bci()) return false; if (sp() != (int)jvms()->sp()) return false; return true; @@ -140,33 +140,33 @@ bool GraphKit::jvms_in_sync() const { // Such merge points must never "escape" into the parser at large, // until they have been handed to gvn.transform. static bool is_hidden_merge(Node* reg) { - if (reg == NULL) return false; + if (reg == nullptr) return false; if (reg->is_Phi()) { reg = reg->in(0); - if (reg == NULL) return false; + if (reg == nullptr) return false; } - return reg->is_Region() && reg->in(0) != NULL && reg->in(0)->is_Root(); + return reg->is_Region() && reg->in(0) != nullptr && reg->in(0)->is_Root(); } void GraphKit::verify_map() const { - if (map() == NULL) return; // null map is OK + if (map() == nullptr) return; // null map is OK assert(map()->req() <= jvms()->endoff(), "no extra garbage on map"); assert(!map()->has_exceptions(), "call add_exception_states_from 1st"); assert(!is_hidden_merge(control()), "call use_exception_state, not set_map"); } void GraphKit::verify_exception_state(SafePointNode* ex_map) { - assert(ex_map->next_exception() == NULL, "not already part of a chain"); + assert(ex_map->next_exception() == nullptr, "not already part of a chain"); assert(has_saved_ex_oop(ex_map), "every exception state has an ex_oop"); } #endif //---------------------------stop_and_kill_map--------------------------------- -// Set _map to NULL, signalling a stop to further bytecode execution. +// Set _map to null, signalling a stop to further bytecode execution. // First smash the current map's control to a constant, to mark it dead. void GraphKit::stop_and_kill_map() { SafePointNode* dead_map = stop(); - if (dead_map != NULL) { + if (dead_map != nullptr) { dead_map->disconnect_inputs(C); // Mark the map as killed. assert(dead_map->is_killed(), "must be so marked"); } @@ -174,9 +174,9 @@ void GraphKit::stop_and_kill_map() { //--------------------------------stopped-------------------------------------- -// Tell if _map is NULL, or control is top. +// Tell if _map is null, or control is top. bool GraphKit::stopped() { - if (map() == NULL) return true; + if (map() == nullptr) return true; else if (control() == top()) return true; else return false; } @@ -185,7 +185,7 @@ bool GraphKit::stopped() { //-----------------------------has_ex_handler---------------------------------- // Tell if this method or any caller method has exception handlers. bool GraphKit::has_ex_handler() { - for (JVMState* jvmsp = jvms(); jvmsp != NULL; jvmsp = jvmsp->caller()) { + for (JVMState* jvmsp = jvms(); jvmsp != nullptr; jvmsp = jvmsp->caller()) { if (jvmsp->has_method() && jvmsp->method()->has_exception_handlers()) { return true; } @@ -241,7 +241,7 @@ SafePointNode* GraphKit::make_exception_state(Node* ex_oop) { //--------------------------add_exception_state-------------------------------- // Add an exception to my list of exceptions. void GraphKit::add_exception_state(SafePointNode* ex_map) { - if (ex_map == NULL || ex_map->control() == top()) { + if (ex_map == nullptr || ex_map->control() == top()) { return; } #ifdef ASSERT @@ -260,7 +260,7 @@ void GraphKit::add_exception_state(SafePointNode* ex_map) { return; } assert(ex_type->isa_instptr(), "exception must be an instance"); - for (SafePointNode* e2 = _exceptions; e2 != NULL; e2 = e2->next_exception()) { + for (SafePointNode* e2 = _exceptions; e2 != nullptr; e2 = e2->next_exception()) { const Type* ex_type2 = _gvn.type(saved_ex_oop(e2)); // We check sp also because call bytecodes can generate exceptions // both before and after arguments are popped! @@ -278,11 +278,11 @@ void GraphKit::add_exception_state(SafePointNode* ex_map) { //-----------------------add_exception_states_from----------------------------- void GraphKit::add_exception_states_from(JVMState* jvms) { SafePointNode* ex_map = jvms->map()->next_exception(); - if (ex_map != NULL) { - jvms->map()->set_next_exception(NULL); - for (SafePointNode* next_map; ex_map != NULL; ex_map = next_map) { + if (ex_map != nullptr) { + jvms->map()->set_next_exception(nullptr); + for (SafePointNode* next_map; ex_map != nullptr; ex_map = next_map) { next_map = ex_map->next_exception(); - ex_map->set_next_exception(NULL); + ex_map->set_next_exception(nullptr); add_exception_state(ex_map); } } @@ -290,18 +290,18 @@ void GraphKit::add_exception_states_from(JVMState* jvms) { //-----------------------transfer_exceptions_into_jvms------------------------- JVMState* GraphKit::transfer_exceptions_into_jvms() { - if (map() == NULL) { + if (map() == nullptr) { // We need a JVMS to carry the exceptions, but the map has gone away. // Create a scratch JVMS, cloned from any of the exception states... if (has_exceptions()) { _map = _exceptions; _map = clone_map(); - _map->set_next_exception(NULL); + _map->set_next_exception(nullptr); clear_saved_ex_oop(_map); debug_only(verify_map()); } else { // ...or created from scratch - JVMState* jvms = new (C) JVMState(_method, NULL); + JVMState* jvms = new (C) JVMState(_method, nullptr); jvms->set_bci(_bci); jvms->set_sp(_sp); jvms->set_map(new SafePointNode(TypeFunc::Parms, jvms)); @@ -316,7 +316,7 @@ JVMState* GraphKit::transfer_exceptions_into_jvms() { JVMState* jvms = sync_jvms(); assert(!jvms->map()->has_exceptions(), "no exceptions on this map yet"); jvms->map()->set_next_exception(_exceptions); - _exceptions = NULL; // done with this set of exceptions + _exceptions = nullptr; // done with this set of exceptions return jvms; } @@ -338,7 +338,7 @@ static inline void add_one_req(Node* dstphi, Node* src) { // This helper function combines exception states by building phis on a // specially marked state-merging region. These regions and phis are // untransformed, and can build up gradually. The region is marked by -// having a control input of its exception map, rather than NULL. Such +// having a control input of its exception map, rather than null. Such // regions do not appear except in this function, and in use_exception_state. void GraphKit::combine_exception_states(SafePointNode* ex_map, SafePointNode* phi_map) { if (failing()) return; // dying anyway... @@ -490,7 +490,7 @@ Node* GraphKit::use_exception_state(SafePointNode* phi_map) { Bytecodes::Code GraphKit::java_bc() const { ciMethod* method = this->method(); int bci = this->bci(); - if (method != NULL && bci != InvocationEntryBci) + if (method != nullptr && bci != InvocationEntryBci) return method->java_code_at_bci(bci); else return Bytecodes::_illegal; @@ -520,7 +520,7 @@ void GraphKit::uncommon_trap_if_should_post_on_exceptions(Deoptimization::DeoptR // Do not try anything fancy if we're notifying the VM on every throw. // Cf. case Bytecodes::_athrow in parse2.cpp. uncommon_trap(reason, Deoptimization::Action_none, - (ciKlass*)NULL, (char*)NULL, must_throw); + (ciKlass*)nullptr, (char*)nullptr, must_throw); } } @@ -568,7 +568,7 @@ void GraphKit::builtin_throw(Deoptimization::DeoptReason reason, Node* arg) { // for its backtrace. // Fixing this remaining case of 4292742 requires some flavor of // escape analysis. Leave that for the future. - ciInstance* ex_obj = NULL; + ciInstance* ex_obj = nullptr; switch (reason) { case Deoptimization::Reason_null_check: ex_obj = env()->NullPointerException_instance(); @@ -590,7 +590,7 @@ void GraphKit::builtin_throw(Deoptimization::DeoptReason reason, Node* arg) { break; } if (failing()) { stop(); return; } // exception allocation might fail - if (ex_obj != NULL) { + if (ex_obj != nullptr) { if (env()->jvmti_can_post_on_exceptions()) { // check if we must post exception events, take uncommon trap if so uncommon_trap_if_should_post_on_exceptions(reason, must_throw); @@ -599,7 +599,7 @@ void GraphKit::builtin_throw(Deoptimization::DeoptReason reason, Node* arg) { } // Cheat with a preallocated exception object. - if (C->log() != NULL) + if (C->log() != nullptr) C->log()->elem("hot_throw preallocated='1' reason='%s'", Deoptimization::trap_reason_name(reason)); const TypeInstPtr* ex_con = TypeInstPtr::make(ex_obj); @@ -635,13 +635,13 @@ void GraphKit::builtin_throw(Deoptimization::DeoptReason reason, Node* arg) { // Usual case: Bail to interpreter. // Reserve the right to recompile if we haven't seen anything yet. - ciMethod* m = Deoptimization::reason_is_speculate(reason) ? C->method() : NULL; + ciMethod* m = Deoptimization::reason_is_speculate(reason) ? C->method() : nullptr; Deoptimization::DeoptAction action = Deoptimization::Action_maybe_recompile; if (treat_throw_as_hot && (method()->method_data()->trap_recompiled_at(bci(), m) || C->too_many_traps(reason))) { // We cannot afford to take more traps here. Suffer in the interpreter. - if (C->log() != NULL) + if (C->log() != nullptr) C->log()->elem("hot_throw preallocated='0' reason='%s' mcount='%d'", Deoptimization::trap_reason_name(reason), C->trap_count(reason)); @@ -653,7 +653,7 @@ void GraphKit::builtin_throw(Deoptimization::DeoptReason reason, Node* arg) { // allocation time and code size, by drastically reducing the number // of in-edges on the call to the uncommon trap. - uncommon_trap(reason, action, (ciKlass*)NULL, (char*)NULL, must_throw); + uncommon_trap(reason, action, (ciKlass*)nullptr, (char*)nullptr, must_throw); } @@ -663,11 +663,11 @@ PreserveJVMState::PreserveJVMState(GraphKit* kit, bool clone_map) { _kit = kit; _map = kit->map(); // preserve the map _sp = kit->sp(); - kit->set_map(clone_map ? kit->clone_map() : NULL); + kit->set_map(clone_map ? kit->clone_map() : nullptr); #ifdef ASSERT _bci = kit->bci(); Parse* parser = kit->is_Parse(); - int block = (parser == NULL || parser->block() == NULL) ? -1 : parser->block()->rpo(); + int block = (parser == nullptr || parser->block() == nullptr) ? -1 : parser->block()->rpo(); _block = block; #endif } @@ -676,7 +676,7 @@ PreserveJVMState::~PreserveJVMState() { #ifdef ASSERT assert(kit->bci() == _bci, "bci must not shift"); Parse* parser = kit->is_Parse(); - int block = (parser == NULL || parser->block() == NULL) ? -1 : parser->block()->rpo(); + int block = (parser == nullptr || parser->block() == nullptr) ? -1 : parser->block()->rpo(); assert(block == _block, "block must not shift"); #endif kit->set_map(_map); @@ -721,7 +721,7 @@ PreserveReexecuteState::~PreserveReexecuteState() { // function eventually and do it all there. SafePointNode* GraphKit::clone_map() { - if (map() == NULL) return NULL; + if (map() == nullptr) return nullptr; // Clone the memory edge first Node* mem = MergeMemNode::make(map()->memory()); @@ -738,12 +738,35 @@ SafePointNode* GraphKit::clone_map() { return clonemap; } +//-----------------------------destruct_map_clone------------------------------ +// +// Order of destruct is important to increase the likelyhood that memory can be re-used. We need +// to destruct/free/delete in the exact opposite order as clone_map(). +void GraphKit::destruct_map_clone(SafePointNode* sfp) { + if (sfp == nullptr) return; + + Node* mem = sfp->memory(); + JVMState* jvms = sfp->jvms(); + + if (jvms != nullptr) { + delete jvms; + } + + remove_for_igvn(sfp); + gvn().clear_type(sfp); + sfp->destruct(&_gvn); + + if (mem != nullptr) { + gvn().clear_type(mem); + mem->destruct(&_gvn); + } +} //-----------------------------set_map_clone----------------------------------- void GraphKit::set_map_clone(SafePointNode* m) { _map = m; _map = clone_map(); - _map->set_next_exception(NULL); + _map->set_next_exception(nullptr); debug_only(verify_map()); } @@ -766,7 +789,7 @@ void GraphKit::kill_dead_locals() { // bci can be -1 (InvocationEntryBci). We return the entry // liveness for the method. - if (method() == NULL || method()->code_size() == 0) { + if (method() == nullptr || method()->code_size() == 0) { // We are building a graph for a call to a native method. // All locals are live. return; @@ -794,14 +817,14 @@ void GraphKit::kill_dead_locals() { // Return true if all dead locals are set to top in the map. // Used to assert "clean" debug info at various points. bool GraphKit::dead_locals_are_killed() { - if (method() == NULL || method()->code_size() == 0) { + if (method() == nullptr || method()->code_size() == 0) { // No locals need to be dead, so all is as it should be. return true; } // Make sure somebody called kill_dead_locals upstream. ResourceMark rm; - for (JVMState* jvms = this->jvms(); jvms != NULL; jvms = jvms->caller()) { + for (JVMState* jvms = this->jvms(); jvms != nullptr; jvms = jvms->caller()) { if (jvms->loc_size() == 0) continue; // no locals to consult SafePointNode* map = jvms->map(); ciMethod* method = jvms->method(); @@ -834,7 +857,7 @@ bool GraphKit::dead_locals_are_killed() { static bool should_reexecute_implied_by_bytecode(JVMState *jvms, bool is_anewarray) { ciMethod* cur_method = jvms->method(); int cur_bci = jvms->bci(); - if (cur_method != NULL && cur_bci != InvocationEntryBci) { + if (cur_method != nullptr && cur_bci != InvocationEntryBci) { Bytecodes::Code code = cur_method->java_code_at_bci(cur_bci); return Interpreter::bytecode_should_reexecute(code) || (is_anewarray && code == Bytecodes::_multianewarray); @@ -928,7 +951,7 @@ void GraphKit::add_safepoint_edges(SafePointNode* call, bool must_throw) { // Loop over the map input edges associated with jvms, add them // to the call node, & reset all offsets to match call node array. - for (JVMState* in_jvms = youngest_jvms; in_jvms != NULL; ) { + for (JVMState* in_jvms = youngest_jvms; in_jvms != nullptr; ) { uint debug_end = debug_ptr; uint debug_start = debug_ptr - in_jvms->debug_size(); debug_ptr = debug_start; // back up the ptr @@ -1077,9 +1100,9 @@ bool GraphKit::compute_stack_effects(int& inputs, int& depth) { case Bytecodes::_invokeinterface: { bool ignored_will_link; - ciSignature* declared_signature = NULL; + ciSignature* declared_signature = nullptr; ciMethod* ignored_callee = method()->get_method_at_bci(bci(), ignored_will_link, &declared_signature); - assert(declared_signature != NULL, "cannot be null"); + assert(declared_signature != nullptr, "cannot be null"); inputs = declared_signature->arg_size_for_bc(code); int size = declared_signature->return_type()->size(); depth = size - inputs; @@ -1177,9 +1200,9 @@ Node* GraphKit::ConvL2I(Node* offset) { Node* GraphKit::load_object_klass(Node* obj) { // Special-case a fresh allocation to avoid building nodes: Node* akls = AllocateNode::Ideal_klass(obj, &_gvn); - if (akls != NULL) return akls; + if (akls != nullptr) return akls; Node* k_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes()); - return _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), k_adr, TypeInstPtr::KLASS)); + return _gvn.transform(LoadKlassNode::make(_gvn, nullptr, immutable_memory(), k_adr, TypeInstPtr::KLASS)); } //-------------------------load_array_length----------------------------------- @@ -1187,7 +1210,7 @@ Node* GraphKit::load_array_length(Node* array) { // Special-case a fresh allocation to avoid building nodes: AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(array, &_gvn); Node *alen; - if (alloc == NULL) { + if (alloc == nullptr) { Node *r_adr = basic_plus_adr(array, arrayOopDesc::length_offset_in_bytes()); alen = _gvn.transform( new LoadRangeNode(0, immutable_memory(), r_adr, TypeInt::POS)); } else { @@ -1217,8 +1240,8 @@ Node* GraphKit::array_ideal_length(AllocateArrayNode* alloc, } //------------------------------do_null_check---------------------------------- -// Helper function to do a NULL pointer check. Returned value is -// the incoming address with NULL casted away. You are allowed to use the +// Helper function to do a null pointer check. Returned value is +// the incoming address with null casted away. You are allowed to use the // not-null value only if you are control dependent on the test. #ifndef PRODUCT extern int explicit_null_checks_inserted, @@ -1229,12 +1252,12 @@ Node* GraphKit::null_check_common(Node* value, BasicType type, bool assert_null, Node* *null_control, bool speculative) { - assert(!assert_null || null_control == NULL, "not both at once"); + assert(!assert_null || null_control == nullptr, "not both at once"); if (stopped()) return top(); NOT_PRODUCT(explicit_null_checks_inserted++); - // Construct NULL check - Node *chk = NULL; + // Construct null check + Node *chk = nullptr; switch(type) { case T_LONG : chk = new CmpLNode(value, _gvn.zerocon(T_LONG)); break; case T_INT : chk = new CmpINode(value, _gvn.intcon(0)); break; @@ -1244,9 +1267,9 @@ Node* GraphKit::null_check_common(Node* value, BasicType type, const Type *t = _gvn.type( value ); const TypeOopPtr* tp = t->isa_oopptr(); - if (tp != NULL && tp->klass() != NULL && !tp->klass()->is_loaded() + if (tp != nullptr && tp->klass() != nullptr && !tp->klass()->is_loaded() // Only for do_null_check, not any of its siblings: - && !assert_null && null_control == NULL) { + && !assert_null && null_control == nullptr) { // Usually, any field access or invocation on an unloaded oop type // will simply fail to link, since the statically linked class is // likely also to be unloaded. However, in -Xcomp mode, sometimes @@ -1275,8 +1298,8 @@ Node* GraphKit::null_check_common(Node* value, BasicType type, return value; // Elided null assert quickly! } } else { - // See if mixing in the NULL pointer changes type. - // If so, then the NULL pointer was not allowed in the original + // See if mixing in the null pointer changes type. + // If so, then the null pointer was not allowed in the original // type. In other words, "value" was not-null. if (t->meet(TypePtr::NULL_PTR) != t->remove_speculative()) { // same as: if (!TypePtr::NULL_PTR->higher_equal(t)) ... @@ -1291,7 +1314,7 @@ Node* GraphKit::null_check_common(Node* value, BasicType type, default: fatal("unexpected type: %s", type2name(type)); } - assert(chk != NULL, "sanity check"); + assert(chk != nullptr, "sanity check"); chk = _gvn.transform(chk); BoolTest::mask btest = assert_null ? BoolTest::eq : BoolTest::ne; @@ -1324,7 +1347,7 @@ Node* GraphKit::null_check_common(Node* value, BasicType type, return res; } cfg = IfNode::up_one_dom(cfg, /*linear_only=*/ true); - if (cfg == NULL) break; // Quit at region nodes + if (cfg == nullptr) break; // Quit at region nodes depth++; } } @@ -1349,18 +1372,18 @@ Node* GraphKit::null_check_common(Node* value, BasicType type, // To cause an implicit null check, we set the not-null probability // to the maximum (PROB_MAX). For an explicit check the probability // is set to a smaller value. - if (null_control != NULL || too_many_traps(reason)) { + if (null_control != nullptr || too_many_traps(reason)) { // probability is less likely ok_prob = PROB_LIKELY_MAG(3); } else if (!assert_null && (ImplicitNullCheckThreshold > 0) && - method() != NULL && + method() != nullptr && (method()->method_data()->trap_count(reason) >= (uint)ImplicitNullCheckThreshold)) { ok_prob = PROB_LIKELY_MAG(3); } - if (null_control != NULL) { + if (null_control != nullptr) { IfNode* iff = create_and_map_if(control(), tst, ok_prob, COUNT_UNKNOWN); Node* null_true = _gvn.transform( new IfFalseNode(iff)); set_control( _gvn.transform( new IfTrueNode(iff))); @@ -1379,7 +1402,7 @@ Node* GraphKit::null_check_common(Node* value, BasicType type, } else if (assert_null) { uncommon_trap(reason, Deoptimization::Action_make_not_entrant, - NULL, "assert_null"); + nullptr, "assert_null"); } else { replace_in_map(value, zerocon(type)); builtin_throw(reason); @@ -1401,7 +1424,7 @@ Node* GraphKit::null_check_common(Node* value, BasicType type, // (If there is a null_control, a non-null value may come back to haunt us.) if (type == T_OBJECT) { Node* cast = cast_not_null(value, false); - if (null_control == NULL || (*null_control) == top()) + if (null_control == nullptr || (*null_control) == top()) replace_in_map(value, cast); value = cast; } @@ -1498,7 +1521,7 @@ Node* GraphKit::memory(uint alias_idx) { Node* GraphKit::reset_memory() { Node* mem = map()->memory(); // do not use this node for any more parsing! - debug_only( map()->set_memory((Node*)NULL) ); + debug_only( map()->set_memory((Node*)nullptr) ); return _gvn.transform( mem ); } @@ -1534,7 +1557,7 @@ Node* GraphKit::make_load(Node* ctl, Node* adr, const Type* t, BasicType bt, bool unsafe, uint8_t barrier_data) { assert(adr_idx != Compile::AliasIdxTop, "use other make_load factory" ); - const TypePtr* adr_type = NULL; // debug-mode-only argument + const TypePtr* adr_type = nullptr; // debug-mode-only argument debug_only(adr_type = C->get_adr_type(adr_idx)); Node* mem = memory(adr_idx); Node* ld = LoadNode::make(_gvn, ctl, mem, adr, adr_type, t, bt, mo, control_dependency, require_atomic_access, unaligned, mismatched, unsafe, barrier_data); @@ -1554,7 +1577,7 @@ Node* GraphKit::store_to_memory(Node* ctl, Node* adr, Node *val, BasicType bt, bool mismatched, bool unsafe) { assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" ); - const TypePtr* adr_type = NULL; + const TypePtr* adr_type = nullptr; debug_only(adr_type = C->get_adr_type(adr_idx)); Node *mem = memory(adr_idx); Node* st = StoreNode::make(_gvn, ctl, mem, adr, adr_type, val, bt, mo, require_atomic_access); @@ -1584,7 +1607,7 @@ Node* GraphKit::access_store_at(Node* obj, const Type* val_type, BasicType bt, DecoratorSet decorators) { - // Transformation of a value which could be NULL pointer (CastPP #NULL) + // Transformation of a value which could be null pointer (CastPP #null) // could be delayed during Parse (for example, in adjust_map_after_if()). // Execute transformation here to avoid barrier generation in such case. if (_gvn.type(val) == TypePtr::NULL_PTR) { @@ -1595,7 +1618,7 @@ Node* GraphKit::access_store_at(Node* obj, return top(); // Dead path ? } - assert(val != NULL, "not dead path"); + assert(val != nullptr, "not dead path"); C2AccessValuePtr addr(adr, adr_type); C2AccessValue value(val, val_type); @@ -1635,7 +1658,7 @@ Node* GraphKit::access_load(Node* adr, // actual adress to load val at } C2AccessValuePtr addr(adr, adr->bottom_type()->is_ptr()); - C2ParseAccess access(this, decorators | C2_READ_ACCESS, bt, NULL, addr); + C2ParseAccess access(this, decorators | C2_READ_ACCESS, bt, nullptr, addr); if (access.is_raw()) { return _barrier_set->BarrierSetC2::load_at(access, val_type); } else { @@ -1800,7 +1823,7 @@ Node* GraphKit::set_results_for_java_call(CallJavaNode* call, bool separate_io_p // Capture the return value, if any. Node* ret; - if (call->method() == NULL || + if (call->method() == nullptr || call->method()->return_type()->basic_type() == T_VOID) ret = top(); else ret = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); @@ -1833,7 +1856,7 @@ Node* GraphKit::set_results_for_java_call(CallJavaNode* call, bool separate_io_p Node* GraphKit::set_predefined_input_for_runtime_call(SafePointNode* call, Node* narrow_mem) { // Set fixed predefined input arguments Node* memory = reset_memory(); - Node* m = narrow_mem == NULL ? memory : narrow_mem; + Node* m = narrow_mem == nullptr ? memory : narrow_mem; call->init_req( TypeFunc::Control, control() ); call->init_req( TypeFunc::I_O, top() ); // does no i/o call->init_req( TypeFunc::Memory, m ); // may gc ptrs @@ -1844,9 +1867,9 @@ Node* GraphKit::set_predefined_input_for_runtime_call(SafePointNode* call, Node* //-------------------set_predefined_output_for_runtime_call-------------------- // Set control and memory (not i_o) from the call. -// If keep_mem is not NULL, use it for the output state, +// If keep_mem is not null, use it for the output state, // except for the RawPtr output of the call, if hook_mem is TypeRawPtr::BOTTOM. -// If hook_mem is NULL, this call produces no memory effects at all. +// If hook_mem is null, this call produces no memory effects at all. // If hook_mem is a Java-visible memory slice (such as arraycopy operands), // then only that memory slice is taken from the call. // In the last case, we must put an appropriate memory barrier before @@ -1860,7 +1883,7 @@ void GraphKit::set_predefined_output_for_runtime_call(Node* call, if (keep_mem) { // First clone the existing memory state set_all_memory(keep_mem); - if (hook_mem != NULL) { + if (hook_mem != nullptr) { // Make memory for the call Node* mem = _gvn.transform( new ProjNode(call, TypeFunc::Memory) ); // Set the RawPtr memory state only. This covers all the heap top/GC stuff @@ -1874,7 +1897,7 @@ void GraphKit::set_predefined_output_for_runtime_call(Node* call, assert(C->alias_type(call->adr_type()) == C->alias_type(hook_mem), "call node must be constructed correctly"); } else { - assert(hook_mem == NULL, ""); + assert(hook_mem == nullptr, ""); // This is not a "slow path" call; all memory comes from the call. set_all_memory_call(call); } @@ -1895,7 +1918,7 @@ static void add_mergemem_users_to_worklist(Unique_Node_List& wl, Node* mem) { // Replace the call with the current state of the kit. void GraphKit::replace_call(CallNode* call, Node* result, bool do_replaced_nodes) { - JVMState* ejvms = NULL; + JVMState* ejvms = nullptr; if (has_exceptions()) { ejvms = transfer_exceptions_into_jvms(); } @@ -1917,10 +1940,10 @@ void GraphKit::replace_call(CallNode* call, Node* result, bool do_replaced_nodes Node* final_io = final_state->in(TypeFunc::I_O); // Replace all the old call edges with the edges from the inlining result - if (callprojs.fallthrough_catchproj != NULL) { + if (callprojs.fallthrough_catchproj != nullptr) { C->gvn_replace_by(callprojs.fallthrough_catchproj, final_ctl); } - if (callprojs.fallthrough_memproj != NULL) { + if (callprojs.fallthrough_memproj != nullptr) { if (final_mem->is_MergeMem()) { // Parser's exits MergeMem was not transformed but may be optimized final_mem = _gvn.transform(final_mem); @@ -1928,28 +1951,28 @@ void GraphKit::replace_call(CallNode* call, Node* result, bool do_replaced_nodes C->gvn_replace_by(callprojs.fallthrough_memproj, final_mem); add_mergemem_users_to_worklist(wl, final_mem); } - if (callprojs.fallthrough_ioproj != NULL) { + if (callprojs.fallthrough_ioproj != nullptr) { C->gvn_replace_by(callprojs.fallthrough_ioproj, final_io); } // Replace the result with the new result if it exists and is used - if (callprojs.resproj != NULL && result != NULL) { + if (callprojs.resproj != nullptr && result != nullptr) { C->gvn_replace_by(callprojs.resproj, result); } - if (ejvms == NULL) { + if (ejvms == nullptr) { // No exception edges to simply kill off those paths - if (callprojs.catchall_catchproj != NULL) { + if (callprojs.catchall_catchproj != nullptr) { C->gvn_replace_by(callprojs.catchall_catchproj, C->top()); } - if (callprojs.catchall_memproj != NULL) { + if (callprojs.catchall_memproj != nullptr) { C->gvn_replace_by(callprojs.catchall_memproj, C->top()); } - if (callprojs.catchall_ioproj != NULL) { + if (callprojs.catchall_ioproj != nullptr) { C->gvn_replace_by(callprojs.catchall_ioproj, C->top()); } // Replace the old exception object with top - if (callprojs.exobj != NULL) { + if (callprojs.exobj != nullptr) { C->gvn_replace_by(callprojs.exobj, C->top()); } } else { @@ -1961,21 +1984,21 @@ void GraphKit::replace_call(CallNode* call, Node* result, bool do_replaced_nodes Node* ex_oop = ekit.use_exception_state(ex_map); - if (callprojs.catchall_catchproj != NULL) { + if (callprojs.catchall_catchproj != nullptr) { C->gvn_replace_by(callprojs.catchall_catchproj, ekit.control()); ex_ctl = ekit.control(); } - if (callprojs.catchall_memproj != NULL) { + if (callprojs.catchall_memproj != nullptr) { Node* ex_mem = ekit.reset_memory(); C->gvn_replace_by(callprojs.catchall_memproj, ex_mem); add_mergemem_users_to_worklist(wl, ex_mem); } - if (callprojs.catchall_ioproj != NULL) { + if (callprojs.catchall_ioproj != nullptr) { C->gvn_replace_by(callprojs.catchall_ioproj, ekit.i_o()); } // Replace the old exception object with the newly created one - if (callprojs.exobj != NULL) { + if (callprojs.exobj != nullptr) { C->gvn_replace_by(callprojs.exobj, ex_oop); } } @@ -1990,7 +2013,7 @@ void GraphKit::replace_call(CallNode* call, Node* result, bool do_replaced_nodes _gvn.transform(wl.pop()); } - if (callprojs.fallthrough_catchproj != NULL && !final_ctl->is_top() && do_replaced_nodes) { + if (callprojs.fallthrough_catchproj != nullptr && !final_ctl->is_top() && do_replaced_nodes) { replaced_nodes.apply(C, final_ctl); } if (!ex_ctl->is_top() && do_replaced_nodes) { @@ -2058,7 +2081,7 @@ void GraphKit::uncommon_trap(int trap_request, Deoptimization::trap_request_index(trap_request) < 0 && too_many_recompiles(reason)) { // This BCI is causing too many recompilations. - if (C->log() != NULL) { + if (C->log() != nullptr) { C->log()->elem("observe that='trap_action_change' reason='%s' from='%s' to='none'", Deoptimization::trap_reason_name(reason), Deoptimization::trap_action_name(action)); @@ -2090,20 +2113,20 @@ void GraphKit::uncommon_trap(int trap_request, } CompileLog* log = C->log(); - if (log != NULL) { - int kid = (klass == NULL)? -1: log->identify(klass); + if (log != nullptr) { + int kid = (klass == nullptr)? -1: log->identify(klass); log->begin_elem("uncommon_trap bci='%d'", bci()); char buf[100]; log->print(" %s", Deoptimization::format_trap_request(buf, sizeof(buf), trap_request)); if (kid >= 0) log->print(" klass='%d'", kid); - if (comment != NULL) log->print(" comment='%s'", comment); + if (comment != nullptr) log->print(" comment='%s'", comment); log->end_elem(); } // Make sure any guarding test views this path as very unlikely Node *i0 = control()->in(0); - if (i0 != NULL && i0->is_If()) { // Found a guarding if test? + if (i0 != nullptr && i0->is_If()) { // Found a guarding if test? IfNode *iff = i0->as_If(); float f = iff->_prob; // Get prob if (control()->Opcode() == Op_IfTrue) { @@ -2120,7 +2143,7 @@ void GraphKit::uncommon_trap(int trap_request, // Now insert the uncommon trap subroutine call address call_addr = SharedRuntime::uncommon_trap_blob()->entry_point(); - const TypePtr* no_memory_effects = NULL; + const TypePtr* no_memory_effects = nullptr; // Pass the index of the class to be loaded Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON | (must_throw ? RC_MUST_THROW : 0), @@ -2154,14 +2177,14 @@ Node* GraphKit::just_allocated_object(Node* current_control) { // Object:: is invoked after allocation, most of invoke nodes // will be reduced, but a region node is kept in parse time, we check // the pattern and skip the region node if it degraded to a copy. - if (ctrl != NULL && ctrl->is_Region() && ctrl->req() == 2 && + if (ctrl != nullptr && ctrl->is_Region() && ctrl->req() == 2 && ctrl->as_Region()->is_copy()) { ctrl = ctrl->as_Region()->is_copy(); } if (C->recent_alloc_ctl() == ctrl) { return C->recent_alloc_obj(); } - return NULL; + return nullptr; } @@ -2200,7 +2223,7 @@ Node* GraphKit::record_profile_for_speculation(Node* n, ciKlass* exact_kls, Prof } else { assert(ptr_kind == ProfileNeverNull, "nothing else is an improvement"); const TypePtr* ptr = TypePtr::NOTNULL; - if (speculative != NULL) { + if (speculative != nullptr) { speculative = speculative->cast_to_ptr_type(ptr->ptr())->is_ptr(); } else { speculative = ptr; @@ -2244,7 +2267,7 @@ Node* GraphKit::record_profiled_receiver_for_speculation(Node* n) { java_bc() == Bytecodes::_aastore) && method()->method_data()->is_mature()) { ciProfileData* data = method()->method_data()->bci_to_data(bci()); - if (data != NULL) { + if (data != nullptr) { if (!data->as_BitData()->null_seen()) { ptr_kind = ProfileNeverNull; } else { @@ -2253,7 +2276,7 @@ Node* GraphKit::record_profiled_receiver_for_speculation(Node* n) { uint i = 0; for (; i < call->row_limit(); i++) { ciKlass* receiver = call->receiver(i); - if (receiver != NULL) { + if (receiver != nullptr) { break; } } @@ -2282,7 +2305,7 @@ void GraphKit::record_profiled_arguments_for_speculation(ciMethod* dest_method, const Type *targ = tf->domain()->field_at(j + TypeFunc::Parms); if (is_reference_type(targ->basic_type())) { ProfilePtrKind ptr_kind = ProfileMaybeNull; - ciKlass* better_type = NULL; + ciKlass* better_type = nullptr; if (method()->argument_profiled_type(bci(), i, better_type, ptr_kind)) { record_profile_for_speculation(argument(j), better_type, ptr_kind); } @@ -2302,7 +2325,7 @@ void GraphKit::record_profiled_parameters_for_speculation() { for (int i = 0, j = 0; i < method()->arg_size() ; i++) { if (_gvn.type(local(i))->isa_oopptr()) { ProfilePtrKind ptr_kind = ProfileMaybeNull; - ciKlass* better_type = NULL; + ciKlass* better_type = nullptr; if (method()->parameter_profiled_type(j, better_type, ptr_kind)) { record_profile_for_speculation(local(i), better_type, ptr_kind); } @@ -2320,7 +2343,7 @@ void GraphKit::record_profiled_return_for_speculation() { return; } ProfilePtrKind ptr_kind = ProfileMaybeNull; - ciKlass* better_type = NULL; + ciKlass* better_type = nullptr; if (method()->return_profiled_type(bci(), better_type, ptr_kind)) { // If profiling reports a single type for the return value, // feed it to the type system so it can propagate it as a @@ -2393,7 +2416,7 @@ Node* GraphKit::dstore_rounding(Node* n) { // Generate a fast path/slow path idiom. Graph looks like: // [foo] indicates that 'foo' is a parameter // -// [in] NULL +// [in] null // \ / // CmpP // Bool ne @@ -2431,23 +2454,23 @@ Node* GraphKit::null_check_oop(Node* value, Node* *null_control, bool never_see_null, bool safe_for_replace, bool speculative) { - // Initial NULL check taken path + // Initial null check taken path (*null_control) = top(); Node* cast = null_check_common(value, T_OBJECT, false, null_control, speculative); // Generate uncommon_trap: if (never_see_null && (*null_control) != top()) { // If we see an unexpected null at a check-cast we record it and force a - // recompile; the offending check-cast will be compiled to handle NULLs. + // recompile; the offending check-cast will be compiled to handle nulls. // If we see more than one offending BCI, then all checkcasts in the - // method will be compiled to handle NULLs. + // method will be compiled to handle nulls. PreserveJVMState pjvms(this); set_control(*null_control); replace_in_map(value, null()); Deoptimization::DeoptReason reason = Deoptimization::reason_null_check(speculative); uncommon_trap(reason, Deoptimization::Action_make_not_entrant); - (*null_control) = top(); // NULL path is dead + (*null_control) = top(); // null path is dead } if ((*null_control) == top() && safe_for_replace) { replace_in_map(value, cast); @@ -2478,17 +2501,17 @@ Node* GraphKit::make_runtime_call(int flags, const char* call_name, const TypePtr* adr_type, // The following parms are all optional. - // The first NULL ends the list. + // The first null ends the list. Node* parm0, Node* parm1, Node* parm2, Node* parm3, Node* parm4, Node* parm5, Node* parm6, Node* parm7) { - assert(call_addr != NULL, "must not call NULL targets"); + assert(call_addr != nullptr, "must not call null targets"); // Slow-path call bool is_leaf = !(flags & RC_NO_LEAF); bool has_io = (!is_leaf && !(flags & RC_NO_IO)); - if (call_name == NULL) { + if (call_name == nullptr) { assert(!is_leaf, "must supply name for leaf"); call_name = OptoRuntime::stub_name(call_addr); } @@ -2511,7 +2534,7 @@ Node* GraphKit::make_runtime_call(int flags, bool wide_in = !(flags & RC_NARROW_MEM); bool wide_out = (C->get_alias_index(adr_type) == Compile::AliasIdxBot); - Node* prev_mem = NULL; + Node* prev_mem = nullptr; if (wide_in) { prev_mem = set_predefined_input_for_runtime_call(call); } else { @@ -2520,17 +2543,17 @@ Node* GraphKit::make_runtime_call(int flags, prev_mem = set_predefined_input_for_runtime_call(call, narrow_mem); } - // Hook each parm in order. Stop looking at the first NULL. - if (parm0 != NULL) { call->init_req(TypeFunc::Parms+0, parm0); - if (parm1 != NULL) { call->init_req(TypeFunc::Parms+1, parm1); - if (parm2 != NULL) { call->init_req(TypeFunc::Parms+2, parm2); - if (parm3 != NULL) { call->init_req(TypeFunc::Parms+3, parm3); - if (parm4 != NULL) { call->init_req(TypeFunc::Parms+4, parm4); - if (parm5 != NULL) { call->init_req(TypeFunc::Parms+5, parm5); - if (parm6 != NULL) { call->init_req(TypeFunc::Parms+6, parm6); - if (parm7 != NULL) { call->init_req(TypeFunc::Parms+7, parm7); + // Hook each parm in order. Stop looking at the first null. + if (parm0 != nullptr) { call->init_req(TypeFunc::Parms+0, parm0); + if (parm1 != nullptr) { call->init_req(TypeFunc::Parms+1, parm1); + if (parm2 != nullptr) { call->init_req(TypeFunc::Parms+2, parm2); + if (parm3 != nullptr) { call->init_req(TypeFunc::Parms+3, parm3); + if (parm4 != nullptr) { call->init_req(TypeFunc::Parms+4, parm4); + if (parm5 != nullptr) { call->init_req(TypeFunc::Parms+5, parm5); + if (parm6 != nullptr) { call->init_req(TypeFunc::Parms+6, parm6); + if (parm7 != nullptr) { call->init_req(TypeFunc::Parms+7, parm7); /* close each nested if ===> */ } } } } } } } } - assert(call->in(call->req()-1) != NULL, "must initialize all parms"); + assert(call->in(call->req()-1) != nullptr, "must initialize all parms"); if (!is_leaf) { // Non-leaves can block and take safepoints: @@ -2704,7 +2727,7 @@ void GraphKit::merge_memory(Node* new_mem, Node* region, int new_path) { if (mms.is_empty()) { // clone base memory Phi's inputs for this memory slice assert(old_slice == mms.base_memory(), "sanity"); - phi = PhiNode::make(region, NULL, Type::MEMORY, mms.adr_type(C)); + phi = PhiNode::make(region, nullptr, Type::MEMORY, mms.adr_type(C)); _gvn.set_type(phi, Type::MEMORY); for (uint i = 1; i < phi->req(); i++) { phi->init_req(i, old_slice->in(i)); @@ -2730,7 +2753,9 @@ void GraphKit::make_slow_call_ex(Node* call, ciInstanceKlass* ex_klass, bool sep // Make a catch node with just two handlers: fall-through and catch-all Node* i_o = _gvn.transform( new ProjNode(call, TypeFunc::I_O, separate_io_proj) ); Node* catc = _gvn.transform( new CatchNode(control(), i_o, 2) ); - Node* norm = _gvn.transform( new CatchProjNode(catc, CatchProjNode::fall_through_index, CatchProjNode::no_handler_bci) ); + Node* norm = new CatchProjNode(catc, CatchProjNode::fall_through_index, CatchProjNode::no_handler_bci); + _gvn.set_type_bottom(norm); + C->record_for_igvn(norm); Node* excp = _gvn.transform( new CatchProjNode(catc, CatchProjNode::catch_all_index, CatchProjNode::no_handler_bci) ); { PreserveJVMState pjvms(this); @@ -2757,7 +2782,7 @@ void GraphKit::make_slow_call_ex(Node* call, ciInstanceKlass* ex_klass, bool sep } static IfNode* gen_subtype_check_compare(Node* ctrl, Node* in1, Node* in2, BoolTest::mask test, float p, PhaseGVN& gvn, BasicType bt) { - Node* cmp = NULL; + Node* cmp = nullptr; switch(bt) { case T_INT: cmp = new CmpINode(in1, in2); break; case T_ADDRESS: cmp = new CmpPNode(in1, in2); break; @@ -2834,7 +2859,7 @@ Node* Phase::gen_subtype_check(Node* subklass, Node* superklass, Node** ctrl, No // First load the super-klass's check-offset Node *p1 = gvn.transform(new AddPNode(superklass, superklass, gvn.MakeConX(in_bytes(Klass::super_check_offset_offset())))); Node* m = C->immutable_memory(); - Node *chk_off = gvn.transform(new LoadINode(NULL, m, p1, gvn.type(p1)->is_ptr(), TypeInt::INT, MemNode::unordered)); + Node *chk_off = gvn.transform(new LoadINode(nullptr, m, p1, gvn.type(p1)->is_ptr(), TypeInt::INT, MemNode::unordered)); int cacheoff_con = in_bytes(Klass::secondary_super_cache_offset()); bool might_be_cache = (gvn.find_int_con(chk_off, cacheoff_con) == cacheoff_con); @@ -2842,8 +2867,8 @@ Node* Phase::gen_subtype_check(Node* subklass, Node* superklass, Node** ctrl, No // the secondary superclass list, or a failing value with a sentinel offset // if the super-klass is an interface or exceptionally deep in the Java // hierarchy and we have to scan the secondary superclass list the hard way. - // Worst-case type is a little odd: NULL is allowed as a result (usually - // klass loads can never produce a NULL). + // Worst-case type is a little odd: null is allowed as a result (usually + // klass loads can never produce a null). Node *chk_off_X = chk_off; #ifdef _LP64 chk_off_X = gvn.transform(new ConvI2LNode(chk_off_X)); @@ -2858,10 +2883,10 @@ Node* Phase::gen_subtype_check(Node* subklass, Node* superklass, Node** ctrl, No // incorrect/missed optimization of the following Load. // - it's a cache so, worse case, not reading the latest value // wouldn't cause incorrect execution - if (might_be_cache && mem != NULL) { + if (might_be_cache && mem != nullptr) { kmem = mem->is_MergeMem() ? mem->as_MergeMem()->memory_at(C->get_alias_index(gvn.type(p2)->is_ptr())) : mem; } - Node *nkls = gvn.transform(LoadKlassNode::make(gvn, NULL, kmem, p2, gvn.type(p2)->is_ptr(), TypeKlassPtr::OBJECT_OR_NULL)); + Node *nkls = gvn.transform(LoadKlassNode::make(gvn, nullptr, kmem, p2, gvn.type(p2)->is_ptr(), TypeKlassPtr::OBJECT_OR_NULL)); // Compile speed common case: ARE a subtype and we canNOT fail if( superklass == nkls ) @@ -3021,7 +3046,7 @@ Node* GraphKit::subtype_check_receiver(Node* receiver, ciKlass* klass, //------------------------------seems_never_null------------------------------- // Use null_seen information if it is available from the profile. // If we see an unexpected null at a type check we record it and force a -// recompile; the offending check will be recompiled to handle NULLs. +// recompile; the offending check will be recompiled to handle nulls. // If we see several offending BCIs, then all checks in the // method will be recompiled. bool GraphKit::seems_never_null(Node* obj, ciProfileData* data, bool& speculating) { @@ -3034,7 +3059,7 @@ bool GraphKit::seems_never_null(Node* obj, ciProfileData* data, bool& speculatin if (speculating) { return true; } - if (data == NULL) + if (data == nullptr) // Edge case: no mature data. Be optimistic here. return true; // If the profile has not seen a null, assume it won't happen. @@ -3050,7 +3075,7 @@ bool GraphKit::seems_never_null(Node* obj, ciProfileData* data, bool& speculatin void GraphKit::guard_klass_being_initialized(Node* klass) { int init_state_off = in_bytes(InstanceKlass::init_state_offset()); Node* adr = basic_plus_adr(top(), klass, init_state_off); - Node* init_state = LoadNode::make(_gvn, NULL, immutable_memory(), adr, + Node* init_state = LoadNode::make(_gvn, nullptr, immutable_memory(), adr, adr->bottom_type()->is_ptr(), TypeInt::BYTE, T_BYTE, MemNode::unordered); init_state = _gvn.transform(init_state); @@ -3069,7 +3094,7 @@ void GraphKit::guard_init_thread(Node* klass) { int init_thread_off = in_bytes(InstanceKlass::init_thread_offset()); Node* adr = basic_plus_adr(top(), klass, init_thread_off); - Node* init_thread = LoadNode::make(_gvn, NULL, immutable_memory(), adr, + Node* init_thread = LoadNode::make(_gvn, nullptr, immutable_memory(), adr, adr->bottom_type()->is_ptr(), TypePtr::NOTNULL, T_ADDRESS, MemNode::unordered); init_thread = _gvn.transform(init_thread); @@ -3097,7 +3122,7 @@ void GraphKit::clinit_barrier(ciInstanceKlass* ik, ciMethod* context) { } else { uncommon_trap(Deoptimization::Reason_uninitialized, Deoptimization::Action_reinterpret, - NULL); + nullptr); } } @@ -3108,21 +3133,21 @@ Node* GraphKit::maybe_cast_profiled_receiver(Node* not_null_obj, ciKlass* require_klass, ciKlass* spec_klass, bool safe_for_replace) { - if (!UseTypeProfile || !TypeProfileCasts) return NULL; + if (!UseTypeProfile || !TypeProfileCasts) return nullptr; - Deoptimization::DeoptReason reason = Deoptimization::reason_class_check(spec_klass != NULL); + Deoptimization::DeoptReason reason = Deoptimization::reason_class_check(spec_klass != nullptr); // Make sure we haven't already deoptimized from this tactic. if (too_many_traps_or_recompiles(reason)) - return NULL; + return nullptr; // (No, this isn't a call, but it's enough like a virtual call // to use the same ciMethod accessor to get the profile info...) // If we have a speculative type use it instead of profiling (which // may not help us) - ciKlass* exact_kls = spec_klass == NULL ? profile_has_unique_klass() : spec_klass; - if (exact_kls != NULL) {// no cast failures here - if (require_klass == NULL || + ciKlass* exact_kls = spec_klass == nullptr ? profile_has_unique_klass() : spec_klass; + if (exact_kls != nullptr) {// no cast failures here + if (require_klass == nullptr || C->static_subtype_check(require_klass, exact_kls) == Compile::SSC_always_true) { // If we narrow the type to match what the type profile sees or // the speculative type, we can then remove the rest of the @@ -3145,7 +3170,7 @@ Node* GraphKit::maybe_cast_profiled_receiver(Node* not_null_obj, // assert(ssc == Compile::SSC_always_true)... except maybe the profile lied to us. } - return NULL; + return nullptr; } /** @@ -3163,14 +3188,14 @@ Node* GraphKit::maybe_cast_profiled_obj(Node* obj, return obj; } - // type == NULL if profiling tells us this object is always null - if (type != NULL) { + // type is null if profiling tells us this object is always null + if (type != nullptr) { Deoptimization::DeoptReason class_reason = Deoptimization::Reason_speculate_class_check; Deoptimization::DeoptReason null_reason = Deoptimization::Reason_speculate_null_check; if (!too_many_traps_or_recompiles(null_reason) && !too_many_traps_or_recompiles(class_reason)) { - Node* not_null_obj = NULL; + Node* not_null_obj = nullptr; // not_null is true if we know the object is not null and // there's no need for a null check if (!not_null) { @@ -3218,7 +3243,7 @@ Node* GraphKit::gen_instanceof(Node* obj, Node* superklass, bool safe_for_replac Node* phi = new PhiNode(region, TypeInt::BOOL); C->set_has_split_ifs(true); // Has chance for split-if optimization - ciProfileData* data = NULL; + ciProfileData* data = nullptr; if (java_bc() == Bytecodes::_instanceof) { // Only for the bytecode data = method()->method_data()->bci_to_data(bci()); } @@ -3231,7 +3256,7 @@ Node* GraphKit::gen_instanceof(Node* obj, Node* superklass, bool safe_for_replac Node* not_null_obj = null_check_oop(obj, &null_ctl, never_see_null, safe_for_replace, speculative_not_null); // If not_null_obj is dead, only null-path is taken - if (stopped()) { // Doing instance-of on a NULL? + if (stopped()) { // Doing instance-of on a null? set_control(null_ctl); return intcon(0); } @@ -3261,13 +3286,13 @@ Node* GraphKit::gen_instanceof(Node* obj, Node* superklass, bool safe_for_replac // We may not have profiling here or it may not help us. If we // have a speculative type use it to perform an exact cast. ciKlass* spec_obj_type = obj_type->speculative_type(); - if (spec_obj_type != NULL || (ProfileDynamicTypes && data != NULL)) { - Node* cast_obj = maybe_cast_profiled_receiver(not_null_obj, NULL, spec_obj_type, safe_for_replace); + if (spec_obj_type != nullptr || (ProfileDynamicTypes && data != nullptr)) { + Node* cast_obj = maybe_cast_profiled_receiver(not_null_obj, nullptr, spec_obj_type, safe_for_replace); if (stopped()) { // Profile disagrees with this path. set_control(null_ctl); // Null is the only remaining possibility. return intcon(0); } - if (cast_obj != NULL) { + if (cast_obj != nullptr) { not_null_obj = cast_obj; } } @@ -3320,7 +3345,7 @@ Node* GraphKit::gen_checkcast(Node *obj, Node* superklass, // for example, in some objArray manipulations, such as a[i]=a[j].) if (tk->singleton()) { const TypeOopPtr* objtp = _gvn.type(obj)->isa_oopptr(); - if (objtp != NULL && objtp->klass() != NULL) { + if (objtp != nullptr && objtp->klass() != nullptr) { switch (C->static_subtype_check(tk->klass(), objtp->klass())) { case Compile::SSC_always_true: // If we know the type check always succeed then we don't use @@ -3341,9 +3366,9 @@ Node* GraphKit::gen_checkcast(Node *obj, Node* superklass, } } - ciProfileData* data = NULL; + ciProfileData* data = nullptr; bool safe_for_replace = false; - if (failure_control == NULL) { // use MDO in regular case only + if (failure_control == nullptr) { // use MDO in regular case only assert(java_bc() == Bytecodes::_aastore || java_bc() == Bytecodes::_checkcast, "interpreter profiles type checks only for these BCs"); @@ -3359,7 +3384,7 @@ Node* GraphKit::gen_checkcast(Node *obj, Node* superklass, // Use null-cast information if it is available bool speculative_not_null = false; - bool never_see_null = ((failure_control == NULL) // regular case only + bool never_see_null = ((failure_control == nullptr) // regular case only && seems_never_null(obj, data, speculative_not_null)); // Null check; get casted pointer; set region slot 3 @@ -3367,7 +3392,7 @@ Node* GraphKit::gen_checkcast(Node *obj, Node* superklass, Node* not_null_obj = null_check_oop(obj, &null_ctl, never_see_null, safe_for_replace, speculative_not_null); // If not_null_obj is dead, only null-path is taken - if (stopped()) { // Doing instance-of on a NULL? + if (stopped()) { // Doing instance-of on a null? set_control(null_ctl); return null(); } @@ -3381,7 +3406,7 @@ Node* GraphKit::gen_checkcast(Node *obj, Node* superklass, phi ->del_req(_null_path); } - Node* cast_obj = NULL; + Node* cast_obj = nullptr; if (tk->klass_is_exact()) { // The following optimization tries to statically cast the speculative type of the object // (for example obtained during profiling) to the type of the superklass and then do a @@ -3391,10 +3416,10 @@ Node* GraphKit::gen_checkcast(Node *obj, Node* superklass, // We may not have profiling here or it may not help us. If we have // a speculative type use it to perform an exact cast. ciKlass* spec_obj_type = obj_type->speculative_type(); - if (spec_obj_type != NULL || data != NULL) { + if (spec_obj_type != nullptr || data != nullptr) { cast_obj = maybe_cast_profiled_receiver(not_null_obj, tk->klass(), spec_obj_type, safe_for_replace); - if (cast_obj != NULL) { - if (failure_control != NULL) // failure is now impossible + if (cast_obj != nullptr) { + if (failure_control != nullptr) // failure is now impossible (*failure_control) = top(); // adjust the type of the phi to the exact klass: phi->raise_bottom_type(_gvn.type(cast_obj)->meet_speculative(TypePtr::NULL_PTR)); @@ -3402,14 +3427,14 @@ Node* GraphKit::gen_checkcast(Node *obj, Node* superklass, } } - if (cast_obj == NULL) { + if (cast_obj == nullptr) { // Generate the subtype check Node* not_subtype_ctrl = gen_subtype_check(not_null_obj, superklass ); // Plug in success path into the merge cast_obj = _gvn.transform(new CheckCastPPNode(control(), not_null_obj, toop)); // Failure path ends in uncommon trap (or may be dead - failure impossible) - if (failure_control == NULL) { + if (failure_control == nullptr) { if (not_subtype_ctrl != top()) { // If failure is possible PreserveJVMState pjvms(this); set_control(not_subtype_ctrl); @@ -3423,7 +3448,7 @@ Node* GraphKit::gen_checkcast(Node *obj, Node* superklass, region->init_req(_obj_path, control()); phi ->init_req(_obj_path, cast_obj); - // A merge of NULL or Casted-NotNull obj + // A merge of null or Casted-NotNull obj Node* res = _gvn.transform(phi); // Note I do NOT always 'replace_in_map(obj,result)' here. @@ -3507,9 +3532,9 @@ FastLockNode* GraphKit::shared_lock(Node* obj) { assert(SynchronizationEntryBCI == InvocationEntryBci, ""); if( !GenerateSynchronizationCode ) - return NULL; // Not locking things? + return nullptr; // Not locking things? if (stopped()) // Dead monitor? - return NULL; + return nullptr; assert(dead_locals_are_killed(), "should kill locals before sync. point"); @@ -3612,26 +3637,26 @@ void GraphKit::shared_unlock(Node* box, Node* obj) { //-------------------------------get_layout_helper----------------------------- // If the given klass is a constant or known to be an array, // fetch the constant layout helper value into constant_value -// and return (Node*)NULL. Otherwise, load the non-constant +// and return null. Otherwise, load the non-constant // layout helper value, and return the node which represents it. // This two-faced routine is useful because allocation sites // almost always feature constant types. Node* GraphKit::get_layout_helper(Node* klass_node, jint& constant_value) { const TypeKlassPtr* inst_klass = _gvn.type(klass_node)->isa_klassptr(); - if (!StressReflectiveCode && inst_klass != NULL) { + if (!StressReflectiveCode && inst_klass != nullptr) { ciKlass* klass = inst_klass->klass(); bool xklass = inst_klass->klass_is_exact(); if (xklass || klass->is_array_klass()) { jint lhelper = klass->layout_helper(); if (lhelper != Klass::_lh_neutral_value) { constant_value = lhelper; - return (Node*) NULL; + return (Node*) nullptr; } } } constant_value = Klass::_lh_neutral_value; // put in a known value Node* lhp = basic_plus_adr(klass_node, klass_node, in_bytes(Klass::layout_helper_offset())); - return make_load(NULL, lhp, TypeInt::INT, T_INT, MemNode::unordered); + return make_load(nullptr, lhp, TypeInt::INT, T_INT, MemNode::unordered); } // We just put in an allocate/initialize with a big raw-memory effect. @@ -3751,13 +3776,13 @@ Node* GraphKit::new_instance(Node* klass_node, // The layout_helper also encodes (in a low bit) the need for a slow path. jint layout_con = Klass::_lh_neutral_value; Node* layout_val = get_layout_helper(klass_node, layout_con); - int layout_is_con = (layout_val == NULL); + int layout_is_con = (layout_val == nullptr); - if (extra_slow_test == NULL) extra_slow_test = intcon(0); + if (extra_slow_test == nullptr) extra_slow_test = intcon(0); // Generate the initial go-slow test. It's either ALWAYS (return a - // Node for 1) or NEVER (return a NULL) or perhaps (in the reflective + // Node for 1) or NEVER (return a null) or perhaps (in the reflective // case) a computed value derived from the layout_helper. - Node* initial_slow_test = NULL; + Node* initial_slow_test = nullptr; if (layout_is_con) { assert(!StressReflectiveCode, "stress mode does not use these paths"); bool must_go_slow = Klass::layout_helper_needs_slow_path(layout_con); @@ -3776,7 +3801,7 @@ Node* GraphKit::new_instance(Node* klass_node, // Find the size in bytes. This is easy; it's the layout_helper. // The size value must be valid even if the slow path is taken. - Node* size = NULL; + Node* size = nullptr; if (layout_is_con) { size = MakeConX(Klass::layout_helper_size_in_bytes(layout_con)); } else { // reflective case @@ -3788,7 +3813,7 @@ Node* GraphKit::new_instance(Node* klass_node, Node* mask = MakeConX(~ (intptr_t)right_n_bits(LogBytesPerLong)); size = _gvn.transform( new AndXNode(size, mask) ); } - if (return_size_val != NULL) { + if (return_size_val != nullptr) { (*return_size_val) = size; } @@ -3825,7 +3850,7 @@ Node* GraphKit::new_array(Node* klass_node, // array klass (maybe variable) bool deoptimize_on_exception) { jint layout_con = Klass::_lh_neutral_value; Node* layout_val = get_layout_helper(klass_node, layout_con); - int layout_is_con = (layout_val == NULL); + int layout_is_con = (layout_val == nullptr); if (!layout_is_con && !StressReflectiveCode && !too_many_traps(Deoptimization::Reason_class_check)) { @@ -3840,7 +3865,7 @@ Node* GraphKit::new_array(Node* klass_node, // array klass (maybe variable) uncommon_trap(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); } - layout_val = NULL; + layout_val = nullptr; layout_is_con = true; } @@ -3867,7 +3892,7 @@ Node* GraphKit::new_array(Node* klass_node, // array klass (maybe variable) // and align_to(x, y) == ((x + y-1) & ~(y-1)) // The rounding mask is strength-reduced, if possible. int round_mask = MinObjAlignmentInBytes - 1; - Node* header_size = NULL; + Node* header_size = nullptr; int header_size_min = arrayOopDesc::base_offset_in_bytes(T_BYTE); // (T_BYTE has the weakest alignment and size restrictions...) if (layout_is_con) { @@ -3889,7 +3914,7 @@ Node* GraphKit::new_array(Node* klass_node, // array klass (maybe variable) header_size = _gvn.transform( new AddINode(hsize, mask) ); } - Node* elem_shift = NULL; + Node* elem_shift = nullptr; if (layout_is_con) { int eshift = Klass::layout_helper_log2_element_size(layout_con); if (eshift != 0) @@ -3906,7 +3931,7 @@ Node* GraphKit::new_array(Node* klass_node, // array klass (maybe variable) Node* headerx = ConvI2X(header_size); #ifdef _LP64 { const TypeInt* tilen = _gvn.find_int_type(length); - if (tilen != NULL && tilen->_lo < 0) { + if (tilen != nullptr && tilen->_lo < 0) { // Add a manual constraint to a positive range. Cf. array_element_address. jint size_max = fast_size_limit; if (size_max > tilen->_hi) size_max = tilen->_hi; @@ -3943,7 +3968,7 @@ Node* GraphKit::new_array(Node* klass_node, // array klass (maybe variable) // places, one where the length is sharply limited, and the other // after a successful allocation. Node* abody = lengthx; - if (elem_shift != NULL) + if (elem_shift != nullptr) abody = _gvn.transform( new LShiftXNode(lengthx, elem_shift) ); Node* size = _gvn.transform( new AddXNode(headerx, abody) ); if (round_mask != 0) { @@ -3952,7 +3977,7 @@ Node* GraphKit::new_array(Node* klass_node, // array klass (maybe variable) } // else if round_mask == 0, the size computation is self-rounding - if (return_size_val != NULL) { + if (return_size_val != nullptr) { // This is the size (*return_size_val) = size; } @@ -3969,21 +3994,29 @@ Node* GraphKit::new_array(Node* klass_node, // array klass (maybe variable) initial_slow_test = initial_slow_test->as_Bool()->as_int_value(&_gvn); } + const TypeOopPtr* ary_type = _gvn.type(klass_node)->is_klassptr()->as_instance_type(); + Node* valid_length_test = _gvn.intcon(1); + if (ary_type->isa_aryptr()) { + BasicType bt = ary_type->isa_aryptr()->elem()->array_element_basic_type(); + jint max = TypeAryPtr::max_array_length(bt); + Node* valid_length_cmp = _gvn.transform(new CmpUNode(length, intcon(max))); + valid_length_test = _gvn.transform(new BoolNode(valid_length_cmp, BoolTest::le)); + } + // Create the AllocateArrayNode and its result projections AllocateArrayNode* alloc = new AllocateArrayNode(C, AllocateArrayNode::alloc_type(TypeInt::INT), control(), mem, i_o(), size, klass_node, initial_slow_test, - length); + length, valid_length_test); // Cast to correct type. Note that the klass_node may be constant or not, // and in the latter case the actual array type will be inexact also. // (This happens via a non-constant argument to inline_native_newArray.) // In any case, the value of klass_node provides the desired array type. const TypeInt* length_type = _gvn.find_int_type(length); - const TypeOopPtr* ary_type = _gvn.type(klass_node)->is_klassptr()->as_instance_type(); - if (ary_type->isa_aryptr() && length_type != NULL) { + if (ary_type->isa_aryptr() && length_type != nullptr) { // Try to get a better type than POS for the size ary_type = ary_type->is_aryptr()->cast_to_size(length_type); } @@ -4000,8 +4033,8 @@ Node* GraphKit::new_array(Node* klass_node, // array klass (maybe variable) //---------------------------Ideal_allocation---------------------------------- // Given an oop pointer or raw pointer, see if it feeds from an AllocateNode. AllocateNode* AllocateNode::Ideal_allocation(Node* ptr, PhaseTransform* phase) { - if (ptr == NULL) { // reduce dumb test in callers - return NULL; + if (ptr == nullptr) { // reduce dumb test in callers + return nullptr; } BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); @@ -4009,27 +4042,27 @@ AllocateNode* AllocateNode::Ideal_allocation(Node* ptr, PhaseTransform* phase) { if (ptr->is_CheckCastPP()) { // strip only one raw-to-oop cast ptr = ptr->in(1); - if (ptr == NULL) return NULL; + if (ptr == nullptr) return nullptr; } - // Return NULL for allocations with several casts: + // Return null for allocations with several casts: // j.l.reflect.Array.newInstance(jobject, jint) // Object.clone() // to keep more precise type from last cast. if (ptr->is_Proj()) { Node* allo = ptr->in(0); - if (allo != NULL && allo->is_Allocate()) { + if (allo != nullptr && allo->is_Allocate()) { return allo->as_Allocate(); } } // Report failure to match. - return NULL; + return nullptr; } // Fancy version which also strips off an offset (and reports it to caller). AllocateNode* AllocateNode::Ideal_allocation(Node* ptr, PhaseTransform* phase, intptr_t& offset) { Node* base = AddPNode::Ideal_base_and_offset(ptr, phase, offset); - if (base == NULL) return NULL; + if (base == nullptr) return nullptr; return Ideal_allocation(base, phase); } @@ -4042,13 +4075,13 @@ AllocateNode* InitializeNode::allocation() { return alloc->as_Allocate(); } } - return NULL; + return nullptr; } // Trace Allocate -> Proj[Parm] -> Initialize InitializeNode* AllocateNode::initialization() { ProjNode* rawoop = proj_out_or_null(AllocateNode::RawAddress); - if (rawoop == NULL) return NULL; + if (rawoop == nullptr) return nullptr; for (DUIterator_Fast imax, i = rawoop->fast_outs(imax); i < imax; i++) { Node* init = rawoop->fast_out(i); if (init->is_Initialize()) { @@ -4056,7 +4089,7 @@ InitializeNode* AllocateNode::initialization() { return init->as_Initialize(); } } - return NULL; + return nullptr; } //----------------------------- loop predicates --------------------------- @@ -4131,7 +4164,7 @@ Node* GraphKit::load_String_length(Node* str, bool set_ctrl) { Node* GraphKit::load_String_value(Node* str, bool set_ctrl) { int value_offset = java_lang_String::value_offset(); const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(), - false, NULL, 0); + false, nullptr, 0); const TypePtr* value_field_type = string_type->add_offset(value_offset); const TypeAryPtr* value_type = TypeAryPtr::make(TypePtr::NotNull, TypeAry::make(TypeInt::BYTE, TypeInt::POS), @@ -4148,7 +4181,7 @@ Node* GraphKit::load_String_coder(Node* str, bool set_ctrl) { } int coder_offset = java_lang_String::coder_offset(); const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(), - false, NULL, 0); + false, nullptr, 0); const TypePtr* coder_field_type = string_type->add_offset(coder_offset); Node* p = basic_plus_adr(str, str, coder_offset); @@ -4160,7 +4193,7 @@ Node* GraphKit::load_String_coder(Node* str, bool set_ctrl) { void GraphKit::store_String_value(Node* str, Node* value) { int value_offset = java_lang_String::value_offset(); const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(), - false, NULL, 0); + false, nullptr, 0); const TypePtr* value_field_type = string_type->add_offset(value_offset); access_store_at(str, basic_plus_adr(str, value_offset), value_field_type, @@ -4170,7 +4203,7 @@ void GraphKit::store_String_value(Node* str, Node* value) { void GraphKit::store_String_coder(Node* str, Node* value) { int coder_offset = java_lang_String::coder_offset(); const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(), - false, NULL, 0); + false, nullptr, 0); const TypePtr* coder_field_type = string_type->add_offset(coder_offset); access_store_at(str, basic_plus_adr(str, coder_offset), coder_field_type, @@ -4271,19 +4304,19 @@ void GraphKit::inflate_string_slow(Node* src, Node* dst, Node* start, Node* coun Node* GraphKit::make_constant_from_field(ciField* field, Node* obj) { if (!field->is_constant()) { - return NULL; // Field not marked as constant. + return nullptr; // Field not marked as constant. } - ciInstance* holder = NULL; + ciInstance* holder = nullptr; if (!field->is_static()) { ciObject* const_oop = obj->bottom_type()->is_oopptr()->const_oop(); - if (const_oop != NULL && const_oop->is_instance()) { + if (const_oop != nullptr && const_oop->is_instance()) { holder = const_oop->as_instance(); } } const Type* con_type = Type::make_constant_from_field(field, holder, field->layout_type(), /*is_unsigned_load=*/false); - if (con_type != NULL) { + if (con_type != nullptr) { return makecon(con_type); } - return NULL; + return nullptr; } diff --git a/src/hotspot/share/opto/graphKit.hpp b/src/hotspot/share/opto/graphKit.hpp index d815e21956fc1..38e825e30e916 100644 --- a/src/hotspot/share/opto/graphKit.hpp +++ b/src/hotspot/share/opto/graphKit.hpp @@ -72,7 +72,7 @@ class GraphKit : public Phase { private: SafePointNode* map_not_null() const { - assert(_map != NULL, "must call stopped() to test for reset compiler map"); + assert(_map != nullptr, "must call stopped() to test for reset compiler map"); return _map; } @@ -86,14 +86,15 @@ class GraphKit : public Phase { } #endif - virtual Parse* is_Parse() const { return NULL; } - virtual LibraryCallKit* is_LibraryCallKit() const { return NULL; } + virtual Parse* is_Parse() const { return nullptr; } + virtual LibraryCallKit* is_LibraryCallKit() const { return nullptr; } ciEnv* env() const { return _env; } PhaseGVN& gvn() const { return _gvn; } void* barrier_set_state() const { return C->barrier_set_state(); } void record_for_igvn(Node* n) const { C->record_for_igvn(n); } // delegate to Compile + void remove_for_igvn(Node* n) const { C->remove_for_igvn(n); } // Handy well-known nodes: Node* null() const { return zerocon(T_OBJECT); } @@ -131,7 +132,7 @@ class GraphKit : public Phase { // See layout accessors in class JVMState. SafePointNode* map() const { return _map; } - bool has_exceptions() const { return _exceptions != NULL; } + bool has_exceptions() const { return _exceptions != nullptr; } JVMState* jvms() const { return map_not_null()->_jvms; } int sp() const { return _sp; } int bci() const { return _bci; } @@ -142,7 +143,7 @@ class GraphKit : public Phase { assert(jvms == this->jvms(), "sanity"); _sp = jvms->sp(); _bci = jvms->bci(); - _method = jvms->has_method() ? jvms->method() : NULL; } + _method = jvms->has_method() ? jvms->method() : nullptr; } void set_map(SafePointNode* m) { _map = m; debug_only(verify_map()); } void set_sp(int sp) { assert(sp >= 0, "sp must be non-negative: %d", sp); _sp = sp; } void clean_stack(int from_sp); // clear garbage beyond from_sp to top @@ -170,20 +171,25 @@ class GraphKit : public Phase { // Clone the existing map state. (Implements PreserveJVMState.) SafePointNode* clone_map(); + // Reverses the work done by clone_map(). Should only be used when the node returned by + // clone_map() is ultimately not used. Calling Node::destruct directly in the previously + // mentioned circumstance instead of this method may result in use-after-free. + void destruct_map_clone(SafePointNode* sfp); + // Set the map to a clone of the given one. void set_map_clone(SafePointNode* m); // Tell if the compilation is failing. bool failing() const { return C->failing(); } - // Set _map to NULL, signalling a stop to further bytecode execution. + // Set _map to null, signalling a stop to further bytecode execution. // Preserve the map intact for future use, and return it back to the caller. - SafePointNode* stop() { SafePointNode* m = map(); set_map(NULL); return m; } + SafePointNode* stop() { SafePointNode* m = map(); set_map(nullptr); return m; } - // Stop, but first smash the map's inputs to NULL, to mark it dead. + // Stop, but first smash the map's inputs to null, to mark it dead. void stop_and_kill_map(); - // Tell if _map is NULL, or control is top. + // Tell if _map is null, or control is top. bool stopped(); // Tell if this method or any caller method has exception handlers. @@ -215,9 +221,9 @@ class GraphKit : public Phase { // Detach and return an exception state. SafePointNode* pop_exception_state() { SafePointNode* ex_map = _exceptions; - if (ex_map != NULL) { + if (ex_map != nullptr) { _exceptions = ex_map->next_exception(); - ex_map->set_next_exception(NULL); + ex_map->set_next_exception(nullptr); debug_only(verify_exception_state(ex_map)); } return ex_map; @@ -240,10 +246,10 @@ class GraphKit : public Phase { // Combine all exceptions of any sort whatever into a single master state. SafePointNode* combine_and_pop_all_exception_states() { - if (_exceptions == NULL) return NULL; + if (_exceptions == nullptr) return nullptr; SafePointNode* phi_map = pop_exception_state(); SafePointNode* ex_map; - while ((ex_map = pop_exception_state()) != NULL) { + while ((ex_map = pop_exception_state()) != nullptr) { combine_exception_states(ex_map, phi_map); } return phi_map; @@ -351,16 +357,16 @@ class GraphKit : public Phase { bool replace_length_in_map); - // Helper function to do a NULL pointer check or ZERO check based on type. + // Helper function to do a null pointer check or ZERO check based on type. // Throw an exception if a given value is null. // Return the value cast to not-null. // Be clever about equivalent dominating null checks. Node* null_check_common(Node* value, BasicType type, bool assert_null = false, - Node* *null_control = NULL, + Node* *null_control = nullptr, bool speculative = false); Node* null_check(Node* value, BasicType type = T_OBJECT) { - return null_check_common(value, type, false, NULL, !_gvn.type(value)->speculative_maybe_null()); + return null_check_common(value, type, false, nullptr, !_gvn.type(value)->speculative_maybe_null()); } Node* null_check_receiver() { assert(argument(0)->bottom_type()->isa_ptr(), "must be"); @@ -379,7 +385,7 @@ class GraphKit : public Phase { // Throw an uncommon trap if a given value is __not__ null. // Return the value cast to null, and be clever about dominating checks. Node* null_assert(Node* value, BasicType type = T_OBJECT) { - return null_check_common(value, type, true, NULL, _gvn.type(value)->speculative_always_null()); + return null_check_common(value, type, true, nullptr, _gvn.type(value)->speculative_always_null()); } // Check if value is null and abort if it is @@ -412,7 +418,7 @@ class GraphKit : public Phase { profile.morphism() == 1) { return profile.receiver(0); } - return NULL; + return nullptr; } // record type from profiling with the type system @@ -477,7 +483,7 @@ class GraphKit : public Phase { int n_size = type2size[n_type]; if (n_size == 1) return pop(); else if (n_size == 2) return pop_pair(); - else return NULL; + else return nullptr; } Node* control() const { return map_not_null()->control(); } @@ -547,7 +553,7 @@ class GraphKit : public Phase { bool require_atomic_access = false, bool unaligned = false, bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0) { // This version computes alias_index from an address type - assert(adr_type != NULL, "use other make_load factory"); + assert(adr_type != nullptr, "use other make_load factory"); return make_load(ctl, adr, t, bt, C->get_alias_index(adr_type), mo, control_dependency, require_atomic_access, unaligned, mismatched, unsafe, barrier_data); @@ -574,7 +580,7 @@ class GraphKit : public Phase { bool mismatched = false, bool unsafe = false) { // This version computes alias_index from an address type - assert(adr_type != NULL, "use other store_to_memory factory"); + assert(adr_type != nullptr, "use other store_to_memory factory"); return store_to_memory(ctl, adr, val, bt, C->get_alias_index(adr_type), mo, require_atomic_access, @@ -655,9 +661,9 @@ class GraphKit : public Phase { // Return addressing for an array element. Node* array_element_address(Node* ary, Node* idx, BasicType elembt, // Optional constraint on the array size: - const TypeInt* sizetype = NULL, + const TypeInt* sizetype = nullptr, // Optional control dependency (for example, on range check) - Node* ctrl = NULL); + Node* ctrl = nullptr); // Return a load of array element at idx. Node* load_array_element(Node* ary, Node* idx, const TypeAryPtr* arytype, bool set_ctrl); @@ -712,12 +718,12 @@ class GraphKit : public Phase { // Similar to set_edges_for_java_call, but simplified for runtime calls. void set_predefined_output_for_runtime_call(Node* call) { - set_predefined_output_for_runtime_call(call, NULL, NULL); + set_predefined_output_for_runtime_call(call, nullptr, nullptr); } void set_predefined_output_for_runtime_call(Node* call, Node* keep_mem, const TypePtr* hook_mem); - Node* set_predefined_input_for_runtime_call(SafePointNode* call, Node* narrow_mem = NULL); + Node* set_predefined_input_for_runtime_call(SafePointNode* call, Node* narrow_mem = nullptr); // Replace the call with the current state of the kit. Requires // that the call was generated with separate io_projs so that @@ -733,13 +739,13 @@ class GraphKit : public Phase { // The optional reason is debug information written to the compile log. // Optional must_throw is the same as with add_safepoint_edges. void uncommon_trap(int trap_request, - ciKlass* klass = NULL, const char* reason_string = NULL, + ciKlass* klass = nullptr, const char* reason_string = nullptr, bool must_throw = false, bool keep_exact_action = false); // Shorthand, to avoid saying "Deoptimization::" so many times. void uncommon_trap(Deoptimization::DeoptReason reason, Deoptimization::DeoptAction action, - ciKlass* klass = NULL, const char* reason_string = NULL, + ciKlass* klass = nullptr, const char* reason_string = nullptr, bool must_throw = false, bool keep_exact_action = false) { uncommon_trap(Deoptimization::make_trap_request(reason, action), klass, reason_string, must_throw, keep_exact_action); @@ -748,7 +754,7 @@ class GraphKit : public Phase { // Bail out to the interpreter and keep exact action (avoid switching to Action_none). void uncommon_trap_exact(Deoptimization::DeoptReason reason, Deoptimization::DeoptAction action, - ciKlass* klass = NULL, const char* reason_string = NULL, + ciKlass* klass = nullptr, const char* reason_string = nullptr, bool must_throw = false) { uncommon_trap(Deoptimization::make_trap_request(reason, action), klass, reason_string, must_throw, /*keep_exact_action=*/true); @@ -798,11 +804,11 @@ class GraphKit : public Phase { Node* make_runtime_call(int flags, const TypeFunc* call_type, address call_addr, const char* call_name, - const TypePtr* adr_type, // NULL if no memory effects - Node* parm0 = NULL, Node* parm1 = NULL, - Node* parm2 = NULL, Node* parm3 = NULL, - Node* parm4 = NULL, Node* parm5 = NULL, - Node* parm6 = NULL, Node* parm7 = NULL); + const TypePtr* adr_type, // null if no memory effects + Node* parm0 = nullptr, Node* parm1 = nullptr, + Node* parm2 = nullptr, Node* parm3 = nullptr, + Node* parm4 = nullptr, Node* parm5 = nullptr, + Node* parm6 = nullptr, Node* parm7 = nullptr); Node* sign_extend_byte(Node* in); Node* sign_extend_short(Node* in); @@ -826,8 +832,8 @@ class GraphKit : public Phase { // Helper functions to build synchronizations int next_monitor(); - Node* insert_mem_bar(int opcode, Node* precedent = NULL); - Node* insert_mem_bar_volatile(int opcode, int alias_idx, Node* precedent = NULL); + Node* insert_mem_bar(int opcode, Node* precedent = nullptr); + Node* insert_mem_bar_volatile(int opcode, int alias_idx, Node* precedent = nullptr); // Optional 'precedent' is appended as an extra edge, to force ordering. FastLockNode* shared_lock(Node* obj); void shared_unlock(Node* box, Node* obj); @@ -842,7 +848,7 @@ class GraphKit : public Phase { // Generate a check-cast idiom. Used by both the check-cast bytecode // and the array-store bytecode Node* gen_checkcast( Node *subobj, Node* superkls, - Node* *failure_control = NULL ); + Node* *failure_control = nullptr ); Node* gen_subtype_check(Node* obj, Node* superklass); @@ -862,11 +868,11 @@ class GraphKit : public Phase { bool deoptimize_on_exception=false); Node* get_layout_helper(Node* klass_node, jint& constant_value); Node* new_instance(Node* klass_node, - Node* slow_test = NULL, - Node* *return_size_val = NULL, + Node* slow_test = nullptr, + Node* *return_size_val = nullptr, bool deoptimize_on_exception = false); Node* new_array(Node* klass_node, Node* count_val, int nargs, - Node* *return_size_val = NULL, + Node* *return_size_val = nullptr, bool deoptimize_on_exception = false); // java.lang.String helpers diff --git a/src/hotspot/share/opto/idealGraphPrinter.cpp b/src/hotspot/share/opto/idealGraphPrinter.cpp index 8d1595879b79d..757f7a9cee34b 100644 --- a/src/hotspot/share/opto/idealGraphPrinter.cpp +++ b/src/hotspot/share/opto/idealGraphPrinter.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2007, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -77,10 +77,10 @@ int IdealGraphPrinter::_file_count = 0; IdealGraphPrinter *IdealGraphPrinter::printer() { JavaThread *thread = JavaThread::current(); - if (!thread->is_Compiler_thread()) return NULL; + if (!thread->is_Compiler_thread()) return nullptr; CompilerThread *compiler_thread = (CompilerThread *)thread; - if (compiler_thread->ideal_graph_printer() == NULL) { + if (compiler_thread->ideal_graph_printer() == nullptr) { IdealGraphPrinter *printer = new IdealGraphPrinter(); compiler_thread->set_ideal_graph_printer(printer); } @@ -96,15 +96,15 @@ void IdealGraphPrinter::clean_up() { if (printer) { delete printer; } - c->set_ideal_graph_printer(NULL); + c->set_ideal_graph_printer(nullptr); } } IdealGraphPrinter* debug_file_printer = Compile::debug_file_printer(); - if (debug_file_printer != NULL) { + if (debug_file_printer != nullptr) { delete debug_file_printer; } IdealGraphPrinter* debug_network_printer = Compile::debug_network_printer(); - if (debug_network_printer != NULL) { + if (debug_network_printer != nullptr) { delete debug_network_printer; } } @@ -114,11 +114,11 @@ IdealGraphPrinter::IdealGraphPrinter() { init(PrintIdealGraphFile, true, false); } -// Either print methods to the specified file 'file_name' or if NULL over the network to the IGV. If 'append' +// Either print methods to the specified file 'file_name' or if null over the network to the IGV. If 'append' // is set, the next phase is directly appended to the specified file 'file_name'. This is useful when doing // replay compilation with a tool like rr that cannot alter the current program state but only the file. IdealGraphPrinter::IdealGraphPrinter(Compile* compile, const char* file_name, bool append) { - assert(!append || (append && file_name != NULL), "can only use append flag when printing to file"); + assert(!append || (append && file_name != nullptr), "can only use append flag when printing to file"); init(file_name, false, append); C = compile; if (append) { @@ -136,13 +136,13 @@ void IdealGraphPrinter::init(const char* file_name, bool use_multiple_files, boo // appear in the dump. _traverse_outs = true; _should_send_method = true; - _output = NULL; + _output = nullptr; buffer[0] = 0; _depth = 0; - _current_method = NULL; - _network_stream = NULL; + _current_method = nullptr; + _network_stream = nullptr; - if (file_name != NULL) { + if (file_name != nullptr) { init_file_stream(file_name, use_multiple_files, append); } else { init_network_stream(); @@ -163,20 +163,20 @@ IdealGraphPrinter::~IdealGraphPrinter() { if(_xml) { delete _xml; - _xml = NULL; + _xml = nullptr; } if (_network_stream) { delete _network_stream; if (_network_stream == _output) { - _output = NULL; + _output = nullptr; } - _network_stream = NULL; + _network_stream = nullptr; } if (_output) { delete _output; - _output = NULL; + _output = nullptr; } } @@ -255,7 +255,7 @@ void IdealGraphPrinter::print_method(ciMethod *method, int bci, InlineTree *tree _xml->print_cr("]]>"); tail(BYTECODES_ELEMENT); - if (tree != NULL && tree->subtrees().length() > 0) { + if (tree != nullptr && tree->subtrees().length() > 0) { head(INLINE_ELEMENT); GrowableArray subtrees = tree->subtrees(); for (int i = 0; i < subtrees.length(); i++) { @@ -269,7 +269,7 @@ void IdealGraphPrinter::print_method(ciMethod *method, int bci, InlineTree *tree } void IdealGraphPrinter::print_inline_tree(InlineTree *tree) { - if (tree != NULL) { + if (tree != nullptr) { print_method(tree->method(), tree->caller_bci(), tree); } } @@ -279,7 +279,7 @@ void IdealGraphPrinter::print_inlining() { // Print inline tree if (_should_send_method) { InlineTree *inlineTree = C->ilt(); - if (inlineTree != NULL) { + if (inlineTree != nullptr) { print_inline_tree(inlineTree); } else { // print this method only @@ -324,7 +324,7 @@ void IdealGraphPrinter::begin_method() { // Has to be called whenever a method has finished compilation void IdealGraphPrinter::end_method() { tail(GROUP_ELEMENT); - _current_method = NULL; + _current_method = nullptr; _xml->flush(); } @@ -371,14 +371,14 @@ void IdealGraphPrinter::visit_node(Node *n, bool edges, VectorSet* temp_set) { print_prop("debug_idx", node->_debug_idx); #endif - if (C->cfg() != NULL) { + if (C->cfg() != nullptr) { Block* block = C->cfg()->get_block_for_node(node); - if (block == NULL) { + if (block == nullptr) { print_prop("block", C->cfg()->get_block(0)->_pre_order); } else { print_prop("block", block->_pre_order); if (node == block->head()) { - if (block->_idom != NULL) { + if (block->_idom != nullptr) { print_prop("idom", block->_idom->_pre_order); } print_prop("dom_depth", block->_dom_depth); @@ -445,7 +445,7 @@ void IdealGraphPrinter::visit_node(Node *n, bool edges, VectorSet* temp_set) { print_prop("has_call", "true"); } - if (C->matcher() != NULL) { + if (C->matcher() != nullptr) { if (C->matcher()->is_shared(node)) { print_prop("is_shared", "true"); } else { @@ -457,7 +457,7 @@ void IdealGraphPrinter::visit_node(Node *n, bool edges, VectorSet* temp_set) { print_prop("is_dontcare", "false"); } Node* old = C->matcher()->find_old_node(node); - if (old != NULL) { + if (old != nullptr) { print_prop("old_node_idx", old->_idx); } } @@ -474,10 +474,10 @@ void IdealGraphPrinter::visit_node(Node *n, bool edges, VectorSet* temp_set) { stringStream s2(buffer, sizeof(buffer) - 1); node->dump_spec(&s2); - if (t != NULL && (t->isa_instptr() || t->isa_klassptr())) { + if (t != nullptr && (t->isa_instptr() || t->isa_klassptr())) { const TypeInstPtr *toop = t->isa_instptr(); const TypeKlassPtr *tkls = t->isa_klassptr(); - ciKlass* klass = toop ? toop->klass() : (tkls ? tkls->klass() : NULL ); + ciKlass* klass = toop ? toop->klass() : (tkls ? tkls->klass() : nullptr ); if( klass && klass->is_loaded() && klass->is_interface() ) { s2.print(" Interface:"); } else if( toop ) { @@ -563,19 +563,19 @@ void IdealGraphPrinter::visit_node(Node *n, bool edges, VectorSet* temp_set) { } } - JVMState* caller = NULL; + JVMState* caller = nullptr; if (node->is_SafePoint()) { caller = node->as_SafePoint()->jvms(); } else { Node_Notes* notes = C->node_notes_at(node->_idx); - if (notes != NULL) { + if (notes != nullptr) { caller = notes->jvms(); } } - if (caller != NULL) { + if (caller != nullptr) { stringStream bciStream; - ciMethod* last = NULL; + ciMethod* last = nullptr; int last_bci; while(caller) { if (caller->has_method()) { @@ -586,13 +586,13 @@ void IdealGraphPrinter::visit_node(Node *n, bool edges, VectorSet* temp_set) { caller = caller->caller(); } print_prop("bci", bciStream.as_string()); - if (last != NULL && last->has_linenumber_table() && last_bci >= 0) { + if (last != nullptr && last->has_linenumber_table() && last_bci >= 0) { print_prop("line", last->line_number_from_bci(last_bci)); } } #ifdef ASSERT - if (node->debug_orig() != NULL) { + if (node->debug_orig() != nullptr) { stringStream dorigStream; node->dump_orig(&dorigStream, false); print_prop("debug_orig", dorigStream.as_string()); @@ -622,10 +622,10 @@ void IdealGraphPrinter::walk_nodes(Node *start, bool edges, VectorSet* temp_set) VectorSet visited; - GrowableArray nodeStack(Thread::current()->resource_area(), 0, 0, NULL); + GrowableArray nodeStack(Thread::current()->resource_area(), 0, 0, nullptr); nodeStack.push(start); visited.test_set(start->_idx); - if (C->cfg() != NULL) { + if (C->cfg() != nullptr) { // once we have a CFG there are some nodes that aren't really // reachable but are in the CFG so add them here. for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) { @@ -669,7 +669,7 @@ void IdealGraphPrinter::print_method(const char *name, int level) { // Print current ideal graph void IdealGraphPrinter::print(const char *name, Node *node) { - if (!_current_method || !_should_send_method || node == NULL) return; + if (!_current_method || !_should_send_method || node == nullptr) return; // Warning, unsafe cast? _chaitin = (PhaseChaitin *)C->regalloc(); @@ -681,7 +681,7 @@ void IdealGraphPrinter::print(const char *name, Node *node) { VectorSet temp_set; head(NODES_ELEMENT); - if (C->cfg() != NULL) { + if (C->cfg() != nullptr) { // Compute the maximum estimated frequency in the current graph. _max_freq = 1.0e-6; for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) { @@ -697,7 +697,7 @@ void IdealGraphPrinter::print(const char *name, Node *node) { head(EDGES_ELEMENT); walk_nodes(node, true, &temp_set); tail(EDGES_ELEMENT); - if (C->cfg() != NULL) { + if (C->cfg() != nullptr) { head(CONTROL_FLOW_ELEMENT); for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) { Block* block = C->cfg()->get_block(i); @@ -762,7 +762,7 @@ void IdealGraphPrinter::init_network_stream() { tty->print_cr("Client available, but does not want to receive data!"); _network_stream->close(); delete _network_stream; - _network_stream = NULL; + _network_stream = nullptr; return; } _output = _network_stream; @@ -775,11 +775,11 @@ void IdealGraphPrinter::init_network_stream() { } void IdealGraphPrinter::update_compiled_method(ciMethod* current_method) { - assert(C != NULL, "must already be set"); + assert(C != nullptr, "must already be set"); if (current_method != _current_method) { // If a different method, end the old and begin with the new one. end_method(); - _current_method = NULL; + _current_method = nullptr; begin_method(); } } diff --git a/src/hotspot/share/opto/idealGraphPrinter.hpp b/src/hotspot/share/opto/idealGraphPrinter.hpp index 6d8db169b9f36..78fdf01d62c62 100644 --- a/src/hotspot/share/opto/idealGraphPrinter.hpp +++ b/src/hotspot/share/opto/idealGraphPrinter.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2007, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -116,7 +116,7 @@ class IdealGraphPrinter : public CHeapObj { ~IdealGraphPrinter(); public: - IdealGraphPrinter(Compile* compile, const char* file_name = NULL, bool append = false); + IdealGraphPrinter(Compile* compile, const char* file_name = nullptr, bool append = false); static void clean_up(); static IdealGraphPrinter *printer(); diff --git a/src/hotspot/share/opto/idealKit.cpp b/src/hotspot/share/opto/idealKit.cpp index a88cbff290144..79b86f3b84bcc 100644 --- a/src/hotspot/share/opto/idealKit.cpp +++ b/src/hotspot/share/opto/idealKit.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -45,9 +45,9 @@ IdealKit::IdealKit(GraphKit* gkit, bool delay_all_transforms, bool has_declarati _initial_i_o = gkit->i_o(); _delay_all_transforms = delay_all_transforms; _var_ct = 0; - _cvstate = NULL; + _cvstate = nullptr; // We can go memory state free or else we need the entire memory state - assert(_initial_memory == NULL || _initial_memory->Opcode() == Op_MergeMem, "memory must be pre-split"); + assert(_initial_memory == nullptr || _initial_memory->Opcode() == Op_MergeMem, "memory must be pre-split"); assert(!_gvn.is_IterGVN(), "IdealKit can't be used during Optimize phase"); int init_size = 5; _pending_cvstates = new (C->node_arena()) GrowableArray(C->node_arena(), init_size, 0, 0); @@ -73,11 +73,11 @@ void IdealKit::if_then(Node* left, BoolTest::mask relop, Node* right, float prob, float cnt, bool push_new_state) { assert((state() & (BlockS|LoopS|IfThenS|ElseS)), "bad state for new If"); Node* bol; - if (left->bottom_type()->isa_ptr() == NULL) { - if (left->bottom_type()->isa_int() != NULL) { + if (left->bottom_type()->isa_ptr() == nullptr) { + if (left->bottom_type()->isa_int() != nullptr) { bol = Bool(CmpI(left, right), relop); } else { - assert(left->bottom_type()->isa_long() != NULL, "what else?"); + assert(left->bottom_type()->isa_long() != nullptr, "what else?"); bol = Bool(CmpL(left, right), relop); } @@ -202,7 +202,7 @@ void IdealKit::end_loop() { // must be specified (which should be 1 less than // the number of precedessors.) Node* IdealKit::make_label(int goto_ct) { - assert(_cvstate != NULL, "must declare variables before labels"); + assert(_cvstate != nullptr, "must declare variables before labels"); Node* lab = new_cvstate(); int sz = 1 + goto_ct + 1 /* fall thru */; Node* reg = delay_transform(new RegionNode(sz)); @@ -228,7 +228,7 @@ void IdealKit::goto_(Node* lab, bool bind) { Node* reg = lab->in(TypeFunc::Control); // find next empty slot in region uint slot = 1; - while (slot < reg->req() && reg->in(slot) != NULL) slot++; + while (slot < reg->req() && reg->in(slot) != nullptr) slot++; assert(slot < reg->req(), "too many gotos"); // If this is last predecessor, then don't force phi creation if (slot == reg->req() - 1) bind = false; @@ -245,9 +245,9 @@ void IdealKit::goto_(Node* lab, bool bind) { // Get the current value of the var Node* m = _cvstate->in(i); // If the var went unused no need for a phi - if (m == NULL) { + if (m == nullptr) { continue; - } else if (l == NULL || m == l) { + } else if (l == nullptr || m == l) { // Only one unique value "m" is known to reach this label so a phi // is not yet necessary unless: // the label is being bound and all predecessors have not been seen, @@ -326,7 +326,7 @@ Node* IdealKit::copy_cvstate() { //-----------------------------clear----------------------------------- void IdealKit::clear(Node* m) { - for (uint i = 0; i < m->req(); i++) m->set_req(i, NULL); + for (uint i = 0; i < m->req(); i++) m->set_req(i, nullptr); } //-----------------------------IdealVariable---------------------------- @@ -356,7 +356,7 @@ Node* IdealKit::load(Node* ctl, LoadNode::ControlDependency control_dependency) { assert(adr_idx != Compile::AliasIdxTop, "use other make_load factory" ); - const TypePtr* adr_type = NULL; // debug-mode-only argument + const TypePtr* adr_type = nullptr; // debug-mode-only argument debug_only(adr_type = C->get_adr_type(adr_idx)); Node* mem = memory(adr_idx); Node* ld = LoadNode::make(_gvn, ctl, mem, adr, adr_type, t, bt, mo, control_dependency, require_atomic_access); @@ -368,7 +368,7 @@ Node* IdealKit::store(Node* ctl, Node* adr, Node *val, BasicType bt, MemNode::MemOrd mo, bool require_atomic_access, bool mismatched) { assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory"); - const TypePtr* adr_type = NULL; + const TypePtr* adr_type = nullptr; debug_only(adr_type = C->get_adr_type(adr_idx)); Node *mem = memory(adr_idx); Node* st = StoreNode::make(_gvn, ctl, mem, adr, adr_type, val, bt, mo, require_atomic_access); @@ -387,7 +387,7 @@ Node* IdealKit::storeCM(Node* ctl, Node* adr, Node *val, Node* oop_store, int oo BasicType bt, int adr_idx) { assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" ); - const TypePtr* adr_type = NULL; + const TypePtr* adr_type = nullptr; debug_only(adr_type = C->get_adr_type(adr_idx)); Node *mem = memory(adr_idx); @@ -411,11 +411,11 @@ void IdealKit::do_memory_merge(Node* merging, Node* join) { // Get the region for the join state Node* join_region = join->in(TypeFunc::Control); - assert(join_region != NULL, "join region must exist"); - if (join->in(TypeFunc::I_O) == NULL ) { + assert(join_region != nullptr, "join region must exist"); + if (join->in(TypeFunc::I_O) == nullptr ) { join->set_req(TypeFunc::I_O, merging->in(TypeFunc::I_O)); } - if (join->in(TypeFunc::Memory) == NULL ) { + if (join->in(TypeFunc::Memory) == nullptr ) { join->set_req(TypeFunc::Memory, merging->in(TypeFunc::Memory)); return; } @@ -503,10 +503,10 @@ Node* IdealKit::make_leaf_call(const TypeFunc *slow_call_type, call->init_req( TypeFunc::FramePtr, top() /* frameptr() */ ); call->init_req( TypeFunc::ReturnAdr, top() ); - if (parm0 != NULL) call->init_req(TypeFunc::Parms+0, parm0); - if (parm1 != NULL) call->init_req(TypeFunc::Parms+1, parm1); - if (parm2 != NULL) call->init_req(TypeFunc::Parms+2, parm2); - if (parm3 != NULL) call->init_req(TypeFunc::Parms+3, parm3); + if (parm0 != nullptr) call->init_req(TypeFunc::Parms+0, parm0); + if (parm1 != nullptr) call->init_req(TypeFunc::Parms+1, parm1); + if (parm2 != nullptr) call->init_req(TypeFunc::Parms+2, parm2); + if (parm3 != nullptr) call->init_req(TypeFunc::Parms+3, parm3); // Node *c = _gvn.transform(call); call = (CallNode *) _gvn.transform(call); @@ -524,7 +524,7 @@ Node* IdealKit::make_leaf_call(const TypeFunc *slow_call_type, assert(C->alias_type(call->adr_type()) == C->alias_type(adr_type), "call node must be constructed correctly"); - Node* res = NULL; + Node* res = nullptr; if (slow_call_type->range()->cnt() > TypeFunc::Parms) { assert(slow_call_type->range()->cnt() == TypeFunc::Parms+1, "only one return value"); res = transform(new ProjNode(call, TypeFunc::Parms)); @@ -555,10 +555,10 @@ void IdealKit::make_leaf_call_no_fp(const TypeFunc *slow_call_type, call->init_req( TypeFunc::FramePtr, top() /* frameptr() */ ); call->init_req( TypeFunc::ReturnAdr, top() ); - if (parm0 != NULL) call->init_req(TypeFunc::Parms+0, parm0); - if (parm1 != NULL) call->init_req(TypeFunc::Parms+1, parm1); - if (parm2 != NULL) call->init_req(TypeFunc::Parms+2, parm2); - if (parm3 != NULL) call->init_req(TypeFunc::Parms+3, parm3); + if (parm0 != nullptr) call->init_req(TypeFunc::Parms+0, parm0); + if (parm1 != nullptr) call->init_req(TypeFunc::Parms+1, parm1); + if (parm2 != nullptr) call->init_req(TypeFunc::Parms+2, parm2); + if (parm3 != nullptr) call->init_req(TypeFunc::Parms+3, parm3); // Node *c = _gvn.transform(call); call = (CallNode *) _gvn.transform(call); diff --git a/src/hotspot/share/opto/idealKit.hpp b/src/hotspot/share/opto/idealKit.hpp index 55d63b8720f00..20acec4721118 100644 --- a/src/hotspot/share/opto/idealKit.hpp +++ b/src/hotspot/share/opto/idealKit.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -159,7 +159,7 @@ class IdealKit: public StackObj { void set_i_o(Node* c) { _cvstate->set_req(TypeFunc::I_O, c); } void set(IdealVariable& v, Node* rhs) { _cvstate->set_req(first_var + v.id(), rhs); } Node* value(IdealVariable& v) { return _cvstate->in(first_var + v.id()); } - void dead(IdealVariable& v) { set(v, (Node*)NULL); } + void dead(IdealVariable& v) { set(v, (Node*)nullptr); } void if_then(Node* left, BoolTest::mask relop, Node* right, float prob = PROB_FAIR, float cnt = COUNT_UNKNOWN, bool push_new_state = true); @@ -248,9 +248,9 @@ class IdealKit: public StackObj { address slow_call, const char *leaf_name, Node* parm0, - Node* parm1 = NULL, - Node* parm2 = NULL, - Node* parm3 = NULL); + Node* parm1 = nullptr, + Node* parm2 = nullptr, + Node* parm3 = nullptr); void make_leaf_call_no_fp(const TypeFunc *slow_call_type, address slow_call, diff --git a/src/hotspot/share/opto/ifg.cpp b/src/hotspot/share/opto/ifg.cpp index 49192d2586ed7..983e4a490d3b1 100644 --- a/src/hotspot/share/opto/ifg.cpp +++ b/src/hotspot/share/opto/ifg.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -819,7 +819,7 @@ void PhaseChaitin::adjust_high_pressure_index(Block* b, uint& block_hrp_index, P } void PhaseChaitin::print_pressure_info(Pressure& pressure, const char *str) { - if (str != NULL) { + if (str != nullptr) { tty->print_cr("# *** %s ***", str); } tty->print_cr("# start pressure is = %d", pressure.start_pressure()); diff --git a/src/hotspot/share/opto/ifnode.cpp b/src/hotspot/share/opto/ifnode.cpp index d934c5a551fc4..c7316fa3a87e5 100644 --- a/src/hotspot/share/opto/ifnode.cpp +++ b/src/hotspot/share/opto/ifnode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -79,28 +79,33 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) { // Look for a compare of a constant and a merged value Node *i1 = iff->in(1); - if( !i1->is_Bool() ) return NULL; + if( !i1->is_Bool() ) return nullptr; BoolNode *b = i1->as_Bool(); Node *cmp = b->in(1); - if( !cmp->is_Cmp() ) return NULL; + if( !cmp->is_Cmp() ) return nullptr; i1 = cmp->in(1); - if( i1 == NULL || !i1->is_Phi() ) return NULL; + if( i1 == nullptr || !i1->is_Phi() ) return nullptr; PhiNode *phi = i1->as_Phi(); Node *con2 = cmp->in(2); - if( !con2->is_Con() ) return NULL; + if( !con2->is_Con() ) return nullptr; // See that the merge point contains some constants - Node *con1=NULL; + Node *con1=nullptr; uint i4; - for( i4 = 1; i4 < phi->req(); i4++ ) { + RegionNode* phi_region = phi->region(); + for (i4 = 1; i4 < phi->req(); i4++ ) { con1 = phi->in(i4); - if( !con1 ) return NULL; // Do not optimize partially collapsed merges + // Do not optimize partially collapsed merges + if (con1 == nullptr || phi_region->in(i4) == nullptr || igvn->type(phi_region->in(i4)) == Type::TOP) { + igvn->_worklist.push(iff); + return nullptr; + } if( con1->is_Con() ) break; // Found a constant // Also allow null-vs-not-null checks const TypePtr *tp = igvn->type(con1)->isa_ptr(); if( tp && tp->_ptr == TypePtr::NotNull ) break; } - if( i4 >= phi->req() ) return NULL; // Found no constants + if( i4 >= phi->req() ) return nullptr; // Found no constants igvn->C->set_has_split_ifs(true); // Has chance for split-if @@ -111,18 +116,18 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) { const Type *t = cmp2->Value(igvn); // This compare is dead, so whack it! igvn->remove_dead_node(cmp2); - if( !t->singleton() ) return NULL; + if( !t->singleton() ) return nullptr; // No intervening control, like a simple Call Node* r = iff->in(0); - if (!r->is_Region() || r->is_Loop() || phi->region() != r || r->as_Region()->is_copy()) { - return NULL; + if (!r->is_Region() || r->is_Loop() || phi_region != r || r->as_Region()->is_copy()) { + return nullptr; } // No other users of the cmp/bool if (b->outcnt() != 1 || cmp->outcnt() != 1) { //tty->print_cr("many users of cmp/bool"); - return NULL; + return nullptr; } // Make sure we can determine where all the uses of merged values go @@ -139,13 +144,13 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) { tty->print_cr("Region has odd use"); u->dump(2); }*/ - return NULL; + return nullptr; } if( u != phi ) { // CNC - do not allow any other merged value //tty->print_cr("Merging another value"); //u->dump(2); - return NULL; + return nullptr; } // Make sure we can account for all Phi uses for (DUIterator_Fast kmax, k = u->fast_outs(kmax); k < kmax; k++) { @@ -157,8 +162,8 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) { // If the cast is derived from data flow edges, it may not have a control edge. // If so, it should be safe to split. But follow-up code can not deal with // this (l. 359). So skip. - if (v->in(0) == NULL) { - return NULL; + if (v->in(0) == nullptr) { + return nullptr; } if (v->in(0)->in(0) == iff) { continue; // CastPP/II of the IfNode is OK @@ -167,9 +172,9 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) { // Disabled following code because I cannot tell if exactly one // path dominates without a real dominator check. CNC 9/9/1999 //uint vop = v->Opcode(); - //if( vop == Op_Phi ) { // Phi from another merge point might be OK - // Node *r = v->in(0); // Get controlling point - // if( !r ) return NULL; // Degraded to a copy + //if( vop == Op_Phi ) { // Phi from another merge point might be OK + // Node *r = v->in(0); // Get controlling point + // if( !r ) return nullptr; // Degraded to a copy // // Find exactly one path in (either True or False doms, but not IFF) // int cnt = 0; // for( uint i = 1; i < r->req(); i++ ) @@ -190,7 +195,7 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) { } */ } - return NULL; + return nullptr; /* CNC - Cut out all the fancy acceptance tests // Can we clone this use when doing the transformation? @@ -198,14 +203,14 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) { if( !v->in(0) && v != cmp ) { tty->print_cr("Phi has free-floating use"); v->dump(2); - return NULL; + return nullptr; } for( uint l = 1; l < v->req(); l++ ) { if( (!v->in(l)->is_Phi() || v->in(l)->in(0) != r) && !v->in(l)->is_Con() ) { tty->print_cr("Phi has use"); v->dump(2); - return NULL; + return nullptr; } // End of if Phi-use input is neither Phi nor Constant } // End of for all inputs to Phi-use */ @@ -214,7 +219,7 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) { // Only do this if the IF node is in a sane state if (iff->outcnt() != 2) - return NULL; + return nullptr; // Got a hit! Do the Mondo Hack! // @@ -243,17 +248,17 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) { req_c++; } Node* proj = PhaseIdealLoop::find_predicate(r->in(ii)); - if (proj != NULL) { + if (proj != nullptr) { // Bail out if splitting through a region with a predicate input (could // also be a loop header before loop opts creates a LoopNode for it). - return NULL; + return nullptr; } } // If all the defs of the phi are the same constant, we already have the desired end state. // Skip the split that would create empty phi and region nodes. if ((r->req() - req_c) == 1) { - return NULL; + return nullptr; } // At this point we know that we can apply the split if optimization. If the region is still on the worklist, @@ -261,7 +266,7 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) { // This also avoids the creation of dead data loops when rewiring data nodes below when a region is dying. if (igvn->_worklist.member(r)) { igvn->_worklist.push(iff); // retry split if later again - return NULL; + return nullptr; } Node *region_c = new RegionNode(req_c + 1); @@ -336,17 +341,17 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) { igvn->register_new_node_with_optimizer( region_f ); igvn->hash_delete(cmp);// Remove soon-to-be-dead node from hash table. - cmp->set_req(1,NULL); // Whack the inputs to cmp because it will be dead - cmp->set_req(2,NULL); + cmp->set_req(1,nullptr); // Whack the inputs to cmp because it will be dead + cmp->set_req(2,nullptr); // Check for all uses of the Phi and give them a new home. // The 'cmp' got cloned, but CastPP/IIs need to be moved. - Node *phi_s = NULL; // do not construct unless needed - Node *phi_f = NULL; // do not construct unless needed + Node *phi_s = nullptr; // do not construct unless needed + Node *phi_f = nullptr; // do not construct unless needed for (DUIterator_Last i2min, i2 = phi->last_outs(i2min); i2 >= i2min; --i2) { Node* v = phi->last_out(i2);// User of the phi igvn->rehash_node_delayed(v); // Have to fixup other Phi users uint vop = v->Opcode(); - Node *proj = NULL; + Node *proj = nullptr; if( vop == Op_Phi ) { // Remote merge point Node *r = v->in(0); for (uint i3 = 1; i3 < r->req(); i3++) @@ -359,11 +364,11 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) { } else { assert( 0, "do not know how to handle this guy" ); } - guarantee(proj != NULL, "sanity"); + guarantee(proj != nullptr, "sanity"); Node *proj_path_data, *proj_path_ctrl; if( proj->Opcode() == Op_IfTrue ) { - if( phi_s == NULL ) { + if( phi_s == nullptr ) { // Only construct phi_s if needed, otherwise provides // interfering use. phi_s = PhiNode::make_blank(region_s,phi); @@ -375,7 +380,7 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) { proj_path_data = phi_s; proj_path_ctrl = region_s; } else { - if( phi_f == NULL ) { + if( phi_f == nullptr ) { // Only construct phi_f if needed, otherwise provides // interfering use. phi_f = PhiNode::make_blank(region_f,phi); @@ -431,7 +436,7 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) { for (DUIterator_Last lmin, l = r->last_outs(lmin); l >= lmin;) { Node* u = r->last_out(l); if( u == r ) { - r->set_req(0, NULL); + r->set_req(0, nullptr); } else { assert(u->outcnt() == 0, "only dead users"); igvn->remove_dead_node(u); @@ -452,14 +457,14 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) { // for the failed path ProjNode* IfNode::range_check_trap_proj(int& flip_test, Node*& l, Node*& r) { if (outcnt() != 2) { - return NULL; + return nullptr; } Node* b = in(1); - if (b == NULL || !b->is_Bool()) return NULL; + if (b == nullptr || !b->is_Bool()) return nullptr; BoolNode* bn = b->as_Bool(); Node* cmp = bn->in(1); - if (cmp == NULL) return NULL; - if (cmp->Opcode() != Op_CmpU) return NULL; + if (cmp == nullptr) return nullptr; + if (cmp->Opcode() != Op_CmpU) return nullptr; l = cmp->in(1); r = cmp->in(2); @@ -469,10 +474,10 @@ ProjNode* IfNode::range_check_trap_proj(int& flip_test, Node*& l, Node*& r) { r = cmp->in(1); flip_test = 2; } else if (bn->_test._test != BoolTest::lt) { - return NULL; + return nullptr; } - if (l->is_top()) return NULL; // Top input means dead test - if (r->Opcode() != Op_LoadRange && !is_RangeCheck()) return NULL; + if (l->is_top()) return nullptr; // Top input means dead test + if (r->Opcode() != Op_LoadRange && !is_RangeCheck()) return nullptr; // We have recognized one of these forms: // Flip 1: If (Bool[<] CmpU(l, LoadRange)) ... @@ -485,15 +490,15 @@ ProjNode* IfNode::range_check_trap_proj(int& flip_test, Node*& l, Node*& r) { //------------------------------is_range_check--------------------------------- // Return 0 if not a range check. Return 1 if a range check and set index and -// offset. Return 2 if we had to negate the test. Index is NULL if the check +// offset. Return 2 if we had to negate the test. Index is null if the check // is versus a constant. int RangeCheckNode::is_range_check(Node* &range, Node* &index, jint &offset) { int flip_test = 0; - Node* l = NULL; - Node* r = NULL; + Node* l = nullptr; + Node* r = nullptr; ProjNode* iftrap = range_check_trap_proj(flip_test, l, r); - if (iftrap == NULL) { + if (iftrap == nullptr) { return 0; } @@ -501,7 +506,7 @@ int RangeCheckNode::is_range_check(Node* &range, Node* &index, jint &offset) { // along the OOB path. Otherwise, it's possible that the user wrote // something which optimized to look like a range check but behaves // in some other way. - if (iftrap->is_uncommon_trap_proj(Deoptimization::Reason_range_check) == NULL) { + if (iftrap->is_uncommon_trap_proj(Deoptimization::Reason_range_check) == nullptr) { return 0; } @@ -518,7 +523,7 @@ int RangeCheckNode::is_range_check(Node* &range, Node* &index, jint &offset) { } } else if ((off = l->find_int_con(-1)) >= 0) { // constant offset with no variable index - ind = NULL; + ind = nullptr; } else { // variable index with no constant offset (or dead negative index) off = 0; @@ -563,7 +568,7 @@ static void adjust_check(Node* proj, Node* range, Node* index, } //------------------------------up_one_dom------------------------------------- -// Walk up the dominator tree one step. Return NULL at root or true +// Walk up the dominator tree one step. Return null at root or true // complex merges. Skips through small diamonds. Node* IfNode::up_one_dom(Node *curr, bool linear_only) { Node *dom = curr->in(0); @@ -576,10 +581,10 @@ Node* IfNode::up_one_dom(Node *curr, bool linear_only) { // Use linear_only if we are still parsing, since we cannot // trust the regions to be fully filled in. if (linear_only) - return NULL; + return nullptr; if( dom->is_Root() ) - return NULL; + return nullptr; // Else hit a Region. Check for a loop header if( dom->is_Loop() ) @@ -598,12 +603,12 @@ Node* IfNode::up_one_dom(Node *curr, bool linear_only) { if( din4->is_Call() && // Handle a slow-path call on either arm (din4 = din4->in(0)) ) din4 = din4->in(0); - if (din3 != NULL && din3 == din4 && din3->is_If()) // Regions not degraded to a copy + if (din3 != nullptr && din3 == din4 && din3->is_If()) // Regions not degraded to a copy return din3; // Skip around diamonds } // Give up the search at true merges - return NULL; // Dead loop? Or hit root? + return nullptr; // Dead loop? Or hit root? } @@ -620,7 +625,7 @@ const TypeInt* IfNode::filtered_int_type(PhaseGVN* gvn, Node* val, Node* if_proj const CmpNode* cmp = bol->in(1)->as_Cmp(); if (cmp->in(1) == val) { const TypeInt* cmp2_t = gvn->type(cmp->in(2))->isa_int(); - if (cmp2_t != NULL) { + if (cmp2_t != nullptr) { jint lo = cmp2_t->_lo; jint hi = cmp2_t->_hi; BoolTest::mask msk = if_proj->Opcode() == Op_IfTrue ? bol->_test._test : bol->_test.negate(); @@ -628,7 +633,7 @@ const TypeInt* IfNode::filtered_int_type(PhaseGVN* gvn, Node* val, Node* if_proj case BoolTest::ne: { // If val is compared to its lower or upper bound, we can narrow the type const TypeInt* val_t = gvn->type(val)->isa_int(); - if (val_t != NULL && !val_t->singleton() && cmp2_t->is_con()) { + if (val_t != nullptr && !val_t->singleton() && cmp2_t->is_con()) { if (val_t->_lo == lo) { return TypeInt::make(val_t->_lo + 1, val_t->_hi, val_t->_widen); } else if (val_t->_hi == hi) { @@ -636,7 +641,7 @@ const TypeInt* IfNode::filtered_int_type(PhaseGVN* gvn, Node* val, Node* if_proj } } // Can't refine type - return NULL; + return nullptr; } case BoolTest::eq: return cmp2_t; @@ -669,7 +674,7 @@ const TypeInt* IfNode::filtered_int_type(PhaseGVN* gvn, Node* val, Node* if_proj } } } - return NULL; + return nullptr; } //------------------------------fold_compares---------------------------- @@ -712,11 +717,11 @@ const TypeInt* IfNode::filtered_int_type(PhaseGVN* gvn, Node* val, Node* if_proj // Is the comparison for this If suitable for folding? bool IfNode::cmpi_folds(PhaseIterGVN* igvn, bool fold_ne) { - return in(1) != NULL && + return in(1) != nullptr && in(1)->is_Bool() && - in(1)->in(1) != NULL && + in(1)->in(1) != nullptr && in(1)->in(1)->Opcode() == Op_CmpI && - in(1)->in(1)->in(2) != NULL && + in(1)->in(1)->in(2) != nullptr && in(1)->in(1)->in(2) != igvn->C->top() && (in(1)->as_Bool()->_test.is_less() || in(1)->as_Bool()->_test.is_greater() || @@ -725,14 +730,14 @@ bool IfNode::cmpi_folds(PhaseIterGVN* igvn, bool fold_ne) { // Is a dominating control suitable for folding with this if? bool IfNode::is_ctrl_folds(Node* ctrl, PhaseIterGVN* igvn) { - return ctrl != NULL && + return ctrl != nullptr && ctrl->is_Proj() && - ctrl->in(0) != NULL && + ctrl->in(0) != nullptr && ctrl->in(0)->Opcode() == Op_If && ctrl->in(0)->outcnt() == 2 && ctrl->in(0)->as_If()->cmpi_folds(igvn, true) && // Must compare same value - ctrl->in(0)->in(1)->in(1)->in(1) != NULL && + ctrl->in(0)->in(1)->in(1)->in(1) != nullptr && ctrl->in(0)->in(1)->in(1)->in(1) != igvn->C->top() && ctrl->in(0)->in(1)->in(1)->in(1) == in(1)->in(1)->in(1); } @@ -741,23 +746,23 @@ bool IfNode::is_ctrl_folds(Node* ctrl, PhaseIterGVN* igvn) { bool IfNode::has_shared_region(ProjNode* proj, ProjNode*& success, ProjNode*& fail) { ProjNode* otherproj = proj->other_if_proj(); Node* otherproj_ctrl_use = otherproj->unique_ctrl_out(); - RegionNode* region = (otherproj_ctrl_use != NULL && otherproj_ctrl_use->is_Region()) ? otherproj_ctrl_use->as_Region() : NULL; - success = NULL; - fail = NULL; + RegionNode* region = (otherproj_ctrl_use != nullptr && otherproj_ctrl_use->is_Region()) ? otherproj_ctrl_use->as_Region() : nullptr; + success = nullptr; + fail = nullptr; - if (otherproj->outcnt() == 1 && region != NULL && !region->has_phi()) { + if (otherproj->outcnt() == 1 && region != nullptr && !region->has_phi()) { for (int i = 0; i < 2; i++) { ProjNode* proj = proj_out(i); - if (success == NULL && proj->outcnt() == 1 && proj->unique_out() == region) { + if (success == nullptr && proj->outcnt() == 1 && proj->unique_out() == region) { success = proj; - } else if (fail == NULL) { + } else if (fail == nullptr) { fail = proj; } else { - success = fail = NULL; + success = fail = nullptr; } } } - return success != NULL && fail != NULL; + return success != nullptr && fail != nullptr; } bool IfNode::is_dominator_unc(CallStaticJavaNode* dom_unc, CallStaticJavaNode* unc) { @@ -772,11 +777,11 @@ bool IfNode::is_dominator_unc(CallStaticJavaNode* dom_unc, CallStaticJavaNode* u // that the call stacks are equal for both JVMStates. JVMState* dom_caller = dom_unc->jvms()->caller(); JVMState* caller = unc->jvms()->caller(); - if ((dom_caller == NULL) != (caller == NULL)) { + if ((dom_caller == nullptr) != (caller == nullptr)) { // The current method must either be inlined into both dom_caller and // caller or must not be inlined at all (top method). Bail out otherwise. return false; - } else if (dom_caller != NULL && !dom_caller->same_calls_as(caller)) { + } else if (dom_caller != nullptr && !dom_caller->same_calls_as(caller)) { return false; } // Check that the bci of the dominating uncommon trap dominates the bci @@ -796,11 +801,11 @@ bool IfNode::is_dominator_unc(CallStaticJavaNode* dom_unc, CallStaticJavaNode* u ProjNode* IfNode::uncommon_trap_proj(CallStaticJavaNode*& call) const { for (int i = 0; i < 2; i++) { call = proj_out(i)->is_uncommon_trap_proj(Deoptimization::Reason_none); - if (call != NULL) { + if (call != nullptr) { return proj_out(i); } } - return NULL; + return nullptr; } // Do this If and the dominating If both branch out to an uncommon trap @@ -808,22 +813,22 @@ bool IfNode::has_only_uncommon_traps(ProjNode* proj, ProjNode*& success, ProjNod ProjNode* otherproj = proj->other_if_proj(); CallStaticJavaNode* dom_unc = otherproj->is_uncommon_trap_proj(Deoptimization::Reason_none); - if (otherproj->outcnt() == 1 && dom_unc != NULL) { + if (otherproj->outcnt() == 1 && dom_unc != nullptr) { // We need to re-execute the folded Ifs after deoptimization from the merged traps if (!dom_unc->jvms()->should_reexecute()) { return false; } - CallStaticJavaNode* unc = NULL; + CallStaticJavaNode* unc = nullptr; ProjNode* unc_proj = uncommon_trap_proj(unc); - if (unc_proj != NULL && unc_proj->outcnt() == 1) { + if (unc_proj != nullptr && unc_proj->outcnt() == 1) { if (dom_unc == unc) { // Allow the uncommon trap to be shared through a region RegionNode* r = unc->in(0)->as_Region(); if (r->outcnt() != 2 || r->req() != 3 || r->find_edge(otherproj) == -1 || r->find_edge(unc_proj) == -1) { return false; } - assert(r->has_phi() == NULL, "simple region shouldn't have a phi"); + assert(r->has_phi() == nullptr, "simple region shouldn't have a phi"); } else if (dom_unc->in(0) != otherproj || unc->in(0) != unc_proj) { return false; } @@ -891,8 +896,8 @@ bool IfNode::fold_compares_helper(ProjNode* proj, ProjNode* success, ProjNode* f // Figure out which of the two tests sets the upper bound and which // sets the lower bound if any. - Node* adjusted_lim = NULL; - if (lo_type != NULL && hi_type != NULL && hi_type->_lo > lo_type->_hi && + Node* adjusted_lim = nullptr; + if (lo_type != nullptr && hi_type != nullptr && hi_type->_lo > lo_type->_hi && hi_type->_hi == max_jint && lo_type->_lo == min_jint && lo_test != BoolTest::ne) { assert((dom_bool->_test.is_less() && !proj->_con) || (dom_bool->_test.is_greater() && proj->_con), "incorrect test"); @@ -937,7 +942,7 @@ bool IfNode::fold_compares_helper(ProjNode* proj, ProjNode* success, ProjNode* f } // this test was canonicalized assert(this_bool->_test.is_less() && fail->_con, "incorrect test"); - } else if (lo_type != NULL && hi_type != NULL && lo_type->_lo > hi_type->_hi && + } else if (lo_type != nullptr && hi_type != nullptr && lo_type->_lo > hi_type->_hi && lo_type->_hi == max_jint && hi_type->_lo == min_jint && lo_test != BoolTest::ne) { // this_bool = < @@ -997,9 +1002,9 @@ bool IfNode::fold_compares_helper(ProjNode* proj, ProjNode* success, ProjNode* f assert(this_bool->_test.is_less() && !fail->_con, "incorrect test"); } else { const TypeInt* failtype = filtered_int_type(igvn, n, proj); - if (failtype != NULL) { + if (failtype != nullptr) { const TypeInt* type2 = filtered_int_type(igvn, n, fail); - if (type2 != NULL) { + if (type2 != nullptr) { failtype = failtype->join(type2)->is_int(); if (failtype->_lo > failtype->_hi) { // previous if determines the result of this if so @@ -1010,8 +1015,8 @@ bool IfNode::fold_compares_helper(ProjNode* proj, ProjNode* success, ProjNode* f } } } - lo = NULL; - hi = NULL; + lo = nullptr; + hi = nullptr; } if (lo && hi) { @@ -1019,7 +1024,7 @@ bool IfNode::fold_compares_helper(ProjNode* proj, ProjNode* success, ProjNode* f hook->init_req(0, lo); // Add a use to lo to prevent him from dying // Merge the two compares into a single unsigned compare by building (CmpU (n - lo) (hi - lo)) Node* adjusted_val = igvn->transform(new SubINode(n, lo)); - if (adjusted_lim == NULL) { + if (adjusted_lim == nullptr) { adjusted_lim = igvn->transform(new SubINode(hi, lo)); } hook->destruct(igvn); @@ -1094,10 +1099,10 @@ Node* IfNode::merge_uncommon_traps(ProjNode* proj, ProjNode* success, ProjNode* Deoptimization::DeoptAction action = Deoptimization::trap_request_action(trap_request); int flip_test = 0; - Node* l = NULL; - Node* r = NULL; + Node* l = nullptr; + Node* r = nullptr; - if (success->in(0)->as_If()->range_check_trap_proj(flip_test, l, r) != NULL) { + if (success->in(0)->as_If()->range_check_trap_proj(flip_test, l, r) != nullptr) { // If this looks like a range check, change the trap to // Reason_range_check so the compiler recognizes it as a range // check and applies the corresponding optimizations @@ -1152,7 +1157,7 @@ void IfNode::improve_address_types(Node* l, Node* r, ProjNode* fail, PhaseIterGV } } else if (use->is_Mem()) { Node* ctrl = use->in(0); - for (int i = 0; i < 10 && ctrl != NULL && ctrl != fail; i++) { + for (int i = 0; i < 10 && ctrl != nullptr && ctrl != fail; i++) { ctrl = up_one_dom(ctrl); } if (ctrl == fail) { @@ -1182,7 +1187,7 @@ void IfNode::improve_address_types(Node* l, Node* r, ProjNode* fail, PhaseIterGV } } } - } else if (use->in(0) == NULL && (igvn->type(use)->isa_long() || + } else if (use->in(0) == nullptr && (igvn->type(use)->isa_long() || igvn->type(use)->isa_ptr())) { stack.set_index(i+1); stack.push(use, 0); @@ -1197,16 +1202,16 @@ void IfNode::improve_address_types(Node* l, Node* r, ProjNode* fail, PhaseIterGV } bool IfNode::is_cmp_with_loadrange(ProjNode* proj) { - if (in(1) != NULL && - in(1)->in(1) != NULL && - in(1)->in(1)->in(2) != NULL) { + if (in(1) != nullptr && + in(1)->in(1) != nullptr && + in(1)->in(1)->in(2) != nullptr) { Node* other = in(1)->in(1)->in(2); if (other->Opcode() == Op_LoadRange && - ((other->in(0) != NULL && other->in(0) == proj) || - (other->in(0) == NULL && - other->in(2) != NULL && + ((other->in(0) != nullptr && other->in(0) == proj) || + (other->in(0) == nullptr && + other->in(2) != nullptr && other->in(2)->is_AddP() && - other->in(2)->in(1) != NULL && + other->in(2)->in(1) != nullptr && other->in(2)->in(1)->Opcode() == Op_CastPP && other->in(2)->in(1)->in(0) == proj))) { return true; @@ -1217,12 +1222,12 @@ bool IfNode::is_cmp_with_loadrange(ProjNode* proj) { bool IfNode::is_null_check(ProjNode* proj, PhaseIterGVN* igvn) { Node* other = in(1)->in(1)->in(2); - if (other->in(MemNode::Address) != NULL && - proj->in(0)->in(1) != NULL && + if (other->in(MemNode::Address) != nullptr && + proj->in(0)->in(1) != nullptr && proj->in(0)->in(1)->is_Bool() && - proj->in(0)->in(1)->in(1) != NULL && + proj->in(0)->in(1)->in(1) != nullptr && proj->in(0)->in(1)->in(1)->Opcode() == Op_CmpP && - proj->in(0)->in(1)->in(1)->in(2) != NULL && + proj->in(0)->in(1)->in(1)->in(2) != nullptr && proj->in(0)->in(1)->in(1)->in(1) == other->in(MemNode::Address)->in(AddPNode::Address)->uncast() && igvn->type(proj->in(0)->in(1)->in(1)->in(2)) == TypePtr::NULL_PTR) { return true; @@ -1233,17 +1238,17 @@ bool IfNode::is_null_check(ProjNode* proj, PhaseIterGVN* igvn) { // Check that the If that is in between the 2 integer comparisons has // no side effect bool IfNode::is_side_effect_free_test(ProjNode* proj, PhaseIterGVN* igvn) { - if (proj == NULL) { + if (proj == nullptr) { return false; } CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); - if (unc != NULL && proj->outcnt() <= 2) { + if (unc != nullptr && proj->outcnt() <= 2) { if (proj->outcnt() == 1 || // Allow simple null check from LoadRange (is_cmp_with_loadrange(proj) && is_null_check(proj, igvn))) { CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); CallStaticJavaNode* dom_unc = proj->in(0)->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); - assert(dom_unc != NULL, "is_uncommon_trap_if_pattern returned NULL"); + assert(dom_unc != nullptr, "is_uncommon_trap_if_pattern returned null"); // reroute_side_effect_free_unc changes the state of this // uncommon trap to restart execution at the previous @@ -1298,15 +1303,15 @@ void IfNode::reroute_side_effect_free_unc(ProjNode* proj, ProjNode* dom_proj, Ph } Node* IfNode::fold_compares(PhaseIterGVN* igvn) { - if (Opcode() != Op_If) return NULL; + if (Opcode() != Op_If) return nullptr; if (cmpi_folds(igvn)) { Node* ctrl = in(0); if (is_ctrl_folds(ctrl, igvn) && ctrl->outcnt() == 1) { // A integer comparison immediately dominated by another integer // comparison - ProjNode* success = NULL; - ProjNode* fail = NULL; + ProjNode* success = nullptr; + ProjNode* fail = nullptr; ProjNode* dom_cmp = ctrl->as_Proj(); if (has_shared_region(dom_cmp, success, fail) && // Next call modifies graph so must be last @@ -1318,11 +1323,11 @@ Node* IfNode::fold_compares(PhaseIterGVN* igvn) { fold_compares_helper(dom_cmp, success, fail, igvn)) { return merge_uncommon_traps(dom_cmp, success, fail, igvn); } - return NULL; - } else if (ctrl->in(0) != NULL && - ctrl->in(0)->in(0) != NULL) { - ProjNode* success = NULL; - ProjNode* fail = NULL; + return nullptr; + } else if (ctrl->in(0) != nullptr && + ctrl->in(0)->in(0) != nullptr) { + ProjNode* success = nullptr; + ProjNode* fail = nullptr; Node* dom = ctrl->in(0)->in(0); ProjNode* dom_cmp = dom->isa_Proj(); ProjNode* other_cmp = ctrl->isa_Proj(); @@ -1339,7 +1344,7 @@ Node* IfNode::fold_compares(PhaseIterGVN* igvn) { } } } - return NULL; + return nullptr; } //------------------------------remove_useless_bool---------------------------- @@ -1348,33 +1353,33 @@ Node* IfNode::fold_compares(PhaseIterGVN* igvn) { // Replace with if( x < y ) { ... } static Node *remove_useless_bool(IfNode *iff, PhaseGVN *phase) { Node *i1 = iff->in(1); - if( !i1->is_Bool() ) return NULL; + if( !i1->is_Bool() ) return nullptr; BoolNode *bol = i1->as_Bool(); Node *cmp = bol->in(1); - if( cmp->Opcode() != Op_CmpI ) return NULL; + if( cmp->Opcode() != Op_CmpI ) return nullptr; // Must be comparing against a bool const Type *cmp2_t = phase->type( cmp->in(2) ); if( cmp2_t != TypeInt::ZERO && cmp2_t != TypeInt::ONE ) - return NULL; + return nullptr; // Find a prior merge point merging the boolean i1 = cmp->in(1); - if( !i1->is_Phi() ) return NULL; + if( !i1->is_Phi() ) return nullptr; PhiNode *phi = i1->as_Phi(); if( phase->type( phi ) != TypeInt::BOOL ) - return NULL; + return nullptr; // Check for diamond pattern int true_path = phi->is_diamond_phi(); - if( true_path == 0 ) return NULL; + if( true_path == 0 ) return nullptr; // Make sure that iff and the control of the phi are different. This // should really only happen for dead control flow since it requires // an illegal cycle. - if (phi->in(0)->in(1)->in(0) == iff) return NULL; + if (phi->in(0)->in(1)->in(0) == iff) return nullptr; // phi->region->if_proj->ifnode->bool->cmp BoolNode *bol2 = phi->in(0)->in(1)->in(0)->in(1)->as_Bool(); @@ -1383,19 +1388,19 @@ static Node *remove_useless_bool(IfNode *iff, PhaseGVN *phase) { // either iff2->in(1) or its complement. int flip = 0; if( bol->_test._test == BoolTest::ne ) flip = 1-flip; - else if( bol->_test._test != BoolTest::eq ) return NULL; + else if( bol->_test._test != BoolTest::eq ) return nullptr; if( cmp2_t == TypeInt::ZERO ) flip = 1-flip; const Type *phi1_t = phase->type( phi->in(1) ); const Type *phi2_t = phase->type( phi->in(2) ); // Check for Phi(0,1) and flip if( phi1_t == TypeInt::ZERO ) { - if( phi2_t != TypeInt::ONE ) return NULL; + if( phi2_t != TypeInt::ONE ) return nullptr; flip = 1-flip; } else { // Check for Phi(1,0) - if( phi1_t != TypeInt::ONE ) return NULL; - if( phi2_t != TypeInt::ZERO ) return NULL; + if( phi1_t != TypeInt::ONE ) return nullptr; + if( phi2_t != TypeInt::ZERO ) return nullptr; } if( true_path == 2 ) { flip = 1-flip; @@ -1419,25 +1424,25 @@ struct RangeCheck { Node* IfNode::Ideal_common(PhaseGVN *phase, bool can_reshape) { if (remove_dead_region(phase, can_reshape)) return this; // No Def-Use info? - if (!can_reshape) return NULL; + if (!can_reshape) return nullptr; // Don't bother trying to transform a dead if - if (in(0)->is_top()) return NULL; + if (in(0)->is_top()) return nullptr; // Don't bother trying to transform an if with a dead test - if (in(1)->is_top()) return NULL; + if (in(1)->is_top()) return nullptr; // Another variation of a dead test - if (in(1)->is_Con()) return NULL; + if (in(1)->is_Con()) return nullptr; // Another variation of a dead if - if (outcnt() < 2) return NULL; + if (outcnt() < 2) return nullptr; // Canonicalize the test. Node* idt_if = idealize_test(phase, this); - if (idt_if != NULL) return idt_if; + if (idt_if != nullptr) return idt_if; // Try to split the IF PhaseIterGVN *igvn = phase->is_IterGVN(); Node *s = split_if(this, igvn); - if (s != NULL) return s; + if (s != nullptr) return s; return NodeSentinel; } @@ -1457,11 +1462,11 @@ Node* IfNode::Ideal(PhaseGVN *phase, bool can_reshape) { Node* bol2 = remove_useless_bool(this, phase); if (bol2) return bol2; - if (in(0) == NULL) return NULL; // Dead loop? + if (in(0) == nullptr) return nullptr; // Dead loop? PhaseIterGVN* igvn = phase->is_IterGVN(); Node* result = fold_compares(igvn); - if (result != NULL) { + if (result != nullptr) { return result; } @@ -1470,7 +1475,7 @@ Node* IfNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (is_If() && in(1)->is_Bool()) { Node* cmp = in(1)->in(1); if (cmp->Opcode() == Op_CmpP && - cmp->in(2) != NULL && // make sure cmp is not already dead + cmp->in(2) != nullptr && // make sure cmp is not already dead cmp->in(2)->bottom_type() == TypePtr::NULL_PTR) { dist = 64; // Limit for null-pointer scans } @@ -1478,7 +1483,7 @@ Node* IfNode::Ideal(PhaseGVN *phase, bool can_reshape) { Node* prev_dom = search_identical(dist); - if (prev_dom != NULL) { + if (prev_dom != nullptr) { // Replace dominated IfNode return dominated_by(prev_dom, igvn); } @@ -1504,8 +1509,8 @@ Node* IfNode::dominated_by(Node* prev_dom, PhaseIterGVN *igvn) { // be skipped. For example, range check predicate has two checks // for lower and upper bounds. ProjNode* unc_proj = proj_out(1 - prev_dom->as_Proj()->_con)->as_Proj(); - if (unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_predicate) != NULL || - unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_profile_predicate) != NULL) { + if (unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_predicate) != nullptr || + unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_profile_predicate) != nullptr) { prev_dom = idom; } @@ -1560,21 +1565,21 @@ Node* IfNode::search_identical(int dist) { while (dom->Opcode() != op || // Not same opcode? dom->in(1) != in(1) || // Not same input 1? prev_dom->in(0) != dom) { // One path of test does not dominate? - if (dist < 0) return NULL; + if (dist < 0) return nullptr; dist--; prev_dom = dom; dom = up_one_dom(dom); - if (!dom) return NULL; + if (!dom) return nullptr; } // Check that we did not follow a loop back to ourselves if (this == dom) { - return NULL; + return nullptr; } #ifndef PRODUCT - if (dist > 2) { // Add to count of NULL checks elided + if (dist > 2) { // Add to count of null checks elided explicit_null_checks_elided++; } #endif @@ -1618,26 +1623,26 @@ Node* IfNode::simple_subsuming(PhaseIterGVN* igvn) { Node* pre = in(0); if (!pre->is_IfTrue() && !pre->is_IfFalse()) { - return NULL; + return nullptr; } Node* dom = pre->in(0); if (!dom->is_If()) { - return NULL; + return nullptr; } Node* bol = in(1); if (!bol->is_Bool()) { - return NULL; + return nullptr; } Node* cmp = in(1)->in(1); if (!cmp->is_Cmp()) { - return NULL; + return nullptr; } if (!dom->in(1)->is_Bool()) { - return NULL; + return nullptr; } if (dom->in(1)->in(1) != cmp) { // Not same cond? - return NULL; + return nullptr; } int drel = subsuming_bool_test_encode(dom->in(1)); @@ -1645,11 +1650,11 @@ Node* IfNode::simple_subsuming(PhaseIterGVN* igvn) { int bout = pre->is_IfFalse() ? 1 : 0; if (drel < 0 || trel < 0) { - return NULL; + return nullptr; } int br = s_short_circuit_map[trel][2*drel+bout]; if (br == na) { - return NULL; + return nullptr; } #ifndef PRODUCT if (TraceIterativeGVN) { @@ -1726,7 +1731,7 @@ Node* IfProjNode::Identity(PhaseGVN* phase) { // CountedLoopEndNode may be eliminated by if subsuming, replace CountedLoopNode with LoopNode to // avoid mismatching between CountedLoopNode and CountedLoopEndNode in the following optimization. Node* head = unique_ctrl_out(); - if (head != NULL && head->is_BaseCountedLoop() && head->in(LoopNode::LoopBackControl) == this) { + if (head != nullptr && head->is_BaseCountedLoop() && head->in(LoopNode::LoopBackControl) == this) { Node* new_head = new LoopNode(head->in(LoopNode::EntryControl), this); phase->is_IterGVN()->register_new_node_with_optimizer(new_head); phase->is_IterGVN()->replace_node(head, new_head); @@ -1781,27 +1786,27 @@ void IfNode::related(GrowableArray *in_rel, GrowableArray *out // converted to 'ne', 'le' and 'lt' forms. IfTrue/IfFalse get swapped as // needed. static IfNode* idealize_test(PhaseGVN* phase, IfNode* iff) { - assert(iff->in(0) != NULL, "If must be live"); + assert(iff->in(0) != nullptr, "If must be live"); - if (iff->outcnt() != 2) return NULL; // Malformed projections. + if (iff->outcnt() != 2) return nullptr; // Malformed projections. Node* old_if_f = iff->proj_out(false); Node* old_if_t = iff->proj_out(true); // CountedLoopEnds want the back-control test to be TRUE, irregardless of // whether they are testing a 'gt' or 'lt' condition. The 'gt' condition // happens in count-down loops - if (iff->is_BaseCountedLoopEnd()) return NULL; - if (!iff->in(1)->is_Bool()) return NULL; // Happens for partially optimized IF tests + if (iff->is_BaseCountedLoopEnd()) return nullptr; + if (!iff->in(1)->is_Bool()) return nullptr; // Happens for partially optimized IF tests BoolNode *b = iff->in(1)->as_Bool(); BoolTest bt = b->_test; // Test already in good order? if( bt.is_canonical() ) - return NULL; + return nullptr; // Flip test to be canonical. Requires flipping the IfFalse/IfTrue and // cloning the IfNode. Node* new_b = phase->transform( new BoolNode(b->in(1), bt.negate()) ); - if( !new_b->is_Bool() ) return NULL; + if( !new_b->is_Bool() ) return nullptr; b = new_b->as_Bool(); PhaseIterGVN *igvn = phase->is_IterGVN(); @@ -1875,7 +1880,7 @@ Node* RangeCheckNode::Ideal(PhaseGVN *phase, bool can_reshape) { for (int dist = 0; dist < 999; dist++) { // Range-Check scan limit if (dom->Opcode() == Op_RangeCheck && // Not same opcode? prev_dom->in(0) == dom) { // One path of test does dominate? - if (dom == this) return NULL; // dead loop + if (dom == this) return nullptr; // dead loop // See if this is a range check Node* index2; Node* range2; @@ -1912,18 +1917,18 @@ Node* RangeCheckNode::Ideal(PhaseGVN *phase, bool can_reshape) { // ones. Since range checks "fail" by uncommon-trapping to the // interpreter, widening a check can make us speculatively enter // the interpreter. If we see range-check deopt's, do not widen! - if (!phase->C->allow_range_check_smearing()) return NULL; + if (!phase->C->allow_range_check_smearing()) return nullptr; // Didn't find prior covering check, so cannot remove anything. if (nb_checks == 0) { - return NULL; + return nullptr; } // Constant indices only need to check the upper bound. // Non-constant indices must check both low and high. int chk0 = (nb_checks - 1) % NRC; if (index1) { if (nb_checks == 1) { - return NULL; + return nullptr; } else { // If the top range check's constant is the min or max of // all constants we widen the next one to cover the whole @@ -1944,7 +1949,7 @@ Node* RangeCheckNode::Ideal(PhaseGVN *phase, bool can_reshape) { // accesses it protects to successfully read/write out of // bounds. if (nb_checks == 2) { - return NULL; + return nullptr; } int chk2 = (nb_checks - 3) % NRC; RangeCheck rc2 = prev_checks[chk2]; @@ -1991,8 +1996,8 @@ Node* RangeCheckNode::Ideal(PhaseGVN *phase, bool can_reshape) { } else { prev_dom = search_identical(4); - if (prev_dom == NULL) { - return NULL; + if (prev_dom == nullptr) { + return nullptr; } } diff --git a/src/hotspot/share/opto/indexSet.cpp b/src/hotspot/share/opto/indexSet.cpp index eb90151eef084..ccdad7dac416a 100644 --- a/src/hotspot/share/opto/indexSet.cpp +++ b/src/hotspot/share/opto/indexSet.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -93,7 +93,7 @@ IndexSet::BitBlock *IndexSet::alloc_block() { #endif Compile *compile = Compile::current(); BitBlock* free_list = (BitBlock*)compile->indexSet_free_block_list(); - if (free_list == NULL) { + if (free_list == nullptr) { populate_free_list(); free_list = (BitBlock*)compile->indexSet_free_block_list(); } diff --git a/src/hotspot/share/opto/indexSet.hpp b/src/hotspot/share/opto/indexSet.hpp index 7ed34116d1c86..7c1fc256cb0be 100644 --- a/src/hotspot/share/opto/indexSet.hpp +++ b/src/hotspot/share/opto/indexSet.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -174,7 +174,7 @@ class IndexSet : public ResourceObj { // from a new arena. It is essential that this method is called whenever // the Arena being used for BitBlock allocation is reset. static void reset_memory(Compile* compile, Arena *arena) { - compile->set_indexSet_free_block_list(NULL); + compile->set_indexSet_free_block_list(nullptr); compile->set_indexSet_arena(arena); // This should probably be done in a static initializer @@ -401,7 +401,7 @@ class IndexSetIterator { // If the iterator was created from a non-const set, we replace // non-canonical empty blocks with the _empty_block pointer. If - // _set is NULL, we do no replacement. + // _set is null, we do no replacement. IndexSet *_set; // Advance to the next non-empty word and return the next @@ -418,7 +418,7 @@ class IndexSetIterator { _next_word(IndexSet::words_per_block), _next_block(0), _max_blocks(set->is_empty() ? 0 : set->_current_block_limit), - _words(NULL), + _words(nullptr), _blocks(set->_blocks), _set(set) { #ifdef ASSERT @@ -435,9 +435,9 @@ class IndexSetIterator { _next_word(IndexSet::words_per_block), _next_block(0), _max_blocks(set->is_empty() ? 0 : set->_current_block_limit), - _words(NULL), + _words(nullptr), _blocks(set->_blocks), - _set(NULL) + _set(nullptr) { #ifdef ASSERT if (CollectIndexSetStatistics) { diff --git a/src/hotspot/share/opto/intrinsicnode.cpp b/src/hotspot/share/opto/intrinsicnode.cpp index 9f852dd637862..16caeaa3e2c6a 100644 --- a/src/hotspot/share/opto/intrinsicnode.cpp +++ b/src/hotspot/share/opto/intrinsicnode.cpp @@ -39,7 +39,7 @@ uint StrIntrinsicNode::match_edge(uint idx) const { Node* StrIntrinsicNode::Ideal(PhaseGVN* phase, bool can_reshape) { if (remove_dead_region(phase, can_reshape)) return this; // Don't bother trying to transform a dead node - if (in(0) && in(0)->is_top()) return NULL; + if (in(0) && in(0)->is_top()) return nullptr; if (can_reshape) { Node* mem = phase->transform(in(MemNode::Memory)); @@ -51,7 +51,7 @@ Node* StrIntrinsicNode::Ideal(PhaseGVN* phase, bool can_reshape) { return this; } } - return NULL; + return nullptr; } //------------------------------Value------------------------------------------ @@ -67,7 +67,7 @@ uint StrIntrinsicNode::size_of() const { return sizeof(*this); } // Return a node which is more "ideal" than the current node. Strip out // control copies Node* StrCompressedCopyNode::Ideal(PhaseGVN* phase, bool can_reshape) { - return remove_dead_region(phase, can_reshape) ? this : NULL; + return remove_dead_region(phase, can_reshape) ? this : nullptr; } //============================================================================= @@ -75,7 +75,7 @@ Node* StrCompressedCopyNode::Ideal(PhaseGVN* phase, bool can_reshape) { // Return a node which is more "ideal" than the current node. Strip out // control copies Node* StrInflatedCopyNode::Ideal(PhaseGVN* phase, bool can_reshape) { - return remove_dead_region(phase, can_reshape) ? this : NULL; + return remove_dead_region(phase, can_reshape) ? this : nullptr; } //============================================================================= @@ -89,7 +89,7 @@ uint EncodeISOArrayNode::match_edge(uint idx) const { // Return a node which is more "ideal" than the current node. Strip out // control copies Node* EncodeISOArrayNode::Ideal(PhaseGVN* phase, bool can_reshape) { - return remove_dead_region(phase, can_reshape) ? this : NULL; + return remove_dead_region(phase, can_reshape) ? this : nullptr; } //------------------------------Value------------------------------------------ diff --git a/src/hotspot/share/opto/lcm.cpp b/src/hotspot/share/opto/lcm.cpp index 8416c6a04de07..7b2be962f2968 100644 --- a/src/hotspot/share/opto/lcm.cpp +++ b/src/hotspot/share/opto/lcm.cpp @@ -40,13 +40,13 @@ // Optimization - Graph Style // Check whether val is not-null-decoded compressed oop, -// i.e. will grab into the base of the heap if it represents NULL. +// i.e. will grab into the base of the heap if it represents null. static bool accesses_heap_base_zone(Node *val) { - if (CompressedOops::base() != NULL) { // Implies UseCompressedOops. + if (CompressedOops::base() != nullptr) { // Implies UseCompressedOops. if (val && val->is_Mach()) { if (val->as_Mach()->ideal_Opcode() == Op_DecodeN) { // This assumes all Decodes with TypePtr::NotNull are matched to nodes that - // decode NULL to point to the heap base (Decode_NN). + // decode null to point to the heap base (Decode_NN). if (val->bottom_type()->is_oopptr()->ptr() == TypePtr::NotNull) { return true; } @@ -77,8 +77,8 @@ static bool needs_explicit_null_check_for_read(Node *val) { } //------------------------------implicit_null_check---------------------------- -// Detect implicit-null-check opportunities. Basically, find NULL checks -// with suitable memory ops nearby. Use the memory op to do the NULL check. +// Detect implicit-null-check opportunities. Basically, find null checks +// with suitable memory ops nearby. Use the memory op to do the null check. // I can generate a memory op if there is not one nearby. // The proj is the control projection for the not-null case. // The val is the pointer being checked for nullness or @@ -149,12 +149,12 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo bool is_decoden = ((intptr_t)val) & 1; val = (Node*)(((intptr_t)val) & ~1); - assert(!is_decoden || (val->in(0) == NULL) && val->is_Mach() && + assert(!is_decoden || (val->in(0) == nullptr) && val->is_Mach() && (val->as_Mach()->ideal_Opcode() == Op_DecodeN), "sanity"); // Search the successor block for a load or store who's base value is also // the tested value. There may be several. - MachNode *best = NULL; // Best found so far + MachNode *best = nullptr; // Best found so far for (DUIterator i = val->outs(); val->has_out(i); i++) { Node *m = val->out(i); if( !m->is_Mach() ) continue; @@ -222,7 +222,7 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo Node* base; Node* index; const MachOper* oper = mach->memory_inputs(base, index); - if (oper == NULL || oper == (MachOper*)-1) { + if (oper == nullptr || oper == (MachOper*)-1) { continue; // Not an memory op; skip it } if (val == base || @@ -245,7 +245,7 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo // Check that node's control edge is not-null block's head or dominates it, // otherwise we can't hoist it because there are other control dependencies. Node* ctrl = mach->in(0); - if (ctrl != NULL && !(ctrl == not_null_block->head() || + if (ctrl != nullptr && !(ctrl == not_null_block->head() || get_block_for_node(ctrl)->dominates(not_null_block))) { continue; } @@ -253,9 +253,9 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo // check if the offset is not too high for implicit exception { intptr_t offset = 0; - const TypePtr *adr_type = NULL; // Do not need this return value here + const TypePtr *adr_type = nullptr; // Do not need this return value here const Node* base = mach->get_base_and_disp(offset, adr_type); - if (base == NULL || base == NodeSentinel) { + if (base == nullptr || base == NodeSentinel) { // Narrow oop address doesn't have base, only index. // Give up if offset is beyond page size or if heap base is not protected. if (val->bottom_type()->isa_narrowoop() && @@ -350,17 +350,17 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo // Make sure this memory op is not already being used for a NullCheck Node *e = mb->end(); if( e->is_MachNullCheck() && e->in(1) == mach ) - continue; // Already being used as a NULL check + continue; // Already being used as a null check // Found a candidate! Pick one with least dom depth - the highest // in the dom tree should be closest to the null check. - if (best == NULL || get_block_for_node(mach)->_dom_depth < get_block_for_node(best)->_dom_depth) { + if (best == nullptr || get_block_for_node(mach)->_dom_depth < get_block_for_node(best)->_dom_depth) { best = mach; bidx = vidx; } } // No candidate! - if (best == NULL) { + if (best == nullptr) { return; } @@ -411,9 +411,9 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo map_node_to_block(best, block); // Move the control dependence if it is pinned to not-null block. - // Don't change it in other cases: NULL or dominating control. + // Don't change it in other cases: null or dominating control. Node* ctrl = best->in(0); - if (ctrl != NULL && get_block_for_node(ctrl) == not_null_block) { + if (ctrl != nullptr && get_block_for_node(ctrl) == not_null_block) { // Set it to control edge of null check. best->set_req(0, proj->in(0)->in(0)); } @@ -431,10 +431,10 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo // proj==Op_True --> ne test; proj==Op_False --> eq test. // One of two graph shapes got matched: - // (IfTrue (If (Bool NE (CmpP ptr NULL)))) - // (IfFalse (If (Bool EQ (CmpP ptr NULL)))) - // NULL checks are always branch-if-eq. If we see a IfTrue projection - // then we are replacing a 'ne' test with a 'eq' NULL check test. + // (IfTrue (If (Bool NE (CmpP ptr null)))) + // (IfFalse (If (Bool EQ (CmpP ptr null)))) + // null checks are always branch-if-eq. If we see a IfTrue projection + // then we are replacing a 'ne' test with a 'eq' null check test. // We need to flip the projections to keep the same semantics. if( proj->Opcode() == Op_IfTrue ) { // Swap order of projections in basic block to swap branch targets @@ -442,11 +442,11 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo Node *tmp2 = block->get_node(block->end_idx()+2); block->map_node(tmp2, block->end_idx()+1); block->map_node(tmp1, block->end_idx()+2); - Node *tmp = new Node(C->top()); // Use not NULL input + Node *tmp = new Node(C->top()); // Use not null input tmp1->replace_by(tmp); tmp2->replace_by(tmp1); tmp->replace_by(tmp2); - tmp->destruct(NULL); + tmp->destruct(nullptr); } // Remove the existing null check; use a new implicit null check instead. @@ -462,7 +462,7 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo // Clean-up any dead code for (uint i3 = 0; i3 < old_tst->req(); i3++) { Node* in = old_tst->in(i3); - old_tst->set_req(i3, NULL); + old_tst->set_req(i3, nullptr); if (in->outcnt() == 0) { // Remove dead input node in->disconnect_inputs(C); @@ -521,7 +521,7 @@ Node* PhaseCFG::select( uint score = 0; // Bigger is better int idx = -1; // Index in worklist int cand_cnt = 0; // Candidate count - bool block_size_threshold_ok = (recalc_pressure_nodes != NULL) && (block->number_of_nodes() > 10); + bool block_size_threshold_ok = (recalc_pressure_nodes != nullptr) && (block->number_of_nodes() > 10); for( uint i=0; ireq(); i++) { bool lrg_ends = false; Node *src_n = n->in(i); - if (src_n == NULL) continue; + if (src_n == nullptr) continue; if (!src_n->is_Mach()) continue; uint src = _regalloc->_lrg_map.find(src_n); if (src == 0) continue; @@ -750,9 +750,9 @@ void PhaseCFG::adjust_register_pressure(Node* n, Block* block, intptr_t* recalc_ // if none, this live range ends and we can adjust register pressure if (lrg_ends) { if (finalize_mode) { - _regalloc->lower_pressure(block, 0, lrg_src, NULL, _regalloc->_sched_int_pressure, _regalloc->_sched_float_pressure); + _regalloc->lower_pressure(block, 0, lrg_src, nullptr, _regalloc->_sched_int_pressure, _regalloc->_sched_float_pressure); } else { - _regalloc->lower_pressure(block, 0, lrg_src, NULL, _regalloc->_scratch_int_pressure, _regalloc->_scratch_float_pressure); + _regalloc->lower_pressure(block, 0, lrg_src, nullptr, _regalloc->_scratch_int_pressure, _regalloc->_scratch_float_pressure); } } } @@ -800,7 +800,7 @@ void PhaseCFG::set_next_call(Block* block, Node* n, VectorSet& next_call) { // carry lots of stuff live across a call. void PhaseCFG::needed_for_next_call(Block* block, Node* this_call, VectorSet& next_call) { // Find the next control-defining Node in this block - Node* call = NULL; + Node* call = nullptr; for (DUIterator_Fast imax, i = this_call->fast_outs(imax); i < imax; i++) { Node* m = this_call->fast_out(i); if (get_block_for_node(m) == block && // Local-block user @@ -810,7 +810,7 @@ void PhaseCFG::needed_for_next_call(Block* block, Node* this_call, VectorSet& ne break; } } - if (call == NULL) return; // No next call (e.g., block end is near) + if (call == nullptr) return; // No next call (e.g., block end is near) // Set next-call for all inputs to this call set_next_call(block, call, next_call); } @@ -881,7 +881,7 @@ uint PhaseCFG::sched_call(Block* block, uint node_cnt, Node_List& worklist, Grow block->insert_node(proj, node_cnt++); // Select the right register save policy. - const char *save_policy = NULL; + const char *save_policy = nullptr; switch (op) { case Op_CallRuntime: case Op_CallLeaf: @@ -964,7 +964,7 @@ bool PhaseCFG::schedule_local(Block* block, GrowableArray& ready_cnt, Vecto return true; } - bool block_size_threshold_ok = (recalc_pressure_nodes != NULL) && (block->number_of_nodes() > 10); + bool block_size_threshold_ok = (recalc_pressure_nodes != nullptr) && (block->number_of_nodes() > 10); // We track the uses of local definitions as input dependences so that // we know when a given instruction is avialable to be scheduled. @@ -1010,7 +1010,7 @@ bool PhaseCFG::schedule_local(Block* block, GrowableArray& ready_cnt, Vecto // Check the precedence edges for (uint prec = n->req(); prec < n->len(); prec++) { Node* oop_store = n->in(prec); - if (oop_store != NULL) { + if (oop_store != nullptr) { assert(get_block_for_node(oop_store)->_dom_depth <= block->_dom_depth, "oop_store must dominate card-mark"); } } @@ -1030,7 +1030,7 @@ bool PhaseCFG::schedule_local(Block* block, GrowableArray& ready_cnt, Vecto // and the edge will be lost. This is why this code should be // executed only when Precedent (== TypeFunc::Parms) edge is present. Node *x = n->in(TypeFunc::Parms); - if (x != NULL && get_block_for_node(x) == block && n->find_prec_edge(x) != -1) { + if (x != nullptr && get_block_for_node(x) == block && n->find_prec_edge(x) != -1) { // Old edge to node within same block will get removed, but no precedence // edge will get added because it already exists. Update ready count. int cnt = ready_cnt.at(n->_idx); @@ -1268,7 +1268,7 @@ Node* PhaseCFG::catch_cleanup_find_cloned_def(Block *use_blk, Node *def, Block * use_blk = use_blk->_idom; // Find the successor - Node *fixup = NULL; + Node *fixup = nullptr; uint j; for( j = 0; j < def_blk->_num_succs; j++ ) @@ -1293,14 +1293,14 @@ Node* PhaseCFG::catch_cleanup_find_cloned_def(Block *use_blk, Node *def, Block * for (uint k = 1; k < use_blk->num_preds(); k++) { if (phi->in(k) != inputs[k]) { // Not a match - fixup = NULL; + fixup = nullptr; break; } } } // If an existing PhiNode was not found, make a new one. - if (fixup == NULL) { + if (fixup == nullptr) { Node *new_phi = PhiNode::make(use_blk->head(), def); use_blk->insert_node(new_phi, 1); map_node_to_block(new_phi, use_blk); diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index b5970545c4eda..c52f248345afd 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -68,7 +68,7 @@ CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) { if (!m->is_loaded()) { // Do not attempt to inline unloaded methods. - return NULL; + return nullptr; } C2Compiler* compiler = (C2Compiler*)CompileBroker::compiler(CompLevel_full_optimization); @@ -80,7 +80,7 @@ CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) { // methods access VM-internal data. VM_ENTRY_MARK; methodHandle mh(THREAD, m->get_Method()); - is_available = compiler != NULL && compiler->is_intrinsic_supported(mh, is_virtual) && + is_available = compiler != nullptr && compiler->is_intrinsic_supported(mh, is_virtual) && !C->directive()->is_intrinsic_disabled(mh) && !vmIntrinsics::is_disabled_by_flags(mh); @@ -94,7 +94,7 @@ CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) { vmIntrinsics::does_virtual_dispatch(id), id); } else { - return NULL; + return nullptr; } } @@ -167,7 +167,7 @@ JVMState* LibraryIntrinsic::generate(JVMState* jvms) { C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed); C->print_inlining_update(this); - return NULL; + return nullptr; } Node* LibraryIntrinsic::generate_predicate(JVMState* jvms, int predicate) { @@ -201,7 +201,7 @@ Node* LibraryIntrinsic::generate_predicate(JVMState* jvms, int predicate) { (is_virtual() ? " virtual='1'" : ""), C->unique() - nodes); } - return slow_ctl; // Could be NULL if the check folds. + return slow_ctl; // Could be null if the check folds. } // The intrinsic bailed out @@ -226,7 +226,7 @@ Node* LibraryIntrinsic::generate_predicate(JVMState* jvms, int predicate) { } } C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed); - return NULL; + return nullptr; } bool LibraryCallKit::try_to_inline(int predicate) { @@ -466,6 +466,7 @@ bool LibraryCallKit::try_to_inline(int predicate) { case vmIntrinsics::_loadFence: case vmIntrinsics::_storeFence: + case vmIntrinsics::_storeStoreFence: case vmIntrinsics::_fullFence: return inline_unsafe_fence(intrinsic_id()); case vmIntrinsics::_onSpinWait: return inline_onspinwait(); @@ -747,20 +748,20 @@ void LibraryCallKit::set_result(RegionNode* region, PhiNode* value) { // In all cases, GraphKit::control() is updated to the fast path. // The returned value represents the control for the slow path. // The return value is never 'top'; it is either a valid control -// or NULL if it is obvious that the slow path can never be taken. -// Also, if region and the slow control are not NULL, the slow edge +// or null if it is obvious that the slow path can never be taken. +// Also, if region and the slow control are not null, the slow edge // is appended to the region. Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_prob) { if (stopped()) { // Already short circuited. - return NULL; + return nullptr; } // Build an if node and its projections. // If test is true we take the slow path, which we assume is uncommon. if (_gvn.type(test) == TypeInt::ZERO) { // The slow branch is never taken. No need to build this guard. - return NULL; + return nullptr; } IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN); @@ -768,10 +769,10 @@ Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_ Node* if_slow = _gvn.transform(new IfTrueNode(iff)); if (if_slow == top()) { // The slow branch is never taken. No need to build this guard. - return NULL; + return nullptr; } - if (region != NULL) + if (region != nullptr) region->add_req(if_slow); Node* if_fast = _gvn.transform(new IfFalseNode(iff)); @@ -790,13 +791,13 @@ inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region) inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region, Node* *pos_index) { if (stopped()) - return NULL; // already stopped + return nullptr; // already stopped if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint] - return NULL; // index is already adequately typed + return nullptr; // index is already adequately typed Node* cmp_lt = _gvn.transform(new CmpINode(index, intcon(0))); Node* bol_lt = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt)); Node* is_neg = generate_guard(bol_lt, region, PROB_MIN); - if (is_neg != NULL && pos_index != NULL) { + if (is_neg != nullptr && pos_index != nullptr) { // Emulate effect of Parse::adjust_map_after_if. Node* ccast = new CastIINode(index, TypeInt::POS); ccast->set_req(0, control()); @@ -824,10 +825,10 @@ inline Node* LibraryCallKit::generate_limit_guard(Node* offset, Node* array_length, RegionNode* region) { if (stopped()) - return NULL; // already stopped + return nullptr; // already stopped bool zero_offset = _gvn.type(offset) == TypeInt::ZERO; if (zero_offset && subseq_length->eqv_uncast(array_length)) - return NULL; // common case of whole-array copy + return nullptr; // common case of whole-array copy Node* last = subseq_length; if (!zero_offset) // last += offset last = _gvn.transform(new AddINode(last, offset)); @@ -870,7 +871,7 @@ Node* LibraryCallKit::generate_current_thread(Node* &tls_output) { Node* thread = _gvn.transform(new ThreadLocalNode()); Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset())); tls_output = thread; - Node* thread_obj_handle = LoadNode::make(_gvn, NULL, immutable_memory(), p, p->bottom_type()->is_ptr(), TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered); + Node* thread_obj_handle = LoadNode::make(_gvn, nullptr, immutable_memory(), p, p->bottom_type()->is_ptr(), TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered); thread_obj_handle = _gvn.transform(thread_obj_handle); return access_load(thread_obj_handle, thread_type, T_OBJECT, IN_NATIVE | C2_IMMUTABLE_MEMORY); } @@ -882,7 +883,7 @@ Node* LibraryCallKit::generate_current_thread(Node* &tls_output) { // characters (depending on 'is_byte'). cnt1 and cnt2 are pointing to Int nodes // containing the lengths of str1 and str2. Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2, StrIntrinsicNode::ArgEnc ae) { - Node* result = NULL; + Node* result = nullptr; switch (opcode) { case Op_StrIndexOf: result = new StrIndexOfNode(control(), memory(TypeAryPtr::BYTES), @@ -900,7 +901,7 @@ Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1_start, Node break; default: ShouldNotReachHere(); - return NULL; + return nullptr; } // All these intrinsics have checks. @@ -956,8 +957,8 @@ bool LibraryCallKit::inline_string_equals(StrIntrinsicNode::ArgEnc ae) { // Check for arg1_cnt != arg2_cnt Node* cmp = _gvn.transform(new CmpINode(arg1_cnt, arg2_cnt)); Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::ne)); - Node* if_ne = generate_slow_guard(bol, NULL); - if (if_ne != NULL) { + Node* if_ne = generate_slow_guard(bol, nullptr); + if (if_ne != nullptr) { phi->init_req(2, intcon(0)); region->init_req(2, if_ne); } @@ -1109,7 +1110,7 @@ bool LibraryCallKit::inline_string_indexOf(StrIntrinsicNode::ArgEnc ae) { } Node* result = make_indexOf_node(src_start, src_count, tgt_start, tgt_count, result_rgn, result_phi, ae); - if (result != NULL) { + if (result != nullptr) { result_phi->init_req(3, result); result_rgn->init_req(3, control()); } @@ -1155,14 +1156,14 @@ bool LibraryCallKit::inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae) { Node* phi = new PhiNode(region, TypeInt::INT); Node* result = make_indexOf_node(src_start, src_count, tgt_start, tgt_count, region, phi, ae); - if (result != NULL) { + if (result != nullptr) { // The result is index relative to from_index if substring was found, -1 otherwise. // Generate code which will fold into cmove. Node* cmp = _gvn.transform(new CmpINode(result, intcon(0))); Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::lt)); - Node* if_lt = generate_slow_guard(bol, NULL); - if (if_lt != NULL) { + Node* if_lt = generate_slow_guard(bol, nullptr); + if (if_lt != nullptr) { // result == -1 phi->init_req(3, result); region->init_req(3, if_lt); @@ -1188,8 +1189,8 @@ Node* LibraryCallKit::make_indexOf_node(Node* src_start, Node* src_count, Node* // Check for substr count > string count Node* cmp = _gvn.transform(new CmpINode(tgt_count, src_count)); Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::gt)); - Node* if_gt = generate_slow_guard(bol, NULL); - if (if_gt != NULL) { + Node* if_gt = generate_slow_guard(bol, nullptr); + if (if_gt != nullptr) { phi->init_req(1, intcon(-1)); region->init_req(1, if_gt); } @@ -1197,8 +1198,8 @@ Node* LibraryCallKit::make_indexOf_node(Node* src_start, Node* src_count, Node* // Check for substr count == 0 cmp = _gvn.transform(new CmpINode(tgt_count, intcon(0))); bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq)); - Node* if_zero = generate_slow_guard(bol, NULL); - if (if_zero != NULL) { + Node* if_zero = generate_slow_guard(bol, nullptr); + if (if_zero != nullptr) { phi->init_req(2, intcon(0)); region->init_req(2, if_zero); } @@ -1206,7 +1207,7 @@ Node* LibraryCallKit::make_indexOf_node(Node* src_start, Node* src_count, Node* if (!stopped()) { return make_string_method_node(Op_StrIndexOf, src_start, src_count, tgt_start, tgt_count, ae); } - return NULL; + return nullptr; } //-----------------------------inline_string_indexOfChar----------------------- @@ -1219,7 +1220,7 @@ bool LibraryCallKit::inline_string_indexOfChar(StrIntrinsicNode::ArgEnc ae) { } assert(callee()->signature()->size() == 4, "String.indexOfChar() has 4 arguments"); Node* src = argument(0); // byte[] - Node* tgt = argument(1); // tgt is int ch + Node* int_ch = argument(1); Node* from_index = argument(2); Node* max = argument(3); @@ -1231,6 +1232,15 @@ bool LibraryCallKit::inline_string_indexOfChar(StrIntrinsicNode::ArgEnc ae) { // Range checks generate_string_range_check(src, src_offset, src_count, ae == StrIntrinsicNode::U); + + // Check for int_ch >= 0 + Node* int_ch_cmp = _gvn.transform(new CmpINode(int_ch, intcon(0))); + Node* int_ch_bol = _gvn.transform(new BoolNode(int_ch_cmp, BoolTest::ge)); + { + BuildCutout unless(this, int_ch_bol, PROB_MAX); + uncommon_trap(Deoptimization::Reason_intrinsic, + Deoptimization::Action_maybe_recompile); + } if (stopped()) { return true; } @@ -1238,15 +1248,15 @@ bool LibraryCallKit::inline_string_indexOfChar(StrIntrinsicNode::ArgEnc ae) { RegionNode* region = new RegionNode(3); Node* phi = new PhiNode(region, TypeInt::INT); - Node* result = new StrIndexOfCharNode(control(), memory(TypeAryPtr::BYTES), src_start, src_count, tgt, ae); + Node* result = new StrIndexOfCharNode(control(), memory(TypeAryPtr::BYTES), src_start, src_count, int_ch, ae); C->set_has_split_ifs(true); // Has chance for split-if optimization _gvn.transform(result); Node* cmp = _gvn.transform(new CmpINode(result, intcon(0))); Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::lt)); - Node* if_lt = generate_slow_guard(bol, NULL); - if (if_lt != NULL) { + Node* if_lt = generate_slow_guard(bol, nullptr); + if (if_lt != nullptr) { // result == -1 phi->init_req(2, result); region->init_req(2, if_lt); @@ -1287,10 +1297,13 @@ bool LibraryCallKit::inline_string_copy(bool compress) { AllocateArrayNode* alloc = tightly_coupled_allocation(dst); // Figure out the size and type of the elements we will be copying. - const Type* src_type = src->Value(&_gvn); - const Type* dst_type = dst->Value(&_gvn); - BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); - BasicType dst_elem = dst_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); + const TypeAryPtr* src_type = src->Value(&_gvn)->isa_aryptr(); + const TypeAryPtr* dst_type = dst->Value(&_gvn)->isa_aryptr(); + if (src_type == nullptr || dst_type == nullptr) { + return false; + } + BasicType src_elem = src_type->klass()->as_array_klass()->element_type()->basic_type(); + BasicType dst_elem = dst_type->klass()->as_array_klass()->element_type()->basic_type(); assert((compress && dst_elem == T_BYTE && (src_elem == T_BYTE || src_elem == T_CHAR)) || (!compress && src_elem == T_BYTE && (dst_elem == T_BYTE || dst_elem == T_CHAR)), "Unsupported array types for inline_string_copy"); @@ -1318,14 +1331,14 @@ bool LibraryCallKit::inline_string_copy(bool compress) { Node* dst_start = array_element_address(dst, dst_offset, dst_elem); // 'src_start' points to src array + scaled offset // 'dst_start' points to dst array + scaled offset - Node* count = NULL; + Node* count = nullptr; if (compress) { count = compress_string(src_start, TypeAryPtr::get_array_body_type(src_elem), dst_start, length); } else { inflate_string(src_start, dst_start, TypeAryPtr::get_array_body_type(dst_elem), length); } - if (alloc != NULL) { + if (alloc != nullptr) { if (alloc->maybe_set_complete(&_gvn)) { // "You break it, you buy it." InitializeNode* init = alloc->initialization(); @@ -1368,7 +1381,7 @@ bool LibraryCallKit::inline_string_toBytesU() { Node* offset = argument(1); Node* length = argument(2); - Node* newcopy = NULL; + Node* newcopy = nullptr; // Set the original stack and the reexecute bit for the interpreter to reexecute // the bytecode that invokes StringUTF16.toBytes() if deoptimization happens. @@ -1402,7 +1415,7 @@ bool LibraryCallKit::inline_string_toBytesU() { Node* klass_node = makecon(TypeKlassPtr::make(ciTypeArrayKlass::make(T_BYTE))); newcopy = new_array(klass_node, size, 0); // no arguments to push AllocateArrayNode* alloc = tightly_coupled_allocation(newcopy); - guarantee(alloc != NULL, "created above"); + guarantee(alloc != nullptr, "created above"); // Calculate starting addresses. Node* src_start = array_element_address(value, offset, T_CHAR); @@ -1502,7 +1515,7 @@ bool LibraryCallKit::inline_string_getCharsU() { copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM, src_start, dst_start, ConvI2X(length) XTOP); // Do not let reads from the cloned object float above the arraycopy. - if (alloc != NULL) { + if (alloc != nullptr) { if (alloc->maybe_set_complete(&_gvn)) { // "You break it, you buy it." InitializeNode* init = alloc->initialization(); @@ -1535,7 +1548,7 @@ bool LibraryCallKit::inline_string_getCharsU() { bool LibraryCallKit::inline_string_char_access(bool is_store) { Node* value = argument(0); Node* index = argument(1); - Node* ch = is_store ? argument(2) : NULL; + Node* ch = is_store ? argument(2) : nullptr; // This intrinsic accesses byte[] array as char[] array. Computing the offsets // correctly requires matched array shapes. @@ -1563,7 +1576,7 @@ bool LibraryCallKit::inline_string_char_access(bool is_store) { set_sp(old_sp); return false; } - old_map->destruct(&_gvn); + destruct_map_clone(old_map); if (is_store) { access_store_at(value, adr, TypeAryPtr::BYTES, ch, TypeInt::CHAR, T_CHAR, IN_HEAP | MO_UNORDERED | C2_MISMATCHED); } else { @@ -1579,7 +1592,7 @@ Node* LibraryCallKit::round_double_node(Node* n) { if (Matcher::strict_fp_requires_explicit_rounding) { #ifdef IA32 if (UseSSE < 2) { - n = _gvn.transform(new RoundDoubleNode(NULL, n)); + n = _gvn.transform(new RoundDoubleNode(nullptr, n)); } #else Unimplemented(); @@ -1595,7 +1608,7 @@ Node* LibraryCallKit::round_double_node(Node* n) { // public static double Math.log10(double) bool LibraryCallKit::inline_double_math(vmIntrinsics::ID id) { Node* arg = round_double_node(argument(0)); - Node* n = NULL; + Node* n = nullptr; switch (id) { case vmIntrinsics::_dabs: n = new AbsDNode( arg); break; case vmIntrinsics::_dsqrt: n = new SqrtDNode(C, control(), arg); break; @@ -1616,7 +1629,7 @@ bool LibraryCallKit::inline_double_math(vmIntrinsics::ID id) { // public static long Math.abs(long) bool LibraryCallKit::inline_math(vmIntrinsics::ID id) { Node* arg = argument(0); - Node* n = NULL; + Node* n = nullptr; switch (id) { case vmIntrinsics::_fabs: n = new AbsFNode( arg); break; case vmIntrinsics::_iabs: n = new AbsINode( arg); break; @@ -1636,12 +1649,12 @@ bool LibraryCallKit::runtime_math(const TypeFunc* call_type, address funcAddr, c // Inputs Node* a = round_double_node(argument(0)); - Node* b = (call_type == OptoRuntime::Math_DD_D_Type()) ? round_double_node(argument(2)) : NULL; + Node* b = (call_type == OptoRuntime::Math_DD_D_Type()) ? round_double_node(argument(2)) : nullptr; - const TypePtr* no_memory_effects = NULL; + const TypePtr* no_memory_effects = nullptr; Node* trig = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName, no_memory_effects, - a, top(), b, b ? top() : NULL); + a, top(), b, b ? top() : nullptr); Node* value = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+0)); #ifdef ASSERT Node* value_top = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+1)); @@ -1656,7 +1669,7 @@ bool LibraryCallKit::runtime_math(const TypeFunc* call_type, address funcAddr, c bool LibraryCallKit::inline_math_pow() { Node* exp = round_double_node(argument(2)); const TypeD* d = _gvn.type(exp)->isa_double_constant(); - if (d != NULL) { + if (d != nullptr) { if (d->getd() == 2.0) { // Special case: pow(x, 2.0) => x * x Node* base = round_double_node(argument(0)); @@ -1676,16 +1689,16 @@ bool LibraryCallKit::inline_math_pow() { // -0.0/+0.0 are both excluded since floating-point comparison doesn't distinguish -0.0 from +0.0. Node* test = _gvn.transform(new BoolNode(cmp, BoolTest::le)); - Node* if_pow = generate_slow_guard(test, NULL); + Node* if_pow = generate_slow_guard(test, nullptr); Node* value_sqrt = _gvn.transform(new SqrtDNode(C, control(), base)); phi->init_req(1, value_sqrt); region->init_req(1, control()); - if (if_pow != NULL) { + if (if_pow != nullptr) { set_control(if_pow); - address target = StubRoutines::dpow() != NULL ? StubRoutines::dpow() : + address target = StubRoutines::dpow() != nullptr ? StubRoutines::dpow() : CAST_FROM_FN_PTR(address, SharedRuntime::dpow); - const TypePtr* no_memory_effects = NULL; + const TypePtr* no_memory_effects = nullptr; Node* trig = make_runtime_call(RC_LEAF, OptoRuntime::Math_DD_D_Type(), target, "POW", no_memory_effects, base, top(), exp, top()); Node* value_pow = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+0)); @@ -1706,7 +1719,7 @@ bool LibraryCallKit::inline_math_pow() { } } - return StubRoutines::dpow() != NULL ? + return StubRoutines::dpow() != nullptr ? runtime_math(OptoRuntime::Math_DD_D_Type(), StubRoutines::dpow(), "dpow") : runtime_math(OptoRuntime::Math_DD_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dpow), "POW"); } @@ -1717,23 +1730,23 @@ bool LibraryCallKit::inline_math_native(vmIntrinsics::ID id) { switch (id) { // These intrinsics are not properly supported on all hardware case vmIntrinsics::_dsin: - return StubRoutines::dsin() != NULL ? + return StubRoutines::dsin() != nullptr ? runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dsin(), "dsin") : runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dsin), "SIN"); case vmIntrinsics::_dcos: - return StubRoutines::dcos() != NULL ? + return StubRoutines::dcos() != nullptr ? runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dcos(), "dcos") : runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dcos), "COS"); case vmIntrinsics::_dtan: - return StubRoutines::dtan() != NULL ? + return StubRoutines::dtan() != nullptr ? runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dtan(), "dtan") : runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dtan), "TAN"); case vmIntrinsics::_dlog: - return StubRoutines::dlog() != NULL ? + return StubRoutines::dlog() != nullptr ? runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dlog(), "dlog") : runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog), "LOG"); case vmIntrinsics::_dlog10: - return StubRoutines::dlog10() != NULL ? + return StubRoutines::dlog10() != nullptr ? runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dlog10(), "dlog10") : runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog10), "LOG10"); @@ -1748,7 +1761,7 @@ bool LibraryCallKit::inline_math_native(vmIntrinsics::ID id) { case vmIntrinsics::_labs: return Matcher::match_rule_supported(Op_AbsL) ? inline_math(id) : false; case vmIntrinsics::_dexp: - return StubRoutines::dexp() != NULL ? + return StubRoutines::dexp() != nullptr ? runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dexp(), "dexp") : runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dexp), "EXP"); #undef FN_PTR @@ -1786,7 +1799,7 @@ bool LibraryCallKit::inline_notify(vmIntrinsics::ID id) { } else { func = OptoRuntime::monitor_notifyAll_Java(); } - Node* call = make_runtime_call(RC_NO_LEAF, ftype, func, NULL, TypeRawPtr::BOTTOM, argument(0)); + Node* call = make_runtime_call(RC_NO_LEAF, ftype, func, nullptr, TypeRawPtr::BOTTOM, argument(0)); make_slow_call_ex(call, env()->Throwable_klass(), false); return true; } @@ -2048,12 +2061,12 @@ LibraryCallKit::generate_min_max(vmIntrinsics::ID id, Node* x0, Node* y0) { inline int LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset, BasicType type) { const TypePtr* base_type = TypePtr::NULL_PTR; - if (base != NULL) base_type = _gvn.type(base)->isa_ptr(); - if (base_type == NULL) { + if (base != nullptr) base_type = _gvn.type(base)->isa_ptr(); + if (base_type == nullptr) { // Unknown type. return Type::AnyPtr; } else if (base_type == TypePtr::NULL_PTR) { - // Since this is a NULL+long form, we have to switch to a rawptr. + // Since this is a null+long form, we have to switch to a rawptr. base = _gvn.transform(new CastX2PNode(offset)); offset = MakeConX(0); return Type::RawPtr; @@ -2066,7 +2079,7 @@ LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset, BasicType type) } // Offset is small => always a heap address. const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t(); - if (offset_type != NULL && + if (offset_type != nullptr && base_type->offset() == 0 && // (should always be?) offset_type->_lo >= 0 && !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) { @@ -2076,7 +2089,7 @@ LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset, BasicType type) // heap. return Type::OopPtr; } - // Otherwise, it might either be oop+off or NULL+addr. + // Otherwise, it might either be oop+off or null+addr. return Type::AnyPtr; } else { // No information: @@ -2141,7 +2154,7 @@ Node* LibraryCallKit::make_unsafe_address(Node*& base, Node* offset, BasicType t // inline long Long.reverseBytes(long) bool LibraryCallKit::inline_number_methods(vmIntrinsics::ID id) { Node* arg = argument(0); - Node* n = NULL; + Node* n = nullptr; switch (id) { case vmIntrinsics::_numberOfLeadingZeros_i: n = new CountLeadingZerosINode( arg); break; case vmIntrinsics::_numberOfLeadingZeros_l: n = new CountLeadingZerosLNode( arg); break; @@ -2163,10 +2176,10 @@ bool LibraryCallKit::inline_number_methods(vmIntrinsics::ID id) { const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type) { // Attempt to infer a sharper value type from the offset and base type. - ciKlass* sharpened_klass = NULL; + ciKlass* sharpened_klass = nullptr; // See if it is an instance field, with an object type. - if (alias_type->field() != NULL) { + if (alias_type->field() != nullptr) { if (alias_type->field()->type()->is_klass()) { sharpened_klass = alias_type->field()->type()->as_klass(); } @@ -2176,7 +2189,7 @@ const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_ if (adr_type->isa_aryptr()) { if (adr_type->offset() >= objArrayOopDesc::base_offset_in_bytes()) { const TypeOopPtr *elem_type = adr_type->is_aryptr()->elem()->isa_oopptr(); - if (elem_type != NULL) { + if (elem_type != nullptr) { sharpened_klass = elem_type->klass(); } } @@ -2184,7 +2197,7 @@ const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_ // The sharpened class might be unloaded if there is no class loader // contraint in place. - if (sharpened_klass != NULL && sharpened_klass->is_loaded()) { + if (sharpened_klass != nullptr && sharpened_klass->is_loaded()) { const TypeOopPtr* tjp = TypeOopPtr::make_from_klass(sharpened_klass); #ifndef PRODUCT @@ -2196,7 +2209,7 @@ const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_ // Sharpen the value type. return tjp; } - return NULL; + return nullptr; } DecoratorSet LibraryCallKit::mo_decorator_for_access_kind(AccessKind kind) { @@ -2295,14 +2308,14 @@ bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, c heap_base_oop = base; // on-heap or mixed access } - // Can base be NULL? Otherwise, always on-heap access. + // Can base be null? Otherwise, always on-heap access. bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(_gvn.type(base)); if (!can_access_non_heap) { decorators |= IN_HEAP; } - Node* val = is_store ? argument(4) : NULL; + Node* val = is_store ? argument(4) : nullptr; const TypePtr* adr_type = _gvn.type(adr)->isa_ptr(); if (adr_type == TypePtr::NULL_PTR) { @@ -2346,7 +2359,7 @@ bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, c mismatched = true; // conservatively mark all "wide" on-heap accesses as mismatched } - old_map->destruct(&_gvn); + destruct_map_clone(old_map); assert(!mismatched || alias_type->adr_type()->is_oopptr(), "off-heap access can't be mismatched"); if (mismatched) { @@ -2361,7 +2374,7 @@ bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, c if (!is_store && type == T_OBJECT) { const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type); - if (tjp != NULL) { + if (tjp != nullptr) { value_type = tjp; } } @@ -2376,21 +2389,21 @@ bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, c // from intended ones in this API. if (!is_store) { - Node* p = NULL; + Node* p = nullptr; // Try to constant fold a load from a constant field ciField* field = alias_type->field(); - if (heap_base_oop != top() && field != NULL && field->is_constant() && !mismatched) { + if (heap_base_oop != top() && field != nullptr && field->is_constant() && !mismatched) { // final or stable field p = make_constant_from_field(field, heap_base_oop); } - if (p == NULL) { // Could not constant fold the load + if (p == nullptr) { // Could not constant fold the load p = access_load_at(heap_base_oop, adr, adr_type, value_type, type, decorators); // Normalize the value returned by getBoolean in the following cases if (type == T_BOOLEAN && (mismatched || - heap_base_oop == top() || // - heap_base_oop is NULL or - (can_access_non_heap && field == NULL)) // - heap_base_oop is potentially NULL + heap_base_oop == top() || // - heap_base_oop is null or + (can_access_non_heap && field == nullptr)) // - heap_base_oop is potentially null // and the unsafe access is made to large offset // (i.e., larger than the maximum offset necessary for any // field access) @@ -2409,7 +2422,7 @@ bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, c } } if (type == T_ADDRESS) { - p = gvn().transform(new CastP2XNode(NULL, p)); + p = gvn().transform(new CastP2XNode(nullptr, p)); p = ConvX2UL(p); } // The load node has the control of the preceding MemBarCPUOrder. All @@ -2544,11 +2557,11 @@ bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadSt C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". // Get arguments: - Node* receiver = NULL; - Node* base = NULL; - Node* offset = NULL; - Node* oldval = NULL; - Node* newval = NULL; + Node* receiver = nullptr; + Node* base = nullptr; + Node* offset = nullptr; + Node* oldval = nullptr; + Node* newval = nullptr; switch(kind) { case LS_cmp_swap: case LS_cmp_swap_weak: @@ -2566,7 +2579,7 @@ bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadSt receiver = argument(0); // type: oop base = argument(1); // type: oop offset = argument(2); // type: long - oldval = NULL; + oldval = nullptr; newval = argument(4); // type: oop, int, or long break; } @@ -2597,7 +2610,7 @@ bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadSt return false; } - old_map->destruct(&_gvn); + destruct_map_clone(old_map); // For CAS, unlike inline_unsafe_access, there seems no point in // trying to refine types. Just use the coarse types here. @@ -2609,7 +2622,7 @@ bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadSt case LS_cmp_exchange: { if (type == T_OBJECT) { const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type); - if (tjp != NULL) { + if (tjp != nullptr) { value_type = tjp; } } @@ -2634,19 +2647,19 @@ bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadSt if (is_reference_type(type)) { decorators |= IN_HEAP | ON_UNKNOWN_OOP_REF; - // Transformation of a value which could be NULL pointer (CastPP #NULL) + // Transformation of a value which could be null pointer (CastPP #null) // could be delayed during Parse (for example, in adjust_map_after_if()). // Execute transformation here to avoid barrier generation in such case. if (_gvn.type(newval) == TypePtr::NULL_PTR) newval = _gvn.makecon(TypePtr::NULL_PTR); - if (oldval != NULL && _gvn.type(oldval) == TypePtr::NULL_PTR) { + if (oldval != nullptr && _gvn.type(oldval) == TypePtr::NULL_PTR) { // Refine the value to a null constant, when it is known to be null oldval = _gvn.makecon(TypePtr::NULL_PTR); } } - Node* result = NULL; + Node* result = nullptr; switch (kind) { case LS_cmp_exchange: { result = access_atomic_cmpxchg_val_at(base, adr, adr_type, alias_idx, @@ -2690,6 +2703,9 @@ bool LibraryCallKit::inline_unsafe_fence(vmIntrinsics::ID id) { case vmIntrinsics::_storeFence: insert_mem_bar(Op_StoreFence); return true; + case vmIntrinsics::_storeStoreFence: + insert_mem_bar(Op_StoreStoreFence); + return true; case vmIntrinsics::_fullFence: insert_mem_bar(Op_MemBarVolatile); return true; @@ -2709,7 +2725,7 @@ bool LibraryCallKit::klass_needs_init_guard(Node* kls) { return true; } const TypeKlassPtr* klsptr = kls->bottom_type()->isa_klassptr(); - if (klsptr == NULL) { + if (klsptr == nullptr) { return true; } ciInstanceKlass* ik = klsptr->klass()->as_instance_klass(); @@ -2775,11 +2791,11 @@ bool LibraryCallKit::inline_unsafe_allocate() { Node* cls = null_check(argument(1)); if (stopped()) return true; - Node* kls = load_klass_from_mirror(cls, false, NULL, 0); + Node* kls = load_klass_from_mirror(cls, false, nullptr, 0); kls = null_check(kls); if (stopped()) return true; // argument was like int.class - Node* test = NULL; + Node* test = nullptr; if (LibraryCallKit::klass_needs_init_guard(kls)) { // Note: The argument might still be an illegal value like // Serializable.class or Object[].class. The runtime will handle it. @@ -2787,7 +2803,7 @@ bool LibraryCallKit::inline_unsafe_allocate() { Node* insp = basic_plus_adr(kls, in_bytes(InstanceKlass::init_state_offset())); // Use T_BOOLEAN for InstanceKlass::_init_state so the compiler // can generate code to load it as unsigned byte. - Node* inst = make_load(NULL, insp, TypeInt::UBYTE, T_BOOLEAN, MemNode::unordered); + Node* inst = make_load(nullptr, insp, TypeInt::UBYTE, T_BOOLEAN, MemNode::unordered); Node* bits = intcon(InstanceKlass::fully_initialized); test = _gvn.transform(new SubINode(inst, bits)); // The 'test' is non-zero if we need to take a slow path. @@ -2803,7 +2819,7 @@ bool LibraryCallKit::inline_unsafe_allocate() { // these have the same type and signature bool LibraryCallKit::inline_native_time_funcs(address funcAddr, const char* funcName) { const TypeFunc* tf = OptoRuntime::void_long_Type(); - const TypePtr* no_memory_effects = NULL; + const TypePtr* no_memory_effects = nullptr; Node* time = make_runtime_call(RC_LEAF, tf, funcAddr, funcName, no_memory_effects); Node* value = _gvn.transform(new ProjNode(time, TypeFunc::Parms+0)); #ifdef ASSERT @@ -2839,7 +2855,7 @@ bool LibraryCallKit::inline_native_classID() { IdealKit ideal(this); #define __ ideal. IdealVariable result(ideal); __ declarations_done(); - Node* kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), + Node* kls = _gvn.transform(LoadKlassNode::make(_gvn, nullptr, immutable_memory(), basic_plus_adr(cls, java_lang_Class::klass_offset()), TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL)); @@ -2869,7 +2885,7 @@ bool LibraryCallKit::inline_native_classID() { ideal.set(result, _gvn.transform(new URShiftLNode(kls_trace_id_raw, ideal.ConI(TRACE_ID_SHIFT)))); } __ else_(); { - Node* array_kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), + Node* array_kls = _gvn.transform(LoadKlassNode::make(_gvn, nullptr, immutable_memory(), basic_plus_adr(cls, java_lang_Class::array_klass_offset()), TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL)); __ if_then(array_kls, BoolTest::ne, null()); { @@ -2936,7 +2952,7 @@ bool LibraryCallKit::inline_native_getEventWriter() { //------------------------inline_native_currentThread------------------ bool LibraryCallKit::inline_native_currentThread() { - Node* junk = NULL; + Node* junk = nullptr; set_result(generate_current_thread(junk)); return true; } @@ -2945,7 +2961,7 @@ bool LibraryCallKit::inline_native_currentThread() { // Given a klass oop, load its java mirror (a java.lang.Class oop). Node* LibraryCallKit::load_mirror_from_klass(Node* klass) { Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset())); - Node* load = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered); + Node* load = make_load(nullptr, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered); // mirror = ((OopHandle)mirror)->resolve(); return access_load(load, TypeInstPtr::MIRROR, T_OBJECT, IN_NATIVE); } @@ -2956,19 +2972,19 @@ Node* LibraryCallKit::load_mirror_from_klass(Node* klass) { // and branch to the given path on the region. // If never_see_null, take an uncommon trap on null, so we can optimistically // compile for the non-null case. -// If the region is NULL, force never_see_null = true. +// If the region is null, force never_see_null = true. Node* LibraryCallKit::load_klass_from_mirror_common(Node* mirror, bool never_see_null, RegionNode* region, int null_path, int offset) { - if (region == NULL) never_see_null = true; + if (region == nullptr) never_see_null = true; Node* p = basic_plus_adr(mirror, offset); const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL; - Node* kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type)); + Node* kls = _gvn.transform(LoadKlassNode::make(_gvn, nullptr, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type)); Node* null_ctl = top(); kls = null_check_oop(kls, &null_ctl, never_see_null); - if (region != NULL) { + if (region != nullptr) { // Set region->in(null_path) if the mirror is a primitive (e.g, int.class). region->init_req(null_path, null_ctl); } else { @@ -2984,7 +3000,7 @@ Node* LibraryCallKit::generate_access_flags_guard(Node* kls, int modifier_mask, // Branch around if the given klass has the given modifier bit set. // Like generate_guard, adds a new path onto the region. Node* modp = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset())); - Node* mods = make_load(NULL, modp, TypeInt::INT, T_INT, MemNode::unordered); + Node* mods = make_load(nullptr, modp, TypeInt::INT, T_INT, MemNode::unordered); Node* mask = intcon(modifier_mask); Node* bits = intcon(modifier_bits); Node* mbit = _gvn.transform(new AndINode(mods, mask)); @@ -3050,7 +3066,7 @@ bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) { } const TypeInstPtr* mirror_con = _gvn.type(mirror)->isa_instptr(); - if (mirror_con == NULL) return false; // cannot happen? + if (mirror_con == nullptr) return false; // cannot happen? #ifndef PRODUCT if (C->print_intrinsics() || C->print_inlining()) { @@ -3103,12 +3119,12 @@ bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) { case vmIntrinsics::_getModifiers: p = basic_plus_adr(kls, in_bytes(Klass::modifier_flags_offset())); - query_value = make_load(NULL, p, TypeInt::INT, T_INT, MemNode::unordered); + query_value = make_load(nullptr, p, TypeInt::INT, T_INT, MemNode::unordered); break; case vmIntrinsics::_isInterface: // (To verify this code sequence, check the asserts in JVM_IsInterface.) - if (generate_interface_guard(kls, region) != NULL) + if (generate_interface_guard(kls, region) != nullptr) // A guard was added. If the guard is taken, it was an interface. phi->add_req(intcon(1)); // If we fall through, it's a plain class. @@ -3117,7 +3133,7 @@ bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) { case vmIntrinsics::_isArray: // (To verify this code sequence, check the asserts in JVM_IsArrayClass.) - if (generate_array_guard(kls, region) != NULL) + if (generate_array_guard(kls, region) != nullptr) // A guard was added. If the guard is taken, it was an array. phi->add_req(intcon(1)); // If we fall through, it's a plain class. @@ -3130,7 +3146,7 @@ bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) { case vmIntrinsics::_isHidden: // (To verify this code sequence, check the asserts in JVM_IsHiddenClass.) - if (generate_hidden_class_guard(kls, region) != NULL) + if (generate_hidden_class_guard(kls, region) != nullptr) // A guard was added. If the guard is taken, it was an hidden class. phi->add_req(intcon(1)); // If we fall through, it's a plain class. @@ -3145,15 +3161,15 @@ bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) { // Arrays store an intermediate super as _super, but must report Object. // Other types can report the actual _super. // (To verify this code sequence, check the asserts in JVM_IsInterface.) - if (generate_interface_guard(kls, region) != NULL) + if (generate_interface_guard(kls, region) != nullptr) // A guard was added. If the guard is taken, it was an interface. phi->add_req(null()); - if (generate_array_guard(kls, region) != NULL) + if (generate_array_guard(kls, region) != nullptr) // A guard was added. If the guard is taken, it was an array. phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror()))); // If we fall through, it's a plain class. Get its _super. p = basic_plus_adr(kls, in_bytes(Klass::super_offset())); - kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL)); + kls = _gvn.transform(LoadKlassNode::make(_gvn, nullptr, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL)); null_ctl = top(); kls = null_check_oop(kls, &null_ctl); if (null_ctl != top()) { @@ -3168,7 +3184,7 @@ bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) { case vmIntrinsics::_getClassAccessFlags: p = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset())); - query_value = make_load(NULL, p, TypeInt::INT, T_INT, MemNode::unordered); + query_value = make_load(nullptr, p, TypeInt::INT, T_INT, MemNode::unordered); break; default: @@ -3190,10 +3206,10 @@ bool LibraryCallKit::inline_Class_cast() { Node* mirror = argument(0); // Class Node* obj = argument(1); const TypeInstPtr* mirror_con = _gvn.type(mirror)->isa_instptr(); - if (mirror_con == NULL) { + if (mirror_con == nullptr) { return false; // dead path (mirror->is_top()). } - if (obj == NULL || obj->is_top()) { + if (obj == nullptr || obj->is_top()) { return false; // dead path } const TypeOopPtr* tp = _gvn.type(obj)->isa_oopptr(); @@ -3201,7 +3217,7 @@ bool LibraryCallKit::inline_Class_cast() { // First, see if Class.cast() can be folded statically. // java_mirror_type() returns non-null for compile-time Class constants. ciType* tm = mirror_con->java_mirror_type(); - if (tm != NULL && tm->is_klass() && + if (tm != nullptr && tm->is_klass() && tp != NULL && tp->klass() != NULL) { if (!tp->klass()->is_loaded()) { // Don't use intrinsic when class is not loaded. @@ -3236,7 +3252,7 @@ bool LibraryCallKit::inline_Class_cast() { return true; } - // Not-subtype or the mirror's klass ptr is NULL (in case it is a primitive). + // Not-subtype or the mirror's klass ptr is null (in case it is a primitive). enum { _bad_type_path = 1, _prim_path = 2, PATH_LIMIT }; RegionNode* region = new RegionNode(PATH_LIMIT); record_for_igvn(region); @@ -3307,7 +3323,7 @@ bool LibraryCallKit::inline_native_subtype_check() { args[which_arg] = arg; Node* p = basic_plus_adr(arg, class_klass_offset); - Node* kls = LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, adr_type, kls_type); + Node* kls = LoadKlassNode::make(_gvn, nullptr, immutable_memory(), p, adr_type, kls_type); klasses[which_arg] = _gvn.transform(kls); } @@ -3357,10 +3373,10 @@ bool LibraryCallKit::inline_native_subtype_check() { assert(region->req() == PATH_LIMIT, "sane region"); for (uint i = 1; i < region->req(); i++) { Node* ctl = region->in(i); - if (ctl == NULL || ctl == top()) { + if (ctl == nullptr || ctl == top()) { region->set_req(i, top()); phi ->set_req(i, top()); - } else if (phi->in(i) == NULL) { + } else if (phi->in(i) == nullptr) { phi->set_req(i, intcon(0)); // all other paths produce 'false' } } @@ -3375,7 +3391,7 @@ Node* LibraryCallKit::generate_array_guard_common(Node* kls, RegionNode* region, bool obj_array, bool not_array) { if (stopped()) { - return NULL; + return nullptr; } // If obj_array/non_array==false/false: @@ -3390,15 +3406,15 @@ Node* LibraryCallKit::generate_array_guard_common(Node* kls, RegionNode* region, // Like generate_guard, adds a new path onto the region. jint layout_con = 0; Node* layout_val = get_layout_helper(kls, layout_con); - if (layout_val == NULL) { + if (layout_val == nullptr) { bool query = (obj_array ? Klass::layout_helper_is_objArray(layout_con) : Klass::layout_helper_is_array(layout_con)); if (query == not_array) { - return NULL; // never a branch + return nullptr; // never a branch } else { // always a branch Node* always_branch = control(); - if (region != NULL) + if (region != nullptr) region->add_req(always_branch); set_control(top()); return always_branch; @@ -3456,12 +3472,12 @@ bool LibraryCallKit::inline_unsafe_newArray(bool uninitialized) { // ensuing call will throw an exception, or else it // will cache the array klass for next time. PreserveJVMState pjvms(this); - CallJavaNode* slow_call = NULL; + CallJavaNode* slow_call = nullptr; if (uninitialized) { // Generate optimized virtual call (holder class 'Unsafe' is final) - slow_call = generate_method_call(vmIntrinsics::_allocateUninitializedArray, false, false); + slow_call = generate_method_call(vmIntrinsics::_allocateUninitializedArray, false, false, true); } else { - slow_call = generate_method_call_static(vmIntrinsics::_newArray); + slow_call = generate_method_call_static(vmIntrinsics::_newArray, true); } Node* slow_result = set_results_for_java_call(slow_call); // this->control() comes from set_results_for_java_call @@ -3508,9 +3524,9 @@ bool LibraryCallKit::inline_native_getLength() { if (stopped()) return true; // Deoptimize if it is a non-array. - Node* non_array = generate_non_array_guard(load_object_klass(array), NULL); + Node* non_array = generate_non_array_guard(load_object_klass(array), nullptr); - if (non_array != NULL) { + if (non_array != nullptr) { PreserveJVMState pjvms(this); set_control(non_array); uncommon_trap(Deoptimization::Reason_intrinsic, @@ -3541,7 +3557,7 @@ bool LibraryCallKit::inline_array_copyOf(bool is_copyOfRange) { Node* end = is_copyOfRange? argument(2): argument(1); Node* array_type_mirror = is_copyOfRange? argument(3): argument(2); - Node* newcopy = NULL; + Node* newcopy = nullptr; // Set the original stack and the reexecute bit for the interpreter to reexecute // the bytecode that invokes Arrays.copyOf if deoptimization happens. @@ -3556,7 +3572,7 @@ bool LibraryCallKit::inline_array_copyOf(bool is_copyOfRange) { Node* orig_length = load_array_length(original); - Node* klass_node = load_klass_from_mirror(array_type_mirror, false, NULL, 0); + Node* klass_node = load_klass_from_mirror(array_type_mirror, false, nullptr, 0); klass_node = null_check(klass_node); RegionNode* bailout = new RegionNode(1); @@ -3565,7 +3581,7 @@ bool LibraryCallKit::inline_array_copyOf(bool is_copyOfRange) { // Despite the generic type of Arrays.copyOf, the mirror might be int, int[], etc. // Bail out if that is so. Node* not_objArray = generate_non_objArray_guard(klass_node, bailout); - if (not_objArray != NULL) { + if (not_objArray != nullptr) { // Improve the klass node's type from the new optimistic assumption: ciKlass* ak = ciArrayKlass::make(env()->Object_klass()); const Type* akls = TypeKlassPtr::make(TypePtr::NotNull, ak, 0/*offset*/); @@ -3622,7 +3638,7 @@ bool LibraryCallKit::inline_array_copyOf(bool is_copyOfRange) { int test = C->static_subtype_check(superk, subk); if (test != Compile::SSC_always_true && test != Compile::SSC_always_false) { const TypeOopPtr* t_original = _gvn.type(original)->is_oopptr(); - if (t_original->speculative_type() != NULL) { + if (t_original->speculative_type() != nullptr) { original = maybe_cast_profiled_obj(original, t_original->speculative_type(), true); } } @@ -3686,7 +3702,7 @@ Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass, vtable_index*vtableEntry::size_in_bytes() + vtableEntry::method_offset_in_bytes(); Node* entry_addr = basic_plus_adr(obj_klass, entry_offset); - Node* target_call = make_load(NULL, entry_addr, TypePtr::NOTNULL, T_ADDRESS, MemNode::unordered); + Node* target_call = make_load(nullptr, entry_addr, TypePtr::NOTNULL, T_ADDRESS, MemNode::unordered); // Compare the target method with the expected method (e.g., Object.hashCode). const TypePtr* native_call_addr = TypeMetadataPtr::make(method); @@ -3706,7 +3722,7 @@ Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass, // not another intrinsic. (E.g., don't use this for making an // arraycopy call inside of the copyOf intrinsic.) CallJavaNode* -LibraryCallKit::generate_method_call(vmIntrinsics::ID method_id, bool is_virtual, bool is_static) { +LibraryCallKit::generate_method_call(vmIntrinsicID method_id, bool is_virtual, bool is_static, bool res_not_null) { // When compiling the intrinsic method itself, do not use this technique. guarantee(callee() != C->method(), "cannot make slow-call to self"); @@ -3715,6 +3731,14 @@ LibraryCallKit::generate_method_call(vmIntrinsics::ID method_id, bool is_virtual guarantee(method_id == method->intrinsic_id(), "must match"); const TypeFunc* tf = TypeFunc::make(method); + if (res_not_null) { + assert(tf->return_type() == T_OBJECT, ""); + const TypeTuple* range = tf->range(); + const Type** fields = TypeTuple::fields(range->cnt()); + fields[TypeFunc::Parms] = range->field_at(TypeFunc::Parms)->filter_speculative(TypePtr::NOTNULL); + const TypeTuple* new_range = TypeTuple::make(range->cnt(), fields); + tf = TypeFunc::make(tf->domain(), new_range); + } CallJavaNode* slow_call; if (is_static) { assert(!is_virtual, ""); @@ -3770,7 +3794,7 @@ bool LibraryCallKit::inline_native_hashcode(bool is_virtual, bool is_static) { PhiNode* result_val = new PhiNode(result_reg, TypeInt::INT); PhiNode* result_io = new PhiNode(result_reg, Type::ABIO); PhiNode* result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM); - Node* obj = NULL; + Node* obj = nullptr; if (!is_static) { // Check for hashing null object obj = null_check_receiver(); @@ -3814,9 +3838,9 @@ bool LibraryCallKit::inline_native_hashcode(bool is_virtual, bool is_static) { // Get the header out of the object, use LoadMarkNode when available Node* header_addr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes()); - // The control of the load must be NULL. Otherwise, the load can move before + // The control of the load must be null. Otherwise, the load can move before // the null check after castPP removal. - Node* no_ctrl = NULL; + Node* no_ctrl = nullptr; Node* header = make_load(no_ctrl, header_addr, TypeX_X, TypeX_X->basic_type(), MemNode::unordered); // Test the header to see if it is unlocked. @@ -3864,7 +3888,7 @@ bool LibraryCallKit::inline_native_hashcode(bool is_virtual, bool is_static) { // No need for PreserveJVMState, because we're using up the present state. set_all_memory(init_mem); vmIntrinsics::ID hashCode_id = is_static ? vmIntrinsics::_identityHashCode : vmIntrinsics::_hashCode; - CallJavaNode* slow_call = generate_method_call(hashCode_id, is_virtual, is_static); + CallJavaNode* slow_call = generate_method_call(hashCode_id, is_virtual, is_static, false); Node* slow_result = set_results_for_java_call(slow_call); // this->control() comes from set_results_for_java_call result_reg->init_req(_slow_path, control()); @@ -3923,7 +3947,7 @@ bool LibraryCallKit::inline_native_Reflection_getCallerClass() { // Cf. JVM_GetCallerClass // NOTE: Start the loop at depth 1 because the current JVM state does // not include the Reflection.getCallerClass() frame. - for (int n = 1; caller_jvms != NULL; caller_jvms = caller_jvms->caller(), n++) { + for (int n = 1; caller_jvms != nullptr; caller_jvms = caller_jvms->caller(), n++) { ciMethod* m = caller_jvms->method(); switch (n) { case 0: @@ -3980,7 +4004,7 @@ bool LibraryCallKit::inline_native_Reflection_getCallerClass() { bool LibraryCallKit::inline_fp_conversions(vmIntrinsics::ID id) { Node* arg = argument(0); - Node* result = NULL; + Node* result = nullptr; switch (id) { case vmIntrinsics::_floatToRawIntBits: result = new MoveF2INode(arg); break; @@ -4132,17 +4156,17 @@ bool LibraryCallKit::inline_unsafe_copyMemory() { //------------------------clone_coping----------------------------------- // Helper function for inline_native_clone. void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array) { - assert(obj_size != NULL, ""); + assert(obj_size != nullptr, ""); Node* raw_obj = alloc_obj->in(1); assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); - AllocateNode* alloc = NULL; + AllocateNode* alloc = nullptr; if (ReduceBulkZeroing) { // We will be completely responsible for initializing this object - // mark Initialize node as complete. alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); // The object was just allocated - there should be no any stores! - guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), ""); + guarantee(alloc != nullptr && alloc->maybe_set_complete(&_gvn), ""); // Mark as complete_with_arraycopy so that on AllocateNode // expansion, we know this AllocateNode is initialized by an array // copy and a StoreStore barrier exists after the array copy. @@ -4153,7 +4177,7 @@ void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, b access_clone(obj, alloc_obj, size, is_array); // Do not let reads from the cloned object float above the arraycopy. - if (alloc != NULL) { + if (alloc != nullptr) { // Do not let stores that initialize this object be reordered with // a subsequent store that would make this object accessible by // other threads. @@ -4202,7 +4226,7 @@ bool LibraryCallKit::inline_native_clone(bool is_virtual) { // know the number and types of fields to convert the clone to // loads/stores. Maybe a speculative type can help us. if (!obj_type->klass_is_exact() && - obj_type->speculative_type() != NULL && + obj_type->speculative_type() != nullptr && obj_type->speculative_type()->is_instance_klass()) { ciInstanceKlass* spec_ik = obj_type->speculative_type()->as_instance_klass(); if (spec_ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem && @@ -4235,21 +4259,21 @@ bool LibraryCallKit::inline_native_clone(bool is_virtual) { record_for_igvn(result_reg); Node* obj_klass = load_object_klass(obj); - Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL); - if (array_ctl != NULL) { + Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)nullptr); + if (array_ctl != nullptr) { // It's an array. PreserveJVMState pjvms(this); set_control(array_ctl); Node* obj_length = load_array_length(obj); - Node* obj_size = NULL; + Node* obj_size = nullptr; Node* alloc_obj = new_array(obj_klass, obj_length, 0, &obj_size, /*deoptimize_on_exception=*/true); BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); if (bs->array_copy_requires_gc_barriers(true, T_OBJECT, true, false, BarrierSetC2::Parsing)) { // If it is an oop array, it requires very special treatment, // because gc barriers are required when accessing the array. - Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL); - if (is_obja != NULL) { + Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)nullptr); + if (is_obja != nullptr) { PreserveJVMState pjvms2(this); set_control(is_obja); // Generate a direct call to the right arraycopy function(s). @@ -4315,11 +4339,11 @@ bool LibraryCallKit::inline_native_clone(bool is_virtual) { if (!stopped()) { // It's an instance, and it passed the slow-path tests. PreserveJVMState pjvms(this); - Node* obj_size = NULL; + Node* obj_size = nullptr; // Need to deoptimize on exception from allocation since Object.clone intrinsic // is reexecuted if deoptimization occurs and there could be problems when merging // exception state between multiple Object.clone versions (reexecute=true vs reexecute=false). - Node* alloc_obj = new_instance(obj_klass, NULL, &obj_size, /*deoptimize_on_exception=*/true); + Node* alloc_obj = new_instance(obj_klass, nullptr, &obj_size, /*deoptimize_on_exception=*/true); copy_to_clone(obj, alloc_obj, obj_size, false); @@ -4334,7 +4358,7 @@ bool LibraryCallKit::inline_native_clone(bool is_virtual) { set_control(_gvn.transform(slow_region)); if (!stopped()) { PreserveJVMState pjvms(this); - CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual); + CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual, false, true); // We need to deoptimize on exception (see comment above) Node* slow_result = set_results_for_java_call(slow_call, false, /* deoptimize */ true); // this->control() comes from set_results_for_java_call @@ -4363,7 +4387,7 @@ bool LibraryCallKit::inline_native_clone(bool is_virtual) { // deoptimization, we'll reexecute the allocation and the // initialization. JVMState* LibraryCallKit::arraycopy_restore_alloc_state(AllocateArrayNode* alloc, int& saved_reexecute_sp) { - if (alloc != NULL) { + if (alloc != nullptr) { ciMethod* trap_method = alloc->jvms()->method(); int trap_bci = alloc->jvms()->bci(); @@ -4395,24 +4419,7 @@ JVMState* LibraryCallKit::arraycopy_restore_alloc_state(AllocateArrayNode* alloc } if (no_interfering_store) { - JVMState* old_jvms = alloc->jvms()->clone_shallow(C); - uint size = alloc->req(); - SafePointNode* sfpt = new SafePointNode(size, old_jvms); - old_jvms->set_map(sfpt); - for (uint i = 0; i < size; i++) { - sfpt->init_req(i, alloc->in(i)); - } - // re-push array length for deoptimization - sfpt->ins_req(old_jvms->stkoff() + old_jvms->sp(), alloc->in(AllocateNode::ALength)); - old_jvms->set_sp(old_jvms->sp()+1); - old_jvms->set_monoff(old_jvms->monoff()+1); - old_jvms->set_scloff(old_jvms->scloff()+1); - old_jvms->set_endoff(old_jvms->endoff()+1); - old_jvms->set_should_reexecute(true); - - sfpt->set_i_o(map()->i_o()); - sfpt->set_memory(map()->memory()); - sfpt->set_control(map()->control()); + SafePointNode* sfpt = create_safepoint_with_state_before_array_allocation(alloc); JVMState* saved_jvms = jvms(); saved_reexecute_sp = _reexecute_sp; @@ -4424,7 +4431,31 @@ JVMState* LibraryCallKit::arraycopy_restore_alloc_state(AllocateArrayNode* alloc } } } - return NULL; + return nullptr; +} + +// Clone the JVMState of the array allocation and create a new safepoint with it. Re-push the array length to the stack +// such that uncommon traps can be emitted to re-execute the array allocation in the interpreter. +SafePointNode* LibraryCallKit::create_safepoint_with_state_before_array_allocation(const AllocateArrayNode* alloc) const { + JVMState* old_jvms = alloc->jvms()->clone_shallow(C); + uint size = alloc->req(); + SafePointNode* sfpt = new SafePointNode(size, old_jvms); + old_jvms->set_map(sfpt); + for (uint i = 0; i < size; i++) { + sfpt->init_req(i, alloc->in(i)); + } + // re-push array length for deoptimization + sfpt->ins_req(old_jvms->stkoff() + old_jvms->sp(), alloc->in(AllocateNode::ALength)); + old_jvms->set_sp(old_jvms->sp()+1); + old_jvms->set_monoff(old_jvms->monoff()+1); + old_jvms->set_scloff(old_jvms->scloff()+1); + old_jvms->set_endoff(old_jvms->endoff()+1); + old_jvms->set_should_reexecute(true); + + sfpt->set_i_o(map()->i_o()); + sfpt->set_memory(map()->memory()); + sfpt->set_control(map()->control()); + return sfpt; } // In case of a deoptimization, we restart execution at the @@ -4434,18 +4465,20 @@ JVMState* LibraryCallKit::arraycopy_restore_alloc_state(AllocateArrayNode* alloc // deoptimize. This is possible because tightly_coupled_allocation() // guarantees there's no observer of the allocated array at this point // and the control flow is simple enough. -void LibraryCallKit::arraycopy_move_allocation_here(AllocateArrayNode* alloc, Node* dest, JVMState* saved_jvms, +void LibraryCallKit::arraycopy_move_allocation_here(AllocateArrayNode* alloc, Node* dest, JVMState* saved_jvms_before_guards, int saved_reexecute_sp, uint new_idx) { - if (saved_jvms != NULL && !stopped()) { - assert(alloc != NULL, "only with a tightly coupled allocation"); + if (saved_jvms_before_guards != nullptr && !stopped()) { + replace_unrelated_uncommon_traps_with_alloc_state(alloc, saved_jvms_before_guards); + + assert(alloc != nullptr, "only with a tightly coupled allocation"); // restore JVM state to the state at the arraycopy - saved_jvms->map()->set_control(map()->control()); - assert(saved_jvms->map()->memory() == map()->memory(), "memory state changed?"); - assert(saved_jvms->map()->i_o() == map()->i_o(), "IO state changed?"); + saved_jvms_before_guards->map()->set_control(map()->control()); + assert(saved_jvms_before_guards->map()->memory() == map()->memory(), "memory state changed?"); + assert(saved_jvms_before_guards->map()->i_o() == map()->i_o(), "IO state changed?"); // If we've improved the types of some nodes (null check) while // emitting the guards, propagate them to the current state - map()->replaced_nodes().apply(saved_jvms->map(), new_idx); - set_jvms(saved_jvms); + map()->replaced_nodes().apply(saved_jvms_before_guards->map(), new_idx); + set_jvms(saved_jvms_before_guards); _reexecute_sp = saved_reexecute_sp; // Remove the allocation from above the guards @@ -4466,13 +4499,13 @@ void LibraryCallKit::arraycopy_move_allocation_here(AllocateArrayNode* alloc, No Node* init_control = init->proj_out(TypeFunc::Control); Node* alloc_length = alloc->Ideal_length(); #ifdef ASSERT - Node* prev_cast = NULL; + Node* prev_cast = nullptr; #endif for (uint i = 0; i < init_control->outcnt(); i++) { Node* init_out = init_control->raw_out(i); if (init_out->is_CastII() && init_out->in(TypeFunc::Control) == init_control && init_out->in(1) == alloc_length) { #ifdef ASSERT - if (prev_cast == NULL) { + if (prev_cast == nullptr) { prev_cast = init_out; } else { if (prev_cast->cmp(*init_out) == false) { @@ -4500,7 +4533,7 @@ void LibraryCallKit::arraycopy_move_allocation_here(AllocateArrayNode* alloc, No // Update memory as done in GraphKit::set_output_for_allocation() const TypeInt* length_type = _gvn.find_int_type(alloc->in(AllocateNode::ALength)); const TypeOopPtr* ary_type = _gvn.type(alloc->in(AllocateNode::KlassNode))->is_klassptr()->as_instance_type(); - if (ary_type->isa_aryptr() && length_type != NULL) { + if (ary_type->isa_aryptr() && length_type != nullptr) { ary_type = ary_type->is_aryptr()->cast_to_size(length_type); } const TypePtr* telemref = ary_type->add_offset(Type::OffsetBot); @@ -4521,6 +4554,58 @@ void LibraryCallKit::arraycopy_move_allocation_here(AllocateArrayNode* alloc, No } } +// Unrelated UCTs between the array allocation and the array copy, which are considered safe by tightly_coupled_allocation(), +// need to be replaced by an UCT with a state before the array allocation (including the array length). This is necessary +// because we could hit one of these UCTs (which are executed before the emitted array copy guards and the actual array +// allocation which is moved down in arraycopy_move_allocation_here()). When later resuming execution in the interpreter, +// we would have wrongly skipped the array allocation. To prevent this, we resume execution at the array allocation in +// the interpreter similar to what we are doing for the newly emitted guards for the array copy. +void LibraryCallKit::replace_unrelated_uncommon_traps_with_alloc_state(AllocateArrayNode* alloc, + JVMState* saved_jvms_before_guards) { + if (saved_jvms_before_guards->map()->control()->is_IfProj()) { + // There is at least one unrelated uncommon trap which needs to be replaced. + SafePointNode* sfpt = create_safepoint_with_state_before_array_allocation(alloc); + + JVMState* saved_jvms = jvms(); + const int saved_reexecute_sp = _reexecute_sp; + set_jvms(sfpt->jvms()); + _reexecute_sp = jvms()->sp(); + + replace_unrelated_uncommon_traps_with_alloc_state(saved_jvms_before_guards); + + // Restore state + set_jvms(saved_jvms); + _reexecute_sp = saved_reexecute_sp; + } +} + +// Replace the unrelated uncommon traps with new uncommon trap nodes by reusing the action and reason. The new uncommon +// traps will have the state of the array allocation. Let the old uncommon trap nodes die. +void LibraryCallKit::replace_unrelated_uncommon_traps_with_alloc_state(JVMState* saved_jvms_before_guards) { + Node* if_proj = saved_jvms_before_guards->map()->control(); // Start the search right before the newly emitted guards + while (if_proj->is_IfProj()) { + CallStaticJavaNode* uncommon_trap = get_uncommon_trap_from_success_proj(if_proj); + if (uncommon_trap != nullptr) { + create_new_uncommon_trap(uncommon_trap); + } + assert(if_proj->in(0)->is_If(), "must be If"); + if_proj = if_proj->in(0)->in(0); + } + assert(if_proj->is_Proj() && if_proj->in(0)->is_Initialize(), + "must have reached control projection of init node"); +} + +void LibraryCallKit::create_new_uncommon_trap(CallStaticJavaNode* uncommon_trap_call) { + const int trap_request = uncommon_trap_call->uncommon_trap_request(); + assert(trap_request != 0, "no valid UCT trap request"); + PreserveJVMState pjvms(this); + set_control(uncommon_trap_call->in(0)); + uncommon_trap(Deoptimization::trap_request_reason(trap_request), + Deoptimization::trap_request_action(trap_request)); + assert(stopped(), "Should be stopped"); + _gvn.hash_delete(uncommon_trap_call); + uncommon_trap_call->set_req(0, top()); // not used anymore, kill it +} //------------------------------inline_arraycopy----------------------- // public static native void java.lang.System.arraycopy(Object src, int srcPos, @@ -4541,12 +4626,12 @@ bool LibraryCallKit::inline_arraycopy() { AllocateArrayNode* alloc = tightly_coupled_allocation(dest); int saved_reexecute_sp = -1; - JVMState* saved_jvms = arraycopy_restore_alloc_state(alloc, saved_reexecute_sp); + JVMState* saved_jvms_before_guards = arraycopy_restore_alloc_state(alloc, saved_reexecute_sp); // See arraycopy_restore_alloc_state() comment - // if alloc == NULL we don't have to worry about a tightly coupled allocation so we can emit all needed guards - // if saved_jvms != NULL (then alloc != NULL) then we can handle guards and a tightly coupled allocation - // if saved_jvms == NULL and alloc != NULL, we can't emit any guards - bool can_emit_guards = (alloc == NULL || saved_jvms != NULL); + // if alloc == null we don't have to worry about a tightly coupled allocation so we can emit all needed guards + // if saved_jvms_before_guards is not null (then alloc is not null) then we can handle guards and a tightly coupled allocation + // if saved_jvms_before_guards is null and alloc is not null, we can't emit any guards + bool can_emit_guards = (alloc == nullptr || saved_jvms_before_guards != nullptr); // The following tests must be performed // (1) src and dest are arrays. @@ -4562,12 +4647,12 @@ bool LibraryCallKit::inline_arraycopy() { // (3) src and dest must not be null. // always do this here because we need the JVM state for uncommon traps Node* null_ctl = top(); - src = saved_jvms != NULL ? null_check_oop(src, &null_ctl, true, true) : null_check(src, T_ARRAY); + src = saved_jvms_before_guards != nullptr ? null_check_oop(src, &null_ctl, true, true) : null_check(src, T_ARRAY); assert(null_ctl->is_top(), "no null control here"); dest = null_check(dest, T_ARRAY); if (!can_emit_guards) { - // if saved_jvms == NULL and alloc != NULL, we don't emit any + // if saved_jvms_before_guards is null and alloc is not null, we don't emit any // guards but the arraycopy node could still take advantage of a // tightly allocated allocation. tightly_coupled_allocation() is // called again to make sure it takes the null check above into @@ -4585,9 +4670,9 @@ bool LibraryCallKit::inline_arraycopy() { const TypeAryPtr* top_dest = dest_type->isa_aryptr(); // Do we have the type of src? - bool has_src = (top_src != NULL && top_src->klass() != NULL); + bool has_src = (top_src != nullptr && top_src->klass() != nullptr); // Do we have the type of dest? - bool has_dest = (top_dest != NULL && top_dest->klass() != NULL); + bool has_dest = (top_dest != nullptr && top_dest->klass() != nullptr); // Is the type for src from speculation? bool src_spec = false; // Is the type for dest from speculation? @@ -4603,18 +4688,18 @@ bool LibraryCallKit::inline_arraycopy() { // Do we already have or could we have type information for dest bool could_have_dest = has_dest; - ciKlass* src_k = NULL; + ciKlass* src_k = nullptr; if (!has_src) { src_k = src_type->speculative_type_not_null(); - if (src_k != NULL && src_k->is_array_klass()) { + if (src_k != nullptr && src_k->is_array_klass()) { could_have_src = true; } } - ciKlass* dest_k = NULL; + ciKlass* dest_k = nullptr; if (!has_dest) { dest_k = dest_type->speculative_type_not_null(); - if (dest_k != NULL && dest_k->is_array_klass()) { + if (dest_k != nullptr && dest_k->is_array_klass()) { could_have_dest = true; } } @@ -4625,14 +4710,14 @@ bool LibraryCallKit::inline_arraycopy() { src = maybe_cast_profiled_obj(src, src_k, true); src_type = _gvn.type(src); top_src = src_type->isa_aryptr(); - has_src = (top_src != NULL && top_src->klass() != NULL); + has_src = (top_src != nullptr && top_src->klass() != nullptr); src_spec = true; } if (!has_dest) { dest = maybe_cast_profiled_obj(dest, dest_k, true); dest_type = _gvn.type(dest); top_dest = dest_type->isa_aryptr(); - has_dest = (top_dest != NULL && top_dest->klass() != NULL); + has_dest = (top_dest != nullptr && top_dest->klass() != nullptr); dest_spec = true; } } @@ -4657,13 +4742,13 @@ bool LibraryCallKit::inline_arraycopy() { ciKlass* dest_k = top_dest->klass(); if (!src_spec) { src_k = src_type->speculative_type_not_null(); - if (src_k != NULL && src_k->is_array_klass()) { + if (src_k != nullptr && src_k->is_array_klass()) { could_have_src = true; } } if (!dest_spec) { dest_k = dest_type->speculative_type_not_null(); - if (dest_k != NULL && dest_k->is_array_klass()) { + if (dest_k != nullptr && dest_k->is_array_klass()) { could_have_dest = true; } } @@ -4681,7 +4766,7 @@ bool LibraryCallKit::inline_arraycopy() { ciMethod* trap_method = method(); int trap_bci = bci(); - if (saved_jvms != NULL) { + if (saved_jvms_before_guards != nullptr) { trap_method = alloc->jvms()->method(); trap_bci = alloc->jvms()->bci(); } @@ -4753,15 +4838,14 @@ bool LibraryCallKit::inline_arraycopy() { const TypeKlassPtr* dest_klass_t = _gvn.type(dest_klass)->is_klassptr(); const Type *toop = TypeOopPtr::make_from_klass(dest_klass_t->klass()); src = _gvn.transform(new CheckCastPPNode(control(), src, toop)); + arraycopy_move_allocation_here(alloc, dest, saved_jvms_before_guards, saved_reexecute_sp, new_idx); } - arraycopy_move_allocation_here(alloc, dest, saved_jvms, saved_reexecute_sp, new_idx); - if (stopped()) { return true; } - ArrayCopyNode* ac = ArrayCopyNode::make(this, true, src, src_offset, dest, dest_offset, length, alloc != NULL, negative_length_guard_generated, + ArrayCopyNode* ac = ArrayCopyNode::make(this, true, src, src_offset, dest, dest_offset, length, alloc != nullptr, negative_length_guard_generated, // Create LoadRange and LoadKlass nodes for use during macro expansion here // so the compiler has a chance to eliminate them: during macro expansion, // we have to set their control (CastPP nodes are eliminated). @@ -4788,29 +4872,29 @@ bool LibraryCallKit::inline_arraycopy() { // an allocation, with no intervening tests or other escapes for the object. AllocateArrayNode* LibraryCallKit::tightly_coupled_allocation(Node* ptr) { - if (stopped()) return NULL; // no fast path - if (C->AliasLevel() == 0) return NULL; // no MergeMems around + if (stopped()) return nullptr; // no fast path + if (C->AliasLevel() == 0) return nullptr; // no MergeMems around AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn); - if (alloc == NULL) return NULL; + if (alloc == nullptr) return nullptr; Node* rawmem = memory(Compile::AliasIdxRaw); // Is the allocation's memory state untouched? if (!(rawmem->is_Proj() && rawmem->in(0)->is_Initialize())) { // Bail out if there have been raw-memory effects since the allocation. // (Example: There might have been a call or safepoint.) - return NULL; + return nullptr; } rawmem = rawmem->in(0)->as_Initialize()->memory(Compile::AliasIdxRaw); if (!(rawmem->is_Proj() && rawmem->in(0) == alloc)) { - return NULL; + return nullptr; } // There must be no unexpected observers of this allocation. for (DUIterator_Fast imax, i = ptr->fast_outs(imax); i < imax; i++) { Node* obs = ptr->fast_out(i); if (obs != this->map()) { - return NULL; + return nullptr; } } @@ -4821,28 +4905,15 @@ LibraryCallKit::tightly_coupled_allocation(Node* ptr) { // There may be guards which feed into the slow_region. // Any other control flow means that we might not get a chance // to finish initializing the allocated object. - if ((ctl->is_IfFalse() || ctl->is_IfTrue()) && ctl->in(0)->is_If()) { - IfNode* iff = ctl->in(0)->as_If(); - Node* not_ctl = iff->proj_out_or_null(1 - ctl->as_Proj()->_con); - assert(not_ctl != NULL && not_ctl != ctl, "found alternate"); - // One more try: Various low-level checks bottom out in - // uncommon traps. If the debug-info of the trap omits - // any reference to the allocation, as we've already - // observed, then there can be no objection to the trap. - bool found_trap = false; - for (DUIterator_Fast jmax, j = not_ctl->fast_outs(jmax); j < jmax; j++) { - Node* obs = not_ctl->fast_out(j); - if (obs->in(0) == not_ctl && obs->is_Call() && - (obs->as_Call()->entry_point() == SharedRuntime::uncommon_trap_blob()->entry_point())) { - found_trap = true; break; - } - } - if (found_trap) { - ctl = iff->in(0); // This test feeds a harmless uncommon trap. - continue; - } + // Various low-level checks bottom out in uncommon traps. These + // are considered safe since we've already checked above that + // there is no unexpected observer of this allocation. + if (get_uncommon_trap_from_success_proj(ctl) != nullptr) { + assert(ctl->in(0)->is_If(), "must be If"); + ctl = ctl->in(0)->in(0); + } else { + return nullptr; } - return NULL; } // If we get this far, we have an allocation which immediately @@ -4853,6 +4924,20 @@ LibraryCallKit::tightly_coupled_allocation(Node* ptr) { return alloc; } +CallStaticJavaNode* LibraryCallKit::get_uncommon_trap_from_success_proj(Node* node) { + if (node->is_IfProj()) { + Node* other_proj = node->as_IfProj()->other_if_proj(); + for (DUIterator_Fast jmax, j = other_proj->fast_outs(jmax); j < jmax; j++) { + Node* obs = other_proj->fast_out(j); + if (obs->in(0) == other_proj && obs->is_CallStaticJava() && + (obs->as_CallStaticJava()->entry_point() == SharedRuntime::uncommon_trap_blob()->entry_point())) { + return obs->as_CallStaticJava(); + } + } + } + return nullptr; +} + //-------------inline_encodeISOArray----------------------------------- // encode char[] to byte[] in ISO_8859_1 or ASCII bool LibraryCallKit::inline_encodeISOArray(bool ascii) { @@ -4878,8 +4963,8 @@ bool LibraryCallKit::inline_encodeISOArray(bool ascii) { } // Figure out the size and type of the elements we will be copying. - BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); - BasicType dst_elem = dst_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); + BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type(); + BasicType dst_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type(); if (!((src_elem == T_CHAR) || (src_elem== T_BYTE)) || dst_elem != T_BYTE) { return false; } @@ -4905,7 +4990,7 @@ bool LibraryCallKit::inline_multiplyToLen() { assert(UseMultiplyToLenIntrinsic, "not implemented on this platform"); address stubAddr = StubRoutines::multiplyToLen(); - if (stubAddr == NULL) { + if (stubAddr == nullptr) { return false; // Intrinsic's stub is not implemented on this platform } const char* stubName = "multiplyToLen"; @@ -4932,8 +5017,8 @@ bool LibraryCallKit::inline_multiplyToLen() { return false; } - BasicType x_elem = x_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); - BasicType y_elem = y_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); + BasicType x_elem = top_x->klass()->as_array_klass()->element_type()->basic_type(); + BasicType y_elem = top_y->klass()->as_array_klass()->element_type()->basic_type(); if (x_elem != T_INT || y_elem != T_INT) { return false; } @@ -5015,7 +5100,7 @@ bool LibraryCallKit::inline_squareToLen() { assert(UseSquareToLenIntrinsic, "not implemented on this platform"); address stubAddr = StubRoutines::squareToLen(); - if (stubAddr == NULL) { + if (stubAddr == nullptr) { return false; // Intrinsic's stub is not implemented on this platform } const char* stubName = "squareToLen"; @@ -5040,8 +5125,8 @@ bool LibraryCallKit::inline_squareToLen() { return false; } - BasicType x_elem = x_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); - BasicType z_elem = z_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); + BasicType x_elem = top_x->klass()->as_array_klass()->element_type()->basic_type(); + BasicType z_elem = top_z->klass()->as_array_klass()->element_type()->basic_type(); if (x_elem != T_INT || z_elem != T_INT) { return false; } @@ -5064,7 +5149,7 @@ bool LibraryCallKit::inline_mulAdd() { assert(UseMulAddIntrinsic, "not implemented on this platform"); address stubAddr = StubRoutines::mulAdd(); - if (stubAddr == NULL) { + if (stubAddr == nullptr) { return false; // Intrinsic's stub is not implemented on this platform } const char* stubName = "mulAdd"; @@ -5089,8 +5174,8 @@ bool LibraryCallKit::inline_mulAdd() { return false; } - BasicType out_elem = out_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); - BasicType in_elem = in_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); + BasicType out_elem = top_out->klass()->as_array_klass()->element_type()->basic_type(); + BasicType in_elem = top_in->klass()->as_array_klass()->element_type()->basic_type(); if (out_elem != T_INT || in_elem != T_INT) { return false; } @@ -5112,7 +5197,7 @@ bool LibraryCallKit::inline_mulAdd() { //-------------inline_montgomeryMultiply----------------------------------- bool LibraryCallKit::inline_montgomeryMultiply() { address stubAddr = StubRoutines::montgomeryMultiply(); - if (stubAddr == NULL) { + if (stubAddr == nullptr) { return false; // Intrinsic's stub is not implemented on this platform } @@ -5144,10 +5229,10 @@ bool LibraryCallKit::inline_montgomeryMultiply() { return false; } - BasicType a_elem = a_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); - BasicType b_elem = b_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); - BasicType n_elem = n_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); - BasicType m_elem = m_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); + BasicType a_elem = top_a->klass()->as_array_klass()->element_type()->basic_type(); + BasicType b_elem = top_b->klass()->as_array_klass()->element_type()->basic_type(); + BasicType n_elem = top_n->klass()->as_array_klass()->element_type()->basic_type(); + BasicType m_elem = top_m->klass()->as_array_klass()->element_type()->basic_type(); if (a_elem != T_INT || b_elem != T_INT || n_elem != T_INT || m_elem != T_INT) { return false; } @@ -5172,7 +5257,7 @@ bool LibraryCallKit::inline_montgomeryMultiply() { bool LibraryCallKit::inline_montgomerySquare() { address stubAddr = StubRoutines::montgomerySquare(); - if (stubAddr == NULL) { + if (stubAddr == nullptr) { return false; // Intrinsic's stub is not implemented on this platform } @@ -5200,9 +5285,9 @@ bool LibraryCallKit::inline_montgomerySquare() { return false; } - BasicType a_elem = a_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); - BasicType n_elem = n_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); - BasicType m_elem = m_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); + BasicType a_elem = top_a->klass()->as_array_klass()->element_type()->basic_type(); + BasicType n_elem = top_n->klass()->as_array_klass()->element_type()->basic_type(); + BasicType m_elem = top_m->klass()->as_array_klass()->element_type()->basic_type(); if (a_elem != T_INT || n_elem != T_INT || m_elem != T_INT) { return false; } @@ -5225,11 +5310,11 @@ bool LibraryCallKit::inline_montgomerySquare() { } bool LibraryCallKit::inline_bigIntegerShift(bool isRightShift) { - address stubAddr = NULL; - const char* stubName = NULL; + address stubAddr = nullptr; + const char* stubName = nullptr; stubAddr = isRightShift? StubRoutines::bigIntegerRightShift(): StubRoutines::bigIntegerLeftShift(); - if (stubAddr == NULL) { + if (stubAddr == nullptr) { return false; // Intrinsic's stub is not implemented on this platform } @@ -5252,8 +5337,8 @@ bool LibraryCallKit::inline_bigIntegerShift(bool isRightShift) { return false; } - BasicType newArr_elem = newArr_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); - BasicType oldArr_elem = oldArr_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); + BasicType newArr_elem = top_newArr->klass()->as_array_klass()->element_type()->basic_type(); + BasicType oldArr_elem = top_oldArr->klass()->as_array_klass()->element_type()->basic_type(); if (newArr_elem != T_INT || oldArr_elem != T_INT) { return false; } @@ -5361,7 +5446,7 @@ bool LibraryCallKit::inline_vectorizedMismatch() { Node* cmp_length = _gvn.transform(new CmpINode(length, intcon(inline_limit))); Node* bol_gt = _gvn.transform(new BoolNode(cmp_length, BoolTest::gt)); - call_stub_path = generate_guard(bol_gt, NULL, PROB_MIN); + call_stub_path = generate_guard(bol_gt, nullptr, PROB_MIN); if (!stopped()) { Node* casted_length = _gvn.transform(new CastIINode(control(), length, TypeInt::make(0, inline_limit, Type::WidenMin))); @@ -5386,7 +5471,7 @@ bool LibraryCallKit::inline_vectorizedMismatch() { } } - if (call_stub_path != NULL) { + if (call_stub_path != nullptr) { set_control(call_stub_path); Node* call = make_runtime_call(RC_LEAF, @@ -5466,7 +5551,7 @@ bool LibraryCallKit::inline_updateBytesCRC32() { } // Figure out the size and type of the elements we will be copying. - BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); + BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type(); if (src_elem != T_BYTE) { return false; } @@ -5524,8 +5609,8 @@ bool LibraryCallKit::inline_updateByteBufferCRC32() { //------------------------------get_table_from_crc32c_class----------------------- Node * LibraryCallKit::get_table_from_crc32c_class(ciInstanceKlass *crc32c_class) { - Node* table = load_field_from_object(NULL, "byteTable", "[I", /*decorators*/ IN_HEAP, /*is_static*/ true, crc32c_class); - assert (table != NULL, "wrong version of java.util.zip.CRC32C"); + Node* table = load_field_from_object(nullptr, "byteTable", "[I", /*decorators*/ IN_HEAP, /*is_static*/ true, crc32c_class); + assert (table != nullptr, "wrong version of java.util.zip.CRC32C"); return table; } @@ -5555,7 +5640,7 @@ bool LibraryCallKit::inline_updateBytesCRC32C() { } // Figure out the size and type of the elements we will be copying. - BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); + BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type(); if (src_elem != T_BYTE) { return false; } @@ -5648,7 +5733,7 @@ bool LibraryCallKit::inline_updateBytesAdler32() { } // Figure out the size and type of the elements we will be copying. - BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); + BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type(); if (src_elem != T_BYTE) { return false; } @@ -5717,8 +5802,8 @@ bool LibraryCallKit::inline_reference_get() { DecoratorSet decorators = IN_HEAP | ON_WEAK_OOP_REF; Node* result = load_field_from_object(reference_obj, "referent", "Ljava/lang/Object;", - decorators, /*is_static*/ false, NULL); - if (result == NULL) return false; + decorators, /*is_static*/ false, nullptr); + if (result == nullptr) return false; // Add memory barrier to prevent commoning reads from this field // across safepoint since GC can change its value. @@ -5740,8 +5825,8 @@ bool LibraryCallKit::inline_reference_refersTo0(bool is_phantom) { DecoratorSet decorators = IN_HEAP | AS_NO_KEEPALIVE; decorators |= (is_phantom ? ON_PHANTOM_OOP_REF : ON_WEAK_OOP_REF); Node* referent = load_field_from_object(reference_obj, "referent", "Ljava/lang/Object;", - decorators, /*is_static*/ false, NULL); - if (referent == NULL) return false; + decorators, /*is_static*/ false, nullptr); + if (referent == nullptr) return false; // Add memory barrier to prevent commoning reads from this field // across safepoint since GC can change its value. @@ -5772,9 +5857,9 @@ bool LibraryCallKit::inline_reference_refersTo0(bool is_phantom) { Node* LibraryCallKit::load_field_from_object(Node* fromObj, const char* fieldName, const char* fieldTypeString, DecoratorSet decorators = IN_HEAP, bool is_static = false, ciInstanceKlass* fromKls = NULL) { - if (fromKls == NULL) { + if (fromKls == nullptr) { const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr(); - assert(tinst != NULL, "obj is null"); + assert(tinst != nullptr, "obj is null"); assert(tinst->klass()->is_loaded(), "obj is not loaded"); fromKls = tinst->klass()->as_instance_klass(); } else { @@ -5785,7 +5870,7 @@ Node* LibraryCallKit::load_field_from_object(Node* fromObj, const char* fieldNam is_static); assert (field != NULL, "undefined field"); - if (field == NULL) return (Node *) NULL; + if (field == nullptr) return (Node *) nullptr; if (is_static) { const TypeInstPtr* tip = TypeInstPtr::make(fromKls->java_mirror()); @@ -5821,9 +5906,9 @@ Node* LibraryCallKit::load_field_from_object(Node* fromObj, const char* fieldNam Node * LibraryCallKit::field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact = true, bool is_static = false, ciInstanceKlass * fromKls = NULL) { - if (fromKls == NULL) { + if (fromKls == nullptr) { const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr(); - assert(tinst != NULL, "obj is null"); + assert(tinst != nullptr, "obj is null"); assert(tinst->klass()->is_loaded(), "obj is not loaded"); assert(!is_exact || tinst->klass_is_exact(), "klass not exact"); fromKls = tinst->klass()->as_instance_klass(); @@ -5835,7 +5920,7 @@ Node * LibraryCallKit::field_address_from_object(Node * fromObj, const char * fi ciSymbol::make(fieldTypeString), is_static); - assert(field != NULL, "undefined field"); + assert(field != nullptr, "undefined field"); assert(!field->is_volatile(), "not defined for volatile fields"); if (is_static) { @@ -5854,7 +5939,7 @@ Node * LibraryCallKit::field_address_from_object(Node * fromObj, const char * fi //------------------------------inline_aescrypt_Block----------------------- bool LibraryCallKit::inline_aescrypt_Block(vmIntrinsics::ID id) { - address stubAddr = NULL; + address stubAddr = nullptr; const char *stubName; assert(UseAES, "need AES instruction support"); @@ -5870,7 +5955,7 @@ bool LibraryCallKit::inline_aescrypt_Block(vmIntrinsics::ID id) { default: break; } - if (stubAddr == NULL) return false; + if (stubAddr == nullptr) return false; Node* aescrypt_object = argument(0); Node* src = argument(1); @@ -5892,8 +5977,8 @@ bool LibraryCallKit::inline_aescrypt_Block(vmIntrinsics::ID id) { // we are just trying to get the call to be generated. Node* src_start = src; Node* dest_start = dest; - if (src_offset != NULL || dest_offset != NULL) { - assert(src_offset != NULL && dest_offset != NULL, ""); + if (src_offset != nullptr || dest_offset != nullptr) { + assert(src_offset != nullptr && dest_offset != nullptr, ""); src_start = array_element_address(src, src_offset, T_BYTE); dest_start = array_element_address(dest, dest_offset, T_BYTE); } @@ -5901,7 +5986,7 @@ bool LibraryCallKit::inline_aescrypt_Block(vmIntrinsics::ID id) { // now need to get the start of its expanded key array // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object); - if (k_start == NULL) return false; + if (k_start == nullptr) return false; // Call the stub. make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::aescrypt_block_Type(), @@ -5913,8 +5998,8 @@ bool LibraryCallKit::inline_aescrypt_Block(vmIntrinsics::ID id) { //------------------------------inline_cipherBlockChaining_AESCrypt----------------------- bool LibraryCallKit::inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id) { - address stubAddr = NULL; - const char *stubName = NULL; + address stubAddr = nullptr; + const char *stubName = nullptr; assert(UseAES, "need AES instruction support"); @@ -5930,7 +6015,7 @@ bool LibraryCallKit::inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id) { default: break; } - if (stubAddr == NULL) return false; + if (stubAddr == nullptr) return false; Node* cipherBlockChaining_object = argument(0); Node* src = argument(1); @@ -5953,8 +6038,8 @@ bool LibraryCallKit::inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id) { // checks are the responsibility of the caller Node* src_start = src; Node* dest_start = dest; - if (src_offset != NULL || dest_offset != NULL) { - assert(src_offset != NULL && dest_offset != NULL, ""); + if (src_offset != nullptr || dest_offset != nullptr) { + assert(src_offset != nullptr && dest_offset != nullptr, ""); src_start = array_element_address(src, src_offset, T_BYTE); dest_start = array_element_address(dest, dest_offset, T_BYTE); } @@ -5965,11 +6050,11 @@ bool LibraryCallKit::inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id) { // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java Node* embeddedCipherObj = load_field_from_object(cipherBlockChaining_object, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); - if (embeddedCipherObj == NULL) return false; + if (embeddedCipherObj == nullptr) return false; // cast it to what we know it will be at runtime const TypeInstPtr* tinst = _gvn.type(cipherBlockChaining_object)->isa_instptr(); - assert(tinst != NULL, "CBC obj is null"); + assert(tinst != nullptr, "CBC obj is null"); assert(tinst->klass()->is_loaded(), "CBC obj is not loaded"); ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); assert(klass_AESCrypt->is_loaded(), "predicate checks that this class is loaded"); @@ -5982,11 +6067,11 @@ bool LibraryCallKit::inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id) { // we need to get the start of the aescrypt_object's expanded key array Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object); - if (k_start == NULL) return false; + if (k_start == nullptr) return false; // similarly, get the start address of the r vector Node* objRvec = load_field_from_object(cipherBlockChaining_object, "r", "[B"); - if (objRvec == NULL) return false; + if (objRvec == nullptr) return false; Node* r_start = array_element_address(objRvec, intcon(0), T_BYTE); // Call the stub, passing src_start, dest_start, k_start, r_start and src_len @@ -6003,8 +6088,8 @@ bool LibraryCallKit::inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id) { //------------------------------inline_electronicCodeBook_AESCrypt----------------------- bool LibraryCallKit::inline_electronicCodeBook_AESCrypt(vmIntrinsics::ID id) { - address stubAddr = NULL; - const char *stubName = NULL; + address stubAddr = nullptr; + const char *stubName = nullptr; assert(UseAES, "need AES instruction support"); @@ -6021,7 +6106,7 @@ bool LibraryCallKit::inline_electronicCodeBook_AESCrypt(vmIntrinsics::ID id) { break; } - if (stubAddr == NULL) return false; + if (stubAddr == nullptr) return false; Node* electronicCodeBook_object = argument(0); Node* src = argument(1); @@ -6041,8 +6126,8 @@ bool LibraryCallKit::inline_electronicCodeBook_AESCrypt(vmIntrinsics::ID id) { // checks are the responsibility of the caller Node* src_start = src; Node* dest_start = dest; - if (src_offset != NULL || dest_offset != NULL) { - assert(src_offset != NULL && dest_offset != NULL, ""); + if (src_offset != nullptr || dest_offset != nullptr) { + assert(src_offset != nullptr && dest_offset != nullptr, ""); src_start = array_element_address(src, src_offset, T_BYTE); dest_start = array_element_address(dest, dest_offset, T_BYTE); } @@ -6053,11 +6138,11 @@ bool LibraryCallKit::inline_electronicCodeBook_AESCrypt(vmIntrinsics::ID id) { // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java Node* embeddedCipherObj = load_field_from_object(electronicCodeBook_object, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); - if (embeddedCipherObj == NULL) return false; + if (embeddedCipherObj == nullptr) return false; // cast it to what we know it will be at runtime const TypeInstPtr* tinst = _gvn.type(electronicCodeBook_object)->isa_instptr(); - assert(tinst != NULL, "ECB obj is null"); + assert(tinst != nullptr, "ECB obj is null"); assert(tinst->klass()->is_loaded(), "ECB obj is not loaded"); ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); assert(klass_AESCrypt->is_loaded(), "predicate checks that this class is loaded"); @@ -6070,7 +6155,7 @@ bool LibraryCallKit::inline_electronicCodeBook_AESCrypt(vmIntrinsics::ID id) { // we need to get the start of the aescrypt_object's expanded key array Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object); - if (k_start == NULL) return false; + if (k_start == nullptr) return false; // Call the stub, passing src_start, dest_start, k_start, r_start and src_len Node* ecbCrypt = make_runtime_call(RC_LEAF | RC_NO_FP, @@ -6089,13 +6174,13 @@ bool LibraryCallKit::inline_counterMode_AESCrypt(vmIntrinsics::ID id) { assert(UseAES, "need AES instruction support"); if (!UseAESCTRIntrinsics) return false; - address stubAddr = NULL; - const char *stubName = NULL; + address stubAddr = nullptr; + const char *stubName = nullptr; if (id == vmIntrinsics::_counterMode_AESCrypt) { stubAddr = StubRoutines::counterMode_AESCrypt(); stubName = "counterMode_AESCrypt"; } - if (stubAddr == NULL) return false; + if (stubAddr == nullptr) return false; Node* counterMode_object = argument(0); Node* src = argument(1); @@ -6115,8 +6200,8 @@ bool LibraryCallKit::inline_counterMode_AESCrypt(vmIntrinsics::ID id) { // checks are the responsibility of the caller Node* src_start = src; Node* dest_start = dest; - if (src_offset != NULL || dest_offset != NULL) { - assert(src_offset != NULL && dest_offset != NULL, ""); + if (src_offset != nullptr || dest_offset != nullptr) { + assert(src_offset != nullptr && dest_offset != nullptr, ""); src_start = array_element_address(src, src_offset, T_BYTE); dest_start = array_element_address(dest, dest_offset, T_BYTE); } @@ -6126,10 +6211,10 @@ bool LibraryCallKit::inline_counterMode_AESCrypt(vmIntrinsics::ID id) { // so we cast it here safely. // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java Node* embeddedCipherObj = load_field_from_object(counterMode_object, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); - if (embeddedCipherObj == NULL) return false; + if (embeddedCipherObj == nullptr) return false; // cast it to what we know it will be at runtime const TypeInstPtr* tinst = _gvn.type(counterMode_object)->isa_instptr(); - assert(tinst != NULL, "CTR obj is null"); + assert(tinst != nullptr, "CTR obj is null"); assert(tinst->klass()->is_loaded(), "CTR obj is not loaded"); ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); assert(klass_AESCrypt->is_loaded(), "predicate checks that this class is loaded"); @@ -6140,14 +6225,14 @@ bool LibraryCallKit::inline_counterMode_AESCrypt(vmIntrinsics::ID id) { aescrypt_object = _gvn.transform(aescrypt_object); // we need to get the start of the aescrypt_object's expanded key array Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object); - if (k_start == NULL) return false; + if (k_start == nullptr) return false; // similarly, get the start address of the r vector Node* obj_counter = load_field_from_object(counterMode_object, "counter", "[B"); - if (obj_counter == NULL) return false; + if (obj_counter == nullptr) return false; Node* cnt_start = array_element_address(obj_counter, intcon(0), T_BYTE); Node* saved_encCounter = load_field_from_object(counterMode_object, "encryptedCounter", "[B"); - if (saved_encCounter == NULL) return false; + if (saved_encCounter == nullptr) return false; Node* saved_encCounter_start = array_element_address(saved_encCounter, intcon(0), T_BYTE); Node* used = field_address_from_object(counterMode_object, "used", "I", /*is_exact*/ false); @@ -6171,16 +6256,16 @@ Node * LibraryCallKit::get_key_start_from_aescrypt_object(Node *aescrypt_object) // However, ppc64 vncipher processes MixColumns and requires the same round keys with encryption. // The ppc64 stubs of encryption and decryption use the same round keys (sessionK[0]). Node* objSessionK = load_field_from_object(aescrypt_object, "sessionK", "[[I"); - assert (objSessionK != NULL, "wrong version of com.sun.crypto.provider.AESCrypt"); - if (objSessionK == NULL) { - return (Node *) NULL; + assert (objSessionK != nullptr, "wrong version of com.sun.crypto.provider.AESCrypt"); + if (objSessionK == nullptr) { + return (Node *) nullptr; } Node* objAESCryptKey = load_array_element(objSessionK, intcon(0), TypeAryPtr::OOPS, /* set_ctrl */ true); #else Node* objAESCryptKey = load_field_from_object(aescrypt_object, "K", "[I"); #endif // PPC64 - assert (objAESCryptKey != NULL, "wrong version of com.sun.crypto.provider.AESCrypt"); - if (objAESCryptKey == NULL) return (Node *) NULL; + assert (objAESCryptKey != nullptr, "wrong version of com.sun.crypto.provider.AESCrypt"); + if (objAESCryptKey == nullptr) return (Node *) nullptr; // now have the array, need to get the start address of the K array Node* k_start = array_element_address(objAESCryptKey, intcon(0), T_INT); @@ -6197,7 +6282,7 @@ Node * LibraryCallKit::get_key_start_from_aescrypt_object(Node *aescrypt_object) // note cipher==plain is more conservative than the original java code but that's OK // Node* LibraryCallKit::inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting) { - // The receiver was checked for NULL already. + // The receiver was checked for null already. Node* objCBC = argument(0); Node* src = argument(1); @@ -6210,7 +6295,7 @@ Node* LibraryCallKit::inline_cipherBlockChaining_AESCrypt_predicate(bool decrypt // AESCrypt might not be loaded yet if some other SymmetricCipher got us to this compile point // will have same classloader as CipherBlockChaining object const TypeInstPtr* tinst = _gvn.type(objCBC)->isa_instptr(); - assert(tinst != NULL, "CBCobj is null"); + assert(tinst != nullptr, "CBCobj is null"); assert(tinst->klass()->is_loaded(), "CBCobj is not loaded"); // we want to do an instanceof comparison against the AESCrypt class @@ -6232,11 +6317,11 @@ Node* LibraryCallKit::inline_cipherBlockChaining_AESCrypt_predicate(bool decrypt Node* cmp_instof = _gvn.transform(new CmpINode(instof, intcon(1))); Node* bool_instof = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne)); - Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN); + Node* instof_false = generate_guard(bool_instof, nullptr, PROB_MIN); // for encryption, we are done if (!decrypting) - return instof_false; // even if it is NULL + return instof_false; // even if it is null // for decryption, we need to add a further check to avoid // taking the intrinsic path when cipher and plain are the same @@ -6246,7 +6331,7 @@ Node* LibraryCallKit::inline_cipherBlockChaining_AESCrypt_predicate(bool decrypt Node* cmp_src_dest = _gvn.transform(new CmpPNode(src, dest)); Node* bool_src_dest = _gvn.transform(new BoolNode(cmp_src_dest, BoolTest::eq)); - Node* src_dest_conjoint = generate_guard(bool_src_dest, NULL, PROB_MIN); + Node* src_dest_conjoint = generate_guard(bool_src_dest, nullptr, PROB_MIN); region->init_req(2, src_dest_conjoint); record_for_igvn(region); @@ -6263,7 +6348,7 @@ Node* LibraryCallKit::inline_cipherBlockChaining_AESCrypt_predicate(bool decrypt // note cipher==plain is more conservative than the original java code but that's OK // Node* LibraryCallKit::inline_electronicCodeBook_AESCrypt_predicate(bool decrypting) { - // The receiver was checked for NULL already. + // The receiver was checked for null already. Node* objECB = argument(0); // Load embeddedCipher field of ElectronicCodeBook object. @@ -6273,7 +6358,7 @@ Node* LibraryCallKit::inline_electronicCodeBook_AESCrypt_predicate(bool decrypti // AESCrypt might not be loaded yet if some other SymmetricCipher got us to this compile point // will have same classloader as ElectronicCodeBook object const TypeInstPtr* tinst = _gvn.type(objECB)->isa_instptr(); - assert(tinst != NULL, "ECBobj is null"); + assert(tinst != nullptr, "ECBobj is null"); assert(tinst->klass()->is_loaded(), "ECBobj is not loaded"); // we want to do an instanceof comparison against the AESCrypt class @@ -6290,11 +6375,11 @@ Node* LibraryCallKit::inline_electronicCodeBook_AESCrypt_predicate(bool decrypti Node* cmp_instof = _gvn.transform(new CmpINode(instof, intcon(1))); Node* bool_instof = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne)); - Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN); + Node* instof_false = generate_guard(bool_instof, nullptr, PROB_MIN); // for encryption, we are done if (!decrypting) - return instof_false; // even if it is NULL + return instof_false; // even if it is null // for decryption, we need to add a further check to avoid // taking the intrinsic path when cipher and plain are the same @@ -6305,7 +6390,7 @@ Node* LibraryCallKit::inline_electronicCodeBook_AESCrypt_predicate(bool decrypti Node* dest = argument(4); Node* cmp_src_dest = _gvn.transform(new CmpPNode(src, dest)); Node* bool_src_dest = _gvn.transform(new BoolNode(cmp_src_dest, BoolTest::eq)); - Node* src_dest_conjoint = generate_guard(bool_src_dest, NULL, PROB_MIN); + Node* src_dest_conjoint = generate_guard(bool_src_dest, nullptr, PROB_MIN); region->init_req(2, src_dest_conjoint); record_for_igvn(region); @@ -6323,7 +6408,7 @@ Node* LibraryCallKit::inline_electronicCodeBook_AESCrypt_predicate(bool decrypti // Node* LibraryCallKit::inline_counterMode_AESCrypt_predicate() { - // The receiver was checked for NULL already. + // The receiver was checked for null already. Node* objCTR = argument(0); // Load embeddedCipher field of CipherBlockChaining object. @@ -6333,7 +6418,7 @@ Node* LibraryCallKit::inline_counterMode_AESCrypt_predicate() { // AESCrypt might not be loaded yet if some other SymmetricCipher got us to this compile point // will have same classloader as CipherBlockChaining object const TypeInstPtr* tinst = _gvn.type(objCTR)->isa_instptr(); - assert(tinst != NULL, "CTRobj is null"); + assert(tinst != nullptr, "CTRobj is null"); assert(tinst->klass()->is_loaded(), "CTRobj is not loaded"); // we want to do an instanceof comparison against the AESCrypt class @@ -6349,9 +6434,9 @@ Node* LibraryCallKit::inline_counterMode_AESCrypt_predicate() { Node* instof = gen_instanceof(embeddedCipherObj, makecon(TypeKlassPtr::make(instklass_AESCrypt))); Node* cmp_instof = _gvn.transform(new CmpINode(instof, intcon(1))); Node* bool_instof = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne)); - Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN); + Node* instof_false = generate_guard(bool_instof, nullptr, PROB_MIN); - return instof_false; // even if it is NULL + return instof_false; // even if it is null } //------------------------------inline_ghash_processBlocks @@ -6374,11 +6459,11 @@ bool LibraryCallKit::inline_ghash_processBlocks() { data = must_be_not_null(data, true); Node* state_start = array_element_address(state, intcon(0), T_LONG); - assert(state_start, "state is NULL"); + assert(state_start, "state is null"); Node* subkeyH_start = array_element_address(subkeyH, intcon(0), T_LONG); - assert(subkeyH_start, "subkeyH is NULL"); + assert(subkeyH_start, "subkeyH is null"); Node* data_start = array_element_address(data, offset, T_BYTE); - assert(data_start, "data is NULL"); + assert(data_start, "data is null"); Node* ghash = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::ghash_processBlocks_Type(), @@ -6408,9 +6493,9 @@ bool LibraryCallKit::inline_base64_encodeBlock() { dest = must_be_not_null(dest, true); Node* src_start = array_element_address(src, intcon(0), T_BYTE); - assert(src_start, "source array is NULL"); + assert(src_start, "source array is null"); Node* dest_start = array_element_address(dest, intcon(0), T_BYTE); - assert(dest_start, "destination array is NULL"); + assert(dest_start, "destination array is null"); Node* base64 = make_runtime_call(RC_LEAF, OptoRuntime::base64_encodeBlock_Type(), @@ -6441,9 +6526,9 @@ bool LibraryCallKit::inline_base64_decodeBlock() { dest = must_be_not_null(dest, true); Node* src_start = array_element_address(src, intcon(0), T_BYTE); - assert(src_start, "source array is NULL"); + assert(src_start, "source array is null"); Node* dest_start = array_element_address(dest, intcon(0), T_BYTE); - assert(dest_start, "destination array is NULL"); + assert(dest_start, "destination array is null"); Node* call = make_runtime_call(RC_LEAF, OptoRuntime::base64_decodeBlock_Type(), @@ -6485,63 +6570,63 @@ bool LibraryCallKit::inline_digestBase_implCompress(vmIntrinsics::ID id) { return false; } // Figure out the size and type of the elements we will be copying. - BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); + BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type(); if (src_elem != T_BYTE) { return false; } // 'src_start' points to src array + offset src = must_be_not_null(src, true); Node* src_start = array_element_address(src, ofs, src_elem); - Node* state = NULL; - Node* digest_length = NULL; + Node* state = nullptr; + Node* digest_length = nullptr; address stubAddr; const char *stubName; switch(id) { case vmIntrinsics::_md5_implCompress: assert(UseMD5Intrinsics, "need MD5 instruction support"); - state = get_state_from_digest_object(digestBase_obj, "[I"); + state = get_state_from_digest_object(digestBase_obj, T_INT); stubAddr = StubRoutines::md5_implCompress(); stubName = "md5_implCompress"; break; case vmIntrinsics::_sha_implCompress: assert(UseSHA1Intrinsics, "need SHA1 instruction support"); - state = get_state_from_digest_object(digestBase_obj, "[I"); + state = get_state_from_digest_object(digestBase_obj, T_INT); stubAddr = StubRoutines::sha1_implCompress(); stubName = "sha1_implCompress"; break; case vmIntrinsics::_sha2_implCompress: assert(UseSHA256Intrinsics, "need SHA256 instruction support"); - state = get_state_from_digest_object(digestBase_obj, "[I"); + state = get_state_from_digest_object(digestBase_obj, T_INT); stubAddr = StubRoutines::sha256_implCompress(); stubName = "sha256_implCompress"; break; case vmIntrinsics::_sha5_implCompress: assert(UseSHA512Intrinsics, "need SHA512 instruction support"); - state = get_state_from_digest_object(digestBase_obj, "[J"); + state = get_state_from_digest_object(digestBase_obj, T_LONG); stubAddr = StubRoutines::sha512_implCompress(); stubName = "sha512_implCompress"; break; case vmIntrinsics::_sha3_implCompress: assert(UseSHA3Intrinsics, "need SHA3 instruction support"); - state = get_state_from_digest_object(digestBase_obj, "[B"); + state = get_state_from_digest_object(digestBase_obj, T_BYTE); stubAddr = StubRoutines::sha3_implCompress(); stubName = "sha3_implCompress"; digest_length = get_digest_length_from_digest_object(digestBase_obj); - if (digest_length == NULL) return false; + if (digest_length == nullptr) return false; break; default: fatal_unexpected_iid(id); return false; } - if (state == NULL) return false; + if (state == nullptr) return false; - assert(stubAddr != NULL, "Stub is generated"); - if (stubAddr == NULL) return false; + assert(stubAddr != nullptr, "Stub is generated"); + if (stubAddr == nullptr) return false; // Call the stub. Node* call; - if (digest_length == NULL) { + if (digest_length == nullptr) { call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::digestBase_implCompress_Type(false), stubAddr, stubName, TypePtr::BOTTOM, src_start, state); @@ -6565,7 +6650,7 @@ bool LibraryCallKit::inline_digestBase_implCompressMB(int predicate) { assert((uint)predicate < 5, "sanity"); assert(callee()->signature()->size() == 3, "digestBase_implCompressMB has 3 parameters"); - Node* digestBase_obj = argument(0); // The receiver was checked for NULL already. + Node* digestBase_obj = argument(0); // The receiver was checked for null already. Node* src = argument(1); // byte[] array Node* ofs = argument(2); // type int Node* limit = argument(3); // type int @@ -6577,7 +6662,7 @@ bool LibraryCallKit::inline_digestBase_implCompressMB(int predicate) { return false; } // Figure out the size and type of the elements we will be copying. - BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); + BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type(); if (src_elem != T_BYTE) { return false; } @@ -6585,10 +6670,10 @@ bool LibraryCallKit::inline_digestBase_implCompressMB(int predicate) { src = must_be_not_null(src, false); Node* src_start = array_element_address(src, ofs, src_elem); - const char* klass_digestBase_name = NULL; - const char* stub_name = NULL; - address stub_addr = NULL; - const char* state_type = "[I"; + const char* klass_digestBase_name = nullptr; + const char* stub_name = nullptr; + address stub_addr = nullptr; + BasicType elem_type = T_INT; switch (predicate) { case 0: @@ -6617,7 +6702,7 @@ bool LibraryCallKit::inline_digestBase_implCompressMB(int predicate) { klass_digestBase_name = "sun/security/provider/SHA5"; stub_name = "sha512_implCompressMB"; stub_addr = StubRoutines::sha512_implCompressMB(); - state_type = "[J"; + elem_type = T_LONG; } break; case 4: @@ -6625,50 +6710,50 @@ bool LibraryCallKit::inline_digestBase_implCompressMB(int predicate) { klass_digestBase_name = "sun/security/provider/SHA3"; stub_name = "sha3_implCompressMB"; stub_addr = StubRoutines::sha3_implCompressMB(); - state_type = "[B"; + elem_type = T_BYTE; } break; default: fatal("unknown DigestBase intrinsic predicate: %d", predicate); } - if (klass_digestBase_name != NULL) { - assert(stub_addr != NULL, "Stub is generated"); - if (stub_addr == NULL) return false; + if (klass_digestBase_name != nullptr) { + assert(stub_addr != nullptr, "Stub is generated"); + if (stub_addr == nullptr) return false; // get DigestBase klass to lookup for SHA klass const TypeInstPtr* tinst = _gvn.type(digestBase_obj)->isa_instptr(); - assert(tinst != NULL, "digestBase_obj is not instance???"); + assert(tinst != nullptr, "digestBase_obj is not instance???"); assert(tinst->klass()->is_loaded(), "DigestBase is not loaded"); ciKlass* klass_digestBase = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make(klass_digestBase_name)); assert(klass_digestBase->is_loaded(), "predicate checks that this class is loaded"); ciInstanceKlass* instklass_digestBase = klass_digestBase->as_instance_klass(); - return inline_digestBase_implCompressMB(digestBase_obj, instklass_digestBase, state_type, stub_addr, stub_name, src_start, ofs, limit); + return inline_digestBase_implCompressMB(digestBase_obj, instklass_digestBase, elem_type, stub_addr, stub_name, src_start, ofs, limit); } return false; } //------------------------------inline_digestBase_implCompressMB----------------------- bool LibraryCallKit::inline_digestBase_implCompressMB(Node* digestBase_obj, ciInstanceKlass* instklass_digestBase, - const char* state_type, address stubAddr, const char *stubName, + BasicType elem_type, address stubAddr, const char *stubName, Node* src_start, Node* ofs, Node* limit) { const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_digestBase); const TypeOopPtr* xtype = aklass->as_instance_type(); Node* digest_obj = new CheckCastPPNode(control(), digestBase_obj, xtype); digest_obj = _gvn.transform(digest_obj); - Node* state = get_state_from_digest_object(digest_obj, state_type); - if (state == NULL) return false; + Node* state = get_state_from_digest_object(digest_obj, elem_type); + if (state == nullptr) return false; - Node* digest_length = NULL; + Node* digest_length = nullptr; if (strcmp("sha3_implCompressMB", stubName) == 0) { digest_length = get_digest_length_from_digest_object(digest_obj); - if (digest_length == NULL) return false; + if (digest_length == nullptr) return false; } // Call the stub. Node* call; - if (digest_length == NULL) { + if (digest_length == nullptr) { call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::digestBase_implCompressMB_Type(false), stubAddr, stubName, TypePtr::BOTTOM, @@ -6688,20 +6773,27 @@ bool LibraryCallKit::inline_digestBase_implCompressMB(Node* digestBase_obj, ciIn } //------------------------------get_state_from_digest_object----------------------- -Node * LibraryCallKit::get_state_from_digest_object(Node *digest_object, const char *state_type) { +Node * LibraryCallKit::get_state_from_digest_object(Node *digest_object, BasicType elem_type) { + const char* state_type; + switch (elem_type) { + case T_BYTE: state_type = "[B"; break; + case T_INT: state_type = "[I"; break; + case T_LONG: state_type = "[J"; break; + default: ShouldNotReachHere(); + } Node* digest_state = load_field_from_object(digest_object, "state", state_type); - assert (digest_state != NULL, "wrong version of sun.security.provider.MD5/SHA/SHA2/SHA5/SHA3"); - if (digest_state == NULL) return (Node *) NULL; + assert (digest_state != nullptr, "wrong version of sun.security.provider.MD5/SHA/SHA2/SHA5/SHA3"); + if (digest_state == nullptr) return (Node *) nullptr; // now have the array, need to get the start address of the state array - Node* state = array_element_address(digest_state, intcon(0), T_INT); + Node* state = array_element_address(digest_state, intcon(0), elem_type); return state; } //------------------------------get_digest_length_from_sha3_object---------------------------------- Node * LibraryCallKit::get_digest_length_from_digest_object(Node *digest_object) { Node* digest_length = load_field_from_object(digest_object, "digestLength", "I"); - assert (digest_length != NULL, "sanity"); + assert (digest_length != nullptr, "sanity"); return digest_length; } @@ -6715,15 +6807,15 @@ Node* LibraryCallKit::inline_digestBase_implCompressMB_predicate(int predicate) "need MD5/SHA1/SHA256/SHA512/SHA3 instruction support"); assert((uint)predicate < 5, "sanity"); - // The receiver was checked for NULL already. + // The receiver was checked for null already. Node* digestBaseObj = argument(0); // get DigestBase klass for instanceOf check const TypeInstPtr* tinst = _gvn.type(digestBaseObj)->isa_instptr(); - assert(tinst != NULL, "digestBaseObj is null"); + assert(tinst != nullptr, "digestBaseObj is null"); assert(tinst->klass()->is_loaded(), "DigestBase is not loaded"); - const char* klass_name = NULL; + const char* klass_name = nullptr; switch (predicate) { case 0: if (UseMD5Intrinsics) { @@ -6759,11 +6851,11 @@ Node* LibraryCallKit::inline_digestBase_implCompressMB_predicate(int predicate) fatal("unknown SHA intrinsic predicate: %d", predicate); } - ciKlass* klass = NULL; - if (klass_name != NULL) { + ciKlass* klass = nullptr; + if (klass_name != nullptr) { klass = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make(klass_name)); } - if ((klass == NULL) || !klass->is_loaded()) { + if ((klass == nullptr) || !klass->is_loaded()) { // if none of MD5/SHA/SHA2/SHA5 is loaded, we never take the intrinsic fast path Node* ctrl = control(); set_control(top()); // no intrinsic path @@ -6774,17 +6866,17 @@ Node* LibraryCallKit::inline_digestBase_implCompressMB_predicate(int predicate) Node* instof = gen_instanceof(digestBaseObj, makecon(TypeKlassPtr::make(instklass))); Node* cmp_instof = _gvn.transform(new CmpINode(instof, intcon(1))); Node* bool_instof = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne)); - Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN); + Node* instof_false = generate_guard(bool_instof, nullptr, PROB_MIN); - return instof_false; // even if it is NULL + return instof_false; // even if it is null } //-------------inline_fma----------------------------------- bool LibraryCallKit::inline_fma(vmIntrinsics::ID id) { - Node *a = NULL; - Node *b = NULL; - Node *c = NULL; - Node* result = NULL; + Node *a = nullptr; + Node *b = nullptr; + Node *c = nullptr; + Node* result = nullptr; switch (id) { case vmIntrinsics::_fmaD: assert(callee()->signature()->size() == 6, "fma has 3 parameters of size 2 each."); @@ -6811,7 +6903,7 @@ bool LibraryCallKit::inline_fma(vmIntrinsics::ID id) { bool LibraryCallKit::inline_character_compare(vmIntrinsics::ID id) { // argument(0) is receiver Node* codePoint = argument(1); - Node* n = NULL; + Node* n = nullptr; switch (id) { case vmIntrinsics::_isDigit : @@ -6844,7 +6936,7 @@ bool LibraryCallKit::inline_fp_min_max(vmIntrinsics::ID id) { ciMethodData *md = callee()->method_data(); - if ( md != NULL && md->is_mature() && md->invocation_count() > 0 ) { + if ( md != nullptr && md->is_mature() && md->invocation_count() > 0 ) { ciCallProfile cp = caller()->call_profile_at_bci(bci()); if ( ((double)cp.count()) / ((double)md->invocation_count()) < 0.8 ) { @@ -6870,9 +6962,9 @@ bool LibraryCallKit::inline_fp_min_max(vmIntrinsics::ID id) { } */ - Node *a = NULL; - Node *b = NULL; - Node *n = NULL; + Node *a = nullptr; + Node *b = nullptr; + Node *n = nullptr; switch (id) { case vmIntrinsics::_maxF: case vmIntrinsics::_minF: @@ -6903,17 +6995,17 @@ bool LibraryCallKit::inline_fp_min_max(vmIntrinsics::ID id) { bool LibraryCallKit::inline_profileBoolean() { Node* counts = argument(1); - const TypeAryPtr* ary = NULL; - ciArray* aobj = NULL; + const TypeAryPtr* ary = nullptr; + ciArray* aobj = nullptr; if (counts->is_Con() - && (ary = counts->bottom_type()->isa_aryptr()) != NULL - && (aobj = ary->const_oop()->as_array()) != NULL + && (ary = counts->bottom_type()->isa_aryptr()) != nullptr + && (aobj = ary->const_oop()->as_array()) != nullptr && (aobj->length() == 2)) { // Profile is int[2] where [0] and [1] correspond to false and true value occurrences respectively. jint false_cnt = aobj->element_value(0).as_int(); jint true_cnt = aobj->element_value(1).as_int(); - if (C->log() != NULL) { + if (C->log() != nullptr) { C->log()->elem("observe source='profileBoolean' false='%d' true='%d'", false_cnt, true_cnt); } @@ -6994,7 +7086,7 @@ bool LibraryCallKit::inline_getObjectSize() { jint layout_con = Klass::_lh_neutral_value; Node* layout_val = get_layout_helper(klass_node, layout_con); - int layout_is_con = (layout_val == NULL); + int layout_is_con = (layout_val == nullptr); if (layout_is_con) { // Layout helper is constant, can figure out things at compile time. @@ -7042,8 +7134,8 @@ bool LibraryCallKit::inline_getObjectSize() { PhiNode* result_val = new PhiNode(result_reg, TypeLong::LONG); record_for_igvn(result_reg); - Node* array_ctl = generate_array_guard(klass_node, NULL); - if (array_ctl != NULL) { + Node* array_ctl = generate_array_guard(klass_node, nullptr); + if (array_ctl != nullptr) { // Array case: size is round(header + element_size*arraylength). // Since arraylength is different for every array instance, we have to // compute the whole thing at runtime. @@ -7109,8 +7201,15 @@ bool LibraryCallKit::inline_blackhole() { assert(callee()->is_empty(), "Should have been checked before: only empty methods here"); assert(callee()->holder()->is_loaded(), "Should have been checked before: only methods for loaded classes here"); + // Blackhole node pinches only the control, not memory. This allows + // the blackhole to be pinned in the loop that computes blackholed + // values, but have no other side effects, like breaking the optimizations + // across the blackhole. + + Node* bh = _gvn.transform(new BlackholeNode(control())); + set_control(_gvn.transform(new ProjNode(bh, TypeFunc::Control))); + // Bind call arguments as blackhole arguments to keep them alive - Node* bh = insert_mem_bar(Op_Blackhole); uint nargs = callee()->arg_size(); for (uint i = 0; i < nargs; i++) { bh->add_req(argument(i)); diff --git a/src/hotspot/share/opto/library_call.hpp b/src/hotspot/share/opto/library_call.hpp index 15e2de48becb1..01d13fcad4b32 100644 --- a/src/hotspot/share/opto/library_call.hpp +++ b/src/hotspot/share/opto/library_call.hpp @@ -75,7 +75,7 @@ class LibraryCallKit : public GraphKit { LibraryCallKit(JVMState* jvms, LibraryIntrinsic* intrinsic) : GraphKit(jvms), _intrinsic(intrinsic), - _result(NULL) + _result(nullptr) { // Check if this is a root compile. In that case we don't have a caller. if (!jvms->has_method()) { @@ -85,7 +85,7 @@ class LibraryCallKit : public GraphKit { // and save the stack pointer value so it can used by uncommon_trap. // We find the argument count by looking at the declared signature. bool ignored_will_link; - ciSignature* declared_signature = NULL; + ciSignature* declared_signature = nullptr; ciMethod* ignored_callee = caller()->get_method_at_bci(bci(), ignored_will_link, &declared_signature); const int nargs = declared_signature->arg_size_for_bc(caller()->java_code_at_bci(bci())); _reexecute_sp = sp() + nargs; // "push" arguments back on stack @@ -105,7 +105,7 @@ class LibraryCallKit : public GraphKit { void push_result() { // Push the result onto the stack. - if (!stopped() && result() != NULL) { + if (!stopped() && result() != nullptr) { BasicType bt = result()->bottom_type()->basic_type(); push_node(bt, result()); } @@ -116,7 +116,7 @@ class LibraryCallKit : public GraphKit { fatal("unexpected intrinsic %d: %s", vmIntrinsics::as_int(iid), vmIntrinsics::name_at(iid)); } - void set_result(Node* n) { assert(_result == NULL, "only set once"); _result = n; } + void set_result(Node* n) { assert(_result == nullptr, "only set once"); _result = n; } void set_result(RegionNode* region, PhiNode* value); Node* result() { return _result; } @@ -128,7 +128,7 @@ class LibraryCallKit : public GraphKit { Node* generate_fair_guard(Node* test, RegionNode* region); Node* generate_negative_guard(Node* index, RegionNode* region, // resulting CastII of index: - Node* *pos_index = NULL); + Node* *pos_index = nullptr); Node* generate_limit_guard(Node* offset, Node* subseq_length, Node* array_length, RegionNode* region); @@ -173,13 +173,9 @@ class LibraryCallKit : public GraphKit { Node* generate_array_guard_common(Node* kls, RegionNode* region, bool obj_array, bool not_array); Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region); - CallJavaNode* generate_method_call(vmIntrinsics::ID method_id, - bool is_virtual = false, bool is_static = false); - CallJavaNode* generate_method_call_static(vmIntrinsics::ID method_id) { - return generate_method_call(method_id, false, true); - } - CallJavaNode* generate_method_call_virtual(vmIntrinsics::ID method_id) { - return generate_method_call(method_id, true, false); + CallJavaNode* generate_method_call(vmIntrinsicID method_id, bool is_virtual, bool is_static, bool res_not_null); + CallJavaNode* generate_method_call_static(vmIntrinsicID method_id, bool res_not_null) { + return generate_method_call(method_id, false, true, res_not_null); } Node* load_field_from_object(Node* fromObj, const char* fieldName, const char* fieldTypeString, DecoratorSet decorators, bool is_static, ciInstanceKlass* fromKls); Node* field_address_from_object(Node* fromObj, const char* fieldName, const char* fieldTypeString, bool is_exact, bool is_static, ciInstanceKlass* fromKls); @@ -253,8 +249,13 @@ class LibraryCallKit : public GraphKit { // Helper functions for inlining arraycopy bool inline_arraycopy(); AllocateArrayNode* tightly_coupled_allocation(Node* ptr); + static CallStaticJavaNode* get_uncommon_trap_from_success_proj(Node* node); + SafePointNode* create_safepoint_with_state_before_array_allocation(const AllocateArrayNode* alloc) const; + void replace_unrelated_uncommon_traps_with_alloc_state(AllocateArrayNode* alloc, JVMState* saved_jvms_before_guards); + void replace_unrelated_uncommon_traps_with_alloc_state(JVMState* saved_jvms_before_guards); + void create_new_uncommon_trap(CallStaticJavaNode* uncommon_trap_call); JVMState* arraycopy_restore_alloc_state(AllocateArrayNode* alloc, int& saved_reexecute_sp); - void arraycopy_move_allocation_here(AllocateArrayNode* alloc, Node* dest, JVMState* saved_jvms, int saved_reexecute_sp, + void arraycopy_move_allocation_here(AllocateArrayNode* alloc, Node* dest, JVMState* saved_jvms_before_guards, int saved_reexecute_sp, uint new_idx); typedef enum { LS_get_add, LS_get_set, LS_cmp_swap, LS_cmp_swap_weak, LS_cmp_exchange } LoadStoreKind; @@ -280,9 +281,9 @@ class LibraryCallKit : public GraphKit { bool inline_digestBase_implCompress(vmIntrinsics::ID id); bool inline_digestBase_implCompressMB(int predicate); bool inline_digestBase_implCompressMB(Node* digestBaseObj, ciInstanceKlass* instklass, - const char* state_type, address stubAddr, const char *stubName, + BasicType elem_type, address stubAddr, const char *stubName, Node* src_start, Node* ofs, Node* limit); - Node* get_state_from_digest_object(Node *digestBase_object, const char* state_type); + Node* get_state_from_digest_object(Node *digestBase_object, BasicType elem_type); Node* get_digest_length_from_digest_object(Node *digestBase_object); Node* inline_digestBase_implCompressMB_predicate(int predicate); bool inline_encodeISOArray(bool ascii); diff --git a/src/hotspot/share/opto/live.cpp b/src/hotspot/share/opto/live.cpp index 7101994b4e3c0..ea1722702cedf 100644 --- a/src/hotspot/share/opto/live.cpp +++ b/src/hotspot/share/opto/live.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -86,7 +86,7 @@ void PhaseLive::compute(uint maxlrg) { _deltas = NEW_RESOURCE_ARRAY(IndexSet*,_cfg.number_of_blocks()); memset(_deltas, 0, sizeof(IndexSet*)* _cfg.number_of_blocks()); - _free_IndexSet = NULL; + _free_IndexSet = nullptr; // Blocks having done pass-1 VectorSet first_pass; @@ -176,7 +176,7 @@ void PhaseLive::compute(uint maxlrg) { } } IndexSet *free = _free_IndexSet; - while (free != NULL) { + while (free != nullptr) { IndexSet *temp = free; free = free->next(); temp->clear(); @@ -223,7 +223,7 @@ void PhaseLive::freeset(Block *p) { } f->set_next(_free_IndexSet); _free_IndexSet = f; // Drop onto free list - _deltas[p->_pre_order-1] = NULL; + _deltas[p->_pre_order-1] = nullptr; } // Add a live-out value to a given blocks live-out set. If it is new, then diff --git a/src/hotspot/share/opto/live.hpp b/src/hotspot/share/opto/live.hpp index 8512a1f85f493..d5ff1570fd135 100644 --- a/src/hotspot/share/opto/live.hpp +++ b/src/hotspot/share/opto/live.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -78,7 +78,7 @@ class PhaseLive : public Phase { // Compute liveness info void compute(uint maxlrg); // Reset arena storage - void reset() { _live = NULL; } + void reset() { _live = nullptr; } // Return the live-out set for this block IndexSet *live( const Block * b ) { return &_live[b->_pre_order-1]; } diff --git a/src/hotspot/share/opto/locknode.cpp b/src/hotspot/share/opto/locknode.cpp index d2fd92ca7fd0e..f5fe99d170620 100644 --- a/src/hotspot/share/opto/locknode.cpp +++ b/src/hotspot/share/opto/locknode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -86,7 +86,7 @@ OptoReg::Name BoxLockNode::reg(Node* box) { // Is BoxLock node used for one simple lock region (same box and obj)? bool BoxLockNode::is_simple_lock_region(LockNode** unique_lock, Node* obj, Node** bad_lock) { - LockNode* lock = NULL; + LockNode* lock = nullptr; bool has_one_lock = false; for (uint i = 0; i < this->outcnt(); i++) { Node* n = this->raw_out(i); @@ -96,19 +96,19 @@ bool BoxLockNode::is_simple_lock_region(LockNode** unique_lock, Node* obj, Node* // Check lock's box since box could be referenced by Lock's debug info. if (alock->box_node() == this) { if (alock->obj_node()->eqv_uncast(obj)) { - if ((unique_lock != NULL) && alock->is_Lock()) { - if (lock == NULL) { + if ((unique_lock != nullptr) && alock->is_Lock()) { + if (lock == nullptr) { lock = alock->as_Lock(); has_one_lock = true; } else if (lock != alock->as_Lock()) { has_one_lock = false; - if (bad_lock != NULL) { + if (bad_lock != nullptr) { *bad_lock = alock; } } } } else { - if (bad_lock != NULL) { + if (bad_lock != nullptr) { *bad_lock = alock; } return false; // Different objects @@ -132,7 +132,7 @@ bool BoxLockNode::is_simple_lock_region(LockNode** unique_lock, Node* obj, Node* // unlocks are reference only this one object. } #endif - if (unique_lock != NULL && has_one_lock) { + if (unique_lock != nullptr && has_one_lock) { *unique_lock = lock; } return true; diff --git a/src/hotspot/share/opto/locknode.hpp b/src/hotspot/share/opto/locknode.hpp index f95f7d1acb6e1..61f344abd6803 100644 --- a/src/hotspot/share/opto/locknode.hpp +++ b/src/hotspot/share/opto/locknode.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -83,8 +83,8 @@ class FastLockNode: public CmpNode { init_req(0,ctrl); init_class_id(Class_FastLock); _counters = NULL; - _rtm_counters = NULL; - _stack_rtm_counters = NULL; + _rtm_counters = nullptr; + _stack_rtm_counters = nullptr; } Node* obj_node() const { return in(1); } Node* box_node() const { return in(2); } diff --git a/src/hotspot/share/opto/loopPredicate.cpp b/src/hotspot/share/opto/loopPredicate.cpp index 9ae9184011651..0a189bf492954 100644 --- a/src/hotspot/share/opto/loopPredicate.cpp +++ b/src/hotspot/share/opto/loopPredicate.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -65,7 +65,7 @@ void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred, } set_loop(n, loop); // When called from beautify_loops() idom is not constructed yet. - if (_idom != NULL) { + if (_idom != nullptr) { set_idom(n, pred, dom_depth(pred)); } } @@ -131,7 +131,7 @@ ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node* register_control(rgn, loop, uncommon_proj); _igvn.replace_input_of(call, 0, rgn); // When called from beautify_loops() idom is not constructed yet. - if (_idom != NULL) { + if (_idom != nullptr) { set_idom(call, rgn, dom_depth(rgn)); } // Move nodes pinned on the projection or whose control is set to @@ -145,13 +145,13 @@ ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node* } Node* entry = iff->in(0); - if (new_entry != NULL) { + if (new_entry != nullptr) { // Clonning the predicate to new location. entry = new_entry; } // Create new_iff IdealLoopTree* lp = get_loop(entry); - IfNode* new_iff = NULL; + IfNode* new_iff = nullptr; if (opcode == Op_If) { new_iff = new IfNode(entry, iff->in(1), iff->_prob, iff->_fcnt); } else { @@ -180,7 +180,7 @@ ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node* _igvn.hash_delete(rgn); rgn->add_req(if_uct); // When called from beautify_loops() idom is not constructed yet. - if (_idom != NULL) { + if (_idom != nullptr) { Node* ridom = idom(rgn); Node* nrdom = dom_lca_internal(ridom, new_iff); set_idom(rgn, nrdom, dom_depth(rgn)); @@ -216,10 +216,10 @@ ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node* } assert(!has_phi || rgn->req() > 3, "no phis when region is created"); - if (new_entry == NULL) { + if (new_entry == nullptr) { // Attach if_cont to iff _igvn.replace_input_of(iff, 0, if_cont); - if (_idom != NULL) { + if (_idom != nullptr) { set_idom(iff, if_cont, dom_depth(iff)); } } @@ -263,7 +263,7 @@ Node* PhaseIdealLoop::clone_nodes_with_same_ctrl(Node* node, ProjNode* old_ctrl, Dict old_new_mapping = clone_nodes(nodes_with_same_ctrl); // Cloned but not rewired, yet rewire_cloned_nodes_to_ctrl(old_ctrl, new_ctrl, nodes_with_same_ctrl, old_new_mapping); Node* clone_phi_input = static_cast(old_new_mapping[node]); - assert(clone_phi_input != NULL && clone_phi_input->_idx >= last_idx, "must exist and be a proper clone"); + assert(clone_phi_input != nullptr && clone_phi_input->_idx >= last_idx, "must exist and be a proper clone"); return clone_phi_input; } @@ -304,7 +304,7 @@ void PhaseIdealLoop::rewire_inputs_of_clones_to_clones(Node* new_ctrl, Node* clo if (!in->is_Phi()) { assert(!in->is_CFG(), "must be data node"); Node* in_clone = static_cast(old_new_mapping[in]); - if (in_clone != NULL) { + if (in_clone != nullptr) { _igvn.replace_input_of(clone, i, in_clone); set_ctrl(clone, new_ctrl); } @@ -390,7 +390,7 @@ void PhaseIdealLoop::get_skeleton_predicates(Node* predicate, Unique_Node_List& assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct"); assert(iff->in(1)->in(1)->Opcode() == Op_Opaque1, "unexpected predicate shape"); predicate = iff->in(0); - while (predicate != NULL && predicate->is_Proj() && predicate->in(0)->is_If()) { + while (predicate != nullptr && predicate->is_Proj() && predicate->in(0)->is_If()) { iff = predicate->in(0)->as_If(); uncommon_proj = iff->proj_out(1 - predicate->as_Proj()->_con); if (uncommon_proj->unique_ctrl_out() != rgn) { @@ -415,8 +415,8 @@ void PhaseIdealLoop::get_skeleton_predicates(Node* predicate, Unique_Node_List& ProjNode* PhaseIdealLoop::clone_skeleton_predicate_for_unswitched_loops(Node* iff, ProjNode* predicate, Deoptimization::DeoptReason reason, ProjNode* output_proj) { - Node* bol = clone_skeleton_predicate_bool(iff, NULL, NULL, output_proj); - ProjNode* proj = create_new_if_for_predicate(output_proj, NULL, reason, iff->Opcode(), + Node* bol = clone_skeleton_predicate_bool(iff, nullptr, nullptr, output_proj); + ProjNode* proj = create_new_if_for_predicate(output_proj, nullptr, reason, iff->Opcode(), false, predicate->is_IfTrue()); _igvn.replace_input_of(proj->in(0), 1, bol); _igvn.replace_input_of(output_proj->in(0), 0, proj); @@ -432,23 +432,23 @@ void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, No Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl); // Search original predicates - ProjNode* limit_check_proj = NULL; + ProjNode* limit_check_proj = nullptr; limit_check_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); - if (limit_check_proj != NULL) { + if (limit_check_proj != nullptr) { entry = skip_loop_predicates(entry); } - ProjNode* profile_predicate_proj = NULL; - ProjNode* predicate_proj = NULL; + ProjNode* profile_predicate_proj = nullptr; + ProjNode* predicate_proj = nullptr; if (UseProfiledLoopPredicate) { profile_predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate); - if (profile_predicate_proj != NULL) { + if (profile_predicate_proj != nullptr) { entry = skip_loop_predicates(entry); } } if (UseLoopPredicate) { predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); } - if (predicate_proj != NULL) { // right pattern that can be used by loop predication + if (predicate_proj != nullptr) { // right pattern that can be used by loop predication // clone predicate iffast_pred = clone_predicate_to_unswitched_loop(predicate_proj, iffast_pred, Deoptimization::Reason_predicate,false); ifslow_pred = clone_predicate_to_unswitched_loop(predicate_proj, ifslow_pred, Deoptimization::Reason_predicate,true); @@ -457,7 +457,7 @@ void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, No check_created_predicate_for_unswitching(iffast_pred); check_created_predicate_for_unswitching(ifslow_pred); } - if (profile_predicate_proj != NULL) { // right pattern that can be used by loop predication + if (profile_predicate_proj != nullptr) { // right pattern that can be used by loop predication // clone predicate iffast_pred = clone_predicate_to_unswitched_loop(profile_predicate_proj, iffast_pred,Deoptimization::Reason_profile_predicate, false); ifslow_pred = clone_predicate_to_unswitched_loop(profile_predicate_proj, ifslow_pred,Deoptimization::Reason_profile_predicate, true); @@ -466,7 +466,7 @@ void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, No check_created_predicate_for_unswitching(iffast_pred); check_created_predicate_for_unswitching(ifslow_pred); } - if (limit_check_proj != NULL && clone_limit_check) { + if (limit_check_proj != nullptr && clone_limit_check) { // Clone loop limit check last to insert it before loop. // Don't clone a limit check which was already finalized // for this counted loop (only one limit check is needed). @@ -480,7 +480,7 @@ void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, No #ifndef PRODUCT void PhaseIdealLoop::check_created_predicate_for_unswitching(const Node* new_entry) { - assert(new_entry != NULL, "IfTrue or IfFalse after clone predicate"); + assert(new_entry != nullptr, "IfTrue or IfFalse after clone predicate"); if (TraceLoopPredicate) { tty->print("Loop Predicate cloned: "); debug_only(new_entry->in(0)->dump();); @@ -497,7 +497,7 @@ Node* PhaseIdealLoop::skip_loop_predicates(Node* entry) { Node* rgn = uncommon_proj->unique_ctrl_out(); assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct"); entry = entry->in(0)->in(0); - while (entry != NULL && entry->is_Proj() && entry->in(0)->is_If()) { + while (entry != nullptr && entry->is_Proj() && entry->in(0)->is_If()) { uncommon_proj = entry->in(0)->as_If()->proj_out(1 - entry->as_Proj()->_con); if (uncommon_proj->unique_ctrl_out() != rgn) break; @@ -530,35 +530,35 @@ Node* PhaseIdealLoop::skip_all_loop_predicates(Node* entry) { //--------------------------find_predicate_insertion_point------------------- // Find a good location to insert a predicate ProjNode* PhaseIdealLoop::find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason) { - if (start_c == NULL || !start_c->is_Proj()) - return NULL; + if (start_c == nullptr || !start_c->is_Proj()) + return nullptr; if (start_c->as_Proj()->is_uncommon_trap_if_pattern(reason)) { return start_c->as_Proj(); } - return NULL; + return nullptr; } //--------------------------find_predicate------------------------------------ // Find a predicate Node* PhaseIdealLoop::find_predicate(Node* entry) { - Node* predicate = NULL; + Node* predicate = nullptr; predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); - if (predicate != NULL) { // right pattern that can be used by loop predication + if (predicate != nullptr) { // right pattern that can be used by loop predication return entry; } if (UseLoopPredicate) { predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); - if (predicate != NULL) { // right pattern that can be used by loop predication + if (predicate != nullptr) { // right pattern that can be used by loop predication return entry; } } if (UseProfiledLoopPredicate) { predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate); - if (predicate != NULL) { // right pattern that can be used by loop predication + if (predicate != nullptr) { // right pattern that can be used by loop predication return entry; } } - return NULL; + return nullptr; } //------------------------------Invariance----------------------------------- @@ -571,7 +571,7 @@ class Invariance : public StackObj { Node_List _old_new; // map of old to new (clone) IdealLoopTree* _lpt; PhaseIdealLoop* _phase; - Node* _data_dependency_on; // The projection into the loop on which data nodes are dependent or NULL otherwise + Node* _data_dependency_on; // The projection into the loop on which data nodes are dependent or null otherwise // Helper function to set up the invariance for invariance computation // If n is a known invariant, set up directly. Otherwise, look up the @@ -583,7 +583,7 @@ class Invariance : public StackObj { Node *n_ctrl = _phase->ctrl_or_self(n); Node *u_ctrl = _phase->ctrl_or_self(use); // self if use is a CFG if (_phase->is_dominator(n_ctrl, u_ctrl)) { - _stack.push(n, n->in(0) == NULL ? 1 : 0); + _stack.push(n, n->in(0) == nullptr ? 1 : 0); } } } @@ -602,7 +602,7 @@ class Invariance : public StackObj { bool all_inputs_invariant = true; for (uint i = 0; i < n->req(); i++) { Node* in = n->in(i); - if (in == NULL) continue; + if (in == nullptr) continue; assert(_visited.test(in->_idx), "must have visited input"); if (!_invariant.test(in->_idx)) { // bad guy all_inputs_invariant = false; @@ -614,14 +614,14 @@ class Invariance : public StackObj { // loop, it was marked invariant but n is only invariant if // it depends only on that test. Otherwise, unless that test // is out of the loop, it's not invariant. - if (n->is_CFG() || n->depends_only_on_test() || n->in(0) == NULL || !_phase->is_member(_lpt, n->in(0))) { + if (n->is_CFG() || n->depends_only_on_test() || n->in(0) == nullptr || !_phase->is_member(_lpt, n->in(0))) { _invariant.set(n->_idx); // I am a invariant too } } } else { // process next input _stack.set_index(idx + 1); Node* m = n->in(idx); - if (m != NULL && !_visited.test_set(m->_idx)) { + if (m != nullptr && !_visited.test_set(m->_idx)) { visit(n, m); } } @@ -637,7 +637,7 @@ class Invariance : public StackObj { _old_new.map(n->_idx, n); } else { // to be cloned assert(!n->is_CFG(), "should not see CFG here"); - _stack.push(n, n->in(0) == NULL ? 1 : 0); + _stack.push(n, n->in(0) == nullptr ? 1 : 0); } } @@ -655,13 +655,13 @@ class Invariance : public StackObj { _phase->register_new_node(n_cl, ctrl); for (uint i = 0; i < n->req(); i++) { Node* in = n_cl->in(i); - if (in == NULL) continue; + if (in == nullptr) continue; n_cl->set_req(i, _old_new[in->_idx]); } } else { // process next input _stack.set_index(idx + 1); Node* m = n->in(idx); - if (m != NULL && !_clone_visited.test_set(m->_idx)) { + if (m != nullptr && !_clone_visited.test_set(m->_idx)) { clone_visit(m); // visit the input } } @@ -674,7 +674,7 @@ class Invariance : public StackObj { _stack(area, 10 /* guess */), _clone_visited(area), _old_new(area), _lpt(lpt), _phase(lpt->_phase), - _data_dependency_on(NULL) + _data_dependency_on(nullptr) { LoopNode* head = _lpt->_head->as_Loop(); Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl); @@ -707,7 +707,7 @@ class Invariance : public StackObj { } // Did we explicitly mark some nodes non-loop-invariant? If so, return the entry node on which some data nodes - // are dependent that prevent loop predication. Otherwise, return NULL. + // are dependent that prevent loop predication. Otherwise, return null. Node* data_dependency_on() { return _data_dependency_on; } @@ -741,7 +741,8 @@ class Invariance : public StackObj { // Returns true if the predicate of iff is in "scale*iv + offset u< load_range(ptr)" format // Note: this function is particularly designed for loop predication. We require load_range // and offset to be loop invariant computed on the fly by "invar" -bool IdealLoopTree::is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invariance& invar DEBUG_ONLY(COMMA ProjNode *predicate_proj)) const { +bool IdealLoopTree::is_range_check_if(IfProjNode *if_success_proj, PhaseIdealLoop *phase, Invariance& invar DEBUG_ONLY(COMMA ProjNode *predicate_proj)) const { + IfNode* iff = if_success_proj->in(0)->as_If(); if (!is_loop_exit(iff)) { return false; } @@ -749,7 +750,43 @@ bool IdealLoopTree::is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invari return false; } const BoolNode *bol = iff->in(1)->as_Bool(); - if (bol->_test._test != BoolTest::lt) { + if (bol->_test._test != BoolTest::lt || if_success_proj->is_IfFalse()) { + // We don't have the required range check pattern: + // if (scale*iv + offset =u limit) { + // + // } else { + // trap(); + // } + // + // If we create a Hoisted Range Check Predicate for this wrong pattern, it could succeed at runtime (i.e. true + // for the value of "scale*iv + offset" in the first loop iteration and true for the value of "scale*iv + offset" + // in the last loop iteration) while the check to be hoisted could fail in other loop iterations. + // + // Example: + // Loop: "for (int i = -1; i < 1000; i++)" + // init = "scale*iv + offset" in the first loop iteration = 1*-1 + 0 = -1 + // last = "scale*iv + offset" in the last loop iteration = 1*999 + 0 = 999 + // limit = 100 + // + // Hoisted Range Check Predicate is always true: + // init >=u limit && last >=u limit <=> + // -1 >=u 100 && 999 >= u 100 + // + // But for 0 <= x < 100: x >=u 100 is false. + // We would wrongly skip the branch with the trap() and possibly miss to execute some other statements inside that + // trap() branch. return false; } if (!bol->in(1)->is_Cmp()) { @@ -762,7 +799,7 @@ bool IdealLoopTree::is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invari Node* range = cmp->in(2); if (range->Opcode() != Op_LoadRange && !iff->is_RangeCheck()) { const TypeInt* tint = phase->_igvn.type(range)->isa_int(); - if (tint == NULL || tint->empty() || tint->_lo < 0) { + if (tint == nullptr || tint->empty() || tint->_lo < 0) { // Allow predication on positive values that aren't LoadRanges. // This allows optimization of loops where the length of the // array is a known value and doesn't need to be loaded back @@ -778,16 +815,16 @@ bool IdealLoopTree::is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invari uint old_unique_idx = C->unique(); Node *iv = _head->as_CountedLoop()->phi(); int scale = 0; - Node *offset = NULL; + Node *offset = nullptr; if (!phase->is_scaled_iv_plus_offset(cmp->in(1), iv, &scale, &offset)) { return false; } - if (offset != NULL) { + if (offset != nullptr) { if (!invar.is_invariant(offset)) { // offset must be invariant return false; } Node* data_dependency_on = invar.data_dependency_on(); - if (data_dependency_on != NULL && old_unique_idx < C->unique()) { + if (data_dependency_on != nullptr && old_unique_idx < C->unique()) { // 'offset' node was newly created by is_scaled_iv_plus_offset(). Check that it does not depend on the entry projection // into the loop. If it does, we cannot perform loop predication (see Invariant::Invariant()). assert(!offset->is_CFG(), "offset must be a data node"); @@ -829,25 +866,23 @@ bool IdealLoopTree::is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invari // max(scale*i + offset) = scale*(limit-stride) + offset // (2) stride*scale < 0 // max(scale*i + offset) = scale*init + offset -BoolNode* PhaseIdealLoop::rc_predicate(IdealLoopTree *loop, Node* ctrl, - int scale, Node* offset, - Node* init, Node* limit, jint stride, - Node* range, bool upper, bool &overflow, bool negate) { - jint con_limit = (limit != NULL && limit->is_Con()) ? limit->get_int() : 0; +BoolNode* PhaseIdealLoop::rc_predicate(IdealLoopTree* loop, Node* ctrl, int scale, Node* offset, Node* init, + Node* limit, jint stride, Node* range, bool upper, bool& overflow) { + jint con_limit = (limit != nullptr && limit->is_Con()) ? limit->get_int() : 0; jint con_init = init->is_Con() ? init->get_int() : 0; jint con_offset = offset->is_Con() ? offset->get_int() : 0; - stringStream* predString = NULL; + stringStream* predString = nullptr; if (TraceLoopPredicate) { predString = new stringStream(); predString->print("rc_predicate "); } overflow = false; - Node* max_idx_expr = NULL; + Node* max_idx_expr = nullptr; const TypeInt* idx_type = TypeInt::INT; if ((stride > 0) == (scale > 0) == upper) { - guarantee(limit != NULL, "sanity"); + guarantee(limit != nullptr, "sanity"); if (TraceLoopPredicate) { if (limit->is_Con()) { predString->print("(%d ", con_limit); @@ -948,7 +983,7 @@ BoolNode* PhaseIdealLoop::rc_predicate(IdealLoopTree *loop, Node* ctrl, register_new_node(max_idx_expr, ctrl); } - CmpNode* cmp = NULL; + CmpNode* cmp = nullptr; if (overflow) { // Integer expressions may overflow, do long comparison range = new ConvI2LNode(range); @@ -958,7 +993,7 @@ BoolNode* PhaseIdealLoop::rc_predicate(IdealLoopTree *loop, Node* ctrl, cmp = new CmpUNode(max_idx_expr, range); } register_new_node(cmp, ctrl); - BoolNode* bol = new BoolNode(cmp, negate ? BoolTest::ge : BoolTest::lt); + BoolNode* bol = new BoolNode(cmp, BoolTest::lt); register_new_node(bol, ctrl); if (TraceLoopPredicate) { @@ -976,7 +1011,7 @@ bool PhaseIdealLoop::loop_predication_should_follow_branches(IdealLoopTree *loop return false; } - if (predicate_proj == NULL) { + if (predicate_proj == nullptr) { return false; } @@ -984,15 +1019,15 @@ bool PhaseIdealLoop::loop_predication_should_follow_branches(IdealLoopTree *loop bool follow_branches = true; IdealLoopTree* l = loop->_child; // For leaf loops and loops with a single inner loop - while (l != NULL && follow_branches) { + while (l != nullptr && follow_branches) { IdealLoopTree* child = l; - if (child->_child != NULL && + if (child->_child != nullptr && child->_head->is_OuterStripMinedLoop()) { - assert(child->_child->_next == NULL, "only one inner loop for strip mined loop"); + assert(child->_child->_next == nullptr, "only one inner loop for strip mined loop"); assert(child->_child->_head->is_CountedLoop() && child->_child->_head->as_CountedLoop()->is_strip_mined(), "inner loop should be strip mined"); child = child->_child; } - if (child->_child != NULL || child->_irreducible) { + if (child->_child != nullptr || child->_irreducible) { follow_branches = false; } l = l->_next; @@ -1005,7 +1040,7 @@ bool PhaseIdealLoop::loop_predication_should_follow_branches(IdealLoopTree *loop loop_trip_cnt = head->profile_trip_cnt(); if (head->is_CountedLoop()) { CountedLoopNode* cl = head->as_CountedLoop(); - if (cl->phi() != NULL) { + if (cl->phi() != nullptr) { const TypeInt* t = _igvn.type(cl->phi())->is_int(); float worst_case_trip_cnt = ((float)t->_hi - t->_lo) / ABS(cl->stride_con()); if (worst_case_trip_cnt < loop_trip_cnt) { @@ -1193,7 +1228,7 @@ class PathFrequency { assert(con >= CatchProjNode::catch_all_index, "what else?"); _freqs.at_put_grow(c->_idx, 0, -1); } - } else if (c->unique_ctrl_out() == NULL && !c->is_If() && !c->is_Jump()) { + } else if (c->unique_ctrl_out() == nullptr && !c->is_If() && !c->is_Jump()) { ShouldNotReachHere(); } else { c = c->in(0); @@ -1249,12 +1284,14 @@ void PhaseIdealLoop::loop_predication_follow_branches(Node *n, IdealLoopTree *lo } -bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree *loop, ProjNode* proj, ProjNode *predicate_proj, +bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree *loop, ProjNode* success_proj, ProjNode *predicate_proj, CountedLoopNode *cl, ConNode* zero, Invariance& invar, Deoptimization::DeoptReason reason) { // Following are changed to nonnull when a predicate can be hoisted - ProjNode* new_predicate_proj = NULL; - IfNode* iff = proj->in(0)->as_If(); + ProjNode* new_predicate_proj = nullptr; + assert(success_proj->is_IfProj(), "Expectiong IfProj. Else predecessor might not be an iff."); + IfProjNode* if_success_proj = success_proj->as_IfProj(); + IfNode* iff = if_success_proj->in(0)->as_If(); Node* test = iff->in(1); if (!test->is_Bool()){ //Conv2B, ... return false; @@ -1262,7 +1299,7 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree *loop, ProjNode* BoolNode* bol = test->as_Bool(); if (invar.is_invariant(bol)) { // Invariant test - new_predicate_proj = create_new_if_for_predicate(predicate_proj, NULL, + new_predicate_proj = create_new_if_for_predicate(predicate_proj, nullptr, reason, iff->Opcode()); Node* ctrl = new_predicate_proj->in(0)->as_If()->in(0); @@ -1270,7 +1307,7 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree *loop, ProjNode* // Negate test if necessary bool negated = false; - if (proj->_con != predicate_proj->_con) { + if (if_success_proj->_con != predicate_proj->_con) { new_predicate_bol = new BoolNode(new_predicate_bol->in(1), new_predicate_bol->_test.negate()); register_new_node(new_predicate_bol, ctrl); negated = true; @@ -1287,8 +1324,9 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree *loop, ProjNode* loop->dump_head(); } #endif - } else if (cl != NULL && loop->is_range_check_if(iff, this, invar DEBUG_ONLY(COMMA predicate_proj))) { + } else if (cl != nullptr && loop->is_range_check_if(if_success_proj, this, invar DEBUG_ONLY(COMMA predicate_proj))) { // Range check for counted loops + assert(if_success_proj->is_IfTrue(), "trap must be on false projection for a range check"); const Node* cmp = bol->in(1)->as_Cmp(); Node* idx = cmp->in(1); assert(!invar.is_invariant(idx), "index is variant"); @@ -1322,31 +1360,31 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree *loop, ProjNode* } // If predicate expressions may overflow in the integer range, longs are used. bool overflow = false; - bool negate = (proj->_con != predicate_proj->_con); // Test the lower bound - BoolNode* lower_bound_bol = rc_predicate(loop, ctrl, scale, offset, init, limit, stride, rng, false, overflow, negate); + BoolNode* lower_bound_bol = rc_predicate(loop, ctrl, scale, offset, init, limit, stride, rng, false, overflow); - ProjNode* lower_bound_proj = create_new_if_for_predicate(predicate_proj, NULL, reason, overflow ? Op_If : iff->Opcode()); + ProjNode* lower_bound_proj = create_new_if_for_predicate(predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode()); IfNode* lower_bound_iff = lower_bound_proj->in(0)->as_If(); _igvn.hash_delete(lower_bound_iff); lower_bound_iff->set_req(1, lower_bound_bol); - if (TraceLoopPredicate) tty->print_cr("lower bound check if: %s %d ", negate ? " negated" : "", lower_bound_iff->_idx); + if (TraceLoopPredicate) tty->print_cr("lower bound check if: %d ", lower_bound_iff->_idx); // Test the upper bound - BoolNode* upper_bound_bol = rc_predicate(loop, lower_bound_proj, scale, offset, init, limit, stride, rng, true, overflow, negate); + BoolNode* upper_bound_bol = rc_predicate(loop, lower_bound_proj, scale, offset, init, limit, stride, rng, true, + overflow); - ProjNode* upper_bound_proj = create_new_if_for_predicate(predicate_proj, NULL, reason, overflow ? Op_If : iff->Opcode()); + ProjNode* upper_bound_proj = create_new_if_for_predicate(predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode()); assert(upper_bound_proj->in(0)->as_If()->in(0) == lower_bound_proj, "should dominate"); IfNode* upper_bound_iff = upper_bound_proj->in(0)->as_If(); _igvn.hash_delete(upper_bound_iff); upper_bound_iff->set_req(1, upper_bound_bol); - if (TraceLoopPredicate) tty->print_cr("upper bound check if: %s %d ", negate ? " negated" : "", lower_bound_iff->_idx); + if (TraceLoopPredicate) tty->print_cr("upper bound check if: %d ", lower_bound_iff->_idx); // Fall through into rest of the cleanup code which will move any dependent nodes to the skeleton predicates of the // upper bound test. We always need to create skeleton predicates in order to properly remove dead loops when later // splitting the predicated loop into (unreachable) sub-loops (i.e. done by unrolling, peeling, pre/main/post etc.). - new_predicate_proj = insert_initial_skeleton_predicate(iff, loop, proj, predicate_proj, upper_bound_proj, scale, + new_predicate_proj = insert_initial_skeleton_predicate(iff, loop, if_success_proj, predicate_proj, upper_bound_proj, scale, offset, init, limit, stride, rng, overflow, reason); #ifndef PRODUCT @@ -1360,12 +1398,12 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree *loop, ProjNode* // with uncommon trap. return false; } - assert(new_predicate_proj != NULL, "sanity"); + assert(new_predicate_proj != nullptr, "sanity"); // Success - attach condition (new_predicate_bol) to predicate if - invar.map_ctrl(proj, new_predicate_proj); // so that invariance test can be appropriate + invar.map_ctrl(if_success_proj, new_predicate_proj); // so that invariance test can be appropriate // Eliminate the old If in the loop body - dominated_by( new_predicate_proj, iff, proj->_con != new_predicate_proj->_con ); + dominated_by( new_predicate_proj, iff, if_success_proj->_con != new_predicate_proj->_con ); C->set_major_progress(); return true; @@ -1388,11 +1426,12 @@ ProjNode* PhaseIdealLoop::insert_initial_skeleton_predicate(IfNode* iff, IdealLo Node* opaque_init = new OpaqueLoopInitNode(C, init); register_new_node(opaque_init, upper_bound_proj); bool negate = (proj->_con != predicate_proj->_con); - BoolNode* bol = rc_predicate(loop, upper_bound_proj, scale, offset, opaque_init, limit, stride, rng, (stride > 0) != (scale > 0), overflow, negate); + BoolNode* bol = rc_predicate(loop, upper_bound_proj, scale, offset, opaque_init, limit, stride, rng, + (stride > 0) != (scale > 0), overflow); Node* opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); // This will go away once loop opts are over C->add_skeleton_predicate_opaq(opaque_bol); register_new_node(opaque_bol, upper_bound_proj); - ProjNode* new_proj = create_new_if_for_predicate(predicate_proj, NULL, reason, overflow ? Op_If : iff->Opcode()); + ProjNode* new_proj = create_new_if_for_predicate(predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode()); _igvn.replace_input_of(new_proj->in(0), 1, opaque_bol); assert(opaque_init->outcnt() > 0, "should be used"); @@ -1410,11 +1449,12 @@ ProjNode* PhaseIdealLoop::insert_initial_skeleton_predicate(IfNode* iff, IdealLo max_value = new CastIINode(max_value, loop->_head->as_CountedLoop()->phi()->bottom_type()); register_new_node(max_value, predicate_proj); - bol = rc_predicate(loop, new_proj, scale, offset, max_value, limit, stride, rng, (stride > 0) != (scale > 0), overflow, negate); + bol = rc_predicate(loop, new_proj, scale, offset, max_value, limit, stride, rng, (stride > 0) != (scale > 0), + overflow); opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); C->add_skeleton_predicate_opaq(opaque_bol); register_new_node(opaque_bol, new_proj); - new_proj = create_new_if_for_predicate(predicate_proj, NULL, reason, overflow ? Op_If : iff->Opcode()); + new_proj = create_new_if_for_predicate(predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode()); _igvn.replace_input_of(new_proj->in(0), 1, opaque_bol); assert(max_value->outcnt() > 0, "should be used"); assert(skeleton_predicate_has_opaque(new_proj->in(0)->as_If()), "unexpected"); @@ -1442,7 +1482,7 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) { return false; } - CountedLoopNode *cl = NULL; + CountedLoopNode *cl = nullptr; if (head->is_valid_counted_loop(T_INT)) { cl = head->as_CountedLoop(); // do nothing for iteration-splitted loops @@ -1450,21 +1490,21 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) { // Avoid RCE if Counted loop's test is '!='. BoolTest::mask bt = cl->loopexit()->test_trip(); if (bt != BoolTest::lt && bt != BoolTest::gt) - cl = NULL; + cl = nullptr; } Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl); - ProjNode *loop_limit_proj = NULL; - ProjNode *predicate_proj = NULL; - ProjNode *profile_predicate_proj = NULL; + ProjNode *loop_limit_proj = nullptr; + ProjNode *predicate_proj = nullptr; + ProjNode *profile_predicate_proj = nullptr; // Loop limit check predicate should be near the loop. loop_limit_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); - if (loop_limit_proj != NULL) { + if (loop_limit_proj != nullptr) { entry = skip_loop_predicates(loop_limit_proj); } bool has_profile_predicates = false; profile_predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate); - if (profile_predicate_proj != NULL) { + if (profile_predicate_proj != nullptr) { Node* n = skip_loop_predicates(entry); // Check if predicates were already added to the profile predicate // block @@ -1479,7 +1519,7 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) { bool follow_branches = loop_predication_should_follow_branches(loop, profile_predicate_proj, loop_trip_cnt); assert(!follow_branches || loop_trip_cnt >= 0, "negative trip count?"); - if (predicate_proj == NULL && !follow_branches) { + if (predicate_proj == nullptr && !follow_branches) { #ifndef PRODUCT if (TraceLoopPredicate) { tty->print("missing predicate:"); @@ -1528,7 +1568,7 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) { IfNode* iff = proj->in(0)->as_If(); CallStaticJavaNode* call = proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); - if (call == NULL) { + if (call == nullptr) { if (loop->is_loop_exit(iff)) { // stop processing the remaining projs in the list because the execution of them // depends on the condition of "iff" (iff->in(1)). @@ -1549,7 +1589,7 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) { break; } - if (predicate_proj != NULL) { + if (predicate_proj != nullptr) { hoisted = loop_predication_impl_helper(loop, proj, predicate_proj, cl, zero, invar, Deoptimization::Reason_predicate) | hoisted; } } // end while diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index f7e7073dd901d..6b3c835e47e80 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -44,10 +44,10 @@ #include "runtime/stubRoutines.hpp" //------------------------------is_loop_exit----------------------------------- -// Given an IfNode, return the loop-exiting projection or NULL if both +// Given an IfNode, return the loop-exiting projection or null if both // arms remain in the loop. Node *IdealLoopTree::is_loop_exit(Node *iff) const { - if (iff->outcnt() != 2) return NULL; // Ignore partially dead tests + if (iff->outcnt() != 2) return nullptr; // Ignore partially dead tests PhaseIdealLoop *phase = _phase; // Test is an IfNode, has 2 projections. If BOTH are in the loop // we need loop unswitching instead of peeling. @@ -55,7 +55,7 @@ Node *IdealLoopTree::is_loop_exit(Node *iff) const { return iff->raw_out(0); if (!is_member(phase->get_loop(iff->raw_out(1)))) return iff->raw_out(1); - return NULL; + return nullptr; } @@ -73,19 +73,19 @@ void IdealLoopTree::record_for_igvn() { if (_head->is_CountedLoop() && _head->as_Loop()->is_strip_mined()) { CountedLoopNode* l = _head->as_CountedLoop(); Node* outer_loop = l->outer_loop(); - assert(outer_loop != NULL, "missing piece of strip mined loop"); + assert(outer_loop != nullptr, "missing piece of strip mined loop"); _phase->_igvn._worklist.push(outer_loop); Node* outer_loop_tail = l->outer_loop_tail(); - assert(outer_loop_tail != NULL, "missing piece of strip mined loop"); + assert(outer_loop_tail != nullptr, "missing piece of strip mined loop"); _phase->_igvn._worklist.push(outer_loop_tail); Node* outer_loop_end = l->outer_loop_end(); - assert(outer_loop_end != NULL, "missing piece of strip mined loop"); + assert(outer_loop_end != nullptr, "missing piece of strip mined loop"); _phase->_igvn._worklist.push(outer_loop_end); Node* outer_safepoint = l->outer_safepoint(); - assert(outer_safepoint != NULL, "missing piece of strip mined loop"); + assert(outer_safepoint != nullptr, "missing piece of strip mined loop"); _phase->_igvn._worklist.push(outer_safepoint); Node* cle_out = _head->as_CountedLoop()->loopexit()->proj_out(false); - assert(cle_out != NULL, "missing piece of strip mined loop"); + assert(cle_out != nullptr, "missing piece of strip mined loop"); _phase->_igvn._worklist.push(cle_out); } } @@ -115,7 +115,7 @@ void IdealLoopTree::compute_trip_count(PhaseIdealLoop* phase) { Node* init_n = cl->init_trip(); Node* limit_n = cl->limit(); - if (init_n != NULL && limit_n != NULL) { + if (init_n != nullptr && limit_n != nullptr) { // Use longs to avoid integer overflow. int stride_con = cl->stride_con(); const TypeInt* init_type = phase->_igvn.type(init_n)->is_int(); @@ -208,7 +208,7 @@ void IdealLoopTree::compute_profile_trip_cnt(PhaseIdealLoop *phase) { // Now compute a loop exit count float loop_exit_cnt = 0.0f; - if (_child == NULL) { + if (_child == nullptr) { for (uint i = 0; i < _body.size(); i++) { Node *n = _body[i]; loop_exit_cnt += compute_profile_trip_cnt_helper(n); @@ -264,10 +264,10 @@ int IdealLoopTree::find_invariant(Node* n, PhaseIdealLoop *phase) { //---------------------is_associative----------------------------- // Return TRUE if "n" is an associative binary node. If "base" is -// not NULL, "n" must be re-associative with it. +// not null, "n" must be re-associative with it. bool IdealLoopTree::is_associative(Node* n, Node* base) { int op = n->Opcode(); - if (base != NULL) { + if (base != nullptr) { assert(is_associative(base), "Base node should be associative"); int base_op = base->Opcode(); if (base_op == Op_AddI || base_op == Op_SubI) { @@ -319,7 +319,7 @@ Node* IdealLoopTree::reassociate_add_sub(Node* n1, int inv1_idx, int inv2_idx, P neg_inv2 = !neg_inv2; } - bool is_int = n1->bottom_type()->isa_int() != NULL; + bool is_int = n1->bottom_type()->isa_int() != nullptr; Node* inv1_c = phase->get_ctrl(inv1); Node* n_inv1; if (neg_inv1) { @@ -375,21 +375,21 @@ Node* IdealLoopTree::reassociate_add_sub(Node* n1, int inv1_idx, int inv2_idx, P // inv1 op (x op inv2) => (inv1 op inv2) op x // Node* IdealLoopTree::reassociate(Node* n1, PhaseIdealLoop *phase) { - if (!is_associative(n1) || n1->outcnt() == 0) return NULL; - if (is_invariant(n1)) return NULL; + if (!is_associative(n1) || n1->outcnt() == 0) return nullptr; + if (is_invariant(n1)) return nullptr; // Don't mess with add of constant (igvn moves them to expression tree root.) - if (n1->is_Add() && n1->in(2)->is_Con()) return NULL; + if (n1->is_Add() && n1->in(2)->is_Con()) return nullptr; int inv1_idx = find_invariant(n1, phase); - if (!inv1_idx) return NULL; + if (!inv1_idx) return nullptr; Node* n2 = n1->in(3 - inv1_idx); - if (!is_associative(n2, n1)) return NULL; + if (!is_associative(n2, n1)) return nullptr; int inv2_idx = find_invariant(n2, phase); - if (!inv2_idx) return NULL; + if (!inv2_idx) return nullptr; - if (!phase->may_require_nodes(10, 10)) return NULL; + if (!phase->may_require_nodes(10, 10)) return nullptr; - Node* result = NULL; + Node* result = nullptr; switch (n1->Opcode()) { case Op_AddI: case Op_AddL: @@ -417,7 +417,7 @@ Node* IdealLoopTree::reassociate(Node* n1, PhaseIdealLoop *phase) { ShouldNotReachHere(); } - assert(result != NULL, ""); + assert(result != nullptr, ""); phase->register_new_node(result, phase->get_ctrl(n1)); phase->_igvn.replace_node(n1, result); assert(phase->get_loop(phase->get_ctrl(n1)) == this, ""); @@ -432,7 +432,7 @@ void IdealLoopTree::reassociate_invariants(PhaseIdealLoop *phase) { Node *n = _body.at(i); for (int j = 0; j < 5; j++) { Node* nn = reassociate(n, phase); - if (nn == NULL) break; + if (nn == nullptr) break; n = nn; // again } } @@ -515,12 +515,12 @@ void PhaseIdealLoop::peeled_dom_test_elim(IdealLoopTree* loop, Node_List& old_ne Node* test = prev->in(0); while (test != loop->_head) { // Scan till run off top of loop int p_op = prev->Opcode(); - assert(test != NULL, "test cannot be NULL"); - Node* test_cond = NULL; + assert(test != nullptr, "test cannot be null"); + Node* test_cond = nullptr; if ((p_op == Op_IfFalse || p_op == Op_IfTrue) && test->is_If()) { test_cond = test->in(1); } - if (test_cond != NULL && // Test? + if (test_cond != nullptr && // Test? !test_cond->is_Con() && // And not already obvious? // And condition is not a member of this loop? !loop->is_member(get_loop(get_ctrl(test_cond)))) { @@ -907,14 +907,14 @@ bool IdealLoopTree::policy_unroll(PhaseIdealLoop *phase) { Node *init_n = cl->init_trip(); Node *limit_n = cl->limit(); int stride_con = cl->stride_con(); - if (limit_n == NULL) return false; // We will dereference it below. + if (limit_n == nullptr) return false; // We will dereference it below. // Non-constant bounds. // Protect against over-unrolling when init or/and limit are not constant // (so that trip_count's init value is maxint) but iv range is known. - if (init_n == NULL || !init_n->is_Con() || !limit_n->is_Con()) { + if (init_n == nullptr || !init_n->is_Con() || !limit_n->is_Con()) { Node* phi = cl->phi(); - if (phi != NULL) { + if (phi != nullptr) { assert(phi->is_Phi() && phi->in(0) == _head, "Counted loop should have iv phi."); const TypeInt* iv_type = phase->_igvn.type(phi)->is_int(); int next_stride = stride_con * 2; // stride after this unroll @@ -1113,7 +1113,7 @@ bool IdealLoopTree::policy_range_check(PhaseIdealLoop *phase) const { } } - if (!phase->is_scaled_iv_plus_offset(rc_exp, trip_counter, NULL, NULL)) { + if (!phase->is_scaled_iv_plus_offset(rc_exp, trip_counter, nullptr, nullptr)) { continue; } // Found a test like 'trip+off vs limit'. Test is an IfNode, has two (2) @@ -1161,13 +1161,13 @@ Node *PhaseIdealLoop::clone_up_backedge_goo(Node *back_ctrl, Node *preheader_ctr // Only visit once if (visited.test_set(n->_idx)) { Node *x = clones.find(n->_idx); - return (x != NULL) ? x : n; + return (x != nullptr) ? x : n; } - Node *x = NULL; // If required, a clone of 'n' + Node *x = nullptr; // If required, a clone of 'n' // Check for 'n' being pinned in the backedge. if (n->in(0) && n->in(0) == back_ctrl) { - assert(clones.find(n->_idx) == NULL, "dead loop"); + assert(clones.find(n->_idx) == nullptr, "dead loop"); x = n->clone(); // Clone a copy of 'n' to preheader clones.push(x, n->_idx); x->set_req(0, preheader_ctrl); // Fix x's control input to preheader @@ -1180,7 +1180,7 @@ Node *PhaseIdealLoop::clone_up_backedge_goo(Node *back_ctrl, Node *preheader_ctr Node *g = clone_up_backedge_goo(back_ctrl, preheader_ctrl, n->in(i), visited, clones); if (g != n->in(i)) { if (!x) { - assert(clones.find(n->_idx) == NULL, "dead loop"); + assert(clones.find(n->_idx) == nullptr, "dead loop"); x = n->clone(); clones.push(x, n->_idx); } @@ -1207,19 +1207,19 @@ Node* PhaseIdealLoop::cast_incr_before_loop(Node* incr, Node* ctrl, Node* loop) return castii; } } - return NULL; + return nullptr; } #ifdef ASSERT void PhaseIdealLoop::ensure_zero_trip_guard_proj(Node* node, bool is_main_loop) { assert(node->is_IfProj(), "must be the zero trip guard If node"); Node* zer_bol = node->in(0)->in(1); - assert(zer_bol != NULL && zer_bol->is_Bool(), "must be Bool"); + assert(zer_bol != nullptr && zer_bol->is_Bool(), "must be Bool"); Node* zer_cmp = zer_bol->in(1); - assert(zer_cmp != NULL && zer_cmp->Opcode() == Op_CmpI, "must be CmpI"); + assert(zer_cmp != nullptr && zer_cmp->Opcode() == Op_CmpI, "must be CmpI"); // For the main loop, the opaque node is the second input to zer_cmp, for the post loop it's the first input node Node* zer_opaq = zer_cmp->in(is_main_loop ? 2 : 1); - assert(zer_opaq != NULL && zer_opaq->Opcode() == Op_Opaque1, "must be Opaque1"); + assert(zer_opaq != nullptr && zer_opaq->Opcode() == Op_Opaque1, "must be Opaque1"); } #endif @@ -1236,7 +1236,7 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicat uint dd_main_head, const uint idx_before_pre_post, const uint idx_after_post_before_pre, Node* zero_trip_guard_proj_main, Node* zero_trip_guard_proj_post, const Node_List &old_new) { - if (predicate != NULL) { + if (predicate != nullptr) { #ifdef ASSERT ensure_zero_trip_guard_proj(zero_trip_guard_proj_main, true); ensure_zero_trip_guard_proj(zero_trip_guard_proj_post, false); @@ -1254,7 +1254,7 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicat Node* opaque_stride = new OpaqueLoopStrideNode(C, stride); register_new_node(opaque_stride, outer_main_head->in(LoopNode::EntryControl)); - while (predicate != NULL && predicate->is_Proj() && predicate->in(0)->is_If()) { + while (predicate != nullptr && predicate->is_Proj() && predicate->in(0)->is_If()) { iff = predicate->in(0)->as_If(); uncommon_proj = iff->proj_out(1 - predicate->as_Proj()->_con); if (uncommon_proj->unique_ctrl_out() != rgn) @@ -1264,7 +1264,7 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicat // Clone the skeleton predicate twice and initialize one with the initial // value of the loop induction variable. Leave the other predicate // to be initialized when increasing the stride during loop unrolling. - prev_proj = clone_skeleton_predicate_for_main_or_post_loop(iff, opaque_init, NULL, predicate, uncommon_proj, + prev_proj = clone_skeleton_predicate_for_main_or_post_loop(iff, opaque_init, nullptr, predicate, uncommon_proj, current_proj, outer_loop, prev_proj); assert(skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), ""); @@ -1280,7 +1280,7 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicat // Change the control if 'loop_node' is part of the main loop. If there is an old->new mapping and the index of // 'pre_loop_node' is greater than idx_before_pre_post, then we know that 'loop_node' was cloned and is part of // the main loop (and 'pre_loop_node' is part of the pre loop). - if (!loop_node->is_CFG() && (pre_loop_node != NULL && pre_loop_node->_idx > idx_after_post_before_pre)) { + if (!loop_node->is_CFG() && (pre_loop_node != nullptr && pre_loop_node->_idx > idx_after_post_before_pre)) { // 'loop_node' is a data node and part of the main loop. Rewire the control to the projection of the zero-trip guard if node // of the main loop that is immediately preceding the cloned predicates. _igvn.replace_input_of(loop_node, 0, zero_trip_guard_proj_main); @@ -1288,7 +1288,7 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicat } else if (loop_node->_idx > idx_before_pre_post && loop_node->_idx < idx_after_post_before_pre) { // 'loop_node' is a data node and part of the post loop. Rewire the control to the projection of the zero-trip guard if node // of the post loop that is immediately preceding the post loop header node (there are no cloned predicates for the post loop). - assert(pre_loop_node == NULL, "a node belonging to the post loop should not have an old_new mapping at this stage"); + assert(pre_loop_node == nullptr, "a node belonging to the post loop should not have an old_new mapping at this stage"); _igvn.replace_input_of(loop_node, 0, zero_trip_guard_proj_post); --i; } @@ -1359,7 +1359,7 @@ bool PhaseIdealLoop::skeleton_predicate_has_opaque(IfNode* iff) { } else { for (uint j = 1; j < n->req(); j++) { Node* m = n->in(j); - if (m != NULL) { + if (m != nullptr) { wq.push(m); } } @@ -1383,7 +1383,7 @@ void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride) if (skeleton_follow_inputs(n)) { for (uint j = 1; j < n->req(); j++) { Node* m = n->in(j); - if (m != NULL) { + if (m != nullptr) { wq.push(m); } } @@ -1399,14 +1399,14 @@ void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride) // Clone the skeleton predicate bool for a main or unswitched loop: // Main loop: Set new_init and new_stride nodes as new inputs. -// Unswitched loop: new_init and new_stride are both NULL. Clone OpaqueLoopInit and OpaqueLoopStride instead. +// Unswitched loop: new_init and new_stride are both null. Clone OpaqueLoopInit and OpaqueLoopStride instead. Node* PhaseIdealLoop::clone_skeleton_predicate_bool(Node* iff, Node* new_init, Node* new_stride, Node* control) { Node_Stack to_clone(2); to_clone.push(iff->in(1), 1); uint current = C->unique(); - Node* result = NULL; - bool is_unswitched_loop = new_init == NULL && new_stride == NULL; - assert(new_init != NULL || is_unswitched_loop, "new_init must be set when new_stride is non-null"); + Node* result = nullptr; + bool is_unswitched_loop = new_init == nullptr && new_stride == nullptr; + assert(new_init != nullptr || is_unswitched_loop, "new_init must be set when new_stride is non-null"); // Look for the opaque node to replace with the new value // and clone everything in between. We keep the Opaque4 node // so the duplicated predicates are eliminated once loop @@ -1427,18 +1427,18 @@ Node* PhaseIdealLoop::clone_skeleton_predicate_bool(Node* iff, Node* new_init, N } int op = m->Opcode(); if (op == Op_OpaqueLoopInit) { - if (is_unswitched_loop && m->_idx < current && new_init == NULL) { + if (is_unswitched_loop && m->_idx < current && new_init == nullptr) { new_init = m->clone(); register_new_node(new_init, control); } n->set_req(i, new_init); } else { assert(op == Op_OpaqueLoopStride, "unexpected opaque node"); - if (is_unswitched_loop && m->_idx < current && new_stride == NULL) { + if (is_unswitched_loop && m->_idx < current && new_stride == nullptr) { new_stride = m->clone(); register_new_node(new_stride, control); } - if (new_stride != NULL) { + if (new_stride != nullptr) { n->set_req(i, new_stride); } } @@ -1468,9 +1468,9 @@ Node* PhaseIdealLoop::clone_skeleton_predicate_bool(Node* iff, Node* new_init, N next->set_req(j, cur); } } - } while (result == NULL); + } while (result == nullptr); assert(result->_idx >= current, "new node expected"); - assert(!is_unswitched_loop || new_init != NULL, "new_init must always be found and cloned"); + assert(!is_unswitched_loop || new_init != nullptr, "new_init must always be found and cloned"); return result; } @@ -1506,15 +1506,15 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop(CountedLoopNode* pre_ Node* zero_trip_guard_proj_post, const Node_List &old_new) { if (UseLoopPredicate) { Node* entry = pre_head->in(LoopNode::EntryControl); - Node* predicate = NULL; + Node* predicate = nullptr; predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); - if (predicate != NULL) { + if (predicate != nullptr) { entry = skip_loop_predicates(entry); } - Node* profile_predicate = NULL; + Node* profile_predicate = nullptr; if (UseProfiledLoopPredicate) { profile_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate); - if (profile_predicate != NULL) { + if (profile_predicate != nullptr) { entry = skip_loop_predicates(entry); } } @@ -1575,7 +1575,7 @@ void PhaseIdealLoop::insert_pre_post_loops(IdealLoopTree *loop, Node_List &old_n // Add the post loop const uint idx_before_pre_post = Compile::current()->unique(); - CountedLoopNode *post_head = NULL; + CountedLoopNode *post_head = nullptr; Node* post_incr = incr; Node* main_exit = insert_post_loop(loop, old_new, main_head, main_end, post_incr, limit, post_head); const uint idx_after_post_before_pre = Compile::current()->unique(); @@ -1681,7 +1681,7 @@ void PhaseIdealLoop::insert_pre_post_loops(IdealLoopTree *loop, Node_List &old_n // CastII for the main loop: Node* castii = cast_incr_before_loop(pre_incr, min_taken, main_head); - assert(castii != NULL, "no castII inserted"); + assert(castii != nullptr, "no castII inserted"); assert(post_head->in(1)->is_IfProj(), "must be zero-trip guard If node projection of the post loop"); copy_skeleton_predicates_to_main_loop(pre_head, castii, stride, outer_loop, outer_main_head, dd_main_head, idx_before_pre_post, idx_after_post_before_pre, min_taken, post_head->in(1), old_new); @@ -1806,7 +1806,7 @@ void PhaseIdealLoop::insert_vector_post_loop(IdealLoopTree *loop, Node_List &old Node *limit = main_end->limit(); // In this case we throw away the result as we are not using it to connect anything else. - CountedLoopNode *post_head = NULL; + CountedLoopNode *post_head = nullptr; insert_post_loop(loop, old_new, main_head, main_end, incr, limit, post_head); copy_skeleton_predicates_to_post_loop(main_head->skip_strip_mined(), post_head, incr, main_head->stride()); @@ -1853,7 +1853,7 @@ void PhaseIdealLoop::insert_scalar_rced_post_loop(IdealLoopTree *loop, Node_List Node *limit = main_end->limit(); // In this case we throw away the result as we are not using it to connect anything else. - CountedLoopNode *post_head = NULL; + CountedLoopNode *post_head = nullptr; insert_post_loop(loop, old_new, main_head, main_end, incr, limit, post_head); copy_skeleton_predicates_to_post_loop(main_head->skip_strip_mined(), post_head, incr, main_head->stride()); @@ -1959,7 +1959,7 @@ Node *PhaseIdealLoop::insert_post_loop(IdealLoopTree* loop, Node_List& old_new, // CastII for the new post loop: incr = cast_incr_before_loop(zer_opaq->in(1), zer_taken, post_head); - assert(incr != NULL, "no castII inserted"); + assert(incr != nullptr, "no castII inserted"); return new_main_exit; } @@ -1991,7 +1991,7 @@ void PhaseIdealLoop::update_main_loop_skeleton_predicates(Node* ctrl, CountedLoo Node* max_value = _igvn.intcon(new_stride_con); set_ctrl(max_value, C->root()); - while (entry != NULL && entry->is_Proj() && entry->in(0)->is_If()) { + while (entry != nullptr && entry->is_Proj() && entry->in(0)->is_If()) { IfNode* iff = entry->in(0)->as_If(); ProjNode* proj = iff->proj_out(1 - entry->as_Proj()->_con); if (proj->unique_ctrl_out()->Opcode() != Op_Halt) { @@ -2029,7 +2029,7 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_post_loop(LoopNode* main_loop_h Node* ctrl = main_loop_entry; Node* prev_proj = post_loop_entry; - while (ctrl != NULL && ctrl->is_Proj() && ctrl->in(0)->is_If()) { + while (ctrl != nullptr && ctrl->is_Proj() && ctrl->in(0)->is_If()) { IfNode* iff = ctrl->in(0)->as_If(); ProjNode* proj = iff->proj_out(1 - ctrl->as_Proj()->_con); if (proj->unique_ctrl_out()->Opcode() != Op_Halt) { @@ -2086,7 +2086,7 @@ void PhaseIdealLoop::do_unroll(IdealLoopTree *loop, Node_List &old_new, bool adj Node *init = loop_head->init_trip(); Node *stride = loop_head->stride(); - Node *opaq = NULL; + Node *opaq = nullptr; if (adjust_min_trip) { // If not maximally unrolling, need adjustment // Search for zero-trip guard. @@ -2094,7 +2094,7 @@ void PhaseIdealLoop::do_unroll(IdealLoopTree *loop, Node_List &old_new, bool adj // graph shape is encountered, the compiler bails out loop unrolling; // compilation of the method will still succeed. opaq = loop_head->is_canonical_loop_entry(); - if (opaq == NULL) { + if (opaq == nullptr) { return; } // Zero-trip test uses an 'opaque' node which is not shared. @@ -2103,7 +2103,7 @@ void PhaseIdealLoop::do_unroll(IdealLoopTree *loop, Node_List &old_new, bool adj C->set_major_progress(); - Node* new_limit = NULL; + Node* new_limit = nullptr; int stride_con = stride->get_int(); int stride_p = (stride_con > 0) ? stride_con : -stride_con; uint old_trip_count = loop_head->trip_count(); @@ -2145,12 +2145,12 @@ void PhaseIdealLoop::do_unroll(IdealLoopTree *loop, Node_List &old_new, bool adj // adjustment underflows or overflows, then the main loop is skipped. Node* cmp = loop_end->cmp_node(); assert(cmp->in(2) == limit, "sanity"); - assert(opaq != NULL && opaq->in(1) == limit, "sanity"); + assert(opaq != nullptr && opaq->in(1) == limit, "sanity"); // Verify that policy_unroll result is still valid. const TypeInt* limit_type = _igvn.type(limit)->is_int(); - assert(stride_con > 0 && ((limit_type->_hi - stride_con) < limit_type->_hi) || - stride_con < 0 && ((limit_type->_lo - stride_con) > limit_type->_lo), + assert((stride_con > 0 && ((min_jint + stride_con) <= limit_type->_hi)) || + (stride_con < 0 && ((max_jint + stride_con) >= limit_type->_lo)), "sanity"); if (limit->is_Con()) { @@ -2190,9 +2190,9 @@ void PhaseIdealLoop::do_unroll(IdealLoopTree *loop, Node_List &old_new, bool adj assert(bt == BoolTest::lt || bt == BoolTest::gt, "canonical test is expected"); Node* adj_max = _igvn.intcon((stride_con > 0) ? min_jint : max_jint); set_ctrl(adj_max, C->root()); - Node* old_limit = NULL; - Node* adj_limit = NULL; - Node* bol = limit->is_CMove() ? limit->in(CMoveNode::Condition) : NULL; + Node* old_limit = nullptr; + Node* adj_limit = nullptr; + Node* bol = limit->is_CMove() ? limit->in(CMoveNode::Condition) : nullptr; if (loop_head->unrolled_count() > 1 && limit->is_CMove() && limit->Opcode() == Op_CMoveI && limit->in(CMoveNode::IfTrue) == adj_max && @@ -2210,7 +2210,7 @@ void PhaseIdealLoop::do_unroll(IdealLoopTree *loop, Node_List &old_new, bool adj old_limit = limit; adj_limit = new SubINode(limit, stride); } - assert(old_limit != NULL && adj_limit != NULL, ""); + assert(old_limit != nullptr && adj_limit != nullptr, ""); register_new_node(adj_limit, ctrl); // adjust amount Node* adj_cmp = new CmpINode(old_limit, adj_limit); register_new_node(adj_cmp, ctrl); @@ -2235,7 +2235,7 @@ void PhaseIdealLoop::do_unroll(IdealLoopTree *loop, Node_List &old_new, bool adj } } - assert(new_limit != NULL, ""); + assert(new_limit != nullptr, ""); // Replace in loop test. assert(loop_end->in(1)->in(1) == cmp, "sanity"); if (cmp->outcnt() == 1 && loop_end->in(1)->outcnt() == 1) { @@ -2389,9 +2389,9 @@ void PhaseIdealLoop::mark_reductions(IdealLoopTree *loop) { // For definitions which are loop inclusive and not tripcounts. Node* def_node = phi->in(LoopNode::LoopBackControl); - if (def_node != NULL) { + if (def_node != nullptr) { Node* n_ctrl = get_ctrl(def_node); - if (n_ctrl != NULL && loop->is_member(get_loop(n_ctrl))) { + if (n_ctrl != nullptr && loop->is_member(get_loop(n_ctrl))) { // Now test it to see if it fits the standard pattern for a reduction operator. int opc = def_node->Opcode(); if (opc != ReductionNode::opcode(opc, def_node->bottom_type()->basic_type()) @@ -2442,7 +2442,7 @@ void PhaseIdealLoop::mark_reductions(IdealLoopTree *loop) { Node* PhaseIdealLoop::adjust_limit(bool is_positive_stride, Node* scale, Node* offset, Node* rc_limit, Node* old_limit, Node* pre_ctrl, bool round) { Node* sub = new SubLNode(rc_limit, offset); register_new_node(sub, pre_ctrl); - Node* limit = new DivLNode(NULL, sub, scale); + Node* limit = new DivLNode(nullptr, sub, scale); register_new_node(limit, pre_ctrl); // When the absolute value of scale is greater than one, the division @@ -2498,8 +2498,8 @@ Node* PhaseIdealLoop::adjust_limit(bool is_positive_stride, Node* scale, Node* o // holds true in the main-loop. Stride, scale, offset and limit are all loop // invariant. Further, stride and scale are constants (offset and limit often are). void PhaseIdealLoop::add_constraint(jlong stride_con, jlong scale_con, Node* offset, Node* low_limit, Node* upper_limit, Node* pre_ctrl, Node** pre_limit, Node** main_limit) { - assert(_igvn.type(offset)->isa_long() != NULL && _igvn.type(low_limit)->isa_long() != NULL && - _igvn.type(upper_limit)->isa_long() != NULL, "arguments should be long values"); + assert(_igvn.type(offset)->isa_long() != nullptr && _igvn.type(low_limit)->isa_long() != nullptr && + _igvn.type(upper_limit)->isa_long() != nullptr, "arguments should be long values"); // For a positive stride, we need to reduce the main-loop limit and // increase the pre-loop limit. This is reversed for a negative stride. @@ -2573,7 +2573,7 @@ void PhaseIdealLoop::add_constraint(jlong stride_con, jlong scale_con, Node* off bool PhaseIdealLoop::is_scaled_iv(Node* exp, Node* iv, int* p_scale) { exp = exp->uncast(); if (exp == iv) { - if (p_scale != NULL) { + if (p_scale != nullptr) { *p_scale = 1; } return true; @@ -2581,20 +2581,20 @@ bool PhaseIdealLoop::is_scaled_iv(Node* exp, Node* iv, int* p_scale) { int opc = exp->Opcode(); if (opc == Op_MulI) { if (exp->in(1)->uncast() == iv && exp->in(2)->is_Con()) { - if (p_scale != NULL) { + if (p_scale != nullptr) { *p_scale = exp->in(2)->get_int(); } return true; } if (exp->in(2)->uncast() == iv && exp->in(1)->is_Con()) { - if (p_scale != NULL) { + if (p_scale != nullptr) { *p_scale = exp->in(1)->get_int(); } return true; } } else if (opc == Op_LShiftI) { if (exp->in(1)->uncast() == iv && exp->in(2)->is_Con()) { - if (p_scale != NULL) { + if (p_scale != nullptr) { *p_scale = 1 << exp->in(2)->get_int(); } return true; @@ -2607,7 +2607,7 @@ bool PhaseIdealLoop::is_scaled_iv(Node* exp, Node* iv, int* p_scale) { // Return true if exp is a simple induction variable expression: k1*iv + (invar + k2) bool PhaseIdealLoop::is_scaled_iv_plus_offset(Node* exp, Node* iv, int* p_scale, Node** p_offset, int depth) { if (is_scaled_iv(exp, iv, p_scale)) { - if (p_offset != NULL) { + if (p_offset != nullptr) { Node *zero = _igvn.intcon(0); set_ctrl(zero, C->root()); *p_offset = zero; @@ -2618,23 +2618,23 @@ bool PhaseIdealLoop::is_scaled_iv_plus_offset(Node* exp, Node* iv, int* p_scale, int opc = exp->Opcode(); if (opc == Op_AddI) { if (is_scaled_iv(exp->in(1), iv, p_scale)) { - if (p_offset != NULL) { + if (p_offset != nullptr) { *p_offset = exp->in(2); } return true; } if (is_scaled_iv(exp->in(2), iv, p_scale)) { - if (p_offset != NULL) { + if (p_offset != nullptr) { *p_offset = exp->in(1); } return true; } if (exp->in(2)->is_Con()) { - Node* offset2 = NULL; + Node* offset2 = nullptr; if (depth < 2 && is_scaled_iv_plus_offset(exp->in(1), iv, p_scale, - p_offset != NULL ? &offset2 : NULL, depth+1)) { - if (p_offset != NULL) { + p_offset != nullptr ? &offset2 : nullptr, depth+1)) { + if (p_offset != nullptr) { Node *ctrl_off2 = get_ctrl(offset2); Node* offset = new AddINode(offset2, exp->in(2)); register_new_node(offset, ctrl_off2); @@ -2645,7 +2645,7 @@ bool PhaseIdealLoop::is_scaled_iv_plus_offset(Node* exp, Node* iv, int* p_scale, } } else if (opc == Op_SubI) { if (is_scaled_iv(exp->in(1), iv, p_scale)) { - if (p_offset != NULL) { + if (p_offset != nullptr) { Node *zero = _igvn.intcon(0); set_ctrl(zero, C->root()); Node *ctrl_off = get_ctrl(exp->in(2)); @@ -2656,7 +2656,7 @@ bool PhaseIdealLoop::is_scaled_iv_plus_offset(Node* exp, Node* iv, int* p_scale, return true; } if (is_scaled_iv(exp->in(2), iv, p_scale)) { - if (p_offset != NULL) { + if (p_offset != nullptr) { *p_scale *= -1; *p_offset = exp->in(1); } @@ -2672,10 +2672,10 @@ Node* PhaseIdealLoop::add_range_check_predicate(IdealLoopTree* loop, CountedLoop Node* predicate_proj, int scale_con, Node* offset, Node* limit, jint stride_con, Node* value) { bool overflow = false; - BoolNode* bol = rc_predicate(loop, predicate_proj, scale_con, offset, value, NULL, stride_con, limit, (stride_con > 0) != (scale_con > 0), overflow, false); + BoolNode* bol = rc_predicate(loop, predicate_proj, scale_con, offset, value, nullptr, stride_con, limit, (stride_con > 0) != (scale_con > 0), overflow); Node* opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); register_new_node(opaque_bol, predicate_proj); - IfNode* new_iff = NULL; + IfNode* new_iff = nullptr; if (overflow) { new_iff = new IfNode(predicate_proj, opaque_bol, PROB_MAX, COUNT_UNKNOWN); } else { @@ -2725,7 +2725,7 @@ int PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) { // Check graph shape. Cannot optimize a loop if zero-trip // Opaque1 node is optimized away and then another round // of loop opts attempted. - if (cl->is_canonical_loop_entry() == NULL) { + if (cl->is_canonical_loop_entry() == nullptr) { return closed_range_checks; } @@ -2760,7 +2760,7 @@ int PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) { // Ensure the original loop limit is available from the // pre-loop Opaque1 node. Node *orig_limit = pre_opaq->original_loop_limit(); - if (orig_limit == NULL || _igvn.type(orig_limit) == Type::TOP) { + if (orig_limit == nullptr || _igvn.type(orig_limit) == Type::TOP) { return closed_range_checks; } // Must know if its a count-up or count-down loop @@ -2831,7 +2831,7 @@ int PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) { } // Check for scaled induction variable plus an offset - Node *offset = NULL; + Node *offset = nullptr; if (!is_scaled_iv_plus_offset(rc_exp, trip_counter, &scale_con, &offset)) { continue; @@ -2952,7 +2952,7 @@ int PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) { if (cd->is_Load() && cd->depends_only_on_test()) { // Loads can now float around in the loop // Allow the load to float around in the loop, or before it // but NOT before the pre-loop. - _igvn.replace_input_of(cd, 0, ctrl); // ctrl, not NULL + _igvn.replace_input_of(cd, 0, ctrl); // ctrl, not null --i; --imax; } @@ -3057,7 +3057,7 @@ bool PhaseIdealLoop::multi_version_post_loops(IdealLoopTree *rce_loop, IdealLoop } // Find RCE'd post loop so that we can stage its guard. - if (legacy_cl->is_canonical_loop_entry() == NULL) { + if (legacy_cl->is_canonical_loop_entry() == nullptr) { return multi_version_succeeded; } Node* ctrl = legacy_cl->in(LoopNode::EntryControl); @@ -3065,19 +3065,19 @@ bool PhaseIdealLoop::multi_version_post_loops(IdealLoopTree *rce_loop, IdealLoop // Now we test that both the post loops are connected Node* post_loop_region = iffm->in(0); - if (post_loop_region == NULL) return multi_version_succeeded; + if (post_loop_region == nullptr) return multi_version_succeeded; if (!post_loop_region->is_Region()) return multi_version_succeeded; Node* covering_region = post_loop_region->in(RegionNode::Control+1); - if (covering_region == NULL) return multi_version_succeeded; + if (covering_region == nullptr) return multi_version_succeeded; if (!covering_region->is_Region()) return multi_version_succeeded; Node* p_f = covering_region->in(RegionNode::Control); - if (p_f == NULL) return multi_version_succeeded; + if (p_f == nullptr) return multi_version_succeeded; if (!p_f->is_IfFalse()) return multi_version_succeeded; if (!p_f->in(0)->is_CountedLoopEnd()) return multi_version_succeeded; CountedLoopEndNode* rce_loop_end = p_f->in(0)->as_CountedLoopEnd(); - if (rce_loop_end == NULL) return multi_version_succeeded; + if (rce_loop_end == nullptr) return multi_version_succeeded; CountedLoopNode* rce_cl = rce_loop_end->loopnode(); - if (rce_cl == NULL || !rce_cl->is_post_loop()) return multi_version_succeeded; + if (rce_cl == nullptr || !rce_cl->is_post_loop()) return multi_version_succeeded; CountedLoopNode *known_rce_cl = rce_loop->_head->as_CountedLoop(); if (rce_cl != known_rce_cl) return multi_version_succeeded; @@ -3101,7 +3101,7 @@ bool PhaseIdealLoop::multi_version_post_loops(IdealLoopTree *rce_loop, IdealLoop // we have a work list. Now we will try to transform the if guard to cause // the loop pair to be multi version executed with the determination left to runtime // or the optimizer if full information is known about the given arrays at compile time. - Node *last_min = NULL; + Node *last_min = nullptr; multi_version_succeeded = true; while (worklist.size()) { Node* rc_iffm = worklist.pop(); @@ -3275,7 +3275,7 @@ void IdealLoopTree::remove_main_post_loops(CountedLoopNode *cl, PhaseIdealLoop * } // Can we find the main loop? - if (_next == NULL) { + if (_next == nullptr) { return; } @@ -3327,11 +3327,11 @@ bool IdealLoopTree::do_remove_empty_loop(PhaseIdealLoop *phase) { #ifdef ASSERT // Ensure only one phi which is the iv. - Node* iv = NULL; + Node* iv = nullptr; for (DUIterator_Fast imax, i = cl->fast_outs(imax); i < imax; i++) { Node* n = cl->fast_out(i); if (n->Opcode() == Op_Phi) { - assert(iv == NULL, "Too many phis"); + assert(iv == nullptr, "Too many phis"); iv = n; } } @@ -3402,7 +3402,12 @@ bool IdealLoopTree::do_remove_empty_loop(PhaseIdealLoop *phase) { // counted loop has limit check predicate. Node* phi = cl->phi(); Node* exact_limit = phase->exact_limit(this); - Node* final_iv = new SubINode(exact_limit, cl->stride()); + + // We need to pin the exact limit to prevent it from floating above the zero trip guard. + Node * cast_ii = ConstraintCastNode::make_cast(Op_CastII, cl->in(LoopNode::EntryControl), exact_limit, phase->_igvn.type(exact_limit), ConstraintCastNode::UnconditionalDependency); + phase->register_new_node(cast_ii, cl->in(LoopNode::EntryControl)); + + Node* final_iv = new SubINode(cast_ii, cl->stride()); phase->register_new_node(final_iv, cl->in(LoopNode::EntryControl)); phase->_igvn.replace_node(phi, final_iv); @@ -3644,21 +3649,21 @@ bool PhaseIdealLoop::do_intrinsify_fill() { // value in a unit stride loop, bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& store_value, Node*& shift, Node*& con) { - const char* msg = NULL; - Node* msg_node = NULL; + const char* msg = nullptr; + Node* msg_node = nullptr; - store_value = NULL; - con = NULL; - shift = NULL; + store_value = nullptr; + con = nullptr; + shift = nullptr; // Process the loop looking for stores. If there are multiple // stores or extra control flow give at this point. CountedLoopNode* head = lpt->_head->as_CountedLoop(); - for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) { + for (uint i = 0; msg == nullptr && i < lpt->_body.size(); i++) { Node* n = lpt->_body.at(i); if (n->outcnt() == 0) continue; // Ignore dead if (n->is_Store()) { - if (store != NULL) { + if (store != nullptr) { msg = "multiple stores"; break; } @@ -3681,12 +3686,12 @@ bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& st } } - if (store == NULL) { + if (store == nullptr) { // No store in loop return false; } - if (msg == NULL && head->stride_con() != 1) { + if (msg == nullptr && head->stride_con() != 1) { // could handle negative strides too if (head->stride_con() < 0) { msg = "negative stride"; @@ -3695,12 +3700,12 @@ bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& st } } - if (msg == NULL && !store->in(MemNode::Address)->is_AddP()) { + if (msg == nullptr && !store->in(MemNode::Address)->is_AddP()) { msg = "can't handle store address"; msg_node = store->in(MemNode::Address); } - if (msg == NULL && + if (msg == nullptr && (!store->in(MemNode::Memory)->is_Phi() || store->in(MemNode::Memory)->in(LoopNode::LoopBackControl) != store)) { msg = "store memory isn't proper phi"; @@ -3710,17 +3715,17 @@ bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& st // Make sure there is an appropriate fill routine BasicType t = store->as_Mem()->memory_type(); const char* fill_name; - if (msg == NULL && - StubRoutines::select_fill_function(t, false, fill_name) == NULL) { + if (msg == nullptr && + StubRoutines::select_fill_function(t, false, fill_name) == nullptr) { msg = "unsupported store"; msg_node = store; } - if (msg != NULL) { + if (msg != nullptr) { #ifndef PRODUCT if (TraceOptimizeFill) { tty->print_cr("not fill intrinsic candidate: %s", msg); - if (msg_node != NULL) msg_node->dump(); + if (msg_node != nullptr) msg_node->dump(); } #endif return false; @@ -3729,15 +3734,15 @@ bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& st // Make sure the address expression can be handled. It should be // head->phi * elsize + con. head->phi might have a ConvI2L(CastII()). Node* elements[4]; - Node* cast = NULL; - Node* conv = NULL; + Node* cast = nullptr; + Node* conv = nullptr; bool found_index = false; int count = store->in(MemNode::Address)->as_AddP()->unpack_offsets(elements, ARRAY_SIZE(elements)); for (int e = 0; e < count; e++) { Node* n = elements[e]; - if (n->is_Con() && con == NULL) { + if (n->is_Con() && con == nullptr) { con = n; - } else if (n->Opcode() == Op_LShiftX && shift == NULL) { + } else if (n->Opcode() == Op_LShiftX && shift == nullptr) { Node* value = n->in(1); #ifdef _LP64 if (value->Opcode() == Op_ConvI2L) { @@ -3761,7 +3766,7 @@ bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& st shift = n; } } - } else if (n->Opcode() == Op_ConvI2L && conv == NULL) { + } else if (n->Opcode() == Op_ConvI2L && conv == nullptr) { conv = n; n = n->in(1); if (n->Opcode() == Op_CastII && @@ -3794,16 +3799,16 @@ bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& st } // byte sized items won't have a shift - if (msg == NULL && shift == NULL && t != T_BYTE && t != T_BOOLEAN) { + if (msg == nullptr && shift == nullptr && t != T_BYTE && t != T_BOOLEAN) { msg = "can't find shift"; msg_node = store; } - if (msg != NULL) { + if (msg != nullptr) { #ifndef PRODUCT if (TraceOptimizeFill) { tty->print_cr("not fill intrinsic: %s", msg); - if (msg_node != NULL) msg_node->dump(); + if (msg_node != nullptr) msg_node->dump(); } #endif return false; @@ -3832,7 +3837,7 @@ bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& st if (cast) ok.set(cast->_idx); if (conv) ok.set(conv->_idx); - for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) { + for (uint i = 0; msg == nullptr && i < lpt->_body.size(); i++) { Node* n = lpt->_body.at(i); if (n->outcnt() == 0) continue; // Ignore dead if (ok.test(n->_idx)) continue; @@ -3846,7 +3851,7 @@ bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& st } // Make sure no unexpected values are used outside the loop - for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) { + for (uint i = 0; msg == nullptr && i < lpt->_body.size(); i++) { Node* n = lpt->_body.at(i); // These values can be replaced with other nodes if they are used // outside the loop. @@ -3863,9 +3868,9 @@ bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& st #ifdef ASSERT if (TraceOptimizeFill) { - if (msg != NULL) { + if (msg != nullptr) { tty->print_cr("no fill intrinsic: %s", msg); - if (msg_node != NULL) msg_node->dump(); + if (msg_node != nullptr) msg_node->dump(); } else { tty->print_cr("fill intrinsic for:"); } @@ -3876,7 +3881,7 @@ bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& st } #endif - return msg == NULL; + return msg == nullptr; } @@ -3897,16 +3902,16 @@ bool PhaseIdealLoop::intrinsify_fill(IdealLoopTree* lpt) { // Check that the body only contains a store of a loop invariant // value that is indexed by the loop phi. - Node* store = NULL; - Node* store_value = NULL; - Node* shift = NULL; - Node* offset = NULL; + Node* store = nullptr; + Node* store_value = nullptr; + Node* shift = nullptr; + Node* offset = nullptr; if (!match_fill_loop(lpt, store, store_value, shift, offset)) { return false; } Node* exit = head->loopexit()->proj_out_or_null(0); - if (exit == NULL) { + if (exit == nullptr) { return false; } @@ -3927,7 +3932,7 @@ bool PhaseIdealLoop::intrinsify_fill(IdealLoopTree* lpt) { index = new ConvI2LNode(index); _igvn.register_new_node_with_optimizer(index); #endif - if (shift != NULL) { + if (shift != nullptr) { // byte arrays don't require a shift but others do. index = new LShiftXNode(index, shift->in(2)); _igvn.register_new_node_with_optimizer(index); @@ -3936,10 +3941,10 @@ bool PhaseIdealLoop::intrinsify_fill(IdealLoopTree* lpt) { _igvn.register_new_node_with_optimizer(from); // For normal array fills, C2 uses two AddP nodes for array element // addressing. But for array fills with Unsafe call, there's only one - // AddP node adding an absolute offset, so we do a NULL check here. - assert(offset != NULL || C->has_unsafe_access(), + // AddP node adding an absolute offset, so we do a null check here. + assert(offset != nullptr || C->has_unsafe_access(), "Only array fills with unsafe have no extra offset"); - if (offset != NULL) { + if (offset != nullptr) { from = new AddPNode(base, from, offset); _igvn.register_new_node_with_optimizer(from); } @@ -3947,9 +3952,22 @@ bool PhaseIdealLoop::intrinsify_fill(IdealLoopTree* lpt) { Node* len = new SubINode(head->limit(), head->init_trip()); _igvn.register_new_node_with_optimizer(len); + // If the store is on the backedge, it is not executed in the last + // iteration, and we must subtract 1 from the len. + Node* backedge = head->loopexit()->proj_out(1); + if (store->in(0) == backedge) { + len = new SubINode(len, _igvn.intcon(1)); + _igvn.register_new_node_with_optimizer(len); +#ifndef PRODUCT + if (TraceOptimizeFill) { + tty->print_cr("ArrayFill store on backedge, subtract 1 from len."); + } +#endif + } + BasicType t = store->as_Mem()->memory_type(); bool aligned = false; - if (offset != NULL && head->init_trip()->is_Con()) { + if (offset != nullptr && head->init_trip()->is_Con()) { int element_size = type2aelembytes(t); aligned = (offset->find_intptr_t_type()->get_con() + head->init_trip()->get_int() * element_size) % HeapWordSize == 0; } @@ -3957,7 +3975,7 @@ bool PhaseIdealLoop::intrinsify_fill(IdealLoopTree* lpt) { // Build a call to the fill routine const char* fill_name; address fill = StubRoutines::select_fill_function(t, aligned, fill_name); - assert(fill != NULL, "what?"); + assert(fill != nullptr, "what?"); // Convert float/double to int/long for fill routines if (t == T_FLOAT) { @@ -4001,7 +4019,7 @@ bool PhaseIdealLoop::intrinsify_fill(IdealLoopTree* lpt) { // If this fill is tightly coupled to an allocation and overwrites // the whole body, allow it to take over the zeroing. AllocateNode* alloc = AllocateNode::Ideal_allocation(base, this); - if (alloc != NULL && alloc->is_AllocateArray()) { + if (alloc != nullptr && alloc->is_AllocateArray()) { Node* length = alloc->as_AllocateArray()->Ideal_length(); if (head->limit() == length && head->init_trip() == _igvn.intcon(0)) { @@ -4049,5 +4067,12 @@ bool PhaseIdealLoop::intrinsify_fill(IdealLoopTree* lpt) { _igvn.replace_node(n, C->top()); } +#ifndef PRODUCT + if (TraceOptimizeFill) { + tty->print("ArrayFill call "); + call->dump(); + } +#endif + return true; } diff --git a/src/hotspot/share/opto/loopUnswitch.cpp b/src/hotspot/share/opto/loopUnswitch.cpp index abdc7e66a492b..519cb96a992aa 100644 --- a/src/hotspot/share/opto/loopUnswitch.cpp +++ b/src/hotspot/share/opto/loopUnswitch.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2006, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -74,7 +74,7 @@ bool IdealLoopTree::policy_unswitching( PhaseIdealLoop *phase ) const { if (head->unswitch_count() + 1 > head->unswitch_max()) { return false; } - if (phase->find_unswitching_candidate(this) == NULL) { + if (phase->find_unswitching_candidate(this) == nullptr) { return false; } @@ -88,7 +88,7 @@ IfNode* PhaseIdealLoop::find_unswitching_candidate(const IdealLoopTree *loop) co // Find first invariant test that doesn't exit the loop LoopNode *head = loop->_head->as_Loop(); - IfNode* unswitch_iff = NULL; + IfNode* unswitch_iff = nullptr; Node* n = head->in(LoopNode::LoopBackControl); while (n != head) { Node* n_dom = idom(n); @@ -120,9 +120,9 @@ void PhaseIdealLoop::do_unswitching(IdealLoopTree *loop, Node_List &old_new) { LoopNode *head = loop->_head->as_Loop(); Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl); - if (find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check) != NULL - || (UseProfiledLoopPredicate && find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate) != NULL) - || (UseLoopPredicate && find_predicate_insertion_point(entry, Deoptimization::Reason_predicate) != NULL)) { + if (find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check) != nullptr + || (UseProfiledLoopPredicate && find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate) != nullptr) + || (UseLoopPredicate && find_predicate_insertion_point(entry, Deoptimization::Reason_predicate) != nullptr)) { assert(entry->is_IfProj(), "sanity - must be ifProj since there is at least one predicate"); if (entry->outcnt() > 1) { // Bailout if there are loop predicates from which there are additional control dependencies (i.e. from @@ -133,7 +133,7 @@ void PhaseIdealLoop::do_unswitching(IdealLoopTree *loop, Node_List &old_new) { } // Find first invariant test that doesn't exit the loop IfNode* unswitch_iff = find_unswitching_candidate((const IdealLoopTree *)loop); - assert(unswitch_iff != NULL, "should be at least one"); + assert(unswitch_iff != nullptr, "should be at least one"); #ifndef PRODUCT if (TraceLoopOpts) { @@ -155,7 +155,7 @@ void PhaseIdealLoop::do_unswitching(IdealLoopTree *loop, Node_List &old_new) { assert(proj_true->is_IfTrue(), "must be true projection"); entry = head->skip_strip_mined()->in(LoopNode::EntryControl); Node* predicate = find_predicate(entry); - if (predicate == NULL) { + if (predicate == nullptr) { // No empty predicate Node* uniqc = proj_true->unique_ctrl_out(); assert((uniqc == head && !head->is_strip_mined()) || (uniqc == head->in(LoopNode::EntryControl) @@ -166,13 +166,13 @@ void PhaseIdealLoop::do_unswitching(IdealLoopTree *loop, Node_List &old_new) { Node* proj_before_first_empty_predicate = skip_loop_predicates(entry); if (UseProfiledLoopPredicate) { predicate = find_predicate(proj_before_first_empty_predicate); - if (predicate != NULL) { + if (predicate != nullptr) { proj_before_first_empty_predicate = skip_loop_predicates(predicate); } } if (UseLoopPredicate) { predicate = find_predicate(proj_before_first_empty_predicate); - if (predicate != NULL) { + if (predicate != nullptr) { proj_before_first_empty_predicate = skip_loop_predicates(predicate); } } @@ -343,9 +343,9 @@ LoopNode* PhaseIdealLoop::create_reserve_version_of_loop(IdealLoopTree *loop, Co CountedLoopReserveKit::CountedLoopReserveKit(PhaseIdealLoop* phase, IdealLoopTree *loop, bool active = true) : _phase(phase), _lpt(loop), - _lp(NULL), - _iff(NULL), - _lp_reserved(NULL), + _lp(nullptr), + _iff(nullptr), + _lp_reserved(nullptr), _has_reserved(false), _use_new(false), _active(active) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index f26df6f239b9f..e2a5d26b14db4 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -74,12 +74,12 @@ bool LoopNode::is_valid_counted_loop(BasicType bt) const { if (is_BaseCountedLoop() && operates_on(bt, false)) { BaseCountedLoopNode* l = as_BaseCountedLoop(); BaseCountedLoopEndNode* le = l->loopexit_or_null(); - if (le != NULL && + if (le != nullptr && le->proj_out_or_null(1 /* true */) == l->in(LoopNode::LoopBackControl)) { Node* phi = l->phi(); Node* exit = le->proj_out_or_null(0 /* false */); - if (exit != NULL && exit->Opcode() == Op_IfFalse && - phi != NULL && phi->is_Phi() && + if (exit != nullptr && exit->Opcode() == Op_IfFalse && + phi != nullptr && phi->is_Phi() && phi->in(LoopNode::LoopBackControl) == l->incr() && le->loopnode() == l && le->stride_is_con()) { return true; @@ -173,7 +173,7 @@ Node *PhaseIdealLoop::get_early_ctrl_for_expensive(Node *n, Node* earliest) { // that doesn't branch to an UNC, we stop. The code that process // expensive nodes will notice the loop and skip over it to try to // move the node further up. - if (ctl->is_CountedLoop() && ctl->in(1) != NULL && ctl->in(1)->in(0) != NULL && ctl->in(1)->in(0)->is_If()) { + if (ctl->is_CountedLoop() && ctl->in(1) != nullptr && ctl->in(1)->in(0) != nullptr && ctl->in(1)->in(0)->is_If()) { if (!ctl->in(1)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) { break; } @@ -183,9 +183,9 @@ Node *PhaseIdealLoop::get_early_ctrl_for_expensive(Node *n, Node* earliest) { // the single control projection for its parent: same code path, // if it's a If with UNC or fallthrough of a call. Node* parent_ctl = ctl->in(0); - if (parent_ctl == NULL) { + if (parent_ctl == nullptr) { break; - } else if (parent_ctl->is_CountedLoopEnd() && parent_ctl->as_CountedLoopEnd()->loopnode() != NULL) { + } else if (parent_ctl->is_CountedLoopEnd() && parent_ctl->as_CountedLoopEnd()->loopnode() != nullptr) { next = parent_ctl->as_CountedLoopEnd()->loopnode()->init_control(); } else if (parent_ctl->is_If()) { if (!ctl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) { @@ -249,7 +249,7 @@ void PhaseIdealLoop::set_early_ctrl(Node* n, bool update_body) { // Record earliest legal location set_ctrl(n, early); IdealLoopTree *loop = get_loop(early); - if (update_body && loop->_child == NULL) { + if (update_body && loop->_child == nullptr) { loop->_body.push(n); } } @@ -289,7 +289,7 @@ IdealLoopTree* PhaseIdealLoop::insert_outer_loop(IdealLoopTree* loop, LoopNode* outer_ilt->_child = loop; outer_ilt->_nest = loop->_nest; loop->_parent = outer_ilt; - loop->_next = NULL; + loop->_next = nullptr; loop->_nest++; assert(loop->_nest <= SHRT_MAX, "sanity"); return outer_ilt; @@ -341,7 +341,7 @@ IdealLoopTree* PhaseIdealLoop::create_outer_strip_mined_loop(BoolNode *test, Nod } void PhaseIdealLoop::insert_loop_limit_check(ProjNode* limit_check_proj, Node* cmp_limit, Node* bol) { - Node* new_predicate_proj = create_new_if_for_predicate(limit_check_proj, NULL, + Node* new_predicate_proj = create_new_if_for_predicate(limit_check_proj, nullptr, Deoptimization::Reason_loop_limit_check, Op_If); Node* iff = new_predicate_proj->in(0); @@ -366,19 +366,19 @@ void PhaseIdealLoop::insert_loop_limit_check(ProjNode* limit_check_proj, Node* c } Node* PhaseIdealLoop::loop_exit_control(Node* x, IdealLoopTree* loop) { - // Counted loop head must be a good RegionNode with only 3 not NULL + // Counted loop head must be a good RegionNode with only 3 not null // control input edges: Self, Entry, LoopBack. - if (x->in(LoopNode::Self) == NULL || x->req() != 3 || loop->_irreducible) { - return NULL; + if (x->in(LoopNode::Self) == nullptr || x->req() != 3 || loop->_irreducible) { + return nullptr; } Node *init_control = x->in(LoopNode::EntryControl); Node *back_control = x->in(LoopNode::LoopBackControl); - if (init_control == NULL || back_control == NULL) { // Partially dead - return NULL; + if (init_control == nullptr || back_control == nullptr) { // Partially dead + return nullptr; } // Must also check for TOP when looking for a dead loop if (init_control->is_top() || back_control->is_top()) { - return NULL; + return nullptr; } // Allow funny placement of Safepoint @@ -394,13 +394,13 @@ Node* PhaseIdealLoop::loop_exit_control(Node* x, IdealLoopTree* loop) { // I have a weird back-control. Probably the loop-exit test is in // the middle of the loop and I am looking at some trailing control-flow // merge point. To fix this I would have to partially peel the loop. - return NULL; // Obscure back-control + return nullptr; // Obscure back-control } // Get boolean guarding loop-back test Node *iff = iftrue->in(0); if (get_loop(iff) != loop || !iff->in(1)->is_Bool()) { - return NULL; + return nullptr; } return iftrue; } @@ -419,7 +419,7 @@ Node* PhaseIdealLoop::loop_exit_test(Node* back_control, IdealLoopTree* loop, No // Get backedge compare Node* cmp = test->in(1); if (!cmp->is_Cmp()) { - return NULL; + return nullptr; } // Find the trip-counter increment & limit. Limit must be loop invariant. @@ -437,10 +437,10 @@ Node* PhaseIdealLoop::loop_exit_test(Node* back_control, IdealLoopTree* loop, No bt = BoolTest(bt).commute(); // And commute the exit test } if (is_member(loop, get_ctrl(limit))) { // Limit must be loop-invariant - return NULL; + return nullptr; } if (!is_member(loop, get_ctrl(incr))) { // Trip counter must be loop-variant - return NULL; + return nullptr; } return cmp; } @@ -448,12 +448,12 @@ Node* PhaseIdealLoop::loop_exit_test(Node* back_control, IdealLoopTree* loop, No Node* PhaseIdealLoop::loop_iv_incr(Node* incr, Node* x, IdealLoopTree* loop, Node*& phi_incr) { if (incr->is_Phi()) { if (incr->as_Phi()->region() != x || incr->req() != 3) { - return NULL; // Not simple trip counter expression + return nullptr; // Not simple trip counter expression } phi_incr = incr; incr = phi_incr->in(LoopNode::LoopBackControl); // Assume incr is on backedge of Phi if (!is_member(loop, get_ctrl(incr))) { // Trip counter must be loop-variant - return NULL; + return nullptr; } } return incr; @@ -466,7 +466,7 @@ Node* PhaseIdealLoop::loop_iv_stride(Node* incr, IdealLoopTree* loop, Node*& xph Node *stride = incr->in(2); if (!stride->is_Con()) { // Oops, swap these if (!xphi->is_Con()) { // Is the other guy a constant? - return NULL; // Nope, unknown stride, bail out + return nullptr; // Nope, unknown stride, bail out } Node *tmp = xphi; // 'incr' is commutative, so ok to swap xphi = stride; @@ -477,16 +477,16 @@ Node* PhaseIdealLoop::loop_iv_stride(Node* incr, IdealLoopTree* loop, Node*& xph PhiNode* PhaseIdealLoop::loop_iv_phi(Node* xphi, Node* phi_incr, Node* x, IdealLoopTree* loop) { if (!xphi->is_Phi()) { - return NULL; // Too much math on the trip counter + return nullptr; // Too much math on the trip counter } - if (phi_incr != NULL && phi_incr != xphi) { - return NULL; + if (phi_incr != nullptr && phi_incr != xphi) { + return nullptr; } PhiNode *phi = xphi->as_Phi(); // Phi must be of loop header; backedge must wrap to increment if (phi->region() != x) { - return NULL; + return nullptr; } return phi; } @@ -566,13 +566,13 @@ void PhaseIdealLoop::add_empty_predicate(Deoptimization::DeoptReason reason, Nod int trap_request = Deoptimization::make_trap_request(reason, Deoptimization::Action_maybe_recompile); address call_addr = SharedRuntime::uncommon_trap_blob()->entry_point(); - const TypePtr* no_memory_effects = NULL; + const TypePtr* no_memory_effects = nullptr; JVMState* jvms = sfpt->jvms(); CallNode* unc = new CallStaticJavaNode(OptoRuntime::uncommon_trap_Type(), call_addr, "uncommon_trap", no_memory_effects); - Node* mem = NULL; - Node* i_o = NULL; + Node* mem = nullptr; + Node* i_o = nullptr; if (sfpt->is_Call()) { mem = sfpt->proj_out(TypeFunc::Memory); i_o = sfpt->proj_out(TypeFunc::I_O); @@ -615,7 +615,7 @@ void PhaseIdealLoop::add_empty_predicate(Deoptimization::DeoptReason reason, Nod // SafePointNode so we can use its jvm state to create empty // predicates. static bool no_side_effect_since_safepoint(Compile* C, Node* x, Node* mem, MergeMemNode* mm) { - SafePointNode* safepoint = NULL; + SafePointNode* safepoint = nullptr; for (DUIterator_Fast imax, i = x->fast_outs(imax); i < imax; i++) { Node* u = x->fast_out(i); if (u->is_Phi() && u->bottom_type() == Type::MEMORY) { @@ -664,7 +664,7 @@ static bool no_side_effect_since_safepoint(Compile* C, Node* x, Node* mem, Merge SafePointNode* PhaseIdealLoop::find_safepoint(Node* back_control, Node* x, IdealLoopTree* loop) { IfNode* exit_test = back_control->in(0)->as_If(); - SafePointNode* safepoint = NULL; + SafePointNode* safepoint = nullptr; if (exit_test->in(0)->is_SafePoint() && exit_test->in(0)->outcnt() == 1) { safepoint = exit_test->in(0)->as_SafePoint(); } else { @@ -677,8 +677,8 @@ SafePointNode* PhaseIdealLoop::find_safepoint(Node* back_control, Node* x, Ideal safepoint = c->as_SafePoint(); } - if (safepoint == NULL) { - return NULL; + if (safepoint == nullptr) { + return nullptr; } Node* mem = safepoint->in(TypeFunc::Memory); @@ -686,7 +686,7 @@ SafePointNode* PhaseIdealLoop::find_safepoint(Node* back_control, Node* x, Ideal // We can only use that safepoint if there's no side effect between the backedge and the safepoint. // mm is used for book keeping - MergeMemNode* mm = NULL; + MergeMemNode* mm = nullptr; #ifdef ASSERT if (mem->is_MergeMem()) { mm = mem->clone()->as_MergeMem(); @@ -699,12 +699,12 @@ SafePointNode* PhaseIdealLoop::find_safepoint(Node* back_control, Node* x, Ideal } #endif if (!no_side_effect_since_safepoint(C, x, mem, mm)) { - safepoint = NULL; + safepoint = nullptr; } else { - assert(mm == NULL|| _igvn.transform(mm) == mem->as_MergeMem()->base_memory(), "all memory state should have been processed"); + assert(mm == nullptr|| _igvn.transform(mm) == mem->as_MergeMem()->base_memory(), "all memory state should have been processed"); } #ifdef ASSERT - if (mm != NULL) { + if (mm != nullptr) { _igvn.remove_dead_node(mm); } #endif @@ -763,7 +763,7 @@ SafePointNode* PhaseIdealLoop::find_safepoint(Node* back_control, Node* x, Ideal bool PhaseIdealLoop::transform_long_counted_loop(IdealLoopTree* loop, Node_List &old_new) { Node* x = loop->_head; // Only for inner loops - if (loop->_child != NULL || !x->is_LongCountedLoop() || x->as_Loop()->is_transformed_long_outer_loop()) { + if (loop->_child != nullptr || !x->is_LongCountedLoop() || x->as_Loop()->is_transformed_long_outer_loop()) { return false; } @@ -860,7 +860,7 @@ bool PhaseIdealLoop::transform_long_counted_loop(IdealLoopTree* loop, Node_List outer_phi->set_req(0, outer_head); register_new_node(outer_phi, outer_head); - Node* inner_iters_max = NULL; + Node* inner_iters_max = nullptr; if (stride_con > 0) { inner_iters_max = MaxNode::max_diff_with_zero(limit, outer_phi, TypeLong::LONG, _igvn); } else { @@ -888,7 +888,7 @@ bool PhaseIdealLoop::transform_long_counted_loop(IdealLoopTree* loop, Node_List set_ctrl(int_stride, C->root()); Node* inner_phi = new PhiNode(x->in(0), TypeInt::INT); Node* inner_incr = new AddINode(inner_phi, int_stride); - Node* inner_cmp = NULL; + Node* inner_cmp = nullptr; inner_cmp = new CmpINode(inner_incr, inner_iters_actual_int); Node* inner_bol = new BoolNode(inner_cmp, exit_test->in(1)->as_Bool()->_test._test); inner_phi->set_req(LoopNode::EntryControl, zero); @@ -946,7 +946,7 @@ bool PhaseIdealLoop::transform_long_counted_loop(IdealLoopTree* loop, Node_List // == new IR nodes (just before final peel) => // // entry_control: {...} - // long adjusted_limit = limit + stride; //because phi_incr != NULL + // long adjusted_limit = limit + stride; //because phi_incr != nullptr // assert(!limit_check_required || (extralong)limit + stride == adjusted_limit); // else deopt // ulong inner_iters_limit = max_jint - ABS(stride) - 1; //near 0x7FFFFFF0 // outer_head: @@ -982,14 +982,14 @@ bool PhaseIdealLoop::transform_long_counted_loop(IdealLoopTree* loop, Node_List // of the peeled iteration to insert empty predicates. If no well // positioned safepoint peel to guarantee a safepoint in the outer // loop. - if (safepoint != NULL || !loop->_has_call) { + if (safepoint != nullptr || !loop->_has_call) { old_new.clear(); do_peeling(loop, old_new); } else { C->set_major_progress(); } - if (safepoint != NULL) { + if (safepoint != nullptr) { SafePointNode* cloned_sfpt = old_new[safepoint->_idx]->as_SafePoint(); if (UseLoopPredicate) { @@ -1034,35 +1034,35 @@ LoopNode* PhaseIdealLoop::create_inner_head(IdealLoopTree* loop, LongCountedLoop #ifdef ASSERT void PhaseIdealLoop::check_long_counted_loop(IdealLoopTree* loop, Node* x) { Node* back_control = loop_exit_control(x, loop); - assert(back_control != NULL, "no back control"); + assert(back_control != nullptr, "no back control"); BoolTest::mask bt = BoolTest::illegal; float cl_prob = 0; - Node* incr = NULL; - Node* limit = NULL; + Node* incr = nullptr; + Node* limit = nullptr; Node* cmp = loop_exit_test(back_control, loop, incr, limit, bt, cl_prob); - assert(cmp != NULL && cmp->Opcode() == Op_CmpL, "no exit test"); + assert(cmp != nullptr && cmp->Opcode() == Op_CmpL, "no exit test"); - Node* phi_incr = NULL; + Node* phi_incr = nullptr; incr = loop_iv_incr(incr, x, loop, phi_incr); - assert(incr != NULL && incr->Opcode() == Op_AddL, "no incr"); + assert(incr != nullptr && incr->Opcode() == Op_AddL, "no incr"); - Node* xphi = NULL; + Node* xphi = nullptr; Node* stride = loop_iv_stride(incr, loop, xphi); - assert(stride != NULL, "no stride"); + assert(stride != nullptr, "no stride"); PhiNode* phi = loop_iv_phi(xphi, phi_incr, x, loop); - assert(phi != NULL && phi->in(LoopNode::LoopBackControl) == incr, "No phi"); + assert(phi != nullptr && phi->in(LoopNode::LoopBackControl) == incr, "No phi"); jlong stride_con = stride->get_long(); assert(condition_stride_ok(bt, stride_con), "illegal condition"); assert(bt != BoolTest::ne, "unexpected condition"); - assert(phi_incr == NULL, "bad loop shape"); + assert(phi_incr == nullptr, "bad loop shape"); assert(cmp->in(1) == incr, "bad exit test shape"); // Safepoint on backedge not supported @@ -1088,12 +1088,12 @@ bool PhaseIdealLoop::convert_to_long_loop(Node* cmp, Node* phi, IdealLoopTree* l break; } case Op_CmpI: { - Node* clone = new CmpLNode(NULL, NULL); + Node* clone = new CmpLNode(nullptr, nullptr); old_new.map(n->_idx, clone); break; } case Op_AddI: { - Node* clone = new AddLNode(NULL, NULL); + Node* clone = new AddLNode(nullptr, nullptr); old_new.map(n->_idx, clone); break; } @@ -1108,7 +1108,7 @@ bool PhaseIdealLoop::convert_to_long_loop(Node* cmp, Node* phi, IdealLoopTree* l for (uint i = 1; i < n->req(); i++) { Node* in = n->in(i); - if (in == NULL) { + if (in == nullptr) { continue; } if (loop->is_member(get_loop(get_ctrl(in)))) { @@ -1121,7 +1121,7 @@ bool PhaseIdealLoop::convert_to_long_loop(Node* cmp, Node* phi, IdealLoopTree* l for (uint i = 0; i < iv_nodes.size(); i++) { Node* n = iv_nodes.at(i); Node* clone = old_new[n->_idx]; - if (clone != NULL) { + if (clone != nullptr) { _igvn.remove_dead_node(clone); } } @@ -1133,20 +1133,20 @@ bool PhaseIdealLoop::convert_to_long_loop(Node* cmp, Node* phi, IdealLoopTree* l Node* clone = old_new[n->_idx]; for (uint i = 1; i < n->req(); i++) { Node* in = n->in(i); - if (in == NULL) { + if (in == nullptr) { continue; } Node* in_clone = old_new[in->_idx]; - if (in_clone == NULL) { + if (in_clone == nullptr) { assert(_igvn.type(in)->isa_int(), ""); in_clone = new ConvI2LNode(in); _igvn.register_new_node_with_optimizer(in_clone); set_subtree_ctrl(in_clone, false); } - if (in_clone->in(0) == NULL) { + if (in_clone->in(0) == nullptr) { in_clone->set_req(0, C->top()); clone->set_req(i, in_clone); - in_clone->set_req(0, NULL); + in_clone->set_req(0, nullptr); } else { clone->set_req(i, in_clone); } @@ -1159,13 +1159,13 @@ bool PhaseIdealLoop::convert_to_long_loop(Node* cmp, Node* phi, IdealLoopTree* l Node* n = iv_nodes.at(i); Node* clone = old_new[n->_idx]; set_subtree_ctrl(clone, false); - Node* m = n->Opcode() == Op_CmpI ? clone : NULL; + Node* m = n->Opcode() == Op_CmpI ? clone : nullptr; for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { Node* u = n->fast_out(i); if (iv_nodes.member(u)) { continue; } - if (m == NULL) { + if (m == nullptr) { m = new ConvL2INode(clone); _igvn.register_new_node_with_optimizer(m); set_subtree_ctrl(m, false); @@ -1184,16 +1184,16 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ PhaseGVN *gvn = &_igvn; Node* back_control = loop_exit_control(x, loop); - if (back_control == NULL) { + if (back_control == nullptr) { return false; } BoolTest::mask bt = BoolTest::illegal; float cl_prob = 0; - Node* incr = NULL; - Node* limit = NULL; + Node* incr = nullptr; + Node* limit = nullptr; Node* cmp = loop_exit_test(back_control, loop, incr, limit, bt, cl_prob); - if (cmp == NULL || !(cmp->is_Cmp() && cmp->operates_on(iv_bt, true))) { + if (cmp == nullptr || !(cmp->is_Cmp() && cmp->operates_on(iv_bt, true))) { return false; // Avoid pointer & float & 64-bit compares } @@ -1202,25 +1202,25 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ incr = incr->in(1); } - Node* phi_incr = NULL; + Node* phi_incr = nullptr; incr = loop_iv_incr(incr, x, loop, phi_incr); - if (incr == NULL) { + if (incr == nullptr) { return false; } - Node* trunc1 = NULL; - Node* trunc2 = NULL; - const TypeInteger* iv_trunc_t = NULL; + Node* trunc1 = nullptr; + Node* trunc2 = nullptr; + const TypeInteger* iv_trunc_t = nullptr; Node* orig_incr = incr; if (!(incr = CountedLoopNode::match_incr_with_optional_truncation(incr, &trunc1, &trunc2, &iv_trunc_t, iv_bt))) { return false; // Funny increment opcode } assert(incr->is_Add() && incr->operates_on(iv_bt, false), "wrong increment code"); - Node* xphi = NULL; + Node* xphi = nullptr; Node* stride = loop_iv_stride(incr, loop, xphi); - if (stride == NULL) { + if (stride == nullptr) { return false; } @@ -1234,9 +1234,9 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ PhiNode* phi = loop_iv_phi(xphi, phi_incr, x, loop); - if (phi == NULL || - (trunc1 == NULL && phi->in(LoopNode::LoopBackControl) != incr) || - (trunc1 != NULL && phi->in(LoopNode::LoopBackControl) != trunc1)) { + if (phi == nullptr || + (trunc1 == nullptr && phi->in(LoopNode::LoopBackControl) != incr) || + (trunc1 != nullptr && phi->in(LoopNode::LoopBackControl) != trunc1)) { return false; } @@ -1246,7 +1246,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ BoolNode* test = iff->in(1)->as_Bool(); const TypeInteger* limit_t = gvn->type(limit)->is_integer(iv_bt); - if (trunc1 != NULL) { + if (trunc1 != nullptr) { // When there is a truncation, we must be sure that after the truncation // the trip counter will end up higher than the limit, otherwise we are looking // at an endless loop. Can happen with range checks. @@ -1275,7 +1275,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ // If iv trunc type is smaller than int, check for possible wrap. if (!TypeInteger::bottom(iv_bt)->higher_equal(iv_trunc_t)) { - assert(trunc1 != NULL, "must have found some truncation"); + assert(trunc1 != nullptr, "must have found some truncation"); // Get a better type for the phi (filtered thru if's) const TypeInteger* phi_ft = filtered_type(phi); @@ -1305,7 +1305,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ // No possibility of wrap so truncation can be discarded // Promote iv type to Int } else { - assert(trunc1 == NULL && trunc2 == NULL, "no truncation for int"); + assert(trunc1 == nullptr && trunc2 == nullptr, "no truncation for int"); } if (!condition_stride_ok(bt, stride_con)) { @@ -1324,7 +1324,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ } } - if (phi_incr != NULL && bt != BoolTest::ne) { + if (phi_incr != nullptr && bt != BoolTest::ne) { // check if there is a possiblity of IV overflowing after the first increment if (stride_con > 0) { if (init_t->hi_as_long() > max_signed_integer(iv_bt) - stride_con) { @@ -1367,7 +1367,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ // to be adjusted to keep trip count the same and the // adjusted limit should be checked for int overflow. Node* adjusted_limit = limit; - if (phi_incr != NULL) { + if (phi_incr != nullptr) { stride_m += stride_con; } @@ -1468,13 +1468,13 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ } } - Node* sfpt = NULL; - if (loop->_child == NULL) { + Node* sfpt = nullptr; + if (loop->_child == nullptr) { sfpt = find_safepoint(back_control, x, loop); } else { sfpt = iff->in(0); if (sfpt->Opcode() != Op_SafePoint) { - sfpt = NULL; + sfpt = nullptr; } } @@ -1482,7 +1482,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ Node* backedge_sfpt = x->in(LoopNode::LoopBackControl); if (((iv_bt == T_INT && LoopStripMiningIter != 0) || iv_bt == T_LONG) && - sfpt == NULL) { + sfpt == nullptr) { // Leaving the safepoint on the backedge and creating a // CountedLoop will confuse optimizations. We can't move the // safepoint around because its jvm state wouldn't match a new @@ -1491,7 +1491,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ } if (is_deleteable_safept(backedge_sfpt)) { lazy_replace(backedge_sfpt, iftrue); - if (loop->_safepts != NULL) { + if (loop->_safepts != nullptr) { loop->_safepts->yank(backedge_sfpt); } loop->_tail = iftrue; @@ -1503,13 +1503,13 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ if (iv_bt == T_INT && !x->as_Loop()->is_transformed_long_inner_loop() && StressLongCountedLoop > 0 && - trunc1 == NULL && + trunc1 == nullptr && convert_to_long_loop(cmp, phi, loop)) { return false; } #endif - if (phi_incr != NULL) { + if (phi_incr != nullptr) { // If compare points directly to the phi we need to adjust // the compare so that it points to the incr. Limit have // to be adjusted to keep trip count the same and we @@ -1611,10 +1611,10 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ Node* entry_control = init_control; bool strip_mine_loop = iv_bt == T_INT && LoopStripMiningIter > 1 && - loop->_child == NULL && - sfpt != NULL && + loop->_child == nullptr && + sfpt != nullptr && !loop->_has_call; - IdealLoopTree* outer_ilt = NULL; + IdealLoopTree* outer_ilt = nullptr; if (strip_mine_loop) { outer_ilt = create_outer_strip_mined_loop(test, cmp, init_control, loop, cl_prob, le->_fcnt, entry_control, @@ -1638,7 +1638,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ if (iv_bt == T_INT && (LoopStripMiningIter == 0 || strip_mine_loop)) { // Check for immediately preceding SafePoint and remove - if (sfpt != NULL && (LoopStripMiningIter != 0 || is_deleteable_safept(sfpt))) { + if (sfpt != nullptr && (LoopStripMiningIter != 0 || is_deleteable_safept(sfpt))) { if (strip_mine_loop) { Node* outer_le = outer_ilt->_tail->in(0); Node* sfpt_clone = sfpt->clone(); @@ -1660,7 +1660,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ set_idom(outer_le, sfpt_clone, dom_depth(sfpt_clone)); } lazy_replace(sfpt, sfpt->in(TypeFunc::Control)); - if (loop->_safepts != NULL) { + if (loop->_safepts != nullptr) { loop->_safepts->yank(sfpt); } } @@ -1715,7 +1715,7 @@ Node* PhaseIdealLoop::exact_limit( IdealLoopTree *loop ) { // Loop limit is exact with stride == 1. And loop may already have exact limit. return cl->limit(); } - Node *limit = NULL; + Node *limit = nullptr; #ifdef ASSERT BoolTest::mask bt = cl->loopexit()->test_trip(); assert(bt == BoolTest::lt || bt == BoolTest::gt, "canonical test is expected"); @@ -1738,7 +1738,7 @@ Node* PhaseIdealLoop::exact_limit( IdealLoopTree *loop ) { limit = new LoopLimitNode(C, cl->init_trip(), cl->limit(), cl->stride()); register_new_node(limit, cl->in(LoopNode::EntryControl)); } - assert(limit != NULL, "sanity"); + assert(limit != nullptr, "sanity"); return limit; } @@ -1754,8 +1754,8 @@ Node *LoopNode::Ideal(PhaseGVN *phase, bool can_reshape) { #ifdef ASSERT void LoopNode::verify_strip_mined(int expect_skeleton) const { - const OuterStripMinedLoopNode* outer = NULL; - const CountedLoopNode* inner = NULL; + const OuterStripMinedLoopNode* outer = nullptr; + const CountedLoopNode* inner = nullptr; if (is_strip_mined()) { if (!is_valid_counted_loop(T_INT)) { return; // Skip malformed counted loop @@ -1769,8 +1769,8 @@ void LoopNode::verify_strip_mined(int expect_skeleton) const { assert(inner->is_valid_counted_loop(T_INT) && inner->is_strip_mined(), "OuterStripMinedLoop should have been removed"); assert(!is_strip_mined(), "outer loop shouldn't be marked strip mined"); } - if (inner != NULL || outer != NULL) { - assert(inner != NULL && outer != NULL, "missing loop in strip mined nest"); + if (inner != nullptr || outer != nullptr) { + assert(inner != nullptr && outer != nullptr, "missing loop in strip mined nest"); Node* outer_tail = outer->in(LoopNode::LoopBackControl); Node* outer_le = outer_tail->in(0); assert(outer_le->Opcode() == Op_OuterStripMinedLoopEnd, "tail of outer loop should be an If"); @@ -1891,8 +1891,14 @@ const Type* LoopLimitNode::Value(PhaseGVN* phase) const { int final_int = (int)final_con; // The final value should be in integer range since the loop // is counted and the limit was checked for overflow. - assert(final_con == (jlong)final_int, "final value should be integer"); - return TypeInt::make(final_int); + // Assert checks for overflow only if all input nodes are ConINodes, as during CCP + // there might be a temporary overflow from PhiNodes see JDK-8309266 + assert((in(Init)->is_ConI() && in(Limit)->is_ConI() && in(Stride)->is_ConI()) ? final_con == (jlong)final_int : true, "final value should be integer"); + if (final_con == (jlong)final_int) { + return TypeInt::make(final_int); + } else { + return bottom_type(); + } } return bottom_type(); // TypeInt::INT @@ -1904,19 +1910,19 @@ Node *LoopLimitNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (phase->type(in(Init)) == Type::TOP || phase->type(in(Limit)) == Type::TOP || phase->type(in(Stride)) == Type::TOP) - return NULL; // Dead + return nullptr; // Dead int stride_con = phase->type(in(Stride))->is_int()->get_con(); if (stride_con == 1) - return NULL; // Identity + return nullptr; // Identity if (in(Init)->is_Con() && in(Limit)->is_Con()) - return NULL; // Value + return nullptr; // Value // Delay following optimizations until all loop optimizations // done to keep Ideal graph simple. if (!can_reshape || !phase->C->post_loop_opts_phase()) { - return NULL; + return nullptr; } const TypeInt* init_t = phase->type(in(Init) )->is_int(); @@ -1978,7 +1984,7 @@ Node *LoopLimitNode::Ideal(PhaseGVN *phase, bool can_reshape) { return new AddINode(span_int, in(Init)); // exact limit } - return NULL; // No progress + return nullptr; // No progress } //------------------------------Identity--------------------------------------- @@ -1994,15 +2000,15 @@ Node* LoopLimitNode::Identity(PhaseGVN* phase) { //----------------------match_incr_with_optional_truncation-------------------- // Match increment with optional truncation: // CHAR: (i+1)&0x7fff, BYTE: ((i+1)<<8)>>8, or SHORT: ((i+1)<<16)>>16 -// Return NULL for failure. Success returns the increment node. +// Return null for failure. Success returns the increment node. Node* CountedLoopNode::match_incr_with_optional_truncation(Node* expr, Node** trunc1, Node** trunc2, const TypeInteger** trunc_type, BasicType bt) { // Quick cutouts: - if (expr == NULL || expr->req() != 3) return NULL; + if (expr == nullptr || expr->req() != 3) return nullptr; - Node *t1 = NULL; - Node *t2 = NULL; + Node *t1 = nullptr; + Node *t2 = nullptr; Node* n1 = expr; int n1op = n1->Opcode(); const TypeInteger* trunc_t = TypeInteger::bottom(bt); @@ -2018,7 +2024,7 @@ Node* CountedLoopNode::match_incr_with_optional_truncation(Node* expr, Node** tr n1op = n1->Opcode(); trunc_t = TypeInt::CHAR; } else if (n1op == Op_RShiftI && - n1->in(1) != NULL && + n1->in(1) != nullptr && n1->in(1)->Opcode() == Op_LShiftI && n1->in(2) == n1->in(1)->in(2) && n1->in(2)->is_Con()) { @@ -2047,11 +2053,11 @@ Node* CountedLoopNode::match_incr_with_optional_truncation(Node* expr, Node** tr } // failed - return NULL; + return nullptr; } LoopNode* CountedLoopNode::skip_strip_mined(int expect_skeleton) { - if (is_strip_mined() && in(EntryControl) != NULL && in(EntryControl)->is_OuterStripMinedLoop()) { + if (is_strip_mined() && in(EntryControl) != nullptr && in(EntryControl)->is_OuterStripMinedLoop()) { verify_strip_mined(expect_skeleton); return in(EntryControl)->as_Loop(); } @@ -2061,76 +2067,76 @@ LoopNode* CountedLoopNode::skip_strip_mined(int expect_skeleton) { OuterStripMinedLoopNode* CountedLoopNode::outer_loop() const { assert(is_strip_mined(), "not a strip mined loop"); Node* c = in(EntryControl); - if (c == NULL || c->is_top() || !c->is_OuterStripMinedLoop()) { - return NULL; + if (c == nullptr || c->is_top() || !c->is_OuterStripMinedLoop()) { + return nullptr; } return c->as_OuterStripMinedLoop(); } IfTrueNode* OuterStripMinedLoopNode::outer_loop_tail() const { Node* c = in(LoopBackControl); - if (c == NULL || c->is_top()) { - return NULL; + if (c == nullptr || c->is_top()) { + return nullptr; } return c->as_IfTrue(); } IfTrueNode* CountedLoopNode::outer_loop_tail() const { LoopNode* l = outer_loop(); - if (l == NULL) { - return NULL; + if (l == nullptr) { + return nullptr; } return l->outer_loop_tail(); } OuterStripMinedLoopEndNode* OuterStripMinedLoopNode::outer_loop_end() const { IfTrueNode* proj = outer_loop_tail(); - if (proj == NULL) { - return NULL; + if (proj == nullptr) { + return nullptr; } Node* c = proj->in(0); - if (c == NULL || c->is_top() || c->outcnt() != 2) { - return NULL; + if (c == nullptr || c->is_top() || c->outcnt() != 2) { + return nullptr; } return c->as_OuterStripMinedLoopEnd(); } OuterStripMinedLoopEndNode* CountedLoopNode::outer_loop_end() const { LoopNode* l = outer_loop(); - if (l == NULL) { - return NULL; + if (l == nullptr) { + return nullptr; } return l->outer_loop_end(); } IfFalseNode* OuterStripMinedLoopNode::outer_loop_exit() const { IfNode* le = outer_loop_end(); - if (le == NULL) { - return NULL; + if (le == nullptr) { + return nullptr; } Node* c = le->proj_out_or_null(false); - if (c == NULL) { - return NULL; + if (c == nullptr) { + return nullptr; } return c->as_IfFalse(); } IfFalseNode* CountedLoopNode::outer_loop_exit() const { LoopNode* l = outer_loop(); - if (l == NULL) { - return NULL; + if (l == nullptr) { + return nullptr; } return l->outer_loop_exit(); } SafePointNode* OuterStripMinedLoopNode::outer_safepoint() const { IfNode* le = outer_loop_end(); - if (le == NULL) { - return NULL; + if (le == nullptr) { + return nullptr; } Node* c = le->in(0); - if (c == NULL || c->is_top()) { - return NULL; + if (c == nullptr || c->is_top()) { + return nullptr; } assert(c->Opcode() == Op_SafePoint, "broken outer loop"); return c->as_SafePoint(); @@ -2138,15 +2144,15 @@ SafePointNode* OuterStripMinedLoopNode::outer_safepoint() const { SafePointNode* CountedLoopNode::outer_safepoint() const { LoopNode* l = outer_loop(); - if (l == NULL) { - return NULL; + if (l == nullptr) { + return nullptr; } return l->outer_safepoint(); } Node* CountedLoopNode::skip_predicates_from_entry(Node* ctrl) { - while (ctrl != NULL && ctrl->is_Proj() && ctrl->in(0) != NULL && ctrl->in(0)->is_If() && - (ctrl->in(0)->as_If()->proj_out_or_null(1-ctrl->as_Proj()->_con) == NULL || + while (ctrl != nullptr && ctrl->is_Proj() && ctrl->in(0) != nullptr && ctrl->in(0)->is_If() && + (ctrl->in(0)->as_If()->proj_out_or_null(1-ctrl->as_Proj()->_con) == nullptr || (ctrl->in(0)->as_If()->proj_out(1-ctrl->as_Proj()->_con)->outcnt() == 1 && ctrl->in(0)->as_If()->proj_out(1-ctrl->as_Proj()->_con)->unique_out()->Opcode() == Op_Halt))) { ctrl = ctrl->in(0)->in(0); @@ -2169,12 +2175,12 @@ Node* CountedLoopNode::skip_predicates() { int CountedLoopNode::stride_con() const { CountedLoopEndNode* cle = loopexit_or_null(); - return cle != NULL ? cle->stride_con() : 0; + return cle != nullptr ? cle->stride_con() : 0; } jlong LongCountedLoopNode::stride_con() const { LongCountedLoopEndNode* cle = loopexit_or_null(); - return cle != NULL ? cle->stride_con() : 0; + return cle != nullptr ? cle->stride_con() : 0; } BaseCountedLoopNode* BaseCountedLoopNode::make(Node* entry, Node* backedge, BasicType bt) { @@ -2193,7 +2199,7 @@ void OuterStripMinedLoopNode::adjust_strip_mined_loop(PhaseIterGVN* igvn) { CountedLoopNode* inner_cl = unique_ctrl_out()->as_CountedLoop(); assert(inner_cl->is_strip_mined(), "inner loop should be strip mined"); Node* inner_iv_phi = inner_cl->phi(); - if (inner_iv_phi == NULL) { + if (inner_iv_phi == nullptr) { IfNode* outer_le = outer_loop_end(); Node* iff = igvn->transform(new IfNode(outer_le->in(0), outer_le->in(1), outer_le->_prob, outer_le->_fcnt)); igvn->replace_node(outer_le, iff); @@ -2265,18 +2271,18 @@ void OuterStripMinedLoopNode::adjust_strip_mined_loop(PhaseIterGVN* igvn) { for (uint next = 0; next < backedge_nodes.size(); next++) { Node *n = old_new[backedge_nodes.at(next)->_idx]; for (uint i = 1; i < n->req(); i++) { - if (n->in(i) != NULL && old_new[n->in(i)->_idx] != NULL) { + if (n->in(i) != nullptr && old_new[n->in(i)->_idx] != nullptr) { n->set_req(i, old_new[n->in(i)->_idx]); } } - if (n->in(0) != NULL && n->in(0) == cle_tail) { + if (n->in(0) != nullptr && n->in(0) == cle_tail) { n->set_req(0, le_tail); } igvn->register_new_node_with_optimizer(n); } } - Node* iv_phi = NULL; + Node* iv_phi = nullptr; // Make a clone of each phi in the inner loop // for the outer loop for (uint i = 0; i < inner_cl->outcnt(); i++) { @@ -2286,7 +2292,7 @@ void OuterStripMinedLoopNode::adjust_strip_mined_loop(PhaseIterGVN* igvn) { Node* phi = u->clone(); phi->set_req(0, this); Node* be = old_new[phi->in(LoopNode::LoopBackControl)->_idx]; - if (be != NULL) { + if (be != nullptr) { phi->set_req(LoopNode::LoopBackControl, be); } phi = igvn->transform(phi); @@ -2412,14 +2418,14 @@ void OuterStripMinedLoopNode::adjust_strip_mined_loop(PhaseIterGVN* igvn) { } } - if (iv_phi != NULL) { + if (iv_phi != nullptr) { // Now adjust the inner loop's exit condition Node* limit = inner_cl->limit(); // If limit < init for stride > 0 (or limit > init for stride < 0), // the loop body is run only once. Given limit - init (init - limit resp.) // would be negative, the unsigned comparison below would cause // the loop body to be run for LoopStripMiningIter. - Node* max = NULL; + Node* max = nullptr; if (stride > 0) { max = MaxNode::max_diff_with_zero(limit, iv_phi, TypeInt::INT, *igvn); } else { @@ -2433,7 +2439,7 @@ void OuterStripMinedLoopNode::adjust_strip_mined_loop(PhaseIterGVN* igvn) { // unsigned_min(max(limit - iv_phi, 0), scaled_iters) if stride > 0 // unsigned_min(max(iv_phi - limit, 0), scaled_iters) if stride < 0 - Node* new_limit = NULL; + Node* new_limit = nullptr; if (stride > 0) { new_limit = igvn->transform(new AddINode(min, iv_phi)); } else { @@ -2476,10 +2482,10 @@ bool OuterStripMinedLoopEndNode::is_expanded(PhaseGVN *phase) const { // The outer strip mined loop head only has Phi uses after expansion if (phase->is_IterGVN()) { Node* backedge = proj_out_or_null(true); - if (backedge != NULL) { + if (backedge != nullptr) { Node* head = backedge->unique_ctrl_out(); - if (head != NULL && head->is_OuterStripMinedLoop()) { - if (head->find_out_with(Op_Phi) != NULL) { + if (head != nullptr && head->is_OuterStripMinedLoop()) { + if (head->find_out_with(Op_Phi) != nullptr) { return true; } } @@ -2491,7 +2497,7 @@ bool OuterStripMinedLoopEndNode::is_expanded(PhaseGVN *phase) const { Node *OuterStripMinedLoopEndNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (remove_dead_region(phase, can_reshape)) return this; - return NULL; + return nullptr; } //------------------------------filtered_type-------------------------------- @@ -2512,22 +2518,22 @@ Node *OuterStripMinedLoopEndNode::Ideal(PhaseGVN *phase, bool can_reshape) { // const TypeInt* PhaseIdealLoop::filtered_type( Node *n, Node* n_ctrl) { assert(n && n->bottom_type()->is_int(), "must be int"); - const TypeInt* filtered_t = NULL; + const TypeInt* filtered_t = nullptr; if (!n->is_Phi()) { - assert(n_ctrl != NULL || n_ctrl == C->top(), "valid control"); + assert(n_ctrl != nullptr || n_ctrl == C->top(), "valid control"); filtered_t = filtered_type_from_dominators(n, n_ctrl); } else { Node* phi = n->as_Phi(); Node* region = phi->in(0); - assert(n_ctrl == NULL || n_ctrl == region, "ctrl parameter must be region"); + assert(n_ctrl == nullptr || n_ctrl == region, "ctrl parameter must be region"); if (region && region != C->top()) { for (uint i = 1; i < phi->req(); i++) { Node* val = phi->in(i); Node* use_c = region->in(i); const TypeInt* val_t = filtered_type_from_dominators(val, use_c); - if (val_t != NULL) { - if (filtered_t == NULL) { + if (val_t != nullptr) { + if (filtered_t == nullptr) { filtered_t = val_t; } else { filtered_t = filtered_t->meet(val_t)->is_int(); @@ -2537,7 +2543,7 @@ const TypeInt* PhaseIdealLoop::filtered_type( Node *n, Node* n_ctrl) { } } const TypeInt* n_t = _igvn.type(n)->is_int(); - if (filtered_t != NULL) { + if (filtered_t != nullptr) { n_t = n_t->join(filtered_t)->is_int(); } return n_t; @@ -2551,7 +2557,7 @@ const TypeInt* PhaseIdealLoop::filtered_type_from_dominators( Node* val, Node *u return val->bottom_type()->is_int(); } uint if_limit = 10; // Max number of dominating if's visited - const TypeInt* rtn_t = NULL; + const TypeInt* rtn_t = nullptr; if (use_ctrl && use_ctrl != C->top()) { Node* val_ctrl = get_ctrl(val); @@ -2562,8 +2568,8 @@ const TypeInt* PhaseIdealLoop::filtered_type_from_dominators( Node* val, Node *u if ((pred->Opcode() == Op_IfTrue || pred->Opcode() == Op_IfFalse)) { if_cnt++; const TypeInt* if_t = IfNode::filtered_int_type(&_igvn, val, pred); - if (if_t != NULL) { - if (rtn_t == NULL) { + if (if_t != nullptr) { + if (rtn_t == nullptr) { rtn_t = if_t; } else { rtn_t = rtn_t->join(if_t)->is_int(); @@ -2571,7 +2577,7 @@ const TypeInt* PhaseIdealLoop::filtered_type_from_dominators( Node* val, Node *u } } pred = idom(pred); - if (pred == NULL || pred == C->top()) { + if (pred == nullptr || pred == C->top()) { break; } // Stop if going beyond definition block of val @@ -2588,7 +2594,7 @@ const TypeInt* PhaseIdealLoop::filtered_type_from_dominators( Node* val, Node *u // Dump special per-node info #ifndef PRODUCT void CountedLoopEndNode::dump_spec(outputStream *st) const { - if( in(TestValue) != NULL && in(TestValue)->is_Bool() ) { + if( in(TestValue) != nullptr && in(TestValue)->is_Bool() ) { BoolTest bt( test_trip()); // Added this for g++. st->print("["); @@ -2772,7 +2778,7 @@ static float estimate_path_freq( Node *n ) { ciMethodData* methodData = jvms->method()->method_data(); if (!methodData->is_mature()) return 0.0f; // No call-site data ciProfileData* data = methodData->bci_to_data(jvms->bci()); - if ((data == NULL) || !data->is_CounterData()) { + if ((data == nullptr) || !data->is_CounterData()) { // no call profile available, try call's control input n = n->in(0); continue; @@ -2827,7 +2833,7 @@ void IdealLoopTree::merge_many_backedges( PhaseIdealLoop *phase ) { // them all except optionally hot_idx. PhaseIterGVN &igvn = phase->_igvn; - Node *hot_tail = NULL; + Node *hot_tail = nullptr; // Make a Region for the merge point Node *r = new RegionNode(1); for( i = 2; i < _head->req(); i++ ) { @@ -2847,7 +2853,7 @@ void IdealLoopTree::merge_many_backedges( PhaseIdealLoop *phase ) { if( out->is_Phi() ) { PhiNode* n = out->as_Phi(); igvn.hash_delete(n); // Delete from hash before hacking edges - Node *hot_phi = NULL; + Node *hot_phi = nullptr; Node *phi = new PhiNode(r, n->type(), n->adr_type()); // Check all inputs for the ones to peel out uint j = 1; @@ -2894,7 +2900,7 @@ void IdealLoopTree::merge_many_backedges( PhaseIdealLoop *phase ) { while( *cp ) cp = &(*cp)->_next; // Find end of child list *cp = ilt->_next; // Hang next list at end of child list *pilt = ilt->_child; // Move child up to replace ilt - ilt->_head = NULL; // Flag as a loop UNIONED into parent + ilt->_head = nullptr; // Flag as a loop UNIONED into parent ilt = ilt->_child; // Repeat using new ilt continue; // do not advance over ilt->_child } @@ -3008,7 +3014,7 @@ void IdealLoopTree::allpaths_check_safepts(VectorSet &visited, Node_List &stack) // Terminate this path } else if (n->Opcode() == Op_SafePoint) { if (_phase->get_loop(n) != this) { - if (_required_safept == NULL) _required_safept = new Node_List(); + if (_required_safept == nullptr) _required_safept = new Node_List(); _required_safept->push(n); // save the one closest to the tail } // Terminate this path @@ -3086,10 +3092,10 @@ void IdealLoopTree::check_safepts(VectorSet &visited, Node_List &stack) { if (_child) _child->check_safepts(visited, stack); if (_next) _next ->check_safepts(visited, stack); - if (!_head->is_CountedLoop() && !_has_sfpt && _parent != NULL && !_irreducible) { - bool has_call = false; // call on dom-path - bool has_local_ncsfpt = false; // ncsfpt on dom-path at this loop depth - Node* nonlocal_ncsfpt = NULL; // ncsfpt on dom-path at a deeper depth + if (!_head->is_CountedLoop() && !_has_sfpt && _parent != nullptr && !_irreducible) { + bool has_call = false; // call on dom-path + bool has_local_ncsfpt = false; // ncsfpt on dom-path at this loop depth + Node* nonlocal_ncsfpt = nullptr; // ncsfpt on dom-path at a deeper depth // Scan the dom-path nodes from tail to head for (Node* n = tail(); n != _head; n = _phase->idom(n)) { if (n->is_Call() && n->as_Call()->guaranteed_safepoint()) { @@ -3101,7 +3107,7 @@ void IdealLoopTree::check_safepts(VectorSet &visited, Node_List &stack) { has_local_ncsfpt = true; break; } - if (nonlocal_ncsfpt == NULL) { + if (nonlocal_ncsfpt == nullptr) { nonlocal_ncsfpt = n; // save the one closest to the tail } } else { @@ -3123,15 +3129,26 @@ void IdealLoopTree::check_safepts(VectorSet &visited, Node_List &stack) { // Skip to head of inner loop assert(_phase->is_dominator(_head, nlpt->_head), "inner head dominated by outer head"); n = nlpt->_head; + if (_head == n) { + // this and nlpt (inner loop) have the same loop head. This should not happen because + // during beautify_loops we call merge_many_backedges. However, infinite loops may not + // have been attached to the loop-tree during build_loop_tree before beautify_loops, + // but then attached in the build_loop_tree afterwards, and so still have unmerged + // backedges. Check if we are indeed in an infinite subgraph, and terminate the scan, + // since we have reached the loop head of this. + assert(_head->as_Region()->is_in_infinite_subgraph(), + "only expect unmerged backedges in infinite loops"); + break; + } } } } } // Record safept's that this loop needs preserved when an // inner loop attempts to delete it's safepoints. - if (_child != NULL && !has_call && !has_local_ncsfpt) { - if (nonlocal_ncsfpt != NULL) { - if (_required_safept == NULL) _required_safept = new Node_List(); + if (_child != nullptr && !has_call && !has_local_ncsfpt) { + if (nonlocal_ncsfpt != nullptr) { + if (_required_safept == nullptr) _required_safept = new Node_List(); _required_safept->push(nonlocal_ncsfpt); } else { // Failed to find a suitable safept on the dom-path. Now use @@ -3147,9 +3164,9 @@ void IdealLoopTree::check_safepts(VectorSet &visited, Node_List &stack) { bool PhaseIdealLoop::is_deleteable_safept(Node* sfpt) { assert(sfpt->Opcode() == Op_SafePoint, ""); IdealLoopTree* lp = get_loop(sfpt)->_parent; - while (lp != NULL) { + while (lp != nullptr) { Node_List* sfpts = lp->_required_safept; - if (sfpts != NULL) { + if (sfpts != nullptr) { for (uint i = 0; i < sfpts->size(); i++) { if (sfpt == sfpts->at(i)) return false; @@ -3168,7 +3185,7 @@ void PhaseIdealLoop::replace_parallel_iv(IdealLoopTree *loop) { if (!cl->is_valid_counted_loop(T_INT)) return; // skip malformed counted loop Node *incr = cl->incr(); - if (incr == NULL) + if (incr == nullptr) return; // Dead loop? Node *init = cl->init_trip(); Node *phi = cl->phi(); @@ -3251,7 +3268,7 @@ void PhaseIdealLoop::replace_parallel_iv(IdealLoopTree *loop) { } void IdealLoopTree::remove_safepoints(PhaseIdealLoop* phase, bool keep_one) { - Node* keep = NULL; + Node* keep = nullptr; if (keep_one) { // Look for a safepoint on the idom-path. for (Node* i = tail(); i != _head; i = phase->idom(i)) { @@ -3265,12 +3282,12 @@ void IdealLoopTree::remove_safepoints(PhaseIdealLoop* phase, bool keep_one) { // Don't remove any safepoints if it is requested to keep a single safepoint and // no safepoint was found on idom-path. It is not safe to remove any safepoint // in this case since there's no safepoint dominating all paths in the loop body. - bool prune = !keep_one || keep != NULL; + bool prune = !keep_one || keep != nullptr; // Delete other safepoints in this loop. Node_List* sfpts = _safepts; - if (prune && sfpts != NULL) { - assert(keep == NULL || keep->Opcode() == Op_SafePoint, "not safepoint"); + if (prune && sfpts != nullptr) { + assert(keep == nullptr || keep->Opcode() == Op_SafePoint, "not safepoint"); for (uint i = 0; i < sfpts->size(); i++) { Node* n = sfpts->at(i); assert(phase->get_loop(n) == this, ""); @@ -3310,7 +3327,7 @@ void IdealLoopTree::counted_loop( PhaseIdealLoop *phase ) { remove_safepoints(phase, true); } else { assert(!_head->is_Loop() || !_head->as_Loop()->is_transformed_long_inner_loop(), "transformation to counted loop should not fail"); - if (_parent != NULL && !_irreducible) { + if (_parent != nullptr && !_irreducible) { // Not a counted loop. Keep one safepoint. bool keep_one_sfpt = true; remove_safepoints(phase, keep_one_sfpt); @@ -3319,7 +3336,7 @@ void IdealLoopTree::counted_loop( PhaseIdealLoop *phase ) { // Recursively assert(loop->_child != this || (loop->_head->as_Loop()->is_OuterStripMinedLoop() && _head->as_CountedLoop()->is_strip_mined()), "what kind of loop was added?"); - assert(loop->_child != this || (loop->_child->_child == NULL && loop->_child->_next == NULL), "would miss some loops"); + assert(loop->_child != this || (loop->_child->_child == nullptr && loop->_child->_next == nullptr), "would miss some loops"); if (loop->_child && loop->_child != this) loop->_child->counted_loop(phase); if (loop->_next) loop->_next ->counted_loop(phase); } @@ -3385,14 +3402,14 @@ uint IdealLoopTree::est_loop_flow_merge_sz() const { for (uint k = 0; k < outcnt; k++) { Node* out = node->raw_out(k); - if (out == NULL) continue; + if (out == nullptr) continue; if (out->is_CFG()) { if (!is_member(_phase->get_loop(out))) { ctrl_edge_out_cnt++; } } else if (_phase->has_ctrl(out)) { Node* ctrl = _phase->get_ctrl(out); - assert(ctrl != NULL, "must be"); + assert(ctrl != nullptr, "must be"); assert(ctrl->is_CFG(), "must be"); if (!is_member(_phase->get_loop(ctrl))) { data_edge_out_cnt++; @@ -3418,20 +3435,20 @@ void IdealLoopTree::dump_head() const { if (_irreducible) tty->print(" IRREDUCIBLE"); Node* entry = _head->is_Loop() ? _head->as_Loop()->skip_strip_mined(-1)->in(LoopNode::EntryControl) : _head->in(LoopNode::EntryControl); Node* predicate = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); - if (predicate != NULL ) { + if (predicate != nullptr ) { tty->print(" limit_check"); entry = PhaseIdealLoop::skip_loop_predicates(entry); } if (UseProfiledLoopPredicate) { predicate = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate); - if (predicate != NULL) { + if (predicate != nullptr) { tty->print(" profile_predicated"); entry = PhaseIdealLoop::skip_loop_predicates(entry); } } if (UseLoopPredicate) { predicate = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); - if (predicate != NULL) { + if (predicate != nullptr) { tty->print(" predicated"); } } @@ -3440,12 +3457,12 @@ void IdealLoopTree::dump_head() const { tty->print(" counted"); Node* init_n = cl->init_trip(); - if (init_n != NULL && init_n->is_Con()) + if (init_n != nullptr && init_n->is_Con()) tty->print(" [%d,", cl->init_trip()->get_int()); else tty->print(" [int,"); Node* limit_n = cl->limit(); - if (limit_n != NULL && limit_n->is_Con()) + if (limit_n != nullptr && limit_n->is_Con()) tty->print("%d),", cl->limit()->get_int()); else tty->print("int),"); @@ -3465,10 +3482,10 @@ void IdealLoopTree::dump_head() const { if (_has_call) tty->print(" has_call"); if (_has_sfpt) tty->print(" has_sfpt"); if (_rce_candidate) tty->print(" rce"); - if (_safepts != NULL && _safepts->size() > 0) { + if (_safepts != nullptr && _safepts->size() > 0) { tty->print(" sfpts={"); _safepts->dump_simple(); tty->print(" }"); } - if (_required_safept != NULL && _required_safept->size() > 0) { + if (_required_safept != nullptr && _required_safept->size() > 0) { tty->print(" req={"); _required_safept->dump_simple(); tty->print(" }"); } if (Verbose) { @@ -3492,14 +3509,14 @@ void IdealLoopTree::dump() const { static void log_loop_tree_helper(IdealLoopTree* root, IdealLoopTree* loop, CompileLog* log) { if (loop == root) { - if (loop->_child != NULL) { + if (loop->_child != nullptr) { log->begin_head("loop_tree"); log->end_head(); log_loop_tree_helper(root, loop->_child, log); log->tail("loop_tree"); - assert(loop->_next == NULL, "what?"); + assert(loop->_next == nullptr, "what?"); } - } else if (loop != NULL) { + } else if (loop != nullptr) { Node* head = loop->_head; log->begin_head("loop"); log->print(" idx='%d' ", head->_idx); @@ -3521,7 +3538,7 @@ static void log_loop_tree_helper(IdealLoopTree* root, IdealLoopTree* loop, Compi } void PhaseIdealLoop::log_loop_tree() { - if (C->log() != NULL) { + if (C->log() != nullptr) { log_loop_tree_helper(_ltree_root, _ltree_root, C->log()); } } @@ -3542,14 +3559,14 @@ void PhaseIdealLoop::collect_potentially_useful_predicates(IdealLoopTree* loop, Node* entry = lpn->in(LoopNode::EntryControl); Node* predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); - if (predicate != NULL) { // right pattern that can be used by loop predication + if (predicate != nullptr) { // right pattern that can be used by loop predication assert(entry->in(0)->in(1)->in(1)->Opcode() == Op_Opaque1, "must be"); useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one entry = skip_loop_predicates(entry); } if (UseProfiledLoopPredicate) { predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate); - if (predicate != NULL) { // right pattern that can be used by loop predication + if (predicate != nullptr) { // right pattern that can be used by loop predication useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one get_skeleton_predicates(entry, useful_predicates, true); entry = skip_loop_predicates(entry); @@ -3558,7 +3575,7 @@ void PhaseIdealLoop::collect_potentially_useful_predicates(IdealLoopTree* loop, if (UseLoopPredicate) { predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); - if (predicate != NULL) { // right pattern that can be used by loop predication + if (predicate != nullptr) { // right pattern that can be used by loop predication useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one get_skeleton_predicates(entry, useful_predicates, true); } @@ -3699,35 +3716,12 @@ bool PhaseIdealLoop::only_has_infinite_loops() { ResourceMark rm; Unique_Node_List worklist; // start traversal at all loop heads of first-level loops - for (IdealLoopTree* l = _ltree_root->_child; l != NULL; l = l->_next) { + for (IdealLoopTree* l = _ltree_root->_child; l != nullptr; l = l->_next) { Node* head = l->_head; assert(head->is_Region(), ""); worklist.push(head); } - // BFS traversal down the CFG, except through NeverBranch exits - for (uint i = 0; i < worklist.size(); ++i) { - Node* n = worklist.at(i); - assert(n->is_CFG(), "only traverse CFG"); - if (n->is_Root()) { - // Found root -> there was an exit! - return false; - } else if (n->is_NeverBranch()) { - // Only follow the loop-internal projection, not the NeverBranch exit - ProjNode* proj = n->as_NeverBranch()->proj_out_or_null(0); - assert(proj != nullptr, "must find loop-internal projection of NeverBranch"); - worklist.push(proj); - } else { - // Traverse all CFG outputs - for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { - Node* use = n->fast_out(i); - if (use->is_CFG()) { - worklist.push(use); - } - } - } - } - // No exit found for any loop -> all are infinite - return true; + return RegionNode::are_all_nodes_in_infinite_subgraph(worklist); } #endif @@ -3764,7 +3758,7 @@ void PhaseIdealLoop::build_and_optimize() { VectorSet visited; // Pre-grow the mapping from Nodes to IdealLoopTrees. - _nodes.map(C->unique(), NULL); + _nodes.map(C->unique(), nullptr); memset(_nodes.adr(), 0, wordSize * C->unique()); // Pre-build the top-level outermost loop tree entry @@ -3775,9 +3769,9 @@ void PhaseIdealLoop::build_and_optimize() { // Initialize Dominators. // Checked in clone_loop_predicate() during beautify_loops(). _idom_size = 0; - _idom = NULL; - _dom_depth = NULL; - _dom_stk = NULL; + _idom = nullptr; + _dom_depth = nullptr; + _dom_stk = nullptr; // Empty pre-order array allocate_preorders(); @@ -3796,7 +3790,7 @@ void PhaseIdealLoop::build_and_optimize() { // has_loops() flag but adds NeverBranch nodes so the next loop opts // verification pass finds a non empty loop tree. When the back edge // is an exception edge, parsing doesn't set has_loops(). - assert(_ltree_root->_child == NULL || C->has_loops() || only_has_infinite_loops() || C->has_exception_backedge(), "parsing found no loops but there are some"); + assert(_ltree_root->_child == nullptr || C->has_loops() || only_has_infinite_loops() || C->has_exception_backedge(), "parsing found no loops but there are some"); // No loops after all if( !_ltree_root->_child && !_verify_only ) C->set_has_loops(false); @@ -3829,7 +3823,7 @@ void PhaseIdealLoop::build_and_optimize() { C->print_method(PHASE_BEFORE_BEAUTIFY_LOOPS, 3); if( _ltree_root->_child->beautify_loops( this ) ) { // Re-build loop tree! - _ltree_root->_child = NULL; + _ltree_root->_child = nullptr; _nodes.clear(); reallocate_preorders(); build_loop_tree(); @@ -3844,13 +3838,13 @@ void PhaseIdealLoop::build_and_optimize() { } } - // Build Dominators for elision of NULL checks & loop finding. + // Build Dominators for elision of null checks & loop finding. // Since nodes do not have a slot for immediate dominator, make // a persistent side array for that info indexed on node->_idx. _idom_size = C->unique(); _idom = NEW_RESOURCE_ARRAY( Node*, _idom_size ); _dom_depth = NEW_RESOURCE_ARRAY( uint, _idom_size ); - _dom_stk = NULL; // Allocated on demand in recompute_dom_depth + _dom_stk = nullptr; // Allocated on demand in recompute_dom_depth memset( _dom_depth, 0, _idom_size * sizeof(uint) ); Dominators(); @@ -4156,7 +4150,7 @@ void PhaseIdealLoop::verify() const { verify_compare(C->root(), &loop_verify, visited); assert(fail == 0, "verify loops failed"); // Verify loop structure is the same - _ltree_root->verify_tree(loop_verify._ltree_root, NULL); + _ltree_root->verify_tree(loop_verify._ltree_root, nullptr); // Reset major-progress. It was cleared by creating a verify version of // PhaseIdealLoop. C->restore_major_progress(old_progress); @@ -4315,12 +4309,12 @@ void IdealLoopTree::verify_tree(IdealLoopTree *loop, const IdealLoopTree *parent } - if (_child != NULL) _child->verify_tree(loop->_child, this); - if (_next != NULL) _next ->verify_tree(loop->_next, parent); + if (_child != nullptr) _child->verify_tree(loop->_child, this); + if (_next != nullptr) _next ->verify_tree(loop->_next, parent); // Innermost loops need to verify loop bodies, // but only if no 'major_progress' int fail = 0; - if (!Compile::current()->major_progress() && _child == NULL) { + if (!Compile::current()->major_progress() && _child == nullptr) { for( uint i = 0; i < _body.size(); i++ ) { Node *n = _body.at(i); if (n->outcnt() == 0) continue; // Ignore dead @@ -4388,17 +4382,17 @@ void PhaseIdealLoop::recompute_dom_depth() { uint i; // Initialize depth to "no depth yet" and realize all lazy updates for (i = 0; i < _idom_size; i++) { - // Only indices with a _dom_depth has a Node* or NULL (otherwise uninitalized). - if (_dom_depth[i] > 0 && _idom[i] != NULL) { + // Only indices with a _dom_depth has a Node* or null (otherwise uninitalized). + if (_dom_depth[i] > 0 && _idom[i] != nullptr) { _dom_depth[i] = no_depth_marker; // heal _idom if it has a fwd mapping in _nodes - if (_idom[i]->in(0) == NULL) { + if (_idom[i]->in(0) == nullptr) { idom(i); } } } - if (_dom_stk == NULL) { + if (_dom_stk == nullptr) { uint init_size = C->live_nodes() / 100; // Guess that 1/100 is a reasonable initial size. if (init_size < 10) init_size = 10; _dom_stk = new GrowableArray(init_size); @@ -4463,7 +4457,7 @@ IdealLoopTree *PhaseIdealLoop::sort( IdealLoopTree *loop, IdealLoopTree *innermo //------------------------------build_loop_tree-------------------------------- // I use a modified Vick/Tarjan algorithm. I need pre- and a post- visit -// bits. The _nodes[] array is mapped by Node index and holds a NULL for +// bits. The _nodes[] array is mapped by Node index and holds a null for // not-yet-pre-walked, pre-order # for pre-but-not-post-walked and holds the // tightest enclosing IdealLoopTree for post-walked. // @@ -4563,7 +4557,7 @@ int PhaseIdealLoop::build_loop_tree_impl( Node *n, int pre_order ) { // Pre-walked but not post-walked nodes need a pre_order number. // Tightest enclosing loop for this Node - IdealLoopTree *innermost = NULL; + IdealLoopTree *innermost = nullptr; // For all children, see if any edge is a backedge. If so, make a loop // for it. Then find the tightest enclosing loop for the self Node. @@ -4620,13 +4614,13 @@ int PhaseIdealLoop::build_loop_tree_impl( Node *n, int pre_order ) { _igvn.register_new_node_with_optimizer(if_t); set_loop(if_t, l); - Node* cfg = NULL; // Find the One True Control User of m + Node* cfg = nullptr; // Find the One True Control User of m for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax; j++) { Node* x = m->fast_out(j); if (x->is_CFG() && x != m && x != iff) { cfg = x; break; } } - assert(cfg != NULL, "must find the control user of m"); + assert(cfg != nullptr, "must find the control user of m"); uint k = 0; // Probably cfg->in(0) while( cfg->in(k) != m ) k++; // But check incase cfg is a Region cfg->set_req( k, if_t ); // Now point to NeverBranch @@ -4660,7 +4654,7 @@ int PhaseIdealLoop::build_loop_tree_impl( Node *n, int pre_order ) { l = l->_parent; _has_irreducible_loops = true; // Check for bad CFG here to prevent crash, and bailout of compile - if (l == NULL) { + if (l == nullptr) { C->record_method_not_compilable("unhandled CFG detected during loop optimization"); return pre_order; } @@ -4721,7 +4715,7 @@ int PhaseIdealLoop::build_loop_tree_impl( Node *n, int pre_order ) { innermost->_has_call = 1; // = true } else if (n->Opcode() == Op_SafePoint) { // Record all safepoints in this loop. - if (innermost->_safepts == NULL) innermost->_safepts = new Node_List(); + if (innermost->_safepts == nullptr) innermost->_safepts = new Node_List(); innermost->_safepts->push(n); } } @@ -4756,7 +4750,7 @@ void PhaseIdealLoop::build_loop_early( VectorSet &visited, Node_List &worklist, // into a single loop. This makes the members of the original // loop bodies pointing to dead loops; they need to move up // to the new UNION'd larger loop. I set the _head field of these - // dead loops to NULL and the _parent field points to the owning + // dead loops to null and the _parent field points to the owning // loop. Shades of UNION-FIND algorithm. IdealLoopTree *ilt; while( !(ilt = get_loop(n))->_head ) { @@ -4773,7 +4767,7 @@ void PhaseIdealLoop::build_loop_early( VectorSet &visited, Node_List &worklist, is_deleteable_safept(n)) { Node *in = n->in(TypeFunc::Control); lazy_replace(n,in); // Pull safepoint now - if (ilt->_safepts != NULL) { + if (ilt->_safepts != nullptr) { ilt->_safepts->yank(n); } // Carry on with the recursion "as if" we are walking @@ -4795,7 +4789,7 @@ void PhaseIdealLoop::build_loop_early( VectorSet &visited, Node_List &worklist, while (i < cnt) { Node *in = n->in(i); ++i; - if (in == NULL) continue; + if (in == nullptr) continue; if (in->pinned() && !in->is_CFG()) set_ctrl(in, in->in(0)); int is_visited = visited.test_set( in->_idx ); @@ -4842,7 +4836,7 @@ void PhaseIdealLoop::build_loop_early( VectorSet &visited, Node_List &worklist, //------------------------------dom_lca_internal-------------------------------- // Pair-wise LCA Node *PhaseIdealLoop::dom_lca_internal( Node *n1, Node *n2 ) const { - if( !n1 ) return n2; // Handle NULL original LCA + if( !n1 ) return n2; // Handle null original LCA assert( n1->is_CFG(), "" ); assert( n2->is_CFG(), "" ); // find LCA of all uses @@ -4887,7 +4881,7 @@ Node *PhaseIdealLoop::dom_lca_internal( Node *n1, Node *n2 ) const { // IDOMs are correct. Node *PhaseIdealLoop::compute_idom( Node *region ) const { assert( region->is_Region(), "" ); - Node *LCA = NULL; + Node *LCA = nullptr; for( uint i = 1; i < region->req(); i++ ) { if( region->in(i) != C->top() ) LCA = dom_lca( LCA, region->in(i) ); @@ -4919,10 +4913,10 @@ bool PhaseIdealLoop::verify_dominance(Node* n, Node* use, Node* LCA, Node* early Node* PhaseIdealLoop::compute_lca_of_uses(Node* n, Node* early, bool verify) { // Compute LCA over list of uses bool had_error = false; - Node *LCA = NULL; + Node *LCA = nullptr; for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax && LCA != early; i++) { Node* c = n->fast_out(i); - if (_nodes[c->_idx] == NULL) + if (_nodes[c->_idx] == nullptr) continue; // Skip the occasional dead node if( c->is_Phi() ) { // For Phis, we must land above on the path for( uint j=1; jreq(); j++ ) {// For all inputs @@ -4956,29 +4950,29 @@ Node* PhaseIdealLoop::compute_lca_of_uses(Node* n, Node* early, bool verify) { // be guaranteed anymore. Node* CountedLoopNode::is_canonical_loop_entry() { if (!is_main_loop() && !is_post_loop()) { - return NULL; + return nullptr; } Node* ctrl = skip_predicates(); - if (ctrl == NULL || (!ctrl->is_IfTrue() && !ctrl->is_IfFalse())) { - return NULL; + if (ctrl == nullptr || (!ctrl->is_IfTrue() && !ctrl->is_IfFalse())) { + return nullptr; } Node* iffm = ctrl->in(0); - if (iffm == NULL || !iffm->is_If()) { - return NULL; + if (iffm == nullptr || !iffm->is_If()) { + return nullptr; } Node* bolzm = iffm->in(1); - if (bolzm == NULL || !bolzm->is_Bool()) { - return NULL; + if (bolzm == nullptr || !bolzm->is_Bool()) { + return nullptr; } Node* cmpzm = bolzm->in(1); - if (cmpzm == NULL || !cmpzm->is_Cmp()) { - return NULL; + if (cmpzm == nullptr || !cmpzm->is_Cmp()) { + return nullptr; } uint input = is_main_loop() ? 2 : 1; - if (input >= cmpzm->req() || cmpzm->in(input) == NULL) { - return NULL; + if (input >= cmpzm->req() || cmpzm->in(input) == nullptr) { + return nullptr; } bool res = cmpzm->in(input)->Opcode() == Op_Opaque1; #ifdef ASSERT @@ -4992,13 +4986,13 @@ Node* CountedLoopNode::is_canonical_loop_entry() { } assert(found_opaque == res, "wrong pattern"); #endif - return res ? cmpzm->in(input) : NULL; + return res ? cmpzm->in(input) : nullptr; } //------------------------------get_late_ctrl---------------------------------- // Compute latest legal control. Node *PhaseIdealLoop::get_late_ctrl( Node *n, Node *early ) { - assert(early != NULL, "early control should not be NULL"); + assert(early != nullptr, "early control should not be null"); Node* LCA = compute_lca_of_uses(n, early); #ifdef ASSERT @@ -5045,8 +5039,8 @@ Node* PhaseIdealLoop::get_late_ctrl_with_anti_dep(LoadNode* n, Node* early, Node } } else { Node* sctrl = has_ctrl(s) ? get_ctrl(s) : s->in(0); - assert(sctrl != NULL || !s->is_reachable_from_root(), "must have control"); - if (sctrl != NULL && !sctrl->is_top() && is_dominator(early, sctrl)) { + assert(sctrl != nullptr || !s->is_reachable_from_root(), "must have control"); + if (sctrl != nullptr && !sctrl->is_top() && is_dominator(early, sctrl)) { const TypePtr* adr_type = s->adr_type(); if (s->is_ArrayCopy()) { // Copy to known instance needs destination type to test for aliasing @@ -5203,7 +5197,7 @@ void PhaseIdealLoop::build_loop_late( VectorSet &visited, Node_List &worklist, N // Check for dead uses. Aggressively prune such junk. It might be // dead in the global sense, but still have local uses so I cannot // easily call 'remove_dead_node'. - if( _nodes[use->_idx] != NULL || use->is_top() ) { // Not dead? + if( _nodes[use->_idx] != nullptr || use->is_top() ) { // Not dead? // Due to cycles, we might not hit the same fixed point in the verify // pass as we do in the regular pass. Instead, visit such phis as // simple uses of the loop head. @@ -5262,7 +5256,7 @@ void PhaseIdealLoop::verify_strip_mined_scheduling(Node *n, Node* least) { if (nn == n) { return; } - if (nn != NULL && has_ctrl(nn) && get_loop(get_ctrl(nn)) == loop) { + if (nn != nullptr && has_ctrl(nn) && get_loop(get_ctrl(nn)) == loop) { wq.push(nn); } } @@ -5353,18 +5347,18 @@ void PhaseIdealLoop::build_loop_late_post_work(Node *n, bool pinned) { // Compute latest point this Node can go Node *LCA = get_late_ctrl( n, early ); - // LCA is NULL due to uses being dead - if( LCA == NULL ) { + // LCA is null due to uses being dead + if( LCA == nullptr ) { #ifdef ASSERT for (DUIterator i1 = n->outs(); n->has_out(i1); i1++) { - assert( _nodes[n->out(i1)->_idx] == NULL, "all uses must also be dead"); + assert( _nodes[n->out(i1)->_idx] == nullptr, "all uses must also be dead"); } #endif _nodes.map(n->_idx, 0); // This node is useless _deadlist.push(n); return; } - assert(LCA != NULL && !LCA->is_top(), "no dead nodes"); + assert(LCA != nullptr && !LCA->is_top(), "no dead nodes"); Node *legal = LCA; // Walk 'legal' up the IDOM chain Node *least = legal; // Best legal position so far @@ -5394,7 +5388,7 @@ void PhaseIdealLoop::build_loop_late_post_work(Node *n, bool pinned) { break; } CallStaticJavaNode* call = new_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); - if (call == NULL) { + if (call == nullptr) { break; } int req = call->uncommon_trap_request(); @@ -5462,27 +5456,27 @@ void PhaseIdealLoop::dump_bad_graph(const char* msg, Node* n, Node* early, Node* tty->print_cr("%s", msg); tty->print("n: "); n->dump(); tty->print("early(n): "); early->dump(); - if (n->in(0) != NULL && !n->in(0)->is_top() && + if (n->in(0) != nullptr && !n->in(0)->is_top() && n->in(0) != early && !n->in(0)->is_Root()) { tty->print("n->in(0): "); n->in(0)->dump(); } for (uint i = 1; i < n->req(); i++) { Node* in1 = n->in(i); - if (in1 != NULL && in1 != n && !in1->is_top()) { + if (in1 != nullptr && in1 != n && !in1->is_top()) { tty->print("n->in(%d): ", i); in1->dump(); Node* in1_early = get_ctrl(in1); tty->print("early(n->in(%d)): ", i); in1_early->dump(); - if (in1->in(0) != NULL && !in1->in(0)->is_top() && + if (in1->in(0) != nullptr && !in1->in(0)->is_top() && in1->in(0) != in1_early && !in1->in(0)->is_Root()) { tty->print("n->in(%d)->in(0): ", i); in1->in(0)->dump(); } for (uint j = 1; j < in1->req(); j++) { Node* in2 = in1->in(j); - if (in2 != NULL && in2 != n && in2 != in1 && !in2->is_top()) { + if (in2 != nullptr && in2 != n && in2 != in1 && !in2->is_top()) { tty->print("n->in(%d)->in(%d): ", i, j); in2->dump(); Node* in2_early = get_ctrl(in2); tty->print("early(n->in(%d)->in(%d)): ", i, j); in2_early->dump(); - if (in2->in(0) != NULL && !in2->in(0)->is_top() && + if (in2->in(0) != nullptr && !in2->in(0)->is_top() && in2->in(0) != in2_early && !in2->in(0)->is_Root()) { tty->print("n->in(%d)->in(%d)->in(0): ", i, j); in2->in(0)->dump(); } @@ -5507,7 +5501,7 @@ void PhaseIdealLoop::dump_bad_graph(const char* msg, Node* n, Node* early, Node* } else { Node* u1_later = get_ctrl(u1); tty->print("later(n->out(%d)): ", i); u1_later->dump(); - if (u1->in(0) != NULL && !u1->in(0)->is_top() && + if (u1->in(0) != nullptr && !u1->in(0)->is_top() && u1->in(0) != u1_later && !u1->in(0)->is_Root()) { tty->print("n->out(%d)->in(0): ", i); u1->in(0)->dump(); } @@ -5519,7 +5513,7 @@ void PhaseIdealLoop::dump_bad_graph(const char* msg, Node* n, Node* early, Node* if (!u2->is_CFG()) { Node* u2_later = get_ctrl(u2); tty->print("later(n->out(%d)->out(%d)): ", i, j); u2_later->dump(); - if (u2->in(0) != NULL && !u2->in(0)->is_top() && + if (u2->in(0) != nullptr && !u2->in(0)->is_top() && u2->in(0) != u2_later && !u2->in(0)->is_Root()) { tty->print("n->out(%d)->in(0): ", i); u2->in(0)->dump(); } @@ -5671,7 +5665,7 @@ void PhaseIdealLoop::dump(IdealLoopTree* loop, uint idx, Node_List &rpo_list) co if (m && m->outcnt() > 0) { if (!(has_ctrl(m) && get_ctrl_no_update(m) == n)) { tty->print_cr("*** BROKEN CTRL ACCESSOR! _nodes[k] is %p, ctrl is %p", - _nodes[k], has_ctrl(m) ? get_ctrl_no_update(m) : NULL); + _nodes[k], has_ctrl(m) ? get_ctrl_no_update(m) : nullptr); } tty->sp(2 * loop->_nest + 1); m->dump(); @@ -5745,19 +5739,19 @@ void PhaseIdealLoop::rpo(Node* start, Node_Stack &stk, VectorSet &visited, Node_ // Advance to next loop tree using a preorder, left-to-right traversal. void LoopTreeIterator::next() { assert(!done(), "must not be done."); - if (_curnt->_child != NULL) { + if (_curnt->_child != nullptr) { _curnt = _curnt->_child; - } else if (_curnt->_next != NULL) { + } else if (_curnt->_next != nullptr) { _curnt = _curnt->_next; } else { - while (_curnt != _root && _curnt->_next == NULL) { + while (_curnt != _root && _curnt->_next == nullptr) { _curnt = _curnt->_parent; } if (_curnt == _root) { - _curnt = NULL; + _curnt = nullptr; assert(done(), "must be done."); } else { - assert(_curnt->_next != NULL, "must be more to do"); + assert(_curnt->_next != nullptr, "must be more to do"); _curnt = _curnt->_next; } } diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index ebc3bd1dbf8fb..3d8f8b6c88fc0 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -149,9 +149,9 @@ class LoopNode : public RegionNode { virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); virtual int Opcode() const; bool can_be_counted_loop(PhaseTransform* phase) const { - return req() == 3 && in(0) != NULL && - in(1) != NULL && phase->type(in(1)) != Type::TOP && - in(2) != NULL && phase->type(in(2)) != Type::TOP; + return req() == 3 && in(0) != nullptr && + in(1) != nullptr && phase->type(in(1)) != Type::TOP && + in(2) != nullptr && phase->type(in(2)) != Type::TOP; } bool is_valid_counted_loop(BasicType bt) const; #ifndef PRODUCT @@ -160,10 +160,10 @@ class LoopNode : public RegionNode { void verify_strip_mined(int expect_skeleton) const NOT_DEBUG_RETURN; virtual LoopNode* skip_strip_mined(int expect_skeleton = 1) { return this; } - virtual IfTrueNode* outer_loop_tail() const { ShouldNotReachHere(); return NULL; } - virtual OuterStripMinedLoopEndNode* outer_loop_end() const { ShouldNotReachHere(); return NULL; } - virtual IfFalseNode* outer_loop_exit() const { ShouldNotReachHere(); return NULL; } - virtual SafePointNode* outer_safepoint() const { ShouldNotReachHere(); return NULL; } + virtual IfTrueNode* outer_loop_tail() const { ShouldNotReachHere(); return nullptr; } + virtual OuterStripMinedLoopEndNode* outer_loop_end() const { ShouldNotReachHere(); return nullptr; } + virtual IfFalseNode* outer_loop_exit() const { ShouldNotReachHere(); return nullptr; } + virtual SafePointNode* outer_safepoint() const { ShouldNotReachHere(); return nullptr; } }; //------------------------------Counted Loops---------------------------------- @@ -393,12 +393,12 @@ class BaseCountedLoopEndNode : public IfNode { init_class_id(Class_BaseCountedLoopEnd); } - Node *cmp_node() const { return (in(TestValue)->req() >=2) ? in(TestValue)->in(1) : NULL; } - Node* incr() const { Node* tmp = cmp_node(); return (tmp && tmp->req() == 3) ? tmp->in(1) : NULL; } - Node* limit() const { Node* tmp = cmp_node(); return (tmp && tmp->req() == 3) ? tmp->in(2) : NULL; } - Node* stride() const { Node* tmp = incr(); return (tmp && tmp->req() == 3) ? tmp->in(2) : NULL; } - Node* init_trip() const { Node* tmp = phi(); return (tmp && tmp->req() == 3) ? tmp->in(1) : NULL; } - bool stride_is_con() const { Node *tmp = stride(); return (tmp != NULL && tmp->is_Con()); } + Node *cmp_node() const { return (in(TestValue)->req() >=2) ? in(TestValue)->in(1) : nullptr; } + Node* incr() const { Node* tmp = cmp_node(); return (tmp && tmp->req() == 3) ? tmp->in(1) : nullptr; } + Node* limit() const { Node* tmp = cmp_node(); return (tmp && tmp->req() == 3) ? tmp->in(2) : nullptr; } + Node* stride() const { Node* tmp = incr(); return (tmp && tmp->req() == 3) ? tmp->in(2) : nullptr; } + Node* init_trip() const { Node* tmp = phi(); return (tmp && tmp->req() == 3) ? tmp->in(1) : nullptr; } + bool stride_is_con() const { Node *tmp = stride(); return (tmp != nullptr && tmp->is_Con()); } PhiNode* phi() const { Node* tmp = incr(); @@ -408,7 +408,7 @@ class BaseCountedLoopEndNode : public IfNode { return phi->as_Phi(); } } - return NULL; + return nullptr; } BaseCountedLoopNode* loopnode() const { @@ -416,15 +416,15 @@ class BaseCountedLoopEndNode : public IfNode { // have been optimized out by the IGVN so be cautious with the // pattern matching on the graph PhiNode* iv_phi = phi(); - if (iv_phi == NULL) { - return NULL; + if (iv_phi == nullptr) { + return nullptr; } Node* ln = iv_phi->in(0); if (!ln->is_BaseCountedLoop() || ln->as_BaseCountedLoop()->loopexit_or_null() != this) { - return NULL; + return nullptr; } if (!ln->operates_on(bt(), true)) { - return NULL; + return nullptr; } return ln->as_BaseCountedLoop(); } @@ -491,49 +491,49 @@ class LongCountedLoopEndNode : public BaseCountedLoopEndNode { inline BaseCountedLoopEndNode* BaseCountedLoopNode::loopexit_or_null() const { Node* bctrl = back_control(); - if (bctrl == NULL) return NULL; + if (bctrl == nullptr) return nullptr; Node* lexit = bctrl->in(0); if (!lexit->is_BaseCountedLoopEnd()) { - return NULL; + return nullptr; } BaseCountedLoopEndNode* result = lexit->as_BaseCountedLoopEnd(); if (!result->operates_on(bt(), true)) { - return NULL; + return nullptr; } return result; } inline BaseCountedLoopEndNode* BaseCountedLoopNode::loopexit() const { BaseCountedLoopEndNode* cle = loopexit_or_null(); - assert(cle != NULL, "loopexit is NULL"); + assert(cle != nullptr, "loopexit is nullptr"); return cle; } inline Node* BaseCountedLoopNode::init_trip() const { BaseCountedLoopEndNode* cle = loopexit_or_null(); - return cle != NULL ? cle->init_trip() : NULL; + return cle != nullptr ? cle->init_trip() : nullptr; } inline Node* BaseCountedLoopNode::stride() const { BaseCountedLoopEndNode* cle = loopexit_or_null(); - return cle != NULL ? cle->stride() : NULL; + return cle != nullptr ? cle->stride() : nullptr; } inline bool BaseCountedLoopNode::stride_is_con() const { BaseCountedLoopEndNode* cle = loopexit_or_null(); - return cle != NULL && cle->stride_is_con(); + return cle != nullptr && cle->stride_is_con(); } inline Node* BaseCountedLoopNode::limit() const { BaseCountedLoopEndNode* cle = loopexit_or_null(); - return cle != NULL ? cle->limit() : NULL; + return cle != nullptr ? cle->limit() : nullptr; } inline Node* BaseCountedLoopNode::incr() const { BaseCountedLoopEndNode* cle = loopexit_or_null(); - return cle != NULL ? cle->incr() : NULL; + return cle != nullptr ? cle->incr() : nullptr; } inline Node* BaseCountedLoopNode::phi() const { BaseCountedLoopEndNode* cle = loopexit_or_null(); - return cle != NULL ? cle->phi() : NULL; + return cle != nullptr ? cle->phi() : nullptr; } //------------------------------LoopLimitNode----------------------------- @@ -632,12 +632,12 @@ class IdealLoopTree : public ResourceObj { _phase(phase), _local_loop_unroll_limit(0), _local_loop_unroll_factor(0), _nest(0), _irreducible(0), _has_call(0), _has_sfpt(0), _rce_candidate(0), - _safepts(NULL), - _required_safept(NULL), + _safepts(nullptr), + _required_safept(nullptr), _allow_optimizations(true) { - precond(_head != NULL); - precond(_tail != NULL); + precond(_head != nullptr); + precond(_tail != nullptr); } // Is 'l' a member of 'this'? @@ -737,7 +737,9 @@ class IdealLoopTree : public ResourceObj { bool policy_range_check( PhaseIdealLoop *phase ) const; // Return TRUE if "iff" is a range check. - bool is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invariance& invar DEBUG_ONLY(COMMA ProjNode *predicate_proj)) const; + bool is_range_check_if(IfProjNode* if_success_proj, PhaseIdealLoop *phase, Invariance& invar DEBUG_ONLY(COMMA ProjNode *predicate_proj)) const; + // GLGL bool is_range_check_if(IfProjNode* if_success_proj, PhaseIdealLoop* phase, BasicType bt, Node* iv, Node*& range, Node*& offset, + // GLGL jlong& scale) const; // Estimate the number of nodes required when cloning a loop (body). uint est_loop_clone_sz(uint factor) const; @@ -761,7 +763,7 @@ class IdealLoopTree : public ResourceObj { // are combined with an associative binary. Helper for reassociate_invariants. int find_invariant(Node* n, PhaseIdealLoop *phase); // Return TRUE if "n" is associative. - bool is_associative(Node* n, Node* base=NULL); + bool is_associative(Node* n, Node* base=nullptr); // Return true if n is invariant bool is_invariant(Node* n) const; @@ -769,11 +771,11 @@ class IdealLoopTree : public ResourceObj { // Put loop body on igvn work list void record_for_igvn(); - bool is_root() { return _parent == NULL; } + bool is_root() { return _parent == nullptr; } // A proper/reducible loop w/o any (occasional) dead back-edge. bool is_loop() { return !_irreducible && !tail()->is_top(); } bool is_counted() { return is_loop() && _head->is_CountedLoop(); } - bool is_innermost() { return is_loop() && _child == NULL; } + bool is_innermost() { return is_loop() && _child == nullptr; } void remove_main_post_loops(CountedLoopNode *cl, PhaseIdealLoop *phase); @@ -883,8 +885,8 @@ class PhaseIdealLoop : public PhaseTransform { // 2) a use is the same as the current LCA passed as 'n1' Node *dom_lca_for_get_late_ctrl( Node *lca, Node *n, Node *tag ) { assert( n->is_CFG(), "" ); - // Fast-path NULL lca - if( lca != NULL && lca != n ) { + // Fast-path null lca + if( lca != nullptr && lca != n ) { assert( lca->is_CFG(), "" ); // find LCA of all uses n = dom_lca_for_get_late_ctrl_internal( lca, n, tag ); @@ -895,7 +897,7 @@ class PhaseIdealLoop : public PhaseTransform { // Helper function for directing control inputs away from CFG split points. Node *find_non_split_ctrl( Node *ctrl ) const { - if (ctrl != NULL) { + if (ctrl != nullptr) { if (ctrl->is_MultiBranch()) { ctrl = ctrl->in(0); } @@ -936,8 +938,8 @@ class PhaseIdealLoop : public PhaseTransform { PhaseIterGVN &igvn() const { return _igvn; } bool has_node( Node* n ) const { - guarantee(n != NULL, "No Node."); - return _nodes[n->_idx] != NULL; + guarantee(n != nullptr, "No Node."); + return _nodes[n->_idx] != nullptr; } // check if transform created new nodes that need _ctrl recorded Node *get_late_ctrl( Node *n, Node *early ); @@ -956,8 +958,8 @@ class PhaseIdealLoop : public PhaseTransform { IdealLoopTree* old_loop = get_loop(get_ctrl(n)); IdealLoopTree* new_loop = get_loop(ctrl); if (old_loop != new_loop) { - if (old_loop->_child == NULL) old_loop->_body.yank(n); - if (new_loop->_child == NULL) new_loop->_body.push(n); + if (old_loop->_child == nullptr) old_loop->_body.yank(n); + if (new_loop->_child == nullptr) new_loop->_body.push(n); } set_ctrl(n, ctrl); } @@ -1094,10 +1096,10 @@ class PhaseIdealLoop : public PhaseTransform { Node* idom_no_update(uint didx) const { assert(didx < _idom_size, "oob"); Node* n = _idom[didx]; - assert(n != NULL,"Bad immediate dominator info."); - while (n->in(0) == NULL) { // Skip dead CFG nodes + assert(n != nullptr,"Bad immediate dominator info."); + while (n->in(0) == nullptr) { // Skip dead CFG nodes n = (Node*)(((intptr_t)_nodes[n->_idx]) & ~1); - assert(n != NULL,"Bad immediate dominator info."); + assert(n != nullptr,"Bad immediate dominator info."); } return n; } @@ -1113,7 +1115,7 @@ class PhaseIdealLoop : public PhaseTransform { } uint dom_depth(Node* d) const { - guarantee(d != NULL, "Null dominator info."); + guarantee(d != nullptr, "Null dominator info."); guarantee(d->_idx < _idom_size, ""); return _dom_depth[d->_idx]; } @@ -1163,7 +1165,7 @@ class PhaseIdealLoop : public PhaseTransform { bool _has_irreducible_loops; // Per-Node transform - virtual Node* transform(Node* n) { return NULL; } + virtual Node* transform(Node* n) { return nullptr; } Node* loop_exit_control(Node* x, IdealLoopTree* loop); Node* loop_exit_test(Node* back_control, IdealLoopTree* loop, Node*& incr, Node*& limit, BoolTest::mask& bt, float& cl_prob); @@ -1210,7 +1212,7 @@ class PhaseIdealLoop : public PhaseTransform { // normal "loop-exit" condition. All uses of loop-invariant old-loop values // now come from (one or more) Phis that merge their new-loop equivalents. // Parameter side_by_side_idom: - // When side_by_size_idom is NULL, the dominator tree is constructed for + // When side_by_size_idom is null, the dominator tree is constructed for // the clone loop to dominate the original. Used in construction of // pre-main-post loop sequence. // When nonnull, the clone and original are side-by-side, both are @@ -1225,7 +1227,7 @@ class PhaseIdealLoop : public PhaseTransform { // strip mined loop. }; void clone_loop( IdealLoopTree *loop, Node_List &old_new, int dom_depth, - CloneLoopMode mode, Node* side_by_side_idom = NULL); + CloneLoopMode mode, Node* side_by_side_idom = nullptr); void clone_loop_handle_data_uses(Node* old, Node_List &old_new, IdealLoopTree* loop, IdealLoopTree* companion_loop, Node_List*& split_if_set, Node_List*& split_bool_set, @@ -1306,15 +1308,12 @@ class PhaseIdealLoop : public PhaseTransform { // Find a predicate static Node* find_predicate(Node* entry); // Construct a range check for a predicate if - BoolNode* rc_predicate(IdealLoopTree *loop, Node* ctrl, - int scale, Node* offset, - Node* init, Node* limit, jint stride, - Node* range, bool upper, bool &overflow, - bool negate); + BoolNode* rc_predicate(IdealLoopTree *loop, Node* ctrl, int scale, Node* offset,Node* init, Node* limit, + jint stride, Node* range, bool upper, bool &overflow/* GLGL, bool negate*/); // Implementation of the loop predication to promote checks outside the loop bool loop_predication_impl(IdealLoopTree *loop); - bool loop_predication_impl_helper(IdealLoopTree *loop, ProjNode* proj, ProjNode *predicate_proj, + bool loop_predication_impl_helper(IdealLoopTree *loop, ProjNode* if_success_proj, ProjNode *predicate_proj, CountedLoopNode *cl, ConNode* zero, Invariance& invar, Deoptimization::DeoptReason reason); bool loop_predication_should_follow_branches(IdealLoopTree *loop, ProjNode *predicate_proj, float& loop_trip_cnt); @@ -1491,7 +1490,7 @@ class PhaseIdealLoop : public PhaseTransform { private: // Return a type based on condition control flow const TypeInt* filtered_type( Node *n, Node* n_ctrl); - const TypeInt* filtered_type( Node *n ) { return filtered_type(n, NULL); } + const TypeInt* filtered_type( Node *n ) { return filtered_type(n, nullptr); } // Helpers for filtered type const TypeInt* filtered_type_from_dominators( Node* val, Node *val_ctrl); @@ -1506,6 +1505,9 @@ class PhaseIdealLoop : public PhaseTransform { void try_move_store_after_loop(Node* n); bool identical_backtoback_ifs(Node *n); bool can_split_if(Node *n_ctrl); + bool cannot_split_division(const Node* n, const Node* region) const; + static bool is_divisor_counted_loop_phi(const Node* divisor, const Node* loop); + bool loop_phi_backedge_type_contains_zero(const Node* phi_divisor, const Type* zero) const; // Determine if a method is too big for a/another round of split-if, based on // a magic (approximate) ratio derived from the equally magic constant 35000, @@ -1655,7 +1657,7 @@ class AutoNodeBudget : public StackObj _check_at_final(chk == BUDGET_CHECK), _nodes_at_begin(0) { - precond(_phase != NULL); + precond(_phase != nullptr); _nodes_at_begin = _phase->require_nodes_begin(); } @@ -1754,7 +1756,7 @@ class CountedLoopReserveKit { inline Node* IdealLoopTree::tail() { // Handle lazy update of _tail field. - if (_tail->in(0) == NULL) { + if (_tail->in(0) == nullptr) { _tail = _phase->get_ctrl(_tail); } return _tail; @@ -1762,7 +1764,7 @@ inline Node* IdealLoopTree::tail() { inline Node* IdealLoopTree::head() { // Handle lazy update of _head field. - if (_head->in(0) == NULL) { + if (_head->in(0) == nullptr) { _head = _phase->get_ctrl(_head); } return _head; @@ -1784,7 +1786,7 @@ class LoopTreeIterator : public StackObj { public: LoopTreeIterator(IdealLoopTree* root) : _root(root), _curnt(root) {} - bool done() { return _curnt == NULL; } // Finished iterating? + bool done() { return _curnt == nullptr; } // Finished iterating? void next(); // Advance to next loop tree diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index c0804ed677ce1..ca5dfa89498a1 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -50,7 +50,7 @@ Node* PhaseIdealLoop::split_thru_phi(Node* n, Node* region, int policy) { if (n->Opcode() == Op_ConvI2L && n->bottom_type() != TypeLong::LONG) { // ConvI2L may have type information on it which is unsafe to push up // so disable this for now - return NULL; + return nullptr; } // Splitting range check CastIIs through a loop induction Phi can @@ -58,22 +58,11 @@ Node* PhaseIdealLoop::split_thru_phi(Node* n, Node* region, int policy) { // induction Phi and prevent optimizations (vectorization) if (n->Opcode() == Op_CastII && region->is_CountedLoop() && n->in(1) == region->as_CountedLoop()->phi()) { - return NULL; + return nullptr; } - // Bail out if 'n' is a Div or Mod node whose zero check was removed earlier (i.e. control is NULL) and its divisor is an induction variable - // phi p of a trip-counted (integer) loop whose inputs could be zero (include zero in their type range). p could have a more precise type - // range that does not necessarily include all values of its inputs. Since each of these inputs will be a divisor of the newly cloned nodes - // of 'n', we need to bail out of one of these divisors could be zero (zero in its type range). - if ((n->Opcode() == Op_DivI || n->Opcode() == Op_ModI) && n->in(0) == NULL - && region->is_CountedLoop() && n->in(2) == region->as_CountedLoop()->phi()) { - Node* phi = region->as_CountedLoop()->phi(); - for (uint i = 1; i < phi->req(); i++) { - if (_igvn.type(phi->in(i))->filter_speculative(TypeInt::ZERO) != Type::TOP) { - // Zero could be a possible value but we already removed the zero check. Bail out to avoid a possible division by zero at a later point. - return NULL; - } - } + if (cannot_split_division(n, region)) { + return nullptr; } int wins = 0; @@ -83,18 +72,18 @@ Node* PhaseIdealLoop::split_thru_phi(Node* n, Node* region, int policy) { const Type* type = n->bottom_type(); const TypeOopPtr* t_oop = _igvn.type(n)->isa_oopptr(); Node* phi; - if (t_oop != NULL && t_oop->is_known_instance_field()) { + if (t_oop != nullptr && t_oop->is_known_instance_field()) { int iid = t_oop->instance_id(); int index = C->get_alias_index(t_oop); int offset = t_oop->offset(); - phi = new PhiNode(region, type, NULL, iid, index, offset); + phi = new PhiNode(region, type, nullptr, iid, index, offset); } else { phi = PhiNode::make_blank(region, n); } uint old_unique = C->unique(); for (uint i = 1; i < region->req(); i++) { Node* x; - Node* the_clone = NULL; + Node* the_clone = nullptr; if (region->in(i) == C->top()) { x = C->top(); // Dead path? Use a dead data op } else { @@ -159,14 +148,14 @@ Node* PhaseIdealLoop::split_thru_phi(Node* n, Node* region, int policy) { } } } - if (x != the_clone && the_clone != NULL) + if (x != the_clone && the_clone != nullptr) _igvn.remove_dead_node(the_clone); phi->set_req( i, x ); } // Too few wins? if (wins <= policy) { _igvn.remove_dead_node(phi); - return NULL; + return nullptr; } // Record Phi @@ -187,8 +176,8 @@ Node* PhaseIdealLoop::split_thru_phi(Node* n, Node* region, int policy) { } // The occasional new node if (x->_idx >= old_unique) { // Found a new, unplaced node? - old_ctrl = NULL; - old_loop = NULL; // Not in any prior loop + old_ctrl = nullptr; + old_loop = nullptr; // Not in any prior loop } else { old_ctrl = get_ctrl(x); old_loop = get_loop(old_ctrl); // Get prior loop @@ -205,7 +194,7 @@ Node* PhaseIdealLoop::split_thru_phi(Node* n, Node* region, int policy) { // for each use outside of this loop. IdealLoopTree *use_loop = get_loop(region); if (!new_loop->is_member(use_loop) && - (old_loop == NULL || !new_loop->is_member(old_loop))) { + (old_loop == nullptr || !new_loop->is_member(old_loop))) { // Take early control, later control will be recalculated // during next iteration of loop optimizations. new_ctrl = get_early_ctrl(x); @@ -225,6 +214,42 @@ Node* PhaseIdealLoop::split_thru_phi(Node* n, Node* region, int policy) { return phi; } +// Return true if 'n' is a Div or Mod node (without zero check If node which was removed earlier) with a loop phi divisor +// of a trip-counted (integer or long) loop with a backedge input that could be zero (include zero in its type range). In +// this case, we cannot split the division to the backedge as it could freely float above the loop exit check resulting in +// a division by zero. This situation is possible because the type of an increment node of an iv phi (trip-counter) could +// include zero while the iv phi does not (see PhiNode::Value() for trip-counted loops where we improve types of iv phis). +// We also need to check other loop phis as they could have been created in the same split-if pass when applying +// PhaseIdealLoop::split_thru_phi() to split nodes through an iv phi. +bool PhaseIdealLoop::cannot_split_division(const Node* n, const Node* region) const { + const Type* zero; + switch (n->Opcode()) { + case Op_DivI: + case Op_ModI: + zero = TypeInt::ZERO; + break; + case Op_DivL: + case Op_ModL: + zero = TypeLong::ZERO; + break; + default: + return false; + } + + assert(n->in(0) == nullptr, "divisions with zero check should already have bailed out earlier in split-if"); + Node* divisor = n->in(2); + return is_divisor_counted_loop_phi(divisor, region) && + loop_phi_backedge_type_contains_zero(divisor, zero); +} + +bool PhaseIdealLoop::is_divisor_counted_loop_phi(const Node* divisor, const Node* loop) { + return loop->is_BaseCountedLoop() && divisor->is_Phi() && divisor->in(0) == loop; +} + +bool PhaseIdealLoop::loop_phi_backedge_type_contains_zero(const Node* phi_divisor, const Type* zero) const { + return _igvn.type(phi_divisor->in(LoopNode::LoopBackControl))->filter_speculative(zero) != Type::TOP; +} + //------------------------------dominated_by------------------------------------ // Replace the dominated test with an obvious true or false. Place it on the // IGVN worklist for later cleanup. Move control-dependent data Nodes on the @@ -268,15 +293,15 @@ void PhaseIdealLoop::dominated_by( Node *prevdom, Node *iff, bool flip, bool exc // Loop predicates may have depending checks which should not // be skipped. For example, range check predicate has two checks // for lower and upper bounds. - if (dp == NULL) + if (dp == nullptr) return; ProjNode* dp_proj = dp->as_Proj(); ProjNode* unc_proj = iff->as_If()->proj_out(1 - dp_proj->_con)->as_Proj(); if (exclude_loop_predicate && - (unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_predicate) != NULL || - unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_profile_predicate) != NULL || - unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_range_check) != NULL)) { + (unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_predicate) != nullptr || + unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_profile_predicate) != nullptr || + unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_range_check) != nullptr)) { // If this is a range check (IfNode::is_range_check), do not // reorder because Compile::allow_range_check_smearing might have // changed the check. @@ -321,7 +346,7 @@ Node *PhaseIdealLoop::has_local_phi_input( Node *n ) { break; } if( i >= n->req() ) - return NULL; // No Phi inputs; nowhere to clone thru + return nullptr; // No Phi inputs; nowhere to clone thru // Check for inputs created between 'n' and the Phi input. These // must split as well; they have already been given the chance @@ -347,7 +372,7 @@ Node *PhaseIdealLoop::has_local_phi_input( Node *n ) { set_ctrl_and_loop(m, c); continue; } - return NULL; + return nullptr; } assert(n->is_Phi() || m->is_Phi() || is_dominator(get_ctrl(m), n_ctrl), "m has strange control"); } @@ -360,7 +385,7 @@ Node *PhaseIdealLoop::has_local_phi_input( Node *n ) { // moved out. We'd like to do all associative operators, but it's especially // important (common) to do address expressions. Node *PhaseIdealLoop::remix_address_expressions( Node *n ) { - if (!has_ctrl(n)) return NULL; + if (!has_ctrl(n)) return nullptr; Node *n_ctrl = get_ctrl(n); IdealLoopTree *n_loop = get_loop(n_ctrl); @@ -368,7 +393,7 @@ Node *PhaseIdealLoop::remix_address_expressions( Node *n ) { // itself is loop-varying. // Only interested in binary ops (and AddP) - if( n->req() < 3 || n->req() > 4 ) return NULL; + if( n->req() < 3 || n->req() > 4 ) return nullptr; Node *n1_ctrl = get_ctrl(n->in( 1)); Node *n2_ctrl = get_ctrl(n->in( 2)); @@ -381,13 +406,13 @@ Node *PhaseIdealLoop::remix_address_expressions( Node *n ) { if( (n_loop->is_member( n1_loop ) && n_loop != n1_loop) || (n_loop->is_member( n2_loop ) && n_loop != n2_loop) || (n_loop->is_member( n3_loop ) && n_loop != n3_loop) ) - return NULL; // Leave well enough alone + return nullptr; // Leave well enough alone // Is at least one of my inputs loop-invariant? if( n1_loop == n_loop && n2_loop == n_loop && n3_loop == n_loop ) - return NULL; // No loop-invariant inputs + return nullptr; // No loop-invariant inputs int n_op = n->Opcode(); @@ -484,7 +509,7 @@ Node *PhaseIdealLoop::remix_address_expressions( Node *n ) { n23_loop == n_loop ) { Node *add1 = new AddPNode( n->in(1), n->in(2)->in(2), n->in(3) ); // Stuff new AddP in the loop preheader - register_new_node( add1, n_loop->_head->in(LoopNode::EntryControl) ); + register_new_node( add1, n_loop->_head->as_Loop()->skip_strip_mined(1)->in(LoopNode::EntryControl) ); Node *add2 = new AddPNode( n->in(1), add1, n->in(2)->in(3) ); register_new_node( add2, n_ctrl ); _igvn.replace_node( n, add2 ); @@ -505,7 +530,7 @@ Node *PhaseIdealLoop::remix_address_expressions( Node *n ) { if (!is_member(n_loop,get_ctrl(I))) { Node *add1 = new AddPNode(n->in(1), n->in(2), I); // Stuff new AddP in the loop preheader - register_new_node(add1, n_loop->_head->in(LoopNode::EntryControl)); + register_new_node(add1, n_loop->_head->as_Loop()->skip_strip_mined(1)->in(LoopNode::EntryControl)); Node *add2 = new AddPNode(n->in(1), add1, V); register_new_node(add2, n_ctrl); _igvn.replace_node(n, add2); @@ -515,13 +540,13 @@ Node *PhaseIdealLoop::remix_address_expressions( Node *n ) { } } - return NULL; + return nullptr; } // Optimize ((in1[2*i] * in2[2*i]) + (in1[2*i+1] * in2[2*i+1])) Node *PhaseIdealLoop::convert_add_to_muladd(Node* n) { assert(n->Opcode() == Op_AddI, "sanity"); - Node * nn = NULL; + Node * nn = nullptr; Node * in1 = n->in(1); Node * in2 = n->in(2); if (in1->Opcode() == Op_MulI && in2->Opcode() == Op_MulI) { @@ -584,20 +609,20 @@ Node *PhaseIdealLoop::convert_add_to_muladd(Node* n) { Node *PhaseIdealLoop::conditional_move( Node *region ) { assert(region->is_Region(), "sanity check"); - if (region->req() != 3) return NULL; + if (region->req() != 3) return nullptr; // Check for CFG diamond Node *lp = region->in(1); Node *rp = region->in(2); - if (!lp || !rp) return NULL; + if (!lp || !rp) return nullptr; Node *lp_c = lp->in(0); - if (lp_c == NULL || lp_c != rp->in(0) || !lp_c->is_If()) return NULL; + if (lp_c == nullptr || lp_c != rp->in(0) || !lp_c->is_If()) return nullptr; IfNode *iff = lp_c->as_If(); // Check for ops pinned in an arm of the diamond. // Can't remove the control flow in this case - if (lp->outcnt() > 1) return NULL; - if (rp->outcnt() > 1) return NULL; + if (lp->outcnt() > 1) return nullptr; + if (rp->outcnt() > 1) return nullptr; IdealLoopTree* r_loop = get_loop(region); assert(r_loop == get_loop(iff), "sanity"); @@ -642,12 +667,12 @@ Node *PhaseIdealLoop::conditional_move( Node *region ) { // manufacturing expensive instructions, generally a bad plan. // Just Say No to Conditionally-Moved Derived Pointers. if (tp && tp->offset() != 0) - return NULL; + return nullptr; cost++; break; } default: - return NULL; // In particular, can't do memory or I/O + return nullptr; // In particular, can't do memory or I/O } // Add in cost any speculative ops for (uint j = 1; j < region->req(); j++) { @@ -680,21 +705,21 @@ Node *PhaseIdealLoop::conditional_move( Node *region ) { }//for Node* bol = iff->in(1); if (bol->Opcode() == Op_Opaque4) { - return NULL; // Ignore loop predicate checks (the Opaque4 ensures they will go away) + return nullptr; // Ignore loop predicate checks (the Opaque4 ensures they will go away) } assert(bol->Opcode() == Op_Bool, "Unexpected node"); int cmp_op = bol->in(1)->Opcode(); if (cmp_op == Op_SubTypeCheck) { // SubTypeCheck expansion expects an IfNode - return NULL; + return nullptr; } // It is expensive to generate flags from a float compare. // Avoid duplicated float compare. - if (phis > 1 && (cmp_op == Op_CmpF || cmp_op == Op_CmpD)) return NULL; + if (phis > 1 && (cmp_op == Op_CmpF || cmp_op == Op_CmpD)) return nullptr; float infrequent_prob = PROB_UNLIKELY_MAG(3); // Ignore cost and blocks frequency if CMOVE can be moved outside the loop. if (used_inside_loop) { - if (cost >= ConditionalMoveLimit) return NULL; // Too much goo + if (cost >= ConditionalMoveLimit) return nullptr; // Too much goo // BlockLayoutByFrequency optimization moves infrequent branch // from hot path. No point in CMOV'ing in such case (110 is used @@ -709,7 +734,7 @@ Node *PhaseIdealLoop::conditional_move( Node *region ) { //keep going } else if (iff->_prob < infrequent_prob || iff->_prob > (1.0f - infrequent_prob)) - return NULL; + return nullptr; // -------------- // Now replace all Phis with CMOV's @@ -717,7 +742,7 @@ Node *PhaseIdealLoop::conditional_move( Node *region ) { uint flip = (lp->Opcode() == Op_IfTrue); Node_List wq; while (1) { - PhiNode* phi = NULL; + PhiNode* phi = nullptr; for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) { Node *out = region->fast_out(i); if (out->is_Phi()) { @@ -725,7 +750,7 @@ Node *PhaseIdealLoop::conditional_move( Node *region ) { break; } } - if (phi == NULL || _igvn.type(phi) == Type::TOP) { + if (phi == nullptr || _igvn.type(phi) == Type::TOP) { break; } if (PrintOpto && VerifyLoopOptimizations) { tty->print_cr("CMOV"); } @@ -735,7 +760,7 @@ Node *PhaseIdealLoop::conditional_move( Node *region ) { Node *n = wq.pop(); for (uint j = 1; j < n->req(); j++) { Node* m = n->in(j); - if (m != NULL && !is_dominator(get_ctrl(m), cmov_ctrl)) { + if (m != nullptr && !is_dominator(get_ctrl(m), cmov_ctrl)) { #ifndef PRODUCT if (PrintOpto && VerifyLoopOptimizations) { tty->print(" speculate: "); @@ -790,7 +815,7 @@ Node* PhaseIdealLoop::try_move_store_before_loop(Node* n, Node *n_ctrl) { IdealLoopTree *n_loop = get_loop(n_ctrl); if (n->is_Store() && n_loop != _ltree_root && n_loop->is_loop() && n_loop->_head->is_Loop() && - n->in(0) != NULL) { + n->in(0) != nullptr) { Node* address = n->in(MemNode::Address); Node* value = n->in(MemNode::ValueIn); Node* mem = n->in(MemNode::Memory); @@ -820,7 +845,7 @@ Node* PhaseIdealLoop::try_move_store_before_loop(Node* n, Node *n_ctrl) { mem->outcnt() == 1 && mem->in(LoopNode::LoopBackControl) == n) { - assert(n_loop->_tail != NULL, "need a tail"); + assert(n_loop->_tail != nullptr, "need a tail"); assert(is_dominator(n_ctrl, n_loop->_tail), "store control must not be in a branch in the loop"); // Verify that there's no early exit of the loop before the store. @@ -866,12 +891,12 @@ Node* PhaseIdealLoop::try_move_store_before_loop(Node* n, Node *n_ctrl) { } } } - return NULL; + return nullptr; } // Try moving a store out of a loop, right after the loop void PhaseIdealLoop::try_move_store_after_loop(Node* n) { - if (n->is_Store() && n->in(0) != NULL) { + if (n->is_Store() && n->in(0) != nullptr) { Node *n_ctrl = get_ctrl(n); IdealLoopTree *n_loop = get_loop(n_ctrl); // Store must be in a loop @@ -883,7 +908,7 @@ void PhaseIdealLoop::try_move_store_after_loop(Node* n) { if (!n_loop->is_member(address_loop)) { // Store must be last on this memory slice in the loop and // nothing in the loop must observe it - Node* phi = NULL; + Node* phi = nullptr; for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { Node* u = n->fast_out(i); if (has_ctrl(u)) { // control use? @@ -894,7 +919,7 @@ void PhaseIdealLoop::try_move_store_after_loop(Node* n) { if (u->is_Phi() && u->in(0) == n_loop->_head) { assert(_igvn.type(u) == Type::MEMORY, "bad phi"); // multiple phis on the same slice are possible - if (phi != NULL) { + if (phi != nullptr) { return; } phi = u; @@ -903,7 +928,7 @@ void PhaseIdealLoop::try_move_store_after_loop(Node* n) { } return; } - if (phi != NULL) { + if (phi != nullptr) { // Nothing in the loop before the store (next iteration) // must observe the stored value bool mem_ok = true; @@ -1023,7 +1048,7 @@ Node *PhaseIdealLoop::split_if_with_blocks_pre( Node *n ) { } Node* res = try_move_store_before_loop(n, n_ctrl); - if (res != NULL) { + if (res != nullptr) { return n; } @@ -1042,7 +1067,7 @@ Node *PhaseIdealLoop::split_if_with_blocks_pre( Node *n ) { // Node control inputs don't necessarily agree with loop control info (due to // transformations happened in between), thus additional dominance check is needed // to keep loop info valid. - if (dom_cast != NULL && is_dominator(get_ctrl(dom_cast), get_ctrl(n))) { + if (dom_cast != nullptr && is_dominator(get_ctrl(dom_cast), get_ctrl(n))) { _igvn.replace_node(n, dom_cast); return dom_cast; } @@ -1257,14 +1282,14 @@ bool PhaseIdealLoop::can_split_if(Node* n_ctrl) { } // Detect if the node is the inner strip-mined loop -// Return: NULL if it's not the case, or the exit of outer strip-mined loop +// Return: null if it's not the case, or the exit of outer strip-mined loop static Node* is_inner_of_stripmined_loop(const Node* out) { - Node* out_le = NULL; + Node* out_le = nullptr; if (out->is_CountedLoopEnd()) { const CountedLoopNode* loop = out->as_CountedLoopEnd()->loopnode(); - if (loop != NULL && loop->is_strip_mined()) { + if (loop != nullptr && loop->is_strip_mined()) { out_le = loop->in(LoopNode::EntryControl)->as_OuterStripMinedLoop()->outer_loop_exit(); } } @@ -1309,8 +1334,8 @@ void PhaseIdealLoop::split_if_with_blocks_post(Node *n) { return; // Compare must be in same blk as if } } else if (iff->is_CMove()) { // Trying to split-up a CMOVE - // Can't split CMove with different control edge. - if (iff->in(0) != NULL && iff->in(0) != n_ctrl ) { + // Can't split CMove with different control. + if (get_ctrl(iff) != n_ctrl) { return; } if (get_ctrl(iff->in(2)) == n_ctrl || @@ -1339,7 +1364,7 @@ void PhaseIdealLoop::split_if_with_blocks_post(Node *n) { // Now split the bool up thru the phi Node *bolphi = split_thru_phi(bol, n_ctrl, -1); - guarantee(bolphi != NULL, "null boolean phi node"); + guarantee(bolphi != nullptr, "null boolean phi node"); _igvn.replace_node(bol, bolphi); assert(iff->in(1) == bolphi, ""); @@ -1412,7 +1437,7 @@ void PhaseIdealLoop::split_if_with_blocks_post(Node *n) { // 2) move code with side-effect in strip-mined loop // Move to the exit of outer strip-mined loop in that case. Node* out_le = is_inner_of_stripmined_loop(dom); - if (out_le != NULL) { + if (out_le != nullptr) { prevdom = out_le; } // Replace the dominated test with an obvious true or false. @@ -1449,13 +1474,13 @@ bool PhaseIdealLoop::safe_for_if_replacement(const Node* dom) const { } CountedLoopEndNode* le = dom->as_CountedLoopEnd(); CountedLoopNode* cl = le->loopnode(); - if (cl == NULL) { + if (cl == nullptr) { return true; } if (!cl->is_main_loop()) { return true; } - if (cl->is_canonical_loop_entry() == NULL) { + if (cl->is_canonical_loop_entry() == nullptr) { return true; } // Further unrolling is possible so loop exit condition might change @@ -1478,7 +1503,7 @@ void PhaseIdealLoop::try_sink_out_of_loop(Node* n) { Node *n_ctrl = get_ctrl(n); IdealLoopTree *n_loop = get_loop(n_ctrl); - if (n->in(0) != NULL) { + if (n->in(0) != nullptr) { IdealLoopTree* loop_ctrl = get_loop(n->in(0)); if (n_loop != loop_ctrl && n_loop->is_member(loop_ctrl)) { // n has a control input inside a loop but get_ctrl() is member of an outer loop. This could happen, for example, @@ -1494,12 +1519,12 @@ void PhaseIdealLoop::try_sink_out_of_loop(Node* n) { if (n_loop->is_member(get_loop(early_ctrl)) && // check that this one can't be hoisted now ctrl_of_all_uses_out_of_loop(n, early_ctrl, n_loop)) { // All uses in outer loops! assert(!n->is_Store() && !n->is_LoadStore(), "no node with a side effect"); - Node* outer_loop_clone = NULL; + Node* outer_loop_clone = nullptr; for (DUIterator_Last jmin, j = n->last_outs(jmin); j >= jmin;) { Node* u = n->last_out(j); // Clone private computation per use _igvn.rehash_node_delayed(u); Node* x = n->clone(); // Clone computation - Node* x_ctrl = NULL; + Node* x_ctrl = nullptr; if (u->is_Phi()) { // Replace all uses of normal nodes. Replace Phi uses // individually, so the separate Nodes can sink down @@ -1542,14 +1567,14 @@ void PhaseIdealLoop::try_sink_out_of_loop(Node* n) { Node* x_head = x_loop->_head; if (x_head->is_Loop() && x_head->is_OuterStripMinedLoop()) { // Do not add duplicate LoadNodes to the outer strip mined loop - if (outer_loop_clone != NULL) { + if (outer_loop_clone != nullptr) { _igvn.replace_node(x, outer_loop_clone); continue; } outer_loop_clone = x; } x->set_req(0, x_ctrl); - } else if (n->in(0) != NULL){ + } else if (n->in(0) != nullptr){ x->set_req(0, x_ctrl); } assert(dom_depth(n_ctrl) <= dom_depth(x_ctrl), "n is later than its clone"); @@ -1563,18 +1588,18 @@ void PhaseIdealLoop::try_sink_out_of_loop(Node* n) { assert(!x->is_AddP() || !x->in(AddPNode::Address)->is_AddP() || x->in(AddPNode::Address)->in(AddPNode::Base) == x->in(AddPNode::Base) || !x->in(AddPNode::Address)->in(AddPNode::Base)->eqv_uncast(x->in(AddPNode::Base)), "unexpected AddP shape"); - if (x->in(0) == NULL && !x->is_DecodeNarrowPtr() && + if (x->in(0) == nullptr && !x->is_DecodeNarrowPtr() && !(x->is_AddP() && x->in(AddPNode::Address)->is_AddP() && x->in(AddPNode::Address)->in(AddPNode::Base) == x->in(AddPNode::Base))) { assert(!x->is_Load(), "load should be pinned"); // Use a cast node to pin clone out of loop - Node* cast = NULL; + Node* cast = nullptr; for (uint k = 0; k < x->req(); k++) { Node* in = x->in(k); - if (in != NULL && n_loop->is_member(get_loop(get_ctrl(in)))) { + if (in != nullptr && n_loop->is_member(get_loop(get_ctrl(in)))) { const Type* in_t = _igvn.type(in); cast = ConstraintCastNode::make_cast_for_type(x_ctrl, in, in_t, ConstraintCastNode::UnconditionalDependency); } - if (cast != NULL) { + if (cast != nullptr) { register_new_node(cast, x_ctrl); x->replace_edge(in, cast); // Chain of AddP: @@ -1584,14 +1609,14 @@ void PhaseIdealLoop::try_sink_out_of_loop(Node* n) { Node* u = x->fast_out(i); if (u->is_AddP() && u->in(AddPNode::Base) == n->in(AddPNode::Base)) { _igvn.replace_input_of(u, AddPNode::Base, cast); - assert(u->find_out_with(Op_AddP) == NULL, "more than 2 chained AddP nodes?"); + assert(u->find_out_with(Op_AddP) == nullptr, "more than 2 chained AddP nodes?"); } } } break; } } - assert(cast != NULL, "must have added a cast to pin the node"); + assert(cast != nullptr, "must have added a cast to pin the node"); } } _igvn.remove_dead_node(n); @@ -1604,13 +1629,13 @@ void PhaseIdealLoop::try_sink_out_of_loop(Node* n) { // Compute the early control of a node by following its inputs until we reach // nodes that are pinned. Then compute the LCA of the control of all pinned nodes. Node* PhaseIdealLoop::compute_early_ctrl(Node* n, Node* n_ctrl) { - Node* early_ctrl = NULL; + Node* early_ctrl = nullptr; ResourceMark rm; Unique_Node_List wq; wq.push(n); for (uint i = 0; i < wq.size(); i++) { Node* m = wq.at(i); - Node* c = NULL; + Node* c = nullptr; if (m->is_CFG()) { c = m; } else if (m->pinned()) { @@ -1618,14 +1643,14 @@ Node* PhaseIdealLoop::compute_early_ctrl(Node* n, Node* n_ctrl) { } else { for (uint j = 0; j < m->req(); j++) { Node* in = m->in(j); - if (in != NULL) { + if (in != nullptr) { wq.push(in); } } } - if (c != NULL) { + if (c != nullptr) { assert(is_dominator(c, n_ctrl), "control input must dominate current control"); - if (early_ctrl == NULL || is_dominator(early_ctrl, c)) { + if (early_ctrl == nullptr || is_dominator(early_ctrl, c)) { early_ctrl = c; } } @@ -1747,8 +1772,8 @@ Node* PhaseIdealLoop::clone_iff(PhiNode *phi, IdealLoopTree *loop) { } Node* n = phi->in(1); - Node* sample_opaque = NULL; - Node *sample_bool = NULL; + Node* sample_opaque = nullptr; + Node *sample_bool = nullptr; if (n->Opcode() == Op_Opaque4) { sample_opaque = n; sample_bool = n->in(1); @@ -1762,8 +1787,8 @@ Node* PhaseIdealLoop::clone_iff(PhiNode *phi, IdealLoopTree *loop) { PhiNode *phi1 = new PhiNode(phi->in(0), Type::TOP); PhiNode *phi2 = new PhiNode(phi->in(0), Type::TOP); for (i = 1; i < phi->req(); i++) { - Node *n1 = sample_opaque == NULL ? phi->in(i)->in(1)->in(1) : phi->in(i)->in(1)->in(1)->in(1); - Node *n2 = sample_opaque == NULL ? phi->in(i)->in(1)->in(2) : phi->in(i)->in(1)->in(1)->in(2); + Node *n1 = sample_opaque == nullptr ? phi->in(i)->in(1)->in(1) : phi->in(i)->in(1)->in(1)->in(1); + Node *n2 = sample_opaque == nullptr ? phi->in(i)->in(1)->in(2) : phi->in(i)->in(1)->in(1)->in(2); phi1->set_req(i, n1); phi2->set_req(i, n2); phi1->set_type(phi1->type()->meet_speculative(n1->bottom_type())); @@ -1803,7 +1828,7 @@ Node* PhaseIdealLoop::clone_iff(PhiNode *phi, IdealLoopTree *loop) { _igvn.register_new_node_with_optimizer(b); set_ctrl(b, phi->in(0)); - if (sample_opaque != NULL) { + if (sample_opaque != nullptr) { Node* opaque = sample_opaque->clone(); opaque->set_req(1, b); _igvn.register_new_node_with_optimizer(opaque); @@ -1914,7 +1939,7 @@ void PhaseIdealLoop::clone_loop_handle_data_uses(Node* old, Node_List &old_new, // Check for data-use outside of loop - at least one of OLD or USE // must not be a CFG node. #ifdef ASSERT - if (loop->_head->as_Loop()->is_strip_mined() && outer_loop->is_member(use_loop) && !loop->is_member(use_loop) && old_new[use->_idx] == NULL) { + if (loop->_head->as_Loop()->is_strip_mined() && outer_loop->is_member(use_loop) && !loop->is_member(use_loop) && old_new[use->_idx] == nullptr) { Node* sfpt = loop->_head->as_CountedLoop()->outer_safepoint(); assert(mode != IgnoreStripMined, "incorrect cloning mode"); assert((mode == ControlAroundStripMined && use == sfpt) || !use->is_reachable_from_root(), "missed a node"); @@ -1928,7 +1953,13 @@ void PhaseIdealLoop::clone_loop_handle_data_uses(Node* old, Node_List &old_new, // in the loop to break the loop, then test is again outside of the // loop to determine which way the loop exited. // Loop predicate If node connects to Bool node through Opaque1 node. - if (use->is_If() || use->is_CMove() || C->is_predicate_opaq(use) || use->Opcode() == Op_Opaque4) { + // + // If the use is an AllocateArray through its ValidLengthTest input, + // make sure the Bool/Cmp input is cloned down to avoid a Phi between + // the AllocateArray node and its ValidLengthTest input that could cause + // split if to break. + if (use->is_If() || use->is_CMove() || C->is_predicate_opaq(use) || use->Opcode() == Op_Opaque4 || + (use->Opcode() == Op_AllocateArray && use->in(AllocateNode::ValidLengthTest) == old)) { // Since this code is highly unlikely, we lazily build the worklist // of such Nodes to go split. if (!split_if_set) { @@ -2013,7 +2044,7 @@ void PhaseIdealLoop::clone_loop_handle_data_uses(Node* old, Node_List &old_new, // If inserting a new Phi, check for prior hits if( idx != 0 ) { Node *hit = _igvn.hash_find_insert(phi); - if( hit == NULL ) { + if( hit == nullptr ) { _igvn.register_new_node_with_optimizer(phi); // Register new phi } else { // or // Remove the new phi from the graph and use the hit @@ -2046,8 +2077,8 @@ static void collect_nodes_in_outer_loop_not_reachable_from_sfpt(Node* n, const I bool check_old_new) { for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { Node* u = n->fast_out(j); - assert(check_old_new || old_new[u->_idx] == NULL, "shouldn't have been cloned"); - if (!u->is_CFG() && (!check_old_new || old_new[u->_idx] == NULL)) { + assert(check_old_new || old_new[u->_idx] == nullptr, "shouldn't have been cloned"); + if (!u->is_CFG() && (!check_old_new || old_new[u->_idx] == nullptr)) { Node* c = phase->get_ctrl(u); IdealLoopTree* u_loop = phase->get_loop(c); assert(!loop->is_member(u_loop) || !loop->_body.contains(u), "can be in outer loop or out of both loops only"); @@ -2058,7 +2089,7 @@ static void collect_nodes_in_outer_loop_not_reachable_from_sfpt(Node* n, const I // nodes pinned with control in the outer loop but not referenced from the safepoint must be moved out of // the outer loop too Node* u_c = u->in(0); - if (u_c != NULL) { + if (u_c != nullptr) { IdealLoopTree* u_c_loop = phase->get_loop(u_c); if (outer_loop->is_member(u_c_loop) && !loop->is_member(u_c_loop)) { wq.push(u); @@ -2084,7 +2115,7 @@ void PhaseIdealLoop::clone_outer_loop(LoopNode* head, CloneLoopMode mode, IdealL CountedLoopEndNode* new_cle = new_cl->as_CountedLoop()->loopexit_or_null(); Node* cle_out = cle->proj_out(false); - Node* new_sfpt = NULL; + Node* new_sfpt = nullptr; Node* new_cle_out = cle_out->clone(); old_new.map(cle_out->_idx, new_cle_out); if (mode == CloneIncludesStripMined) { @@ -2139,21 +2170,21 @@ void PhaseIdealLoop::clone_outer_loop(LoopNode* head, CloneLoopMode mode, IdealL Node* n = stack.node(); uint i = stack.index(); while (i < n->req() && - (n->in(i) == NULL || + (n->in(i) == nullptr || !has_ctrl(n->in(i)) || get_loop(get_ctrl(n->in(i))) != outer_loop || - (old_new[n->in(i)->_idx] != NULL && old_new[n->in(i)->_idx]->_idx >= new_counter))) { + (old_new[n->in(i)->_idx] != nullptr && old_new[n->in(i)->_idx]->_idx >= new_counter))) { i++; } if (i < n->req()) { stack.set_index(i+1); stack.push(n->in(i), 0); } else { - assert(old_new[n->_idx] == NULL || n == sfpt || old_new[n->_idx]->_idx < new_counter, "no clone yet"); + assert(old_new[n->_idx] == nullptr || n == sfpt || old_new[n->_idx]->_idx < new_counter, "no clone yet"); Node* m = n == sfpt ? new_sfpt : n->clone(); - if (m != NULL) { + if (m != nullptr) { for (uint i = 0; i < n->req(); i++) { - if (m->in(i) != NULL && old_new[m->in(i)->_idx] != NULL) { + if (m->in(i) != nullptr && old_new[m->in(i)->_idx] != nullptr) { m->set_req(i, old_new[m->in(i)->_idx]); } } @@ -2199,7 +2230,7 @@ void PhaseIdealLoop::clone_outer_loop(LoopNode* head, CloneLoopMode mode, IdealL for (uint i = 0; i < wq.size(); i++) { Node* n = wq.at(i); set_ctrl(n, new_ctrl); - if (n->in(0) != NULL) { + if (n->in(0) != nullptr) { _igvn.replace_input_of(n, 0, new_ctrl); } collect_nodes_in_outer_loop_not_reachable_from_sfpt(n, loop, outer_loop, old_new, wq, this, false); @@ -2231,7 +2262,7 @@ void PhaseIdealLoop::clone_outer_loop(LoopNode* head, CloneLoopMode mode, IdealL // or loop unrolling or iteration splitting (Range-Check-Elimination), etc. // // Parameter side_by_size_idom: -// When side_by_size_idom is NULL, the dominator tree is constructed for +// When side_by_size_idom is null, the dominator tree is constructed for // the clone loop to dominate the original. Used in construction of // pre-main-post loop sequence. // When nonnull, the clone and original are side-by-side, both are @@ -2245,7 +2276,7 @@ void PhaseIdealLoop::clone_loop( IdealLoopTree *loop, Node_List &old_new, int dd if (C->do_vector_loop() && PrintOpto) { const char* mname = C->method()->name()->as_quoted_ascii(); - if (mname != NULL) { + if (mname != nullptr) { tty->print("PhaseIdealLoop::clone_loop: for vectorize method %s\n", mname); } } @@ -2417,9 +2448,9 @@ void PhaseIdealLoop::clone_loop( IdealLoopTree *loop, Node_List &old_new, int dd // Step 4: If loop-invariant use is not control, it must be dominated by a // loop exit IfFalse/IfTrue. Find "proper" loop exit. Make a Region // there if needed. Make a Phi there merging old and new used values. - Node_List *split_if_set = NULL; - Node_List *split_bool_set = NULL; - Node_List *split_cex_set = NULL; + Node_List *split_if_set = nullptr; + Node_List *split_bool_set = nullptr; + Node_List *split_cex_set = nullptr; for( i = 0; i < loop->_body.size(); i++ ) { Node* old = loop->_body.at(i); clone_loop_handle_data_uses(old, old_new, loop, outer_loop, split_if_set, @@ -2442,9 +2473,10 @@ void PhaseIdealLoop::clone_loop( IdealLoopTree *loop, Node_List &old_new, int dd if (split_if_set) { while (split_if_set->size()) { Node *iff = split_if_set->pop(); - if (iff->in(1)->is_Phi()) { - Node *b = clone_iff(iff->in(1)->as_Phi(), loop); - _igvn.replace_input_of(iff, 1, b); + uint input = iff->Opcode() == Op_AllocateArray ? AllocateNode::ValidLengthTest : 1; + if (iff->in(input)->is_Phi()) { + Node *b = clone_iff(iff->in(input)->as_Phi(), loop); + _igvn.replace_input_of(iff, input, b); } } } @@ -2463,7 +2495,7 @@ void PhaseIdealLoop::clone_loop( IdealLoopTree *loop, Node_List &old_new, int dd assert(b->in(0)->is_Region(), ""); assert(b->in(1)->is_Phi(), ""); assert(b->in(0)->in(0) == b->in(1)->in(0), ""); - split_up(b, b->in(0), NULL); + split_up(b, b->in(0), nullptr); } } @@ -2476,10 +2508,10 @@ void PhaseIdealLoop::clone_loop( IdealLoopTree *loop, Node_List &old_new, int dd // with an optional truncation (left-shift followed by a right-shift) // of the add. Returns zero if not an iv. int PhaseIdealLoop::stride_of_possible_iv(Node* iff) { - Node* trunc1 = NULL; - Node* trunc2 = NULL; - const TypeInteger* ttype = NULL; - if (!iff->is_If() || iff->in(1) == NULL || !iff->in(1)->is_Bool()) { + Node* trunc1 = nullptr; + Node* trunc2 = nullptr; + const TypeInteger* ttype = nullptr; + if (!iff->is_If() || iff->in(1) == nullptr || !iff->in(1)->is_Bool()) { return 0; } BoolNode* bl = iff->in(1)->as_Bool(); @@ -2491,7 +2523,7 @@ int PhaseIdealLoop::stride_of_possible_iv(Node* iff) { if (is_member(get_loop(iff), get_ctrl(cmp->in(2)))) { return 0; } - Node* add2 = NULL; + Node* add2 = nullptr; Node* cmp1 = cmp->in(1); if (cmp1->is_Phi()) { // (If (Bool (CmpX phi:(Phi ...(Optional-trunc(AddI phi add2))) ))) @@ -2520,7 +2552,7 @@ int PhaseIdealLoop::stride_of_possible_iv(Node* iff) { } } } - if (add2 != NULL) { + if (add2 != nullptr) { const TypeInt* add2t = _igvn.type(add2)->is_int(); if (add2t->is_con()) { return add2t->get_con(); @@ -2533,13 +2565,13 @@ int PhaseIdealLoop::stride_of_possible_iv(Node* iff) { //---------------------- stay_in_loop ------------------------------------- // Return the (unique) control output node that's in the loop (if it exists.) Node* PhaseIdealLoop::stay_in_loop( Node* n, IdealLoopTree *loop) { - Node* unique = NULL; - if (!n) return NULL; + Node* unique = nullptr; + if (!n) return nullptr; for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { Node* use = n->fast_out(i); if (!has_ctrl(use) && loop->is_member(get_loop(use))) { - if (unique != NULL) { - return NULL; + if (unique != nullptr) { + return nullptr; } unique = use; } @@ -2571,7 +2603,7 @@ ProjNode* PhaseIdealLoop::proj_clone(ProjNode* p, IfNode* iff) { //------------------------------ short_circuit_if ------------------------------------- // Force the iff control output to be the live_proj Node* PhaseIdealLoop::short_circuit_if(IfNode* iff, ProjNode* live_proj) { - guarantee(live_proj != NULL, "null projection"); + guarantee(live_proj != nullptr, "null projection"); int proj_con = live_proj->_con; assert(proj_con == 0 || proj_con == 1, "false or true projection"); Node *con = _igvn.intcon(proj_con); @@ -2613,7 +2645,7 @@ ProjNode* PhaseIdealLoop::insert_if_before_proj(Node* left, bool Signed, BoolTes _igvn.rehash_node_delayed(iff); _igvn.rehash_node_delayed(proj); - proj->set_req(0, NULL); // temporary disconnect + proj->set_req(0, nullptr); // temporary disconnect ProjNode* proj2 = proj_clone(proj, iff); register_node(proj2, loop, iff, ddepth); @@ -2633,7 +2665,7 @@ ProjNode* PhaseIdealLoop::insert_if_before_proj(Node* left, bool Signed, BoolTes set_idom(proj, new_if, ddepth); ProjNode* new_exit = proj_clone(other_proj, new_if)->as_Proj(); - guarantee(new_exit != NULL, "null exit node"); + guarantee(new_exit != nullptr, "null exit node"); register_node(new_exit, get_loop(other_proj), new_if, ddepth); return new_exit; @@ -2674,7 +2706,7 @@ RegionNode* PhaseIdealLoop::insert_region_before_proj(ProjNode* proj) { _igvn.rehash_node_delayed(iff); _igvn.rehash_node_delayed(proj); - proj->set_req(0, NULL); // temporary disconnect + proj->set_req(0, nullptr); // temporary disconnect ProjNode* proj2 = proj_clone(proj, iff); register_node(proj2, loop, iff, ddepth); @@ -2682,7 +2714,7 @@ RegionNode* PhaseIdealLoop::insert_region_before_proj(ProjNode* proj) { reg->set_req(1, proj2); register_node(reg, loop, iff, ddepth); - IfNode* dum_if = new IfNode(reg, short_circuit_if(NULL, proj), iff->_prob, iff->_fcnt); + IfNode* dum_if = new IfNode(reg, short_circuit_if(nullptr, proj), iff->_prob, iff->_fcnt); register_node(dum_if, loop, reg, ddepth); proj->set_req(0, dum_if); // reattach @@ -2735,25 +2767,25 @@ IfNode* PhaseIdealLoop::insert_cmpi_loop_exit(IfNode* if_cmpu, IdealLoopTree *lo const bool Unsigned = false; BoolNode* bol = if_cmpu->in(1)->as_Bool(); - if (bol->_test._test != BoolTest::lt) return NULL; + if (bol->_test._test != BoolTest::lt) return nullptr; CmpNode* cmpu = bol->in(1)->as_Cmp(); - if (cmpu->Opcode() != Op_CmpU) return NULL; + if (cmpu->Opcode() != Op_CmpU) return nullptr; int stride = stride_of_possible_iv(if_cmpu); - if (stride == 0) return NULL; + if (stride == 0) return nullptr; Node* lp_proj = stay_in_loop(if_cmpu, loop); - guarantee(lp_proj != NULL, "null loop node"); + guarantee(lp_proj != nullptr, "null loop node"); ProjNode* lp_continue = lp_proj->as_Proj(); ProjNode* lp_exit = if_cmpu->proj_out(!lp_continue->is_IfTrue())->as_Proj(); if (!lp_exit->is_IfFalse()) { // The loop exit condition is (i (i >= 0 && i < limit). // We therefore can't add a single exit condition. - return NULL; + return nullptr; } // The loop exit condition is !(i (i < 0 || i >= limit). // Split out the exit condition (i < 0) for stride < 0 or (i >= limit) for stride > 0. - Node* limit = NULL; + Node* limit = nullptr; if (stride > 0) { limit = cmpu->in(2); } else { @@ -2762,7 +2794,7 @@ IfNode* PhaseIdealLoop::insert_cmpi_loop_exit(IfNode* if_cmpu, IdealLoopTree *lo } // Create a new region on the exit path RegionNode* reg = insert_region_before_proj(lp_exit); - guarantee(reg != NULL, "null region node"); + guarantee(reg != nullptr, "null region node"); // Clone the if-cmpu-true-false using a signed compare BoolTest::mask rel_i = stride > 0 ? bol->_test._test : BoolTest::ge; @@ -2985,7 +3017,7 @@ void PhaseIdealLoop::insert_phi_for_loop( Node* use, uint idx, Node* lp_entry_va phi->set_req(LoopNode::EntryControl, lp_entry_val); // Use existing phi if it already exists Node *hit = _igvn.hash_find_insert(phi); - if( hit == NULL ) { + if( hit == nullptr ) { _igvn.register_new_node_with_optimizer(phi); set_ctrl(phi, lp); } else { @@ -3321,8 +3353,8 @@ bool PhaseIdealLoop::partial_peel( IdealLoopTree *loop, Node_List &old_new ) { // Walk up dominators to loop head looking for first loop exit // which is executed on every path thru loop. - IfNode *peel_if = NULL; - IfNode *peel_if_cmpu = NULL; + IfNode *peel_if = nullptr; + IfNode *peel_if_cmpu = nullptr; Node *iff = loop->tail(); while (iff != head) { @@ -3346,20 +3378,20 @@ bool PhaseIdealLoop::partial_peel( IdealLoopTree *loop, Node_List &old_new ) { } // Prefer signed compare over unsigned compare. - IfNode* new_peel_if = NULL; - if (peel_if == NULL) { - if (!PartialPeelAtUnsignedTests || peel_if_cmpu == NULL) { + IfNode* new_peel_if = nullptr; + if (peel_if == nullptr) { + if (!PartialPeelAtUnsignedTests || peel_if_cmpu == nullptr) { return false; // No peel point found } new_peel_if = insert_cmpi_loop_exit(peel_if_cmpu, loop); - if (new_peel_if == NULL) { + if (new_peel_if == nullptr) { return false; // No peel point found } peel_if = new_peel_if; } Node* last_peel = stay_in_loop(peel_if, loop); Node* first_not_peeled = stay_in_loop(last_peel, loop); - if (first_not_peeled == NULL || first_not_peeled == head) { + if (first_not_peeled == nullptr || first_not_peeled == head) { return false; } @@ -3466,7 +3498,7 @@ bool PhaseIdealLoop::partial_peel( IdealLoopTree *loop, Node_List &old_new ) { if (!has_use_internal_to_set(n, peel, loop)) { // if not pinned and not a load (which maybe anti-dependent on a store) // and not a CMove (Matcher expects only bool->cmove). - if (n->in(0) == NULL && !n->is_Load() && !n->is_CMove()) { + if (n->in(0) == nullptr && !n->is_Load() && !n->is_CMove()) { int new_clones = clone_for_use_outside_loop(loop, n, worklist); if (new_clones == -1) { too_many_clones = true; @@ -3504,10 +3536,10 @@ bool PhaseIdealLoop::partial_peel( IdealLoopTree *loop, Node_List &old_new ) { #ifndef PRODUCT if (TracePartialPeeling && exceed_phi_limit) { tty->print_cr("\nToo many new phis: %d old %d new cmpi: %c", - new_phi_cnt, old_phi_cnt, new_peel_if != NULL?'T':'F'); + new_phi_cnt, old_phi_cnt, new_peel_if != nullptr?'T':'F'); } #endif - if (new_peel_if != NULL) { + if (new_peel_if != nullptr) { remove_cmpi_loop_exit(new_peel_if, loop); } // Inhibit more partial peeling on this loop @@ -3580,7 +3612,7 @@ bool PhaseIdealLoop::partial_peel( IdealLoopTree *loop, Node_List &old_new ) { if ( loop->is_member(get_loop( use_c )) ) { // use is in loop - if (old_new[use->_idx] != NULL) { // null for dead code + if (old_new[use->_idx] != nullptr) { // null for dead code Node* use_clone = old_new[use->_idx]; _igvn.replace_input_of(use, j, C->top()); insert_phi_for_loop( use_clone, j, old_new[def->_idx], def, new_head_clone ); @@ -3614,7 +3646,7 @@ bool PhaseIdealLoop::partial_peel( IdealLoopTree *loop, Node_List &old_new ) { for (uint i = 0; i < loop->_body.size(); i++) { Node *n = loop->_body.at(i); - if (!n->is_CFG() && n->in(0) != NULL && + if (!n->is_CFG() && n->in(0) != nullptr && not_peel.test(n->_idx) && peel.test(n->in(0)->_idx)) { Node* n_clone = old_new[n->_idx]; _igvn.replace_input_of(n_clone, 0, new_head_clone); @@ -3712,7 +3744,7 @@ void PhaseIdealLoop::reorg_offsets(IdealLoopTree *loop) { if (!has_ctrl(use)) continue; Node *u_ctrl = get_ctrl(use); if (use->is_Phi()) { - u_ctrl = NULL; + u_ctrl = nullptr; for (uint j = 1; j < use->req(); j++) if (use->in(j) == phi) u_ctrl = dom_lca(u_ctrl, use->in(0)->in(j)); diff --git a/src/hotspot/share/opto/machnode.cpp b/src/hotspot/share/opto/machnode.cpp index 09e4cfde42923..6e9a84236852a 100644 --- a/src/hotspot/share/opto/machnode.cpp +++ b/src/hotspot/share/opto/machnode.cpp @@ -46,7 +46,7 @@ relocInfo::relocType MachOper::constant_reloc() const { return relocInfo::none; jdouble MachOper::constantD() const { ShouldNotReachHere(); return 0.0; } jfloat MachOper::constantF() const { ShouldNotReachHere(); return 0.0; } jlong MachOper::constantL() const { ShouldNotReachHere(); return CONST64(0) ; } -TypeOopPtr *MachOper::oop() const { return NULL; } +TypeOopPtr *MachOper::oop() const { return nullptr; } int MachOper::ccode() const { return 0x00; } // A zero, default, indicates this value is not needed. // May need to lookup the base register, as done in int_ and ext_format @@ -78,7 +78,7 @@ const Type *MachOper::type() const { //------------------------------in_RegMask------------------------------------- const RegMask *MachOper::in_RegMask(int index) const { ShouldNotReachHere(); - return NULL; + return nullptr; } //------------------------------dump_spec-------------------------------------- @@ -184,7 +184,7 @@ bool MachNode::cmp( const Node &node ) const { // Return an equivalent instruction using memory for cisc_operand position MachNode *MachNode::cisc_version(int offset) { ShouldNotCallThis(); - return NULL; + return nullptr; } void MachNode::use_cisc_RegMask() { @@ -212,7 +212,7 @@ const RegMask &MachNode::in_RegMask( uint idx ) const { } const RegMask *rm = cisc_RegMask(); - if( rm == NULL || (int)opcnt != cisc_operand() ) { + if( rm == nullptr || (int)opcnt != cisc_operand() ) { rm = _opnds[opcnt]->in_RegMask(idx-skipped); } return *rm; @@ -226,9 +226,9 @@ const MachOper* MachNode::memory_inputs(Node* &base, Node* &index) const { base = NodeSentinel; index = NodeSentinel; } else { - base = NULL; - index = NULL; - if (oper != NULL) { + base = nullptr; + index = nullptr; + if (oper != nullptr) { // It has a unique memory operand. Find its index. int oper_idx = num_opnds(); while (--oper_idx >= 0) { @@ -257,36 +257,36 @@ const Node* MachNode::get_base_and_disp(intptr_t &offset, const TypePtr* &adr_ty Node* index; const MachOper* oper = memory_inputs(base, index); - if (oper == NULL) { - // Base has been set to NULL + if (oper == nullptr) { + // Base has been set to null offset = 0; } else if (oper == (MachOper*)-1) { // Base has been set to NodeSentinel // There is not a unique memory use here. We will fall to AliasIdxBot. offset = Type::OffsetBot; } else { - // Base may be NULL, even if offset turns out to be != 0 + // Base may be null, even if offset turns out to be != 0 intptr_t disp = oper->constant_disp(); int scale = oper->scale(); // Now we have collected every part of the ADLC MEMORY_INTER. // See if it adds up to a base + offset. - if (index != NULL) { + if (index != nullptr) { const Type* t_index = index->bottom_type(); if (t_index->isa_narrowoop() || t_index->isa_narrowklass()) { // EncodeN, LoadN, LoadConN, LoadNKlass, // EncodeNKlass, LoadConNklass. // Memory references through narrow oops have a // funny base so grab the type from the index: // [R12 + narrow_oop_reg<<3 + offset] - assert(base == NULL, "Memory references through narrow oops have no base"); + assert(base == nullptr, "Memory references through narrow oops have no base"); offset = disp; adr_type = t_index->make_ptr()->add_offset(offset); - return NULL; + return nullptr; } else if (!index->is_Con()) { disp = Type::OffsetBot; } else if (disp != Type::OffsetBot) { const TypeX* ti = t_index->isa_intptr_t(); - if (ti == NULL) { + if (ti == nullptr) { disp = Type::OffsetBot; // a random constant?? } else { disp += ti->get_con() << scale; @@ -300,8 +300,8 @@ const Node* MachNode::get_base_and_disp(intptr_t &offset, const TypePtr* &adr_ty // Lookup the TypePtr used by indOffset32X, a compile-time constant oop, // Add the offset determined by the "base", or use Type::OffsetBot. if( adr_type == TYPE_PTR_SENTINAL ) { - const TypePtr *t_disp = oper->disp_as_type(); // only !NULL for indOffset32X - if (t_disp != NULL) { + const TypePtr *t_disp = oper->disp_as_type(); // only not null for indOffset32X + if (t_disp != nullptr) { offset = Type::OffsetBot; const Type* t_base = base->bottom_type(); if (t_base->isa_intptr_t()) { @@ -311,10 +311,10 @@ const Node* MachNode::get_base_and_disp(intptr_t &offset, const TypePtr* &adr_ty } } adr_type = t_disp->add_offset(offset); - } else if( base == NULL && offset != 0 && offset != Type::OffsetBot ) { + } else if( base == nullptr && offset != 0 && offset != Type::OffsetBot ) { // Use ideal type if it is oop ptr. const TypePtr *tp = oper->type()->isa_ptr(); - if( tp != NULL) { + if( tp != nullptr) { adr_type = tp; } } @@ -339,12 +339,12 @@ const class TypePtr *MachNode::adr_type() const { // %%%%% Someday we'd like to allow constant oop offsets which // would let Intel load from static globals in 1 instruction. // Currently Intel requires 2 instructions and a register temp. - if (base == NULL) { - // NULL base, zero offset means no memory at all (a null pointer!) + if (base == nullptr) { + // null base, zero offset means no memory at all (a null pointer!) if (offset == 0) { - return NULL; + return nullptr; } - // NULL base, any offset means any pointer whatever + // null base, any offset means any pointer whatever if (offset == Type::OffsetBot) { return TypePtr::BOTTOM; } @@ -377,7 +377,7 @@ const class TypePtr *MachNode::adr_type() const { const TypePtr *tp = t->isa_ptr(); // be conservative if we do not recognize the type - if (tp == NULL) { + if (tp == nullptr) { assert(false, "this path may produce not optimal code"); return TypePtr::BOTTOM; } @@ -505,7 +505,7 @@ bool MachNode::rematerialize() const { void MachNode::dump_spec(outputStream *st) const { uint cnt = num_opnds(); for( uint i=0; idump_spec(st); } else { st->print(" _"); @@ -529,10 +529,10 @@ void MachNode::dump_format(PhaseRegAlloc *ra, outputStream *st) const { //============================================================================= #ifndef PRODUCT void MachTypeNode::dump_spec(outputStream *st) const { - if (_bottom_type != NULL) { + if (_bottom_type != nullptr) { _bottom_type->dump_on(st); } else { - st->print(" NULL"); + st->print(" null"); } } #endif @@ -602,16 +602,16 @@ const TypePtr *MachProjNode::adr_type() const { if (bottom_type() == Type::MEMORY) { // in(0) might be a narrow MemBar; otherwise we will report TypePtr::BOTTOM Node* ctrl = in(0); - if (ctrl == NULL) return NULL; // node is dead + if (ctrl == nullptr) return nullptr; // node is dead const TypePtr* adr_type = ctrl->adr_type(); #ifdef ASSERT if (!VMError::is_error_reported() && !Node::in_dump()) - assert(adr_type != NULL, "source must have adr_type"); + assert(adr_type != nullptr, "source must have adr_type"); #endif return adr_type; } assert(bottom_type()->base() != Type::Memory, "no other memories?"); - return NULL; + return nullptr; } #ifndef PRODUCT @@ -675,9 +675,9 @@ const Type* MachCallNode::Value(PhaseGVN* phase) const { return tf()->range(); } #ifndef PRODUCT void MachCallNode::dump_spec(outputStream *st) const { st->print("# "); - if (tf() != NULL) tf()->dump_on(st); + if (tf() != nullptr) tf()->dump_on(st); if (_cnt != COUNT_UNKNOWN) st->print(" C=%f",_cnt); - if (jvms() != NULL) jvms()->dump_spec(st); + if (jvms() != nullptr) jvms()->dump_spec(st); } #endif @@ -769,7 +769,7 @@ bool MachCallStaticJavaNode::cmp( const Node &n ) const { //----------------------------uncommon_trap_request---------------------------- // If this is an uncommon trap, return the request code, else zero. int MachCallStaticJavaNode::uncommon_trap_request() const { - if (_name != NULL && !strcmp(_name, "uncommon_trap")) { + if (_name != nullptr && !strcmp(_name, "uncommon_trap")) { return CallStaticJavaNode::extract_uncommon_trap_request(this); } return 0; @@ -789,7 +789,7 @@ void MachCallStaticJavaNode::dump_trap_args(outputStream *st) const { void MachCallStaticJavaNode::dump_spec(outputStream *st) const { st->print("Static "); - if (_name != NULL) { + if (_name != nullptr) { st->print("wrapper for: %s", _name ); dump_trap_args(st); st->print(" "); diff --git a/src/hotspot/share/opto/machnode.hpp b/src/hotspot/share/opto/machnode.hpp index d96879f2dabec..57a72adf3ca29 100644 --- a/src/hotspot/share/opto/machnode.hpp +++ b/src/hotspot/share/opto/machnode.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -160,7 +160,7 @@ class MachOper : public ResourceObj { // Access the TypeKlassPtr of operands with a base==RegI and disp==RegP // Only returns non-null value for x86_32.ad's indOffset32X - virtual const TypePtr *disp_as_type() const { return NULL; } + virtual const TypePtr *disp_as_type() const { return nullptr; } // Return the label virtual Label *label() const; @@ -193,7 +193,7 @@ class MachOper : public ResourceObj { // Check whether o is a valid oper. static bool notAnOper(const MachOper *o) { - if (o == NULL) return true; + if (o == nullptr) return true; if (((intptr_t)o & 1) != 0) return true; if (*(address*)o == badAddress) return true; // kill by Node::destruct return false; @@ -206,7 +206,7 @@ class MachOper : public ResourceObj { // ADLC inherit from this class. class MachNode : public Node { public: - MachNode() : Node((uint)0), _barrier(0), _num_opnds(0), _opnds(NULL) { + MachNode() : Node((uint)0), _barrier(0), _num_opnds(0), _opnds(nullptr) { init_class_id(Class_Mach); } // Required boilerplate @@ -258,7 +258,7 @@ class MachNode : public Node { virtual const RegMask &in_RegMask(uint) const; // cisc-spillable instructions redefine for use by in_RegMask - virtual const RegMask *cisc_RegMask() const { return NULL; } + virtual const RegMask *cisc_RegMask() const { return nullptr; } // If this instruction is a 2-address instruction, then return the // index of the input which must match the output. Not nessecary @@ -328,7 +328,7 @@ class MachNode : public Node { } // If this is a memory op, return the base pointer and fixed offset. - // If there are no such, return NULL. If there are multiple addresses + // If there are no such, return null. If there are multiple addresses // or the address is indeterminate (rare cases) then return (Node*)-1, // which serves as node bottom. // If the offset is not statically determined, set it to Type::OffsetBot. @@ -340,14 +340,14 @@ class MachNode : public Node { // Helper for get_base_and_disp: find the base and index input nodes. // Returns the MachOper as determined by memory_operand(), for use, if // needed by the caller. If (MachOper *)-1 is returned, base and index - // are set to NodeSentinel. If (MachOper *) NULL is returned, base and - // index are set to NULL. + // are set to NodeSentinel. If null is returned, base and + // index are set to null. const MachOper* memory_inputs(Node* &base, Node* &index) const; // Helper for memory_inputs: Which operand carries the necessary info? - // By default, returns NULL, which means there is no such operand. + // By default, returns null, which means there is no such operand. // If it returns (MachOper*)-1, this means there are multiple memories. - virtual const MachOper* memory_operand() const { return NULL; } + virtual const MachOper* memory_operand() const { return nullptr; } // Call "get_base_and_disp" to decide which category of memory is used here. virtual const class TypePtr *adr_type() const; @@ -392,7 +392,7 @@ class MachIdealNode : public MachNode { // Define the following defaults for non-matched machine nodes virtual uint oper_input_base() const { return 0; } virtual uint rule() const { return 9999999; } - virtual const class Type *bottom_type() const { return _opnds == NULL ? Type::CONTROL : MachNode::bottom_type(); } + virtual const class Type *bottom_type() const { return _opnds == nullptr ? Type::CONTROL : MachNode::bottom_type(); } }; //------------------------------MachTypeNode---------------------------- @@ -592,7 +592,7 @@ class MachSpillCopyNode : public MachIdealNode { MachIdealNode(), _in(&in), _out(&out), _type(n->bottom_type()), _spill_type(spill_type) { init_class_id(Class_MachSpillCopy); init_flags(Flag_is_Copy); - add_req(NULL); + add_req(nullptr); add_req(n); } virtual uint size_of() const { return sizeof(*this); } @@ -660,7 +660,7 @@ class MachMergeNode : public MachIdealNode { public: MachMergeNode(Node *n1) { init_class_id(Class_MachMerge); - add_req(NULL); + add_req(nullptr); add_req(n1); } virtual const RegMask &out_RegMask() const { return in(1)->out_RegMask(); } @@ -686,7 +686,7 @@ class MachBranchNode : public MachIdealNode { virtual void save_label(Label** label, uint* block_num) = 0; // Support for short branches - virtual MachNode *short_branch_version() { return NULL; } + virtual MachNode *short_branch_version() { return nullptr; } virtual bool pinned() const { return true; }; }; @@ -832,7 +832,7 @@ class MachSafePointNode : public MachReturnNode { OopMap* oop_map() const { return _oop_map; } void set_oop_map(OopMap* om) { _oop_map = om; } - MachSafePointNode() : MachReturnNode(), _oop_map(NULL), _jvms(NULL), _jvmadj(0), _has_ea_local_in_scope(false) { + MachSafePointNode() : MachReturnNode(), _oop_map(nullptr), _jvms(nullptr), _jvmadj(0), _has_ea_local_in_scope(false) { init_class_id(Class_MachSafePoint); } @@ -945,11 +945,11 @@ class MachCallJavaNode : public MachCallNode { if (_override_symbolic_info) { // Attach corresponding Method* to the call site, so VM can use it during resolution // instead of querying symbolic info from bytecode. - assert(_method != NULL, "method should be set"); + assert(_method != nullptr, "method should be set"); assert(_method->constant_encoding()->is_method(), "should point to a Method"); return cbuf.oop_recorder()->find_index(_method->constant_encoding()); } - return 0; // Use symbolic info from bytecode (resolved_method == NULL). + return 0; // Use symbolic info from bytecode (resolved_method is null). } #ifndef PRODUCT @@ -999,7 +999,7 @@ class MachCallRuntimeNode : public MachCallNode { virtual bool cmp( const Node &n ) const; virtual uint size_of() const; // Size is bigger public: - const char *_name; // Printable name, if _method is NULL + const char *_name; // Printable name, if _method is null bool _leaf_no_fp; // Is this CallLeafNoFP? MachCallRuntimeNode() : MachCallNode() { init_class_id(Class_MachCallRuntime); @@ -1077,7 +1077,7 @@ class MachTempNode : public MachNode { init_class_id(Class_MachTemp); _num_opnds = 1; _opnds = _opnd_array; - add_req(NULL); + add_req(nullptr); _opnds[0] = oper; } virtual uint size_of() const { return sizeof(MachTempNode); } @@ -1109,7 +1109,7 @@ class labelOper : public MachOper { virtual MachOper *clone() const; - virtual Label *label() const { assert(_label != NULL, "need Label"); return _label; } + virtual Label *label() const { assert(_label != nullptr, "need Label"); return _label; } virtual uint opcode() const; diff --git a/src/hotspot/share/opto/macro.cpp b/src/hotspot/share/opto/macro.cpp index 673784d2aab6a..293d630ded4dc 100644 --- a/src/hotspot/share/opto/macro.cpp +++ b/src/hotspot/share/opto/macro.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -74,7 +74,7 @@ int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) { else use->set_prec(j, newref); nreplacements++; - } else if (j >= req && uin == NULL) { + } else if (j >= req && uin == nullptr) { break; } } @@ -82,7 +82,7 @@ int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) { } void PhaseMacroExpand::migrate_outs(Node *old, Node *target) { - assert(old != NULL, "sanity"); + assert(old != nullptr, "sanity"); for (DUIterator_Fast imax, i = old->fast_outs(imax); i < imax; i++) { Node* use = old->fast_out(i); _igvn.rehash_node_delayed(use); @@ -142,9 +142,9 @@ CallNode* PhaseMacroExpand::make_slow_call(CallNode *oldcall, const TypeFunc* sl // Slow path call has no side-effects, uses few values copy_predefined_input_for_runtime_call(slow_path, oldcall, call ); - if (parm0 != NULL) call->init_req(TypeFunc::Parms+0, parm0); - if (parm1 != NULL) call->init_req(TypeFunc::Parms+1, parm1); - if (parm2 != NULL) call->init_req(TypeFunc::Parms+2, parm2); + if (parm0 != nullptr) call->init_req(TypeFunc::Parms+0, parm0); + if (parm1 != nullptr) call->init_req(TypeFunc::Parms+1, parm1); + if (parm2 != nullptr) call->init_req(TypeFunc::Parms+2, parm2); call->copy_call_debug_info(&_igvn, oldcall); call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON. _igvn.replace_node(oldcall, call); @@ -184,9 +184,9 @@ static Node *scan_mem_chain(Node *mem, int alias_idx, int offset, Node *start_me } mem = in->in(TypeFunc::Memory); } else if (in->is_MemBar()) { - ArrayCopyNode* ac = NULL; + ArrayCopyNode* ac = nullptr; if (ArrayCopyNode::may_modify(tinst, in->as_MemBar(), phase, ac)) { - if (ac != NULL) { + if (ac != nullptr) { assert(ac->is_clonebasic(), "Only basic clone is a non escaping clone"); return ac; } @@ -224,7 +224,7 @@ static Node *scan_mem_chain(Node *mem, int alias_idx, int offset, Node *start_me InitializeNode* init = alloc->as_Allocate()->initialization(); // We are looking for stored value, return Initialize node // or memory edge from Allocate node. - if (init != NULL) { + if (init != nullptr) { return init; } else { return alloc->in(TypeFunc::Memory); // It will produce zero value (see callers). @@ -233,7 +233,7 @@ static Node *scan_mem_chain(Node *mem, int alias_idx, int offset, Node *start_me // Otherwise skip it (the call updated 'mem' value). } else if (mem->Opcode() == Op_SCMemProj) { mem = mem->in(0); - Node* adr = NULL; + Node* adr = nullptr; if (mem->is_LoadStore()) { adr = mem->in(MemNode::Address); } else { @@ -246,7 +246,7 @@ static Node *scan_mem_chain(Node *mem, int alias_idx, int offset, Node *start_me if (adr_idx == alias_idx) { DEBUG_ONLY(mem->dump();) assert(false, "Object is not scalar replaceable if a LoadStore node accesses its field"); - return NULL; + return nullptr; } mem = mem->in(MemNode::Memory); } else if (mem->Opcode() == Op_StrInflatedCopy) { @@ -256,7 +256,7 @@ static Node *scan_mem_chain(Node *mem, int alias_idx, int offset, Node *start_me if (adr_idx == alias_idx) { DEBUG_ONLY(mem->dump();) assert(false, "Object is not scalar replaceable if a StrInflatedCopy node accesses its field"); - return NULL; + return nullptr; } mem = mem->in(MemNode::Memory); } else { @@ -275,7 +275,7 @@ Node* PhaseMacroExpand::make_arraycopy_load(ArrayCopyNode* ac, intptr_t offset, bt = T_OBJECT; type = ftype->make_oopptr(); } - Node* res = NULL; + Node* res = nullptr; if (ac->is_clonebasic()) { assert(ac->in(ArrayCopyNode::Src) != ac->in(ArrayCopyNode::Dest), "clone source equals destination"); Node* base = ac->in(ArrayCopyNode::Src); @@ -293,8 +293,8 @@ Node* PhaseMacroExpand::make_arraycopy_load(ArrayCopyNode* ac, intptr_t offset, const TypeInt* src_pos_t = _igvn.type(src_pos)->is_int(); const TypeInt* dest_pos_t = _igvn.type(dest_pos)->is_int(); - Node* adr = NULL; - const TypePtr* adr_type = NULL; + Node* adr = nullptr; + const TypePtr* adr_type = nullptr; if (src_pos_t->is_con() && dest_pos_t->is_con()) { intptr_t off = ((src_pos_t->get_con() - dest_pos_t->get_con()) << shift) + offset; Node* base = ac->in(ArrayCopyNode::Src); @@ -318,7 +318,7 @@ Node* PhaseMacroExpand::make_arraycopy_load(ArrayCopyNode* ac, intptr_t offset, if (ac->in(ArrayCopyNode::Src) == ac->in(ArrayCopyNode::Dest)) { // Non constant offset in the array: we can't statically // determine the value - return NULL; + return nullptr; } } MergeMemNode* mergemen = _igvn.transform(MergeMemNode::make(mem))->as_MergeMem(); @@ -326,21 +326,21 @@ Node* PhaseMacroExpand::make_arraycopy_load(ArrayCopyNode* ac, intptr_t offset, res = ArrayCopyNode::load(bs, &_igvn, ctl, mergemen, adr, adr_type, type, bt); } } - if (res != NULL) { + if (res != nullptr) { if (ftype->isa_narrowoop()) { // PhaseMacroExpand::scalar_replacement adds DecodeN nodes res = _igvn.transform(new EncodePNode(res, ftype)); } return res; } - return NULL; + return nullptr; } // // Given a Memory Phi, compute a value Phi containing the values from stores // on the input paths. // Note: this function is recursive, its depth is limited by the "level" argument -// Returns the computed Phi, or NULL if it cannot compute it. +// Returns the computed Phi, or null if it cannot compute it. Node *PhaseMacroExpand::value_from_mem_phi(Node *mem, BasicType ft, const Type *phi_type, const TypeOopPtr *adr_t, AllocateNode *alloc, Node_Stack *value_phis, int level) { assert(mem->is_Phi(), "sanity"); int alias_idx = C->get_alias_index(adr_t); @@ -358,26 +358,26 @@ Node *PhaseMacroExpand::value_from_mem_phi(Node *mem, BasicType ft, const Type * } // Check if an appropriate new value phi already exists. Node* new_phi = value_phis->find(mem->_idx); - if (new_phi != NULL) + if (new_phi != nullptr) return new_phi; if (level <= 0) { - return NULL; // Give up: phi tree too deep + return nullptr; // Give up: phi tree too deep } Node *start_mem = C->start()->proj_out_or_null(TypeFunc::Memory); Node *alloc_mem = alloc->in(TypeFunc::Memory); uint length = mem->req(); - GrowableArray values(length, length, NULL); + GrowableArray values(length, length, nullptr); // create a new Phi for the value - PhiNode *phi = new PhiNode(mem->in(0), phi_type, NULL, mem->_idx, instance_id, alias_idx, offset); + PhiNode *phi = new PhiNode(mem->in(0), phi_type, nullptr, mem->_idx, instance_id, alias_idx, offset); transform_later(phi); value_phis->push(phi, mem->_idx); for (uint j = 1; j < length; j++) { Node *in = mem->in(j); - if (in == NULL || in->is_top()) { + if (in == nullptr || in->is_top()) { values.at_put(j, in); } else { Node *val = scan_mem_chain(in, alias_idx, offset, start_mem, alloc, &_igvn); @@ -389,8 +389,8 @@ Node *PhaseMacroExpand::value_from_mem_phi(Node *mem, BasicType ft, const Type * if (val->is_Initialize()) { val = val->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn); } - if (val == NULL) { - return NULL; // can't find a value on this path + if (val == nullptr) { + return nullptr; // can't find a value on this path } if (val == mem) { values.at_put(j, mem); @@ -406,8 +406,8 @@ Node *PhaseMacroExpand::value_from_mem_phi(Node *mem, BasicType ft, const Type * values.at_put(j, _igvn.zerocon(ft)); } else if (val->is_Phi()) { val = value_from_mem_phi(val, ft, phi_type, adr_t, alloc, value_phis, level-1); - if (val == NULL) { - return NULL; + if (val == nullptr) { + return nullptr; } values.at_put(j, val); } else if (val->Opcode() == Op_SCMemProj) { @@ -415,17 +415,17 @@ Node *PhaseMacroExpand::value_from_mem_phi(Node *mem, BasicType ft, const Type * val->in(0)->Opcode() == Op_EncodeISOArray || val->in(0)->Opcode() == Op_StrCompressedCopy, "sanity"); assert(false, "Object is not scalar replaceable if a LoadStore node accesses its field"); - return NULL; + return nullptr; } else if (val->is_ArrayCopy()) { Node* res = make_arraycopy_load(val->as_ArrayCopy(), offset, val->in(0), val->in(TypeFunc::Memory), ft, phi_type, alloc); - if (res == NULL) { - return NULL; + if (res == nullptr) { + return nullptr; } values.at_put(j, res); } else { DEBUG_ONLY( val->dump(); ) assert(false, "unknown node on this path"); - return NULL; // unknown node on this path + return nullptr; // unknown node on this path } } } @@ -457,14 +457,14 @@ Node *PhaseMacroExpand::value_from_mem(Node *sfpt_mem, Node *sfpt_ctl, BasicType Node *mem = sfpt_mem; while (!done) { if (visited.test_set(mem->_idx)) { - return NULL; // found a loop, give up + return nullptr; // found a loop, give up } mem = scan_mem_chain(mem, alias_idx, offset, start_mem, alloc, &_igvn); if (mem == start_mem || mem == alloc_mem) { done = true; // hit a sentinel, return appropriate 0 value } else if (mem->is_Initialize()) { mem = mem->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn); - if (mem == NULL) { + if (mem == nullptr) { done = true; // Something go wrong. } else if (mem->is_Store()) { const TypePtr* atype = mem->as_Store()->adr_type(); @@ -473,27 +473,27 @@ Node *PhaseMacroExpand::value_from_mem(Node *sfpt_mem, Node *sfpt_ctl, BasicType } } else if (mem->is_Store()) { const TypeOopPtr* atype = mem->as_Store()->adr_type()->isa_oopptr(); - assert(atype != NULL, "address type must be oopptr"); + assert(atype != nullptr, "address type must be oopptr"); assert(C->get_alias_index(atype) == alias_idx && atype->is_known_instance_field() && atype->offset() == offset && atype->instance_id() == instance_id, "store is correct memory slice"); done = true; } else if (mem->is_Phi()) { // try to find a phi's unique input - Node *unique_input = NULL; + Node *unique_input = nullptr; Node *top = C->top(); for (uint i = 1; i < mem->req(); i++) { Node *n = scan_mem_chain(mem->in(i), alias_idx, offset, start_mem, alloc, &_igvn); - if (n == NULL || n == top || n == mem) { + if (n == nullptr || n == top || n == mem) { continue; - } else if (unique_input == NULL) { + } else if (unique_input == nullptr) { unique_input = n; } else if (unique_input != n) { unique_input = top; break; } } - if (unique_input != NULL && unique_input != top) { + if (unique_input != nullptr && unique_input != top) { mem = unique_input; } else { done = true; @@ -505,7 +505,7 @@ Node *PhaseMacroExpand::value_from_mem(Node *sfpt_mem, Node *sfpt_ctl, BasicType assert(false, "unexpected node"); } } - if (mem != NULL) { + if (mem != nullptr) { if (mem == start_mem || mem == alloc_mem) { // hit a sentinel, return appropriate 0 value return _igvn.zerocon(ft); @@ -518,7 +518,7 @@ Node *PhaseMacroExpand::value_from_mem(Node *sfpt_mem, Node *sfpt_ctl, BasicType // attempt to produce a Phi reflecting the values on the input paths of the Phi Node_Stack value_phis(8); Node* phi = value_from_mem_phi(mem, ft, ftype, adr_t, alloc, &value_phis, ValueSearchLimit); - if (phi != NULL) { + if (phi != nullptr) { return phi; } else { // Kill all new Phis @@ -540,27 +540,27 @@ Node *PhaseMacroExpand::value_from_mem(Node *sfpt_mem, Node *sfpt_ctl, BasicType } } // Something go wrong. - return NULL; + return nullptr; } // Check the possibility of scalar replacement. bool PhaseMacroExpand::can_eliminate_allocation(AllocateNode *alloc, GrowableArray & safepoints) { // Scan the uses of the allocation to check for anything that would // prevent us from eliminating it. - NOT_PRODUCT( const char* fail_eliminate = NULL; ) - DEBUG_ONLY( Node* disq_node = NULL; ) + NOT_PRODUCT( const char* fail_eliminate = nullptr; ) + DEBUG_ONLY( Node* disq_node = nullptr; ) bool can_eliminate = true; Node* res = alloc->result_cast(); - const TypeOopPtr* res_type = NULL; - if (res == NULL) { + const TypeOopPtr* res_type = nullptr; + if (res == nullptr) { // All users were eliminated. } else if (!res->is_CheckCastPP()) { NOT_PRODUCT(fail_eliminate = "Allocation does not have unique CheckCastPP";) can_eliminate = false; } else { res_type = _igvn.type(res)->isa_oopptr(); - if (res_type == NULL) { + if (res_type == nullptr) { NOT_PRODUCT(fail_eliminate = "Neither instance or array allocation";) can_eliminate = false; } else if (res_type->isa_aryptr()) { @@ -572,7 +572,7 @@ bool PhaseMacroExpand::can_eliminate_allocation(AllocateNode *alloc, GrowableArr } } - if (can_eliminate && res != NULL) { + if (can_eliminate && res != nullptr) { BarrierSetC2 *bs = BarrierSet::barrier_set()->barrier_set_c2(); for (DUIterator_Fast jmax, j = res->fast_outs(jmax); j < jmax && can_eliminate; j++) { @@ -616,9 +616,9 @@ bool PhaseMacroExpand::can_eliminate_allocation(AllocateNode *alloc, GrowableArr can_eliminate = false; } Node* sfptMem = sfpt->memory(); - if (sfptMem == NULL || sfptMem->is_top()) { + if (sfptMem == nullptr || sfptMem->is_top()) { DEBUG_ONLY(disq_node = use;) - NOT_PRODUCT(fail_eliminate = "NULL or TOP memory";) + NOT_PRODUCT(fail_eliminate = "null or TOP memory";) can_eliminate = false; } else { safepoints.append_if_missing(sfpt); @@ -648,18 +648,18 @@ bool PhaseMacroExpand::can_eliminate_allocation(AllocateNode *alloc, GrowableArr if (PrintEliminateAllocations) { if (can_eliminate) { tty->print("Scalar "); - if (res == NULL) + if (res == nullptr) alloc->dump(); else res->dump(); } else if (alloc->_is_scalar_replaceable) { tty->print("NotScalar (%s)", fail_eliminate); - if (res == NULL) + if (res == nullptr) alloc->dump(); else res->dump(); #ifdef ASSERT - if (disq_node != NULL) { + if (disq_node != nullptr) { tty->print(" >>>> "); disq_node->dump(); } @@ -675,21 +675,21 @@ bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray safepoints_done; ciKlass* klass = NULL; - ciInstanceKlass* iklass = NULL; + ciInstanceKlass* iklass = nullptr; int nfields = 0; int array_base = 0; int element_size = 0; BasicType basic_elem_type = T_ILLEGAL; - ciType* elem_type = NULL; + ciType* elem_type = nullptr; Node* res = alloc->result_cast(); - assert(res == NULL || res->is_CheckCastPP(), "unexpected AllocateNode result"); - const TypeOopPtr* res_type = NULL; - if (res != NULL) { // Could be NULL when there are no users + assert(res == nullptr || res->is_CheckCastPP(), "unexpected AllocateNode result"); + const TypeOopPtr* res_type = nullptr; + if (res != nullptr) { // Could be null when there are no users res_type = _igvn.type(res)->isa_oopptr(); } - if (res != NULL) { + if (res != nullptr) { klass = res_type->klass(); if (res_type->isa_instptr()) { // find the fields of the class which will be needed for safepoint debug information @@ -713,7 +713,7 @@ bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray memory(); Node* ctl = sfpt->control(); - assert(sfpt->jvms() != NULL, "missed JVMS"); + assert(sfpt->jvms() != nullptr, "missed JVMS"); // Fields of scalar objs are referenced only at the end // of regular debuginfo at the last (youngest) JVMS. // Record relative start index. @@ -729,8 +729,8 @@ bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray nonstatic_field_at(j); offset = field->offset(); elem_type = field->type(); @@ -744,13 +744,13 @@ bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray is_loaded()) { field_type = TypeInstPtr::BOTTOM; - } else if (field != NULL && field->is_static_constant()) { + } else if (field != nullptr && field->is_static_constant()) { // This can happen if the constant oop is non-perm. ciObject* con = field->constant_value().as_object(); // Do not "join" in the previous type; it doesn't add value, // and may yield a vacuous result if the field is of interface type. field_type = TypeOopPtr::make_from_constant(con)->isa_oopptr(); - assert(field_type != NULL, "field singleton type must be consistent"); + assert(field_type != nullptr, "field singleton type must be consistent"); } else { field_type = TypeOopPtr::make_from_klass(elem_type->as_klass()); } @@ -765,7 +765,7 @@ bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray add_offset(offset)->isa_oopptr(); Node *field_val = value_from_mem(mem, ctl, basic_elem_type, field_type, field_addr_type, alloc); - if (field_val == NULL) { + if (field_val == nullptr) { // We weren't able to find a value for this field, // give up on eliminating this allocation. @@ -803,7 +803,7 @@ bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray print("=== At SafePoint node %d can't find value of Field: ", sfpt->_idx); field->print(); @@ -814,7 +814,7 @@ bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray _idx, j); } tty->print(", which prevents elimination of: "); - if (res == NULL) + if (res == nullptr) alloc->dump(); else res->dump(); @@ -849,10 +849,10 @@ bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray proj_out_or_null(TypeFunc::Control); Node* mem_proj = n->proj_out_or_null(TypeFunc::Memory); - if (ctl_proj != NULL) { + if (ctl_proj != nullptr) { igvn.replace_node(ctl_proj, n->in(0)); } - if (mem_proj != NULL) { + if (mem_proj != nullptr) { igvn.replace_node(mem_proj, n->in(TypeFunc::Memory)); } } @@ -860,7 +860,7 @@ static void disconnect_projections(MultiNode* n, PhaseIterGVN& igvn) { // Process users of eliminated allocation. void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) { Node* res = alloc->result_cast(); - if (res != NULL) { + if (res != nullptr) { for (DUIterator_Last jmin, j = res->last_outs(jmin); j >= jmin; ) { Node *use = res->last_out(j); uint oc1 = res->outcnt(); @@ -939,7 +939,7 @@ void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) { // // Process other users of allocation's projections // - if (_callprojs.resproj != NULL && _callprojs.resproj->outcnt() != 0) { + if (_callprojs.resproj != nullptr && _callprojs.resproj->outcnt() != 0) { // First disconnect stores captured by Initialize node. // If Initialize node is eliminated first in the following code, // it will kill such stores and DUIterator_Last will assert. @@ -959,16 +959,16 @@ void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) { InitializeNode *init = use->as_Initialize(); assert(init->outcnt() <= 2, "only a control and memory projection expected"); Node *ctrl_proj = init->proj_out_or_null(TypeFunc::Control); - if (ctrl_proj != NULL) { + if (ctrl_proj != nullptr) { _igvn.replace_node(ctrl_proj, init->in(TypeFunc::Control)); #ifdef ASSERT - // If the InitializeNode has no memory out, it will die, and tmp will become NULL + // If the InitializeNode has no memory out, it will die, and tmp will become null Node* tmp = init->in(TypeFunc::Control); - assert(tmp == NULL || tmp == _callprojs.fallthrough_catchproj, "allocation control projection"); + assert(tmp == nullptr || tmp == _callprojs.fallthrough_catchproj, "allocation control projection"); #endif } Node *mem_proj = init->proj_out_or_null(TypeFunc::Memory); - if (mem_proj != NULL) { + if (mem_proj != nullptr) { Node *mem = init->in(TypeFunc::Memory); #ifdef ASSERT if (mem->is_MergeMem()) { @@ -985,22 +985,22 @@ void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) { j -= (oc1 - _callprojs.resproj->outcnt()); } } - if (_callprojs.fallthrough_catchproj != NULL) { + if (_callprojs.fallthrough_catchproj != nullptr) { _igvn.replace_node(_callprojs.fallthrough_catchproj, alloc->in(TypeFunc::Control)); } - if (_callprojs.fallthrough_memproj != NULL) { + if (_callprojs.fallthrough_memproj != nullptr) { _igvn.replace_node(_callprojs.fallthrough_memproj, alloc->in(TypeFunc::Memory)); } - if (_callprojs.catchall_memproj != NULL) { + if (_callprojs.catchall_memproj != nullptr) { _igvn.replace_node(_callprojs.catchall_memproj, C->top()); } - if (_callprojs.fallthrough_ioproj != NULL) { + if (_callprojs.fallthrough_ioproj != nullptr) { _igvn.replace_node(_callprojs.fallthrough_ioproj, alloc->in(TypeFunc::I_O)); } - if (_callprojs.catchall_ioproj != NULL) { + if (_callprojs.catchall_ioproj != nullptr) { _igvn.replace_node(_callprojs.catchall_ioproj, C->top()); } - if (_callprojs.catchall_catchproj != NULL) { + if (_callprojs.catchall_catchproj != nullptr) { _igvn.replace_node(_callprojs.catchall_catchproj, C->top()); } } @@ -1022,7 +1022,7 @@ bool PhaseMacroExpand::eliminate_allocate_node(AllocateNode *alloc) { bool boxing_alloc = C->eliminate_boxing() && tklass->klass()->is_instance_klass() && tklass->klass()->as_instance_klass()->is_box_klass(); - if (!alloc->_is_scalar_replaceable && (!boxing_alloc || (res != NULL))) { + if (!alloc->_is_scalar_replaceable && (!boxing_alloc || (res != nullptr))) { return false; } @@ -1034,7 +1034,7 @@ bool PhaseMacroExpand::eliminate_allocate_node(AllocateNode *alloc) { } if (!alloc->_is_scalar_replaceable) { - assert(res == NULL, "sanity"); + assert(res == nullptr, "sanity"); // We can only eliminate allocation if all debug info references // are already replaced with SafePointScalarObject because // we can't search for a fields value without instance_id. @@ -1048,11 +1048,11 @@ bool PhaseMacroExpand::eliminate_allocate_node(AllocateNode *alloc) { } CompileLog* log = C->log(); - if (log != NULL) { + if (log != nullptr) { log->head("eliminate_allocation type='%d'", log->identify(tklass->klass())); JVMState* p = alloc->jvms(); - while (p != NULL) { + while (p != nullptr) { log->elem("jvms bci='%d' method='%d'", p->bci(), log->identify(p->method())); p = p->caller(); } @@ -1075,25 +1075,25 @@ bool PhaseMacroExpand::eliminate_allocate_node(AllocateNode *alloc) { bool PhaseMacroExpand::eliminate_boxing_node(CallStaticJavaNode *boxing) { // EA should remove all uses of non-escaping boxing node. - if (!C->eliminate_boxing() || boxing->proj_out_or_null(TypeFunc::Parms) != NULL) { + if (!C->eliminate_boxing() || boxing->proj_out_or_null(TypeFunc::Parms) != nullptr) { return false; } - assert(boxing->result_cast() == NULL, "unexpected boxing node result"); + assert(boxing->result_cast() == nullptr, "unexpected boxing node result"); boxing->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/); const TypeTuple* r = boxing->tf()->range(); assert(r->cnt() > TypeFunc::Parms, "sanity"); const TypeInstPtr* t = r->field_at(TypeFunc::Parms)->isa_instptr(); - assert(t != NULL, "sanity"); + assert(t != nullptr, "sanity"); CompileLog* log = C->log(); - if (log != NULL) { + if (log != nullptr) { log->head("eliminate_boxing type='%d'", log->identify(t->klass())); JVMState* p = boxing->jvms(); - while (p != NULL) { + while (p != nullptr) { log->elem("jvms bci='%d' method='%d'", p->bci(), log->identify(p->method())); p = p->caller(); } @@ -1142,7 +1142,7 @@ Node* PhaseMacroExpand::make_load(Node* ctl, Node* mem, Node* base, int offset, Node* PhaseMacroExpand::make_store(Node* ctl, Node* mem, Node* base, int offset, Node* value, BasicType bt) { Node* adr = basic_plus_adr(base, offset); - mem = StoreNode::make(_igvn, ctl, mem, adr, NULL, value, bt, MemNode::unordered); + mem = StoreNode::make(_igvn, ctl, mem, adr, nullptr, value, bt, MemNode::unordered); transform_later(mem); return mem; } @@ -1205,7 +1205,8 @@ void PhaseMacroExpand::expand_allocate_common( AllocateNode* alloc, // allocation node to be expanded Node* length, // array length for an array allocation const TypeFunc* slow_call_type, // Type of slow call - address slow_call_address // Address of slow call + address slow_call_address, // Address of slow call + Node* valid_length_test // whether length is valid or not ) { Node* ctrl = alloc->in(TypeFunc::Control); @@ -1214,15 +1215,15 @@ void PhaseMacroExpand::expand_allocate_common( Node* size_in_bytes = alloc->in(AllocateNode::AllocSize); Node* klass_node = alloc->in(AllocateNode::KlassNode); Node* initial_slow_test = alloc->in(AllocateNode::InitialTest); - assert(ctrl != NULL, "must have control"); + assert(ctrl != nullptr, "must have control"); // We need a Region and corresponding Phi's to merge the slow-path and fast-path results. // they will not be used if "always_slow" is set enum { slow_result_path = 1, fast_result_path = 2 }; - Node *result_region = NULL; - Node *result_phi_rawmem = NULL; - Node *result_phi_rawoop = NULL; - Node *result_phi_i_o = NULL; + Node *result_region = nullptr; + Node *result_phi_rawmem = nullptr; + Node *result_phi_rawoop = nullptr; + Node *result_phi_i_o = nullptr; // The initial slow comparison is a size check, the comparison // we want to do is a BoolTest::gt @@ -1234,7 +1235,7 @@ void PhaseMacroExpand::expand_allocate_common( // 1 - always too big or negative assert(tv <= 1, "0 or 1 if a constant"); expand_fast_path = (tv == 0); - initial_slow_test = NULL; + initial_slow_test = nullptr; } else { initial_slow_test = BoolNode::make_predicate(initial_slow_test, &_igvn); } @@ -1243,16 +1244,16 @@ void PhaseMacroExpand::expand_allocate_common( (!UseTLAB && !Universe::heap()->supports_inline_contig_alloc())) { // Force slow-path allocation expand_fast_path = false; - initial_slow_test = NULL; + initial_slow_test = nullptr; } - bool allocation_has_use = (alloc->result_cast() != NULL); + bool allocation_has_use = (alloc->result_cast() != nullptr); if (!allocation_has_use) { InitializeNode* init = alloc->initialization(); - if (init != NULL) { + if (init != nullptr) { init->remove(&_igvn); } - if (expand_fast_path && (initial_slow_test == NULL)) { + if (expand_fast_path && (initial_slow_test == nullptr)) { // Remove allocation node and return. // Size is a non-negative constant -> no initial check needed -> directly to fast path. // Also, no usages -> empty fast path -> no fall out to slow path -> nothing left. @@ -1260,7 +1261,7 @@ void PhaseMacroExpand::expand_allocate_common( if (PrintEliminateAllocations) { tty->print("NotUsed "); Node* res = alloc->proj_out_or_null(TypeFunc::Parms); - if (res != NULL) { + if (res != nullptr) { res->dump(); } else { alloc->dump(); @@ -1273,11 +1274,11 @@ void PhaseMacroExpand::expand_allocate_common( } enum { too_big_or_final_path = 1, need_gc_path = 2 }; - Node *slow_region = NULL; + Node *slow_region = nullptr; Node *toobig_false = ctrl; // generate the initial test if necessary - if (initial_slow_test != NULL ) { + if (initial_slow_test != nullptr ) { assert (expand_fast_path, "Only need test if there is a fast path"); slow_region = new RegionNode(3); @@ -1330,16 +1331,16 @@ void PhaseMacroExpand::expand_allocate_common( Node* fast_oop_ctrl; Node* fast_oop_rawmem; if (allocation_has_use) { - Node* needgc_ctrl = NULL; + Node* needgc_ctrl = nullptr; result_phi_rawoop = new PhiNode(result_region, TypeRawPtr::BOTTOM); - intx prefetch_lines = length != NULL ? AllocatePrefetchLines : AllocateInstancePrefetchLines; + intx prefetch_lines = length != nullptr ? AllocatePrefetchLines : AllocateInstancePrefetchLines; BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); Node* fast_oop = bs->obj_allocate(this, mem, toobig_false, size_in_bytes, i_o, needgc_ctrl, fast_oop_ctrl, fast_oop_rawmem, prefetch_lines); - if (initial_slow_test != NULL) { + if (initial_slow_test != nullptr) { // This completes all paths into the slow merge point slow_region->init_req(need_gc_path, needgc_ctrl); transform_later(slow_region); @@ -1358,7 +1359,7 @@ void PhaseMacroExpand::expand_allocate_common( result_phi_rawoop->init_req(fast_result_path, fast_oop); } else { - assert (initial_slow_test != NULL, "sanity"); + assert (initial_slow_test != nullptr, "sanity"); fast_oop_ctrl = toobig_false; fast_oop_rawmem = mem; transform_later(slow_region); @@ -1384,13 +1385,19 @@ void PhaseMacroExpand::expand_allocate_common( call->init_req(TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr)); call->init_req(TypeFunc::Parms+0, klass_node); - if (length != NULL) { + if (length != nullptr) { call->init_req(TypeFunc::Parms+1, length); } // Copy debug information and adjust JVMState information, then replace // allocate node with the call call->copy_call_debug_info(&_igvn, alloc); + // For array allocations, copy the valid length check to the call node so Compile::final_graph_reshaping() can verify + // that the call has the expected number of CatchProj nodes (in case the allocation always fails and the fallthrough + // path dies). + if (valid_length_test != nullptr) { + call->add_req(valid_length_test); + } if (expand_fast_path) { call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON. } else { @@ -1417,13 +1424,13 @@ void PhaseMacroExpand::expand_allocate_common( // the control and i_o paths. Replace the control memory projection with // result_phi_rawmem (unless we are only generating a slow call when // both memory projections are combined) - if (expand_fast_path && _callprojs.fallthrough_memproj != NULL) { + if (expand_fast_path && _callprojs.fallthrough_memproj != nullptr) { migrate_outs(_callprojs.fallthrough_memproj, result_phi_rawmem); } // Now change uses of catchall_memproj to use fallthrough_memproj and delete // catchall_memproj so we end up with a call that has only 1 memory projection. - if (_callprojs.catchall_memproj != NULL ) { - if (_callprojs.fallthrough_memproj == NULL) { + if (_callprojs.catchall_memproj != nullptr ) { + if (_callprojs.fallthrough_memproj == nullptr) { _callprojs.fallthrough_memproj = new ProjNode(call, TypeFunc::Memory); transform_later(_callprojs.fallthrough_memproj); } @@ -1436,13 +1443,13 @@ void PhaseMacroExpand::expand_allocate_common( // otherwise incoming i_o become dead when only a slow call is generated // (it is different from memory projections where both projections are // combined in such case). - if (_callprojs.fallthrough_ioproj != NULL) { + if (_callprojs.fallthrough_ioproj != nullptr) { migrate_outs(_callprojs.fallthrough_ioproj, result_phi_i_o); } // Now change uses of catchall_ioproj to use fallthrough_ioproj and delete // catchall_ioproj so we end up with a call that has only 1 i_o projection. - if (_callprojs.catchall_ioproj != NULL ) { - if (_callprojs.fallthrough_ioproj == NULL) { + if (_callprojs.catchall_ioproj != nullptr ) { + if (_callprojs.fallthrough_ioproj == nullptr) { _callprojs.fallthrough_ioproj = new ProjNode(call, TypeFunc::I_O); transform_later(_callprojs.fallthrough_ioproj); } @@ -1466,7 +1473,7 @@ void PhaseMacroExpand::expand_allocate_common( return; } - if (_callprojs.fallthrough_catchproj != NULL) { + if (_callprojs.fallthrough_catchproj != nullptr) { ctrl = _callprojs.fallthrough_catchproj->clone(); transform_later(ctrl); _igvn.replace_node(_callprojs.fallthrough_catchproj, result_region); @@ -1474,7 +1481,7 @@ void PhaseMacroExpand::expand_allocate_common( ctrl = top(); } Node *slow_result; - if (_callprojs.resproj == NULL) { + if (_callprojs.resproj == nullptr) { // no uses of the allocation result slow_result = top(); } else { @@ -1503,7 +1510,7 @@ void PhaseMacroExpand::yank_alloc_node(AllocateNode* alloc) { Node* i_o = alloc->in(TypeFunc::I_O); alloc->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/); - if (_callprojs.resproj != NULL) { + if (_callprojs.resproj != nullptr) { for (DUIterator_Fast imax, i = _callprojs.resproj->fast_outs(imax); i < imax; i++) { Node* use = _callprojs.resproj->fast_out(i); use->isa_MemBar()->remove(&_igvn); @@ -1513,32 +1520,32 @@ void PhaseMacroExpand::yank_alloc_node(AllocateNode* alloc) { assert(_callprojs.resproj->outcnt() == 0, "all uses must be deleted"); _igvn.remove_dead_node(_callprojs.resproj); } - if (_callprojs.fallthrough_catchproj != NULL) { + if (_callprojs.fallthrough_catchproj != nullptr) { migrate_outs(_callprojs.fallthrough_catchproj, ctrl); _igvn.remove_dead_node(_callprojs.fallthrough_catchproj); } - if (_callprojs.catchall_catchproj != NULL) { + if (_callprojs.catchall_catchproj != nullptr) { _igvn.rehash_node_delayed(_callprojs.catchall_catchproj); _callprojs.catchall_catchproj->set_req(0, top()); } - if (_callprojs.fallthrough_proj != NULL) { + if (_callprojs.fallthrough_proj != nullptr) { Node* catchnode = _callprojs.fallthrough_proj->unique_ctrl_out(); _igvn.remove_dead_node(catchnode); _igvn.remove_dead_node(_callprojs.fallthrough_proj); } - if (_callprojs.fallthrough_memproj != NULL) { + if (_callprojs.fallthrough_memproj != nullptr) { migrate_outs(_callprojs.fallthrough_memproj, mem); _igvn.remove_dead_node(_callprojs.fallthrough_memproj); } - if (_callprojs.fallthrough_ioproj != NULL) { + if (_callprojs.fallthrough_ioproj != nullptr) { migrate_outs(_callprojs.fallthrough_ioproj, i_o); _igvn.remove_dead_node(_callprojs.fallthrough_ioproj); } - if (_callprojs.catchall_memproj != NULL) { + if (_callprojs.catchall_memproj != nullptr) { _igvn.rehash_node_delayed(_callprojs.catchall_memproj); _callprojs.catchall_memproj->set_req(0, top()); } - if (_callprojs.catchall_ioproj != NULL) { + if (_callprojs.catchall_ioproj != nullptr) { _igvn.rehash_node_delayed(_callprojs.catchall_ioproj); _callprojs.catchall_ioproj->set_req(0, top()); } @@ -1572,8 +1579,8 @@ void PhaseMacroExpand::expand_initialize_membar(AllocateNode* alloc, InitializeN // not escape. if (!alloc->does_not_escape_thread() && !alloc->is_allocation_MemBar_redundant() && - (init == NULL || !init->is_complete_with_arraycopy())) { - if (init == NULL || init->req() < InitializeNode::RawStores) { + (init == nullptr || !init->is_complete_with_arraycopy())) { + if (init == nullptr || init->req() < InitializeNode::RawStores) { // No InitializeNode or no stores captured by zeroing // elimination. Simply add the MemBarStoreStore after object // initialization. @@ -1616,10 +1623,10 @@ void PhaseMacroExpand::expand_initialize_membar(AllocateNode* alloc, InitializeN // All nodes that depended on the InitializeNode for control // and memory must now depend on the MemBarNode that itself // depends on the InitializeNode - if (init_ctrl != NULL) { + if (init_ctrl != nullptr) { _igvn.replace_node(init_ctrl, ctrl); } - if (init_mem != NULL) { + if (init_mem != nullptr) { _igvn.replace_node(init_mem, mem); } } @@ -1674,7 +1681,7 @@ PhaseMacroExpand::initialize_object(AllocateNode* alloc, int header_size = alloc->minimum_header_size(); // conservatively small // Array length - if (length != NULL) { // Arrays need length field + if (length != nullptr) { // Arrays need length field rawmem = make_store(control, rawmem, object, arrayOopDesc::length_offset_in_bytes(), length, T_INT); // conservatively small header size: header_size = arrayOopDesc::base_offset_in_bytes(T_BYTE); @@ -1684,7 +1691,7 @@ PhaseMacroExpand::initialize_object(AllocateNode* alloc, } // Clear the object body, if necessary. - if (init == NULL) { + if (init == nullptr) { // The init has somehow disappeared; be cautious and clear everything. // // This can happen if a node is allocated but an uncommon trap occurs @@ -1870,18 +1877,19 @@ Node* PhaseMacroExpand::prefetch_allocation(Node* i_o, Node*& needgc_false, void PhaseMacroExpand::expand_allocate(AllocateNode *alloc) { - expand_allocate_common(alloc, NULL, + expand_allocate_common(alloc, nullptr, OptoRuntime::new_instance_Type(), - OptoRuntime::new_instance_Java()); + OptoRuntime::new_instance_Java(), nullptr); } void PhaseMacroExpand::expand_allocate_array(AllocateArrayNode *alloc) { Node* length = alloc->in(AllocateNode::ALength); + Node* valid_length_test = alloc->in(AllocateNode::ValidLengthTest); InitializeNode* init = alloc->initialization(); Node* klass_node = alloc->in(AllocateNode::KlassNode); ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass(); address slow_call_address; // Address of slow call - if (init != NULL && init->is_complete_with_arraycopy() && + if (init != nullptr && init->is_complete_with_arraycopy() && k->is_type_array_klass()) { // Don't zero type array during slow allocation in VM since // it will be initialized later by arraycopy in compiled code. @@ -1891,7 +1899,7 @@ void PhaseMacroExpand::expand_allocate_array(AllocateArrayNode *alloc) { } expand_allocate_common(alloc, length, OptoRuntime::new_array_Type(), - slow_call_address); + slow_call_address, valid_length_test); } //-------------------mark_eliminated_box---------------------------------- @@ -1915,7 +1923,7 @@ void PhaseMacroExpand::mark_eliminated_box(Node* oldbox, Node* obj) { // eliminated even if different objects are referenced in one locked region // (for example, OSR compilation of nested loop inside locked scope). if (EliminateNestedLocks || - oldbox->as_BoxLock()->is_simple_lock_region(NULL, obj, NULL)) { + oldbox->as_BoxLock()->is_simple_lock_region(nullptr, obj, nullptr)) { // Box is used only in one lock region. Mark this box as eliminated. _igvn.hash_delete(oldbox); oldbox->as_BoxLock()->set_eliminated(); // This changes box's hash value @@ -2007,7 +2015,7 @@ void PhaseMacroExpand::mark_eliminated_locking_nodes(AbstractLockNode *alock) { } else if (!alock->is_non_esc_obj()) { // Not eliminated or coarsened // Only Lock node has JVMState needed here. // Not that preceding claim is documented anywhere else. - if (alock->jvms() != NULL) { + if (alock->jvms() != nullptr) { if (alock->as_Lock()->is_nested_lock_region()) { // Mark eliminated related nested locks and unlocks. Node* obj = alock->obj_node(); @@ -2035,7 +2043,7 @@ void PhaseMacroExpand::mark_eliminated_locking_nodes(AbstractLockNode *alock) { } else { #ifdef ASSERT alock->log_lock_optimization(C, "eliminate_lock_NOT_nested_lock_region"); - if (C->log() != NULL) + if (C->log() != nullptr) alock->as_Lock()->is_nested_lock_region(C); // rerun for debugging output #endif } @@ -2093,14 +2101,14 @@ bool PhaseMacroExpand::eliminate_locking_node(AbstractLockNode *alock) { Node* mem = alock->in(TypeFunc::Memory); Node* ctrl = alock->in(TypeFunc::Control); - guarantee(ctrl != NULL, "missing control projection, cannot replace_node() with NULL"); + guarantee(ctrl != nullptr, "missing control projection, cannot replace_node() with null"); alock->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/); // There are 2 projections from the lock. The lock node will // be deleted when its last use is subsumed below. assert(alock->outcnt() == 2 && - _callprojs.fallthrough_proj != NULL && - _callprojs.fallthrough_memproj != NULL, + _callprojs.fallthrough_proj != nullptr && + _callprojs.fallthrough_memproj != nullptr, "Unexpected projections from Lock/Unlock"); Node* fallthroughproj = _callprojs.fallthrough_proj; @@ -2112,7 +2120,7 @@ bool PhaseMacroExpand::eliminate_locking_node(AbstractLockNode *alock) { if (alock->is_Lock()) { // Seach for MemBarAcquireLock node and delete it also. MemBarNode* membar = fallthroughproj->unique_ctrl_out()->as_MemBar(); - assert(membar != NULL && membar->Opcode() == Op_MemBarAcquireLock, ""); + assert(membar != nullptr && membar->Opcode() == Op_MemBarAcquireLock, ""); Node* ctrlproj = membar->proj_out(TypeFunc::Control); Node* memproj = membar->proj_out(TypeFunc::Memory); _igvn.replace_node(ctrlproj, fallthroughproj); @@ -2355,8 +2363,8 @@ void PhaseMacroExpand::expand_lock_node(LockNode *lock) { // Make slow path call CallNode *call = make_slow_call((CallNode *) lock, OptoRuntime::complete_monitor_enter_Type(), - OptoRuntime::complete_monitor_locking_Java(), NULL, slow_path, - obj, box, NULL); + OptoRuntime::complete_monitor_locking_Java(), nullptr, slow_path, + obj, box, nullptr); call->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/); @@ -2364,8 +2372,8 @@ void PhaseMacroExpand::expand_lock_node(LockNode *lock) { // de-opted. So the compiler thinks the slow-call can never throw an // exception. If it DOES throw an exception we would need the debug // info removed first (since if it throws there is no monitor). - assert(_callprojs.fallthrough_ioproj == NULL && _callprojs.catchall_ioproj == NULL && - _callprojs.catchall_memproj == NULL && _callprojs.catchall_catchproj == NULL, "Unexpected projection from Lock"); + assert(_callprojs.fallthrough_ioproj == nullptr && _callprojs.catchall_ioproj == nullptr && + _callprojs.catchall_memproj == nullptr && _callprojs.catchall_catchproj == nullptr, "Unexpected projection from Lock"); // Capture slow path // disconnect fall-through projection from call and create a new one @@ -2430,8 +2438,8 @@ void PhaseMacroExpand::expand_unlock_node(UnlockNode *unlock) { "complete_monitor_unlocking_C", slow_path, obj, box, thread); call->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/); - assert(_callprojs.fallthrough_ioproj == NULL && _callprojs.catchall_ioproj == NULL && - _callprojs.catchall_memproj == NULL && _callprojs.catchall_catchproj == NULL, "Unexpected projection from Lock"); + assert(_callprojs.fallthrough_ioproj == nullptr && _callprojs.catchall_ioproj == nullptr && + _callprojs.catchall_memproj == nullptr && _callprojs.catchall_catchproj == nullptr, "Unexpected projection from Lock"); // No exceptions for unlocking // Capture slow path @@ -2454,7 +2462,7 @@ void PhaseMacroExpand::expand_unlock_node(UnlockNode *unlock) { } void PhaseMacroExpand::expand_subtypecheck_node(SubTypeCheckNode *check) { - assert(check->in(SubTypeCheckNode::Control) == NULL, "should be pinned"); + assert(check->in(SubTypeCheckNode::Control) == nullptr, "should be pinned"); Node* bol = check->unique_out(); Node* obj_or_subklass = check->in(SubTypeCheckNode::ObjOrSubKlass); Node* superklass = check->in(SubTypeCheckNode::SuperKlass); @@ -2473,15 +2481,15 @@ void PhaseMacroExpand::expand_subtypecheck_node(SubTypeCheckNode *check) { Node* iffalse = iff->as_If()->proj_out(0); Node* ctrl = iff->in(0); - Node* subklass = NULL; + Node* subklass = nullptr; if (_igvn.type(obj_or_subklass)->isa_klassptr()) { subklass = obj_or_subklass; } else { Node* k_adr = basic_plus_adr(obj_or_subklass, oopDesc::klass_offset_in_bytes()); - subklass = _igvn.transform(LoadKlassNode::make(_igvn, NULL, C->immutable_memory(), k_adr, TypeInstPtr::KLASS)); + subklass = _igvn.transform(LoadKlassNode::make(_igvn, nullptr, C->immutable_memory(), k_adr, TypeInstPtr::KLASS)); } - Node* not_subtype_ctrl = Phase::gen_subtype_check(subklass, superklass, &ctrl, NULL, _igvn); + Node* not_subtype_ctrl = Phase::gen_subtype_check(subklass, superklass, &ctrl, nullptr, _igvn); _igvn.replace_input_of(iff, 0, C->top()); _igvn.replace_node(iftrue, not_subtype_ctrl); @@ -2610,7 +2618,7 @@ bool PhaseMacroExpand::expand_macro_nodes() { (bol->_test._test == BoolTest::ne), ""); IfNode* ifn = bol->unique_out()->as_If(); assert((ifn->outcnt() == 2) && - ifn->proj_out(1)->is_uncommon_trap_proj(Deoptimization::Reason_rtm_state_change) != NULL, ""); + ifn->proj_out(1)->is_uncommon_trap_proj(Deoptimization::Reason_rtm_state_change) != nullptr, ""); #endif Node* repl = n->in(1); if (!_has_locks) { @@ -2667,7 +2675,7 @@ bool PhaseMacroExpand::expand_macro_nodes() { int macro_count = C->macro_count(); Node * n = C->macro_node(macro_count-1); assert(n->is_macro(), "only macro nodes expected here"); - if (_igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) { + if (_igvn.type(n) == Type::TOP || (n->in(0) != nullptr && n->in(0)->is_top())) { // node is unreachable, so don't try to expand it C->remove_macro_node(n); continue; @@ -2720,7 +2728,7 @@ bool PhaseMacroExpand::expand_macro_nodes() { int macro_count = C->macro_count(); Node * n = C->macro_node(macro_count-1); assert(n->is_macro(), "only macro nodes expected here"); - if (_igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) { + if (_igvn.type(n) == Type::TOP || (n->in(0) != nullptr && n->in(0)->is_top())) { // node is unreachable, so don't try to expand it C->remove_macro_node(n); continue; diff --git a/src/hotspot/share/opto/macro.hpp b/src/hotspot/share/opto/macro.hpp index 413e8d04c4bf5..ad059802723c2 100644 --- a/src/hotspot/share/opto/macro.hpp +++ b/src/hotspot/share/opto/macro.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -68,10 +68,10 @@ class PhaseMacroExpand : public Phase { const TypeFunc* call_type, address call_addr, const char* call_name, const TypePtr* adr_type, - Node* parm0 = NULL, Node* parm1 = NULL, - Node* parm2 = NULL, Node* parm3 = NULL, - Node* parm4 = NULL, Node* parm5 = NULL, - Node* parm6 = NULL, Node* parm7 = NULL); + Node* parm0 = nullptr, Node* parm1 = nullptr, + Node* parm2 = nullptr, Node* parm3 = nullptr, + Node* parm4 = nullptr, Node* parm5 = nullptr, + Node* parm6 = nullptr, Node* parm7 = nullptr); address basictype2arraycopy(BasicType t, Node* src_offset, @@ -92,8 +92,8 @@ class PhaseMacroExpand : public Phase { void expand_allocate_common(AllocateNode* alloc, Node* length, const TypeFunc* slow_call_type, - address slow_call_address); - void yank_initalize_node(InitializeNode* node); + address slow_call_address, + Node* valid_length_test); void yank_alloc_node(AllocateNode* alloc); Node *value_from_mem(Node *mem, Node *ctl, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, AllocateNode *alloc); Node *value_from_mem_phi(Node *mem, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, AllocateNode *alloc, Node_Stack *value_phis, int level); @@ -112,7 +112,7 @@ class PhaseMacroExpand : public Phase { void expand_unlock_node(UnlockNode *unlock); // More helper methods modeled after GraphKit for array copy - void insert_mem_bar(Node** ctrl, Node** mem, int opcode, Node* precedent = NULL); + void insert_mem_bar(Node** ctrl, Node** mem, int opcode, Node* precedent = nullptr); Node* array_element_address(Node* ary, Node* idx, BasicType elembt); Node* ConvI2L(Node* offset); @@ -140,7 +140,7 @@ class PhaseMacroExpand : public Phase { Node* copy_length, bool disjoint_bases = false, bool length_never_negative = false, - RegionNode* slow_region = NULL); + RegionNode* slow_region = nullptr); void generate_clear_array(Node* ctrl, MergeMemNode* merge_mem, const TypePtr* adr_type, Node* dest, diff --git a/src/hotspot/share/opto/macroArrayCopy.cpp b/src/hotspot/share/opto/macroArrayCopy.cpp index 9fe2f483d8f14..4785e45b798e0 100644 --- a/src/hotspot/share/opto/macroArrayCopy.cpp +++ b/src/hotspot/share/opto/macroArrayCopy.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -83,17 +83,17 @@ Node* PhaseMacroExpand::make_leaf_call(Node* ctrl, Node* mem, call->init_req(TypeFunc::ReturnAdr, top()); call->init_req(TypeFunc::FramePtr, top()); - // Hook each parm in order. Stop looking at the first NULL. - if (parm0 != NULL) { call->init_req(TypeFunc::Parms+0, parm0); - if (parm1 != NULL) { call->init_req(TypeFunc::Parms+1, parm1); - if (parm2 != NULL) { call->init_req(TypeFunc::Parms+2, parm2); - if (parm3 != NULL) { call->init_req(TypeFunc::Parms+3, parm3); - if (parm4 != NULL) { call->init_req(TypeFunc::Parms+4, parm4); - if (parm5 != NULL) { call->init_req(TypeFunc::Parms+5, parm5); - if (parm6 != NULL) { call->init_req(TypeFunc::Parms+6, parm6); - if (parm7 != NULL) { call->init_req(TypeFunc::Parms+7, parm7); + // Hook each parm in order. Stop looking at the first null. + if (parm0 != nullptr) { call->init_req(TypeFunc::Parms+0, parm0); + if (parm1 != nullptr) { call->init_req(TypeFunc::Parms+1, parm1); + if (parm2 != nullptr) { call->init_req(TypeFunc::Parms+2, parm2); + if (parm3 != nullptr) { call->init_req(TypeFunc::Parms+3, parm3); + if (parm4 != nullptr) { call->init_req(TypeFunc::Parms+4, parm4); + if (parm5 != nullptr) { call->init_req(TypeFunc::Parms+5, parm5); + if (parm6 != nullptr) { call->init_req(TypeFunc::Parms+6, parm6); + if (parm7 != nullptr) { call->init_req(TypeFunc::Parms+7, parm7); /* close each nested if ===> */ } } } } } } } } - assert(call->in(call->req()-1) != NULL, "must initialize all parms"); + assert(call->in(call->req()-1) != nullptr, "must initialize all parms"); return call; } @@ -106,19 +106,19 @@ Node* PhaseMacroExpand::make_leaf_call(Node* ctrl, Node* mem, // In all cases, GraphKit::control() is updated to the fast path. // The returned value represents the control for the slow path. // The return value is never 'top'; it is either a valid control -// or NULL if it is obvious that the slow path can never be taken. -// Also, if region and the slow control are not NULL, the slow edge +// or null if it is obvious that the slow path can never be taken. +// Also, if region and the slow control are not null, the slow edge // is appended to the region. Node* PhaseMacroExpand::generate_guard(Node** ctrl, Node* test, RegionNode* region, float true_prob) { if ((*ctrl)->is_top()) { // Already short circuited. - return NULL; + return nullptr; } // Build an if node and its projections. // If test is true we take the slow path, which we assume is uncommon. if (_igvn.type(test) == TypeInt::ZERO) { // The slow branch is never taken. No need to build this guard. - return NULL; + return nullptr; } IfNode* iff = new IfNode(*ctrl, test, true_prob, COUNT_UNKNOWN); @@ -127,7 +127,7 @@ Node* PhaseMacroExpand::generate_guard(Node** ctrl, Node* test, RegionNode* regi Node* if_slow = new IfTrueNode(iff); transform_later(if_slow); - if (region != NULL) { + if (region != nullptr) { region->add_req(if_slow); } @@ -199,11 +199,11 @@ void PhaseMacroExpand::generate_partial_inlining_block(Node** ctrl, MergeMemNode RegionNode** exit_block, Node** result_memory, Node* length, Node* src_start, Node* dst_start, BasicType type) { const TypePtr *src_adr_type = _igvn.type(src_start)->isa_ptr(); - Node* inline_block = NULL; - Node* stub_block = NULL; + Node* inline_block = nullptr; + Node* stub_block = nullptr; int const_len = -1; - const TypeInt* lty = NULL; + const TypeInt* lty = nullptr; uint shift = exact_log2(type2aelembytes(type)); if (length->Opcode() == Op_ConvI2L) { lty = _igvn.type(length->in(1))->isa_int(); @@ -234,7 +234,7 @@ void PhaseMacroExpand::generate_partial_inlining_block(Node** ctrl, MergeMemNode transform_later(cmp_le); Node* bol_le = new BoolNode(cmp_le, BoolTest::le); transform_later(bol_le); - inline_block = generate_guard(ctrl, bol_le, NULL, PROB_FAIR); + inline_block = generate_guard(ctrl, bol_le, nullptr, PROB_FAIR); stub_block = *ctrl; Node* mask_gen = new VectorMaskGenNode(casted_length, TypeVect::VECTMASK, type); @@ -269,16 +269,16 @@ void PhaseMacroExpand::generate_partial_inlining_block(Node** ctrl, MergeMemNode Node* PhaseMacroExpand::generate_nonpositive_guard(Node** ctrl, Node* index, bool never_negative) { - if ((*ctrl)->is_top()) return NULL; + if ((*ctrl)->is_top()) return nullptr; if (_igvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint] - return NULL; // index is already adequately typed + return nullptr; // index is already adequately typed Node* cmp_le = new CmpINode(index, intcon(0)); transform_later(cmp_le); BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le); Node* bol_le = new BoolNode(cmp_le, le_or_eq); transform_later(bol_le); - Node* is_notp = generate_guard(ctrl, bol_le, NULL, PROB_MIN); + Node* is_notp = generate_guard(ctrl, bol_le, nullptr, PROB_MIN); return is_notp; } @@ -318,8 +318,8 @@ address PhaseMacroExpand::basictype2arraycopy(BasicType t, // or they are identical (which we can treat as disjoint.) We can also // treat a copy with a destination index less that the source index // as disjoint since a low->high copy will work correctly in this case. - if (src_offset_inttype != NULL && src_offset_inttype->is_con() && - dest_offset_inttype != NULL && dest_offset_inttype->is_con()) { + if (src_offset_inttype != nullptr && src_offset_inttype->is_con() && + dest_offset_inttype != nullptr && dest_offset_inttype->is_con()) { // both indices are constants int s_offs = src_offset_inttype->get_con(); int d_offs = dest_offset_inttype->get_con(); @@ -327,7 +327,7 @@ address PhaseMacroExpand::basictype2arraycopy(BasicType t, aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) && ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0); if (s_offs >= d_offs) disjoint = true; - } else if (src_offset == dest_offset && src_offset != NULL) { + } else if (src_offset == dest_offset && src_offset != nullptr) { // This can occur if the offsets are identical non-constants. disjoint = true; } @@ -380,7 +380,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* bool disjoint_bases, bool length_never_negative, RegionNode* slow_region) { - if (slow_region == NULL) { + if (slow_region == nullptr) { slow_region = new RegionNode(1); transform_later(slow_region); } @@ -398,7 +398,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* && !(UseTLAB && ZeroTLAB) // pointless if already zeroed && basic_elem_type != T_CONFLICT // avoid corner case && !src->eqv_uncast(dest) - && alloc != NULL + && alloc != nullptr && _igvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0) { assert(ac->is_alloc_tightly_coupled(), "sanity"); // acopy to uninitialized tightly coupled allocations @@ -423,7 +423,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* } } else { // No zeroing elimination needed here. - alloc = NULL; + alloc = nullptr; acopy_to_uninitialized = false; //original_dest = dest; //dest_needs_zeroing = false; @@ -455,9 +455,9 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* // Checked control path: Node* checked_control = top(); - Node* checked_mem = NULL; - Node* checked_i_o = NULL; - Node* checked_value = NULL; + Node* checked_mem = nullptr; + Node* checked_i_o = nullptr; + Node* checked_value = nullptr; if (basic_elem_type == T_CONFLICT) { assert(!dest_needs_zeroing, ""); @@ -465,7 +465,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* adr_type, src, src_offset, dest, dest_offset, copy_length, acopy_to_uninitialized); - if (cv == NULL) cv = intcon(-1); // failure (no stub available) + if (cv == nullptr) cv = intcon(-1); // failure (no stub available) checked_control = *ctrl; checked_i_o = *io; checked_mem = mem->memory_at(alias_idx); @@ -474,7 +474,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* } Node* not_pos = generate_nonpositive_guard(ctrl, copy_length, length_never_negative); - if (not_pos != NULL) { + if (not_pos != nullptr) { Node* local_ctrl = not_pos, *local_io = *io; MergeMemNode* local_mem = MergeMemNode::make(mem); transform_later(local_mem); @@ -495,7 +495,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* // Clear the whole thing since there are no source elements to copy. generate_clear_array(local_ctrl, local_mem, adr_type, dest, basic_elem_type, - intcon(0), NULL, + intcon(0), nullptr, alloc->in(AllocateNode::AllocSize)); // Use a secondary InitializeNode as raw memory barrier. // Currently it is needed only on this path since other @@ -533,7 +533,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* generate_clear_array(*ctrl, mem, adr_type, dest, basic_elem_type, intcon(0), dest_offset, - NULL); + nullptr); } // Next, perform a dynamic check on the tail length. @@ -541,16 +541,16 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* // There are two wins: Avoid generating the ClearArray // with its attendant messy index arithmetic, and upgrade // the copy to a more hardware-friendly word size of 64 bits. - Node* tail_ctl = NULL; + Node* tail_ctl = nullptr; if (!(*ctrl)->is_top() && !dest_tail->eqv_uncast(dest_length)) { Node* cmp_lt = transform_later( new CmpINode(dest_tail, dest_length) ); Node* bol_lt = transform_later( new BoolNode(cmp_lt, BoolTest::lt) ); - tail_ctl = generate_slow_guard(ctrl, bol_lt, NULL); - assert(tail_ctl != NULL || !(*ctrl)->is_top(), "must be an outcome"); + tail_ctl = generate_slow_guard(ctrl, bol_lt, nullptr); + assert(tail_ctl != nullptr || !(*ctrl)->is_top(), "must be an outcome"); } // At this point, let's assume there is no tail. - if (!(*ctrl)->is_top() && alloc != NULL && basic_elem_type != T_OBJECT) { + if (!(*ctrl)->is_top() && alloc != nullptr && basic_elem_type != T_OBJECT) { // There is no tail. Try an upgrade to a 64-bit copy. bool didit = false; { @@ -575,13 +575,13 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* } // Clear the tail, if any. - if (tail_ctl != NULL) { - Node* notail_ctl = (*ctrl)->is_top() ? NULL : *ctrl; + if (tail_ctl != nullptr) { + Node* notail_ctl = (*ctrl)->is_top() ? nullptr : *ctrl; *ctrl = tail_ctl; - if (notail_ctl == NULL) { + if (notail_ctl == nullptr) { generate_clear_array(*ctrl, mem, adr_type, dest, basic_elem_type, - dest_tail, NULL, + dest_tail, nullptr, dest_size); } else { // Make a local merge. @@ -591,7 +591,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* done_mem->init_req(1, mem->memory_at(alias_idx)); generate_clear_array(*ctrl, mem, adr_type, dest, basic_elem_type, - dest_tail, NULL, + dest_tail, nullptr, dest_size); done_ctl->init_req(2, *ctrl); done_mem->init_req(2, mem->memory_at(alias_idx)); @@ -620,7 +620,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* Node* src_klass = ac->in(ArrayCopyNode::SrcKlass); Node* dest_klass = ac->in(ArrayCopyNode::DestKlass); - assert(src_klass != NULL && dest_klass != NULL, "should have klasses"); + assert(src_klass != nullptr && dest_klass != nullptr, "should have klasses"); // Generate the subtype check. // This might fold up statically, or then again it might not. @@ -643,14 +643,14 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* // (At this point we can assume disjoint_bases, since types differ.) int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset()); Node* p1 = basic_plus_adr(dest_klass, ek_offset); - Node* n1 = LoadKlassNode::make(_igvn, NULL, C->immutable_memory(), p1, TypeRawPtr::BOTTOM); + Node* n1 = LoadKlassNode::make(_igvn, nullptr, C->immutable_memory(), p1, TypeRawPtr::BOTTOM); Node* dest_elem_klass = transform_later(n1); Node* cv = generate_checkcast_arraycopy(&local_ctrl, &local_mem, adr_type, dest_elem_klass, src, src_offset, dest, dest_offset, ConvI2X(copy_length), acopy_to_uninitialized); - if (cv == NULL) cv = intcon(-1); // failure (no stub available) + if (cv == nullptr) cv = intcon(-1); // failure (no stub available) checked_control = local_ctrl; checked_i_o = *io; checked_mem = local_mem->memory_at(alias_idx); @@ -660,7 +660,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* // At this point we know we do not need type checks on oop stores. BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); - if (!bs->array_copy_requires_gc_barriers(alloc != NULL, copy_type, false, false, BarrierSetC2::Expansion)) { + if (!bs->array_copy_requires_gc_barriers(alloc != nullptr, copy_type, false, false, BarrierSetC2::Expansion)) { // If we do not need gc barriers, copy using the jint or jlong stub. copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT); assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type), @@ -686,7 +686,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* } // Here are all the slow paths up to this point, in one bundle: - assert(slow_region != NULL, "allocated on entry"); + assert(slow_region != nullptr, "allocated on entry"); slow_control = slow_region; DEBUG_ONLY(slow_region = (RegionNode*)badAddress); @@ -729,7 +729,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* slow_i_o = slow_i_o2; slow_mem = slow_mem2; - if (alloc != NULL) { + if (alloc != nullptr) { // We'll restart from the very beginning, after zeroing the whole thing. // This can cause double writes, but that's OK since dest is brand new. // So we ignore the low 31 bits of the value returned from the stub. @@ -769,7 +769,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* if (dest_needs_zeroing) { generate_clear_array(local_ctrl, local_mem, adr_type, dest, basic_elem_type, - intcon(0), NULL, + intcon(0), nullptr, alloc->in(AllocateNode::AllocSize)); } @@ -789,7 +789,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* // Remove unused edges. for (uint i = 1; i < result_region->req(); i++) { - if (result_region->in(i) == NULL) { + if (result_region->in(i) == nullptr) { result_region->init_req(i, top()); } } @@ -801,7 +801,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* // mem no longer guaranteed to stay a MergeMemNode Node* out_mem = mem; - DEBUG_ONLY(mem = NULL); + DEBUG_ONLY(mem = nullptr); // The memory edges above are precise in order to model effects around // array copies accurately to allow value numbering of field loads around @@ -815,7 +815,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* // the membar also. // // Do not let reads from the cloned object float above the arraycopy. - if (alloc != NULL && !alloc->initialization()->does_not_escape()) { + if (alloc != nullptr && !alloc->initialization()->does_not_escape()) { // Do not let stores that initialize this object be reordered with // a subsequent store that would make this object accessible by // other threads. @@ -831,7 +831,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* } _igvn.replace_node(_callprojs.fallthrough_memproj, out_mem); - if (_callprojs.fallthrough_ioproj != NULL) { + if (_callprojs.fallthrough_ioproj != nullptr) { _igvn.replace_node(_callprojs.fallthrough_ioproj, *io); } _igvn.replace_node(_callprojs.fallthrough_catchproj, *ctrl); @@ -839,9 +839,9 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* #ifdef ASSERT const TypeOopPtr* dest_t = _igvn.type(dest)->is_oopptr(); if (dest_t->is_known_instance() && !is_partial_array_copy) { - ArrayCopyNode* ac = NULL; + ArrayCopyNode* ac = nullptr; assert(ArrayCopyNode::may_modify(dest_t, (*ctrl)->in(0)->as_MemBar(), &_igvn, ac), "dependency on arraycopy lost"); - assert(ac == NULL, "no arraycopy anymore"); + assert(ac == nullptr, "no arraycopy anymore"); } #endif @@ -865,12 +865,12 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* // dest oop of the destination array // basic_elem_type element type of the destination // slice_idx array index of first element to store -// slice_len number of elements to store (or NULL) +// slice_len number of elements to store (or null) // dest_size total size in bytes of the array object // -// Exactly one of slice_len or dest_size must be non-NULL. -// If dest_size is non-NULL, zeroing extends to the end of the object. -// If slice_len is non-NULL, the slice_idx value must be a constant. +// Exactly one of slice_len or dest_size must be non-null. +// If dest_size is non-null, zeroing extends to the end of the object. +// If slice_len is non-null, the slice_idx value must be a constant. void PhaseMacroExpand::generate_clear_array(Node* ctrl, MergeMemNode* merge_mem, const TypePtr* adr_type, Node* dest, @@ -879,9 +879,9 @@ void PhaseMacroExpand::generate_clear_array(Node* ctrl, MergeMemNode* merge_mem, Node* slice_len, Node* dest_size) { // one or the other but not both of slice_len and dest_size: - assert((slice_len != NULL? 1: 0) + (dest_size != NULL? 1: 0) == 1, ""); - if (slice_len == NULL) slice_len = top(); - if (dest_size == NULL) dest_size = top(); + assert((slice_len != nullptr? 1: 0) + (dest_size != nullptr? 1: 0) == 1, ""); + if (slice_len == nullptr) slice_len = top(); + if (dest_size == nullptr) dest_size = top(); uint alias_idx = C->get_alias_index(adr_type); @@ -1041,10 +1041,10 @@ bool PhaseMacroExpand::generate_block_arraycopy(Node** ctrl, MergeMemNode** mem, countx = transform_later(new SubXNode(countx, MakeConX(dest_off))); countx = transform_later(new URShiftXNode(countx, intcon(LogBytesPerLong))); - bool disjoint_bases = true; // since alloc != NULL + bool disjoint_bases = true; // since alloc isn't null generate_unchecked_arraycopy(ctrl, mem, adr_type, T_LONG, disjoint_bases, - sptr, NULL, dptr, NULL, countx, dest_uninitialized); + sptr, nullptr, dptr, nullptr, countx, dest_uninitialized); return true; } @@ -1098,12 +1098,12 @@ MergeMemNode* PhaseMacroExpand::generate_slow_arraycopy(ArrayCopyNode *ac, transform_later(out_mem); // When src is negative and arraycopy is before an infinite loop,_callprojs.fallthrough_ioproj - // could be NULL. Skip clone and update NULL fallthrough_ioproj. - if (_callprojs.fallthrough_ioproj != NULL) { + // could be null. Skip clone and update null fallthrough_ioproj. + if (_callprojs.fallthrough_ioproj != nullptr) { *io = _callprojs.fallthrough_ioproj->clone(); transform_later(*io); } else { - *io = NULL; + *io = nullptr; } return out_mem; @@ -1116,11 +1116,11 @@ Node* PhaseMacroExpand::generate_checkcast_arraycopy(Node** ctrl, MergeMemNode** Node* src, Node* src_offset, Node* dest, Node* dest_offset, Node* copy_length, bool dest_uninitialized) { - if ((*ctrl)->is_top()) return NULL; + if ((*ctrl)->is_top()) return nullptr; address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized); - if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. - return NULL; + if (copyfunc_addr == nullptr) { // Stub was not generated, go slow path. + return nullptr; } // Pick out the parameters required to perform a store-check @@ -1129,7 +1129,7 @@ Node* PhaseMacroExpand::generate_checkcast_arraycopy(Node** ctrl, MergeMemNode** // super_check_offset, for the desired klass. int sco_offset = in_bytes(Klass::super_check_offset_offset()); Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset); - Node* n3 = new LoadINode(NULL, *mem /*memory(p3)*/, p3, _igvn.type(p3)->is_ptr(), TypeInt::INT, MemNode::unordered); + Node* n3 = new LoadINode(nullptr, *mem /*memory(p3)*/, p3, _igvn.type(p3)->is_ptr(), TypeInt::INT, MemNode::unordered); Node* check_offset = ConvI2X(transform_later(n3)); Node* check_value = dest_elem_klass; @@ -1154,12 +1154,12 @@ Node* PhaseMacroExpand::generate_generic_arraycopy(Node** ctrl, MergeMemNode** m Node* src, Node* src_offset, Node* dest, Node* dest_offset, Node* copy_length, bool dest_uninitialized) { - if ((*ctrl)->is_top()) return NULL; + if ((*ctrl)->is_top()) return nullptr; assert(!dest_uninitialized, "Invariant"); address copyfunc_addr = StubRoutines::generic_arraycopy(); - if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. - return NULL; + if (copyfunc_addr == nullptr) { // Stub was not generated, go slow path. + return nullptr; } const TypeFunc* call_type = OptoRuntime::generic_arraycopy_Type(); @@ -1186,7 +1186,7 @@ bool PhaseMacroExpand::generate_unchecked_arraycopy(Node** ctrl, MergeMemNode** Node* src_start = src; Node* dest_start = dest; - if (src_offset != NULL || dest_offset != NULL) { + if (src_offset != nullptr || dest_offset != nullptr) { src_start = array_element_address(src, src_offset, basic_elem_type); dest_start = array_element_address(dest, dest_offset, basic_elem_type); } @@ -1197,8 +1197,8 @@ bool PhaseMacroExpand::generate_unchecked_arraycopy(Node** ctrl, MergeMemNode** basictype2arraycopy(basic_elem_type, src_offset, dest_offset, disjoint_bases, copyfunc_name, dest_uninitialized); - Node* result_memory = NULL; - RegionNode* exit_block = NULL; + Node* result_memory = nullptr; + RegionNode* exit_block = nullptr; if (ArrayOperationPartialInlineSize > 0 && is_subword_type(basic_elem_type) && Matcher::vector_width_in_bytes(basic_elem_type) >= 16) { generate_partial_inlining_block(ctrl, mem, adr_type, &exit_block, &result_memory, @@ -1242,7 +1242,7 @@ void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) { Node* dest = ac->in(ArrayCopyNode::Dest); Node* dest_offset = ac->in(ArrayCopyNode::DestPos); Node* length = ac->in(ArrayCopyNode::Length); - MergeMemNode* merge_mem = NULL; + MergeMemNode* merge_mem = nullptr; if (ac->is_clonebasic()) { BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); @@ -1253,10 +1253,10 @@ void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) { merge_mem = MergeMemNode::make(mem); transform_later(merge_mem); - AllocateArrayNode* alloc = NULL; + AllocateArrayNode* alloc = nullptr; if (ac->is_alloc_tightly_coupled()) { alloc = AllocateArrayNode::Ideal_array_allocation(dest, &_igvn); - assert(alloc != NULL, "expect alloc"); + assert(alloc != nullptr, "expect alloc"); } const TypePtr* adr_type = _igvn.type(dest)->is_oopptr()->add_offset(Type::OffsetBot); @@ -1271,10 +1271,10 @@ void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) { return; } - AllocateArrayNode* alloc = NULL; + AllocateArrayNode* alloc = nullptr; if (ac->is_alloc_tightly_coupled()) { alloc = AllocateArrayNode::Ideal_array_allocation(dest, &_igvn); - assert(alloc != NULL, "expect alloc"); + assert(alloc != nullptr, "expect alloc"); } assert(ac->is_arraycopy() || ac->is_arraycopy_validated(), "should be an arraycopy"); @@ -1292,10 +1292,10 @@ void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) { BasicType src_elem = T_CONFLICT; BasicType dest_elem = T_CONFLICT; - if (top_dest != NULL && top_dest->klass() != NULL) { + if (top_dest != nullptr && top_dest->klass() != nullptr) { dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type(); } - if (top_src != NULL && top_src->klass() != NULL) { + if (top_src != nullptr && top_src->klass() != nullptr) { src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type(); } if (is_reference_type(src_elem)) src_elem = T_OBJECT; @@ -1319,7 +1319,7 @@ void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) { } // Call StubRoutines::generic_arraycopy stub. - Node* mem = generate_arraycopy(ac, NULL, &ctrl, merge_mem, &io, + Node* mem = generate_arraycopy(ac, nullptr, &ctrl, merge_mem, &io, TypeRawPtr::BOTTOM, T_CONFLICT, src, src_offset, dest, dest_offset, length, // If a negative length guard was generated for the ArrayCopyNode, @@ -1341,7 +1341,7 @@ void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) { } _igvn.replace_node(_callprojs.fallthrough_memproj, merge_mem); - if (_callprojs.fallthrough_ioproj != NULL) { + if (_callprojs.fallthrough_ioproj != nullptr) { _igvn.replace_node(_callprojs.fallthrough_ioproj, io); } _igvn.replace_node(_callprojs.fallthrough_catchproj, ctrl); @@ -1390,7 +1390,7 @@ void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) { // (7) src_offset + length must not exceed length of src. Node* alen = ac->in(ArrayCopyNode::SrcLen); - assert(alen != NULL, "need src len"); + assert(alen != nullptr, "need src len"); generate_limit_guard(&ctrl, src_offset, length, alen, @@ -1398,7 +1398,7 @@ void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) { // (8) dest_offset + length must not exceed length of dest. alen = ac->in(ArrayCopyNode::DestLen); - assert(alen != NULL, "need dest len"); + assert(alen != nullptr, "need dest len"); generate_limit_guard(&ctrl, dest_offset, length, alen, @@ -1408,7 +1408,7 @@ void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) { // The generate_arraycopy subroutine checks this. } // This is where the memory effects are placed: - const TypePtr* adr_type = NULL; + const TypePtr* adr_type = nullptr; if (ac->_dest_type != TypeOopPtr::BOTTOM) { adr_type = ac->_dest_type->add_offset(Type::OffsetBot)->is_ptr(); } else { diff --git a/src/hotspot/share/opto/matcher.cpp b/src/hotspot/share/opto/matcher.cpp index 52be4b022cae6..5558393c6d5de 100644 --- a/src/hotspot/share/opto/matcher.cpp +++ b/src/hotspot/share/opto/matcher.cpp @@ -83,52 +83,52 @@ Matcher::Matcher() _register_save_type(register_save_type) { C->set_matcher(this); - idealreg2spillmask [Op_RegI] = NULL; - idealreg2spillmask [Op_RegN] = NULL; - idealreg2spillmask [Op_RegL] = NULL; - idealreg2spillmask [Op_RegF] = NULL; - idealreg2spillmask [Op_RegD] = NULL; - idealreg2spillmask [Op_RegP] = NULL; - idealreg2spillmask [Op_VecA] = NULL; - idealreg2spillmask [Op_VecS] = NULL; - idealreg2spillmask [Op_VecD] = NULL; - idealreg2spillmask [Op_VecX] = NULL; - idealreg2spillmask [Op_VecY] = NULL; - idealreg2spillmask [Op_VecZ] = NULL; - idealreg2spillmask [Op_RegFlags] = NULL; - idealreg2spillmask [Op_RegVectMask] = NULL; - - idealreg2debugmask [Op_RegI] = NULL; - idealreg2debugmask [Op_RegN] = NULL; - idealreg2debugmask [Op_RegL] = NULL; - idealreg2debugmask [Op_RegF] = NULL; - idealreg2debugmask [Op_RegD] = NULL; - idealreg2debugmask [Op_RegP] = NULL; - idealreg2debugmask [Op_VecA] = NULL; - idealreg2debugmask [Op_VecS] = NULL; - idealreg2debugmask [Op_VecD] = NULL; - idealreg2debugmask [Op_VecX] = NULL; - idealreg2debugmask [Op_VecY] = NULL; - idealreg2debugmask [Op_VecZ] = NULL; - idealreg2debugmask [Op_RegFlags] = NULL; - idealreg2debugmask [Op_RegVectMask] = NULL; - - idealreg2mhdebugmask[Op_RegI] = NULL; - idealreg2mhdebugmask[Op_RegN] = NULL; - idealreg2mhdebugmask[Op_RegL] = NULL; - idealreg2mhdebugmask[Op_RegF] = NULL; - idealreg2mhdebugmask[Op_RegD] = NULL; - idealreg2mhdebugmask[Op_RegP] = NULL; - idealreg2mhdebugmask[Op_VecA] = NULL; - idealreg2mhdebugmask[Op_VecS] = NULL; - idealreg2mhdebugmask[Op_VecD] = NULL; - idealreg2mhdebugmask[Op_VecX] = NULL; - idealreg2mhdebugmask[Op_VecY] = NULL; - idealreg2mhdebugmask[Op_VecZ] = NULL; - idealreg2mhdebugmask[Op_RegFlags] = NULL; - idealreg2mhdebugmask[Op_RegVectMask] = NULL; - - debug_only(_mem_node = NULL;) // Ideal memory node consumed by mach node + idealreg2spillmask [Op_RegI] = nullptr; + idealreg2spillmask [Op_RegN] = nullptr; + idealreg2spillmask [Op_RegL] = nullptr; + idealreg2spillmask [Op_RegF] = nullptr; + idealreg2spillmask [Op_RegD] = nullptr; + idealreg2spillmask [Op_RegP] = nullptr; + idealreg2spillmask [Op_VecA] = nullptr; + idealreg2spillmask [Op_VecS] = nullptr; + idealreg2spillmask [Op_VecD] = nullptr; + idealreg2spillmask [Op_VecX] = nullptr; + idealreg2spillmask [Op_VecY] = nullptr; + idealreg2spillmask [Op_VecZ] = nullptr; + idealreg2spillmask [Op_RegFlags] = nullptr; + idealreg2spillmask [Op_RegVectMask] = nullptr; + + idealreg2debugmask [Op_RegI] = nullptr; + idealreg2debugmask [Op_RegN] = nullptr; + idealreg2debugmask [Op_RegL] = nullptr; + idealreg2debugmask [Op_RegF] = nullptr; + idealreg2debugmask [Op_RegD] = nullptr; + idealreg2debugmask [Op_RegP] = nullptr; + idealreg2debugmask [Op_VecA] = nullptr; + idealreg2debugmask [Op_VecS] = nullptr; + idealreg2debugmask [Op_VecD] = nullptr; + idealreg2debugmask [Op_VecX] = nullptr; + idealreg2debugmask [Op_VecY] = nullptr; + idealreg2debugmask [Op_VecZ] = nullptr; + idealreg2debugmask [Op_RegFlags] = nullptr; + idealreg2debugmask [Op_RegVectMask] = nullptr; + + idealreg2mhdebugmask[Op_RegI] = nullptr; + idealreg2mhdebugmask[Op_RegN] = nullptr; + idealreg2mhdebugmask[Op_RegL] = nullptr; + idealreg2mhdebugmask[Op_RegF] = nullptr; + idealreg2mhdebugmask[Op_RegD] = nullptr; + idealreg2mhdebugmask[Op_RegP] = nullptr; + idealreg2mhdebugmask[Op_VecA] = nullptr; + idealreg2mhdebugmask[Op_VecS] = nullptr; + idealreg2mhdebugmask[Op_VecD] = nullptr; + idealreg2mhdebugmask[Op_VecX] = nullptr; + idealreg2mhdebugmask[Op_VecY] = nullptr; + idealreg2mhdebugmask[Op_VecZ] = nullptr; + idealreg2mhdebugmask[Op_RegFlags] = nullptr; + idealreg2mhdebugmask[Op_RegVectMask] = nullptr; + + debug_only(_mem_node = nullptr;) // Ideal memory node consumed by mach node } //------------------------------warp_incoming_stk_arg------------------------ @@ -172,7 +172,7 @@ void Matcher::verify_new_nodes_only(Node* xroot) { assert(C->node_arena()->contains(n), "dead node"); for (uint j = 0; j < n->req(); j++) { Node* in = n->in(j); - if (in != NULL) { + if (in != nullptr) { assert(C->node_arena()->contains(in), "dead node"); if (!visited.test(in->_idx)) { worklist.push(in); @@ -324,7 +324,7 @@ void Matcher::match( ) { C->print_method(PHASE_BEFORE_MATCHING); - // Create new ideal node ConP #NULL even if it does exist in old space + // Create new ideal node ConP #null even if it does exist in old space // to avoid false sharing if the corresponding mach node is not used. // The corresponding mach node is only used in rare cases for derived // pointers. @@ -335,10 +335,10 @@ void Matcher::match( ) { // Save debug and profile information for nodes in old space: _old_node_note_array = C->node_note_array(); - if (_old_node_note_array != NULL) { + if (_old_node_note_array != nullptr) { C->set_node_note_array(new(C->comp_arena()) GrowableArray (C->comp_arena(), _old_node_note_array->length(), - 0, NULL)); + 0, nullptr)); } // Pre-size the new_node table to avoid the need for range checks. @@ -355,7 +355,7 @@ void Matcher::match( ) { C->set_cached_top_node(xform( C->top(), live_nodes )); if (!C->failing()) { Node* xroot = xform( C->root(), 1 ); - if (xroot == NULL) { + if (xroot == nullptr) { Matcher::soft_match_failure(); // recursive matching process failed C->record_method_not_compilable("instruction match failed"); } else { @@ -372,22 +372,22 @@ void Matcher::match( ) { } } - // Generate new mach node for ConP #NULL - assert(new_ideal_null != NULL, "sanity"); + // Generate new mach node for ConP #null + assert(new_ideal_null != nullptr, "sanity"); _mach_null = match_tree(new_ideal_null); // Don't set control, it will confuse GCM since there are no uses. // The control will be set when this node is used first time // in find_base_for_derived(). - assert(_mach_null != NULL, ""); + assert(_mach_null != nullptr, ""); - C->set_root(xroot->is_Root() ? xroot->as_Root() : NULL); + C->set_root(xroot->is_Root() ? xroot->as_Root() : nullptr); #ifdef ASSERT verify_new_nodes_only(xroot); #endif } } - if (C->top() == NULL || C->root() == NULL) { + if (C->top() == nullptr || C->root() == nullptr) { C->record_method_not_compilable("graph lost"); // %%% cannot happen? } if (C->failing()) { @@ -989,7 +989,7 @@ static void match_alias_type(Compile* C, Node* n, Node* m) { for (uint i = 1; i < n->req(); i++) { Node* n1 = n->in(i); const TypePtr* n1at = n1->adr_type(); - if (n1at != NULL) { + if (n1at != nullptr) { nat = n1at; nidx = C->get_alias_index(n1at); } @@ -1039,7 +1039,7 @@ static void match_alias_type(Compile* C, Node* n, Node* m) { case Op_OnSpinWait: case Op_EncodeISOArray: nidx = Compile::AliasIdxTop; - nat = NULL; + nat = nullptr; break; } } @@ -1062,10 +1062,10 @@ Node *Matcher::transform( Node *n ) { ShouldNotCallThis(); return n; } Node *Matcher::xform( Node *n, int max_stack ) { // Use one stack to keep both: child's node/state and parent's node/index MStack mstack(max_stack * 2 * 2); // usually: C->live_nodes() * 2 * 2 - mstack.push(n, Visit, NULL, -1); // set NULL as parent to indicate root + mstack.push(n, Visit, nullptr, -1); // set null as parent to indicate root while (mstack.is_nonempty()) { C->check_node_count(NodeLimitFudgeFactor, "too many nodes matching instructions"); - if (C->failing()) return NULL; + if (C->failing()) return nullptr; n = mstack.node(); // Leave node on stack Node_State nstate = mstack.state(); if (nstate == Visit) { @@ -1082,17 +1082,17 @@ Node *Matcher::xform( Node *n, int max_stack ) { // Calls match special. They match alone with no children. // Their children, the incoming arguments, match normally. m = n->is_SafePoint() ? match_sfpt(n->as_SafePoint()):match_tree(n); - if (C->failing()) return NULL; - if (m == NULL) { Matcher::soft_match_failure(); return NULL; } + if (C->failing()) return nullptr; + if (m == nullptr) { Matcher::soft_match_failure(); return nullptr; } if (n->is_MemBar()) { m->as_MachMemBar()->set_adr_type(n->adr_type()); } } else { // Nothing the matcher cares about - if (n->is_Proj() && n->in(0) != NULL && n->in(0)->is_Multi()) { // Projections? + if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Multi()) { // Projections? // Convert to machine-dependent projection m = n->in(0)->as_Multi()->match( n->as_Proj(), this ); NOT_PRODUCT(record_new2old(m, n);) - if (m->in(0) != NULL) // m might be top + if (m->in(0) != nullptr) // m might be top collect_null_checks(m, n); } else { // Else just a regular 'ol guy m = n->clone(); // So just clone into new-space @@ -1104,7 +1104,7 @@ Node *Matcher::xform( Node *n, int max_stack ) { } set_new_node(n, m); // Map old to new - if (_old_node_note_array != NULL) { + if (_old_node_note_array != nullptr) { Node_Notes* nn = C->locate_node_notes(_old_node_note_array, n->_idx); C->set_node_notes_at(m->_idx, nn); @@ -1122,7 +1122,7 @@ Node *Matcher::xform( Node *n, int max_stack ) { // Put precedence edges on stack first (match them last). for (i = oldn->req(); (uint)i < oldn->len(); i++) { Node *m = oldn->in(i); - if (m == NULL) break; + if (m == nullptr) break; // set -1 to call add_prec() instead of set_req() during Step1 mstack.push(m, Visit, n, -1); } @@ -1130,7 +1130,7 @@ Node *Matcher::xform( Node *n, int max_stack ) { // Handle precedence edges for interior nodes for (i = n->len()-1; (uint)i >= n->req(); i--) { Node *m = n->in(i); - if (m == NULL || C->node_arena()->contains(m)) continue; + if (m == nullptr || C->node_arena()->contains(m)) continue; n->rm_prec(i); // set -1 to call add_prec() instead of set_req() during Step1 mstack.push(m, Visit, n, -1); @@ -1165,7 +1165,7 @@ Node *Matcher::xform( Node *n, int max_stack ) { // And now walk his children, and convert his inputs to new-space. for( ; i >= 0; --i ) { // For all normal inputs do Node *m = n->in(i); // Get input - if(m != NULL) + if(m != nullptr) mstack.push(m, Visit, n, i); } @@ -1173,7 +1173,7 @@ Node *Matcher::xform( Node *n, int max_stack ) { else if (nstate == Post_Visit) { // Set xformed input Node *p = mstack.parent(); - if (p != NULL) { // root doesn't have parent + if (p != nullptr) { // root doesn't have parent int i = (int)mstack.index(); if (i >= 0) p->set_req(i, n); // required input @@ -1219,13 +1219,13 @@ OptoReg::Name Matcher::warp_outgoing_stk_arg( VMReg reg, OptoReg::Name begin_out // They match alone with no children. Their children, the incoming // arguments, match normally. MachNode *Matcher::match_sfpt( SafePointNode *sfpt ) { - MachSafePointNode *msfpt = NULL; - MachCallNode *mcall = NULL; + MachSafePointNode *msfpt = nullptr; + MachCallNode *mcall = nullptr; uint cnt; // Split out case for SafePoint vs Call CallNode *call; const TypeTuple *domain; - ciMethod* method = NULL; + ciMethod* method = nullptr; bool is_method_handle_invoke = false; // for special kill effects if( sfpt->is_Call() ) { call = sfpt->as_Call(); @@ -1234,8 +1234,8 @@ MachNode *Matcher::match_sfpt( SafePointNode *sfpt ) { // Match just the call, nothing else MachNode *m = match_tree(call); - if (C->failing()) return NULL; - if( m == NULL ) { Matcher::soft_match_failure(); return NULL; } + if (C->failing()) return nullptr; + if( m == nullptr ) { Matcher::soft_match_failure(); return nullptr; } // Copy data from the Ideal SafePoint to the machine version mcall = m->as_MachCall(); @@ -1282,10 +1282,10 @@ MachNode *Matcher::match_sfpt( SafePointNode *sfpt ) { } // This is a non-call safepoint else { - call = NULL; - domain = NULL; + call = nullptr; + domain = nullptr; MachNode *mn = match_tree(sfpt); - if (C->failing()) return NULL; + if (C->failing()) return nullptr; msfpt = mn->as_MachSafePoint(); cnt = TypeFunc::Parms; } @@ -1310,7 +1310,7 @@ MachNode *Matcher::match_sfpt( SafePointNode *sfpt ) { OptoReg::Name out_arg_limit_per_call = begin_out_arg_area; // Calls to C may hammer extra stack slots above and beyond any arguments. // These are usually backing store for register arguments for varargs. - if( call != NULL && call->is_CallRuntime() ) + if( call != nullptr && call->is_CallRuntime() ) out_arg_limit_per_call = OptoReg::add(out_arg_limit_per_call,C->varargs_C_out_slots_killed()); if( call != NULL && call->is_CallNative() ) out_arg_limit_per_call = OptoReg::add(out_arg_limit_per_call, call->as_CallNative()->_shadow_space_bytes); @@ -1423,7 +1423,7 @@ MachNode *Matcher::match_sfpt( SafePointNode *sfpt ) { } // Debug inputs begin just after the last incoming parameter - assert((mcall == NULL) || (mcall->jvms() == NULL) || + assert((mcall == nullptr) || (mcall->jvms() == nullptr) || (mcall->jvms()->debug_start() + mcall->_jvmadj == mcall->tf()->domain()->cnt()), ""); // Add additional edges. @@ -1462,18 +1462,18 @@ MachNode *Matcher::match_tree( const Node *n ) { Node *mem = n->is_Store() ? n->in(MemNode::Memory) : (Node*)1 ; #ifdef ASSERT Node* save_mem_node = _mem_node; - _mem_node = n->is_Store() ? (Node*)n : NULL; + _mem_node = n->is_Store() ? (Node*)n : nullptr; #endif // State object for root node of match tree // Allocate it on _states_arena - stack allocation can cause stack overflow. State *s = new (&_states_arena) State; - s->_kids[0] = NULL; - s->_kids[1] = NULL; + s->_kids[0] = nullptr; + s->_kids[1] = nullptr; s->_leaf = (Node*)n; // Label the input tree, allocating labels from top-level arena Node* root_mem = mem; Label_Root(n, s, n->in(0), root_mem); - if (C->failing()) return NULL; + if (C->failing()) return nullptr; // The minimum cost match for the whole tree is found at the root State uint mincost = max_juint; @@ -1493,7 +1493,7 @@ MachNode *Matcher::match_tree( const Node *n ) { s->dump(); #endif Matcher::soft_match_failure(); - return NULL; + return nullptr; } // Reduce input tree based upon the state labels to machine Nodes MachNode *m = ReduceInst(s, s->rule(mincost), mem); @@ -1540,7 +1540,7 @@ static bool match_into_reg( const Node *n, Node *m, Node *control, int i, bool s Node* m_control = m->in(0); // Control of load's memory can post-dominates load's control. // So use it since load can't float above its memory. - Node* mem_control = (m->is_Load()) ? m->in(MemNode::Memory)->in(0) : NULL; + Node* mem_control = (m->is_Load()) ? m->in(MemNode::Memory)->in(0) : nullptr; if (control && m_control && control != m_control && control != mem_control) { // Actually, we can live with the most conservative control we @@ -1594,7 +1594,7 @@ Node* Matcher::Label_Root(const Node* n, State* svec, Node* control, Node*& mem) LabelRootDepth++; if (LabelRootDepth > MaxLabelRootDepth) { C->record_method_not_compilable("Out of stack space, increase MaxLabelRootDepth"); - return NULL; + return nullptr; } uint care = 0; // Edges matcher cares about uint cnt = n->req(); @@ -1604,13 +1604,13 @@ Node* Matcher::Label_Root(const Node* n, State* svec, Node* control, Node*& mem) // Can only subsume a child into your match-tree if that child's memory state // is not modified along the path to another input. // It is unsafe even if the other inputs are separate roots. - Node *input_mem = NULL; + Node *input_mem = nullptr; for( i = 1; i < cnt; i++ ) { if( !n->match_edge(i) ) continue; Node *m = n->in(i); // Get ith input assert( m, "expect non-null children" ); if( m->is_Load() ) { - if( input_mem == NULL ) { + if( input_mem == nullptr ) { input_mem = m->in(MemNode::Memory); if (mem == (Node*)1) { // Save this memory to bail out if there's another memory access @@ -1632,8 +1632,8 @@ Node* Matcher::Label_Root(const Node* n, State* svec, Node* control, Node*& mem) assert( care <= 2, "binary only for now" ); // Recursively label the State tree. - s->_kids[0] = NULL; - s->_kids[1] = NULL; + s->_kids[0] = nullptr; + s->_kids[1] = nullptr; s->_leaf = m; // Check for leaves of the State Tree; things that cannot be a part of @@ -1658,11 +1658,11 @@ Node* Matcher::Label_Root(const Node* n, State* svec, Node* control, Node*& mem) } else { // If match tree has no control and we do, adopt it for entire tree - if( control == NULL && m->in(0) != NULL && m->req() > 1 ) + if( control == nullptr && m->in(0) != nullptr && m->req() > 1 ) control = m->in(0); // Pick up control // Else match as a normal part of the match tree. control = Label_Root(m, s, control, mem); - if (C->failing()) return NULL; + if (C->failing()) return nullptr; } } @@ -1690,36 +1690,36 @@ Node* Matcher::Label_Root(const Node* n, State* svec, Node* control, Node*& mem) // program. The register allocator is free to split uses later to // split live ranges. MachNode* Matcher::find_shared_node(Node* leaf, uint rule) { - if (!leaf->is_Con() && !leaf->is_DecodeNarrowPtr()) return NULL; + if (!leaf->is_Con() && !leaf->is_DecodeNarrowPtr()) return nullptr; // See if this Con has already been reduced using this rule. - if (_shared_nodes.Size() <= leaf->_idx) return NULL; + if (_shared_nodes.Size() <= leaf->_idx) return nullptr; MachNode* last = (MachNode*)_shared_nodes.at(leaf->_idx); - if (last != NULL && rule == last->rule()) { + if (last != nullptr && rule == last->rule()) { // Don't expect control change for DecodeN if (leaf->is_DecodeNarrowPtr()) return last; // Get the new space root. Node* xroot = new_node(C->root()); - if (xroot == NULL) { + if (xroot == nullptr) { // This shouldn't happen give the order of matching. - return NULL; + return nullptr; } // Shared constants need to have their control be root so they // can be scheduled properly. Node* control = last->in(0); if (control != xroot) { - if (control == NULL || control == C->root()) { + if (control == nullptr || control == C->root()) { last->set_req(0, xroot); } else { assert(false, "unexpected control"); - return NULL; + return nullptr; } } return last; } - return NULL; + return nullptr; } @@ -1745,15 +1745,15 @@ MachNode *Matcher::ReduceInst( State *s, int rule, Node *&mem ) { assert( rule >= NUM_OPERANDS, "called with operand rule" ); MachNode* shared_node = find_shared_node(s->_leaf, rule); - if (shared_node != NULL) { + if (shared_node != nullptr) { return shared_node; } // Build the object to represent this state & prepare for recursive calls MachNode *mach = s->MachNodeGenerator(rule); - guarantee(mach != NULL, "Missing MachNode"); + guarantee(mach != nullptr, "Missing MachNode"); mach->_opnds[0] = s->MachOperGenerator(_reduceOp[rule]); - assert( mach->_opnds[0] != NULL, "Missing result operand" ); + assert( mach->_opnds[0] != nullptr, "Missing result operand" ); Node *leaf = s->_leaf; NOT_PRODUCT(record_new2old(mach, leaf);) // Check for instruction or instruction chain rule @@ -1776,14 +1776,14 @@ MachNode *Matcher::ReduceInst( State *s, int rule, Node *&mem ) { #ifdef ASSERT // Verify adr type after matching memory operation const MachOper* oper = mach->memory_operand(); - if (oper != NULL && oper != (MachOper*)-1) { + if (oper != nullptr && oper != (MachOper*)-1) { // It has a unique memory operand. Find corresponding ideal mem node. - Node* m = NULL; + Node* m = nullptr; if (leaf->is_Mem()) { m = leaf; } else { m = _mem_node; - assert(m != NULL && m->is_Mem(), "expecting memory node"); + assert(m != nullptr && m->is_Mem(), "expecting memory node"); } const Type* mach_at = mach->adr_type(); // DecodeN node consumed by an address may have different type @@ -1822,7 +1822,7 @@ MachNode *Matcher::ReduceInst( State *s, int rule, Node *&mem ) { ex->in(1)->set_req(0, C->root()); // Remove old node from the graph for( uint i=0; ireq(); i++ ) { - mach->set_req(i,NULL); + mach->set_req(i,nullptr); } NOT_PRODUCT(record_new2old(ex, s->_leaf);) } @@ -1855,7 +1855,7 @@ MachNode *Matcher::ReduceInst( State *s, int rule, Node *&mem ) { void Matcher::handle_precedence_edges(Node* n, MachNode *mach) { for (uint i = n->req(); i < n->len(); i++) { - if (n->in(i) != NULL) { + if (n->in(i) != nullptr) { mach->add_prec(n->in(i)); } } @@ -1902,15 +1902,15 @@ uint Matcher::ReduceInst_Interior( State *s, int rule, Node *&mem, MachNode *mac debug_only( if( mem == (Node*)1 ) _mem_node = s->_leaf;) mem = mem2; } - if( s->_leaf->in(0) != NULL && s->_leaf->req() > 1) { - if( mach->in(0) == NULL ) + if( s->_leaf->in(0) != nullptr && s->_leaf->req() > 1) { + if( mach->in(0) == nullptr ) mach->set_req(0, s->_leaf->in(0)); } // Now recursively walk the state tree & add operand list. for( uint i=0; i<2; i++ ) { // binary tree State *newstate = s->_kids[i]; - if( newstate == NULL ) break; // Might only have 1 child + if( newstate == nullptr ) break; // Might only have 1 child // 'op' is what I am expecting to receive int op; if( i == 0 ) { @@ -1965,10 +1965,10 @@ uint Matcher::ReduceInst_Interior( State *s, int rule, Node *&mem, MachNode *mac void Matcher::ReduceOper( State *s, int rule, Node *&mem, MachNode *mach ) { assert( rule < _LAST_MACH_OPER, "called with operand rule" ); State *kid = s->_kids[0]; - assert( kid == NULL || s->_leaf->in(0) == NULL, "internal operands have no control" ); + assert( kid == nullptr || s->_leaf->in(0) == nullptr, "internal operands have no control" ); // Leaf? And not subsumed? - if( kid == NULL && !_swallowed[rule] ) { + if( kid == nullptr && !_swallowed[rule] ) { mach->add_req( s->_leaf ); // Add leaf pointer return; // Bail out } @@ -1989,7 +1989,7 @@ void Matcher::ReduceOper( State *s, int rule, Node *&mem, MachNode *mach ) { } } - for (uint i = 0; kid != NULL && i < 2; kid = s->_kids[1], i++) { // binary tree + for (uint i = 0; kid != nullptr && i < 2; kid = s->_kids[1], i++) { // binary tree int newrule; if( i == 0) { newrule = kid->rule(_leftOp[rule]); @@ -2029,7 +2029,7 @@ OptoReg::Name Matcher::find_receiver() { } bool Matcher::is_vshift_con_pattern(Node* n, Node* m) { - if (n != NULL && m != NULL) { + if (n != nullptr && m != nullptr) { return VectorNode::is_vector_shift(n) && VectorNode::is_vector_shift_count(m) && m->in(1)->is_Con(); } @@ -2109,8 +2109,8 @@ void Matcher::find_shared(Node* n) { } for (int i = n->req() - 1; i >= 0; --i) { // For my children Node* m = n->in(i); // Get ith input - if (m == NULL) { - continue; // Ignore NULLs + if (m == nullptr) { + continue; // Ignore nulls } if (clone_node(n, m, mstack)) { continue; @@ -2489,7 +2489,7 @@ void Matcher::collect_null_checks( Node *proj, Node *orig_proj ) { // Look for DecodeN node which should be pinned to orig_proj. // On platforms (Sparc) which can not handle 2 adds // in addressing mode we have to keep a DecodeN node and - // use it to do implicit NULL check in address. + // use it to do implicit null check in address. // // DecodeN node was pinned to non-null path (orig_proj) during // CastPP transformation in final_graph_reshaping_impl(). @@ -2499,9 +2499,9 @@ void Matcher::collect_null_checks( Node *proj, Node *orig_proj ) { Node* d = orig_proj->raw_out(i); if (d->is_DecodeN() && d->in(1) == val) { val = d; - val->set_req(0, NULL); // Unpin now. + val->set_req(0, nullptr); // Unpin now. // Mark this as special case to distinguish from - // a regular case: CmpP(DecodeN, NULL). + // a regular case: CmpP(DecodeN, null). val = (Node*)(((intptr_t)val) | 1); break; } @@ -2515,7 +2515,7 @@ void Matcher::collect_null_checks( Node *proj, Node *orig_proj ) { } //---------------------------validate_null_checks------------------------------ -// Its possible that the value being NULL checked is not the root of a match +// Its possible that the value being null checked is not the root of a match // tree. If so, I cannot use the value in an implicit null check. void Matcher::validate_null_checks( ) { uint cnt = _null_check_tests.size(); @@ -2527,12 +2527,12 @@ void Matcher::validate_null_checks( ) { if (has_new_node(val)) { Node* new_val = new_node(val); if (is_decoden) { - assert(val->is_DecodeNarrowPtr() && val->in(0) == NULL, "sanity"); + assert(val->is_DecodeNarrowPtr() && val->in(0) == nullptr, "sanity"); // Note: new_val may have a control edge if // the original ideal node DecodeN was matched before // it was unpinned in Matcher::collect_null_checks(). // Unpin the mach node and mark it. - new_val->set_req(0, NULL); + new_val->set_req(0, nullptr); new_val = (Node*)(((intptr_t)new_val) | 1); } // Is a match-tree root, so replace with the matched value @@ -2558,15 +2558,15 @@ bool Matcher::gen_narrow_oop_implicit_null_checks() { } return CompressedOops::use_implicit_null_checks() && (narrow_oop_use_complex_address() || - CompressedOops::base() != NULL); + CompressedOops::base() != nullptr); } // Compute RegMask for an ideal register. const RegMask* Matcher::regmask_for_ideal_register(uint ideal_reg, Node* ret) { const Type* t = Type::mreg2type[ideal_reg]; - if (t == NULL) { + if (t == nullptr) { assert(ideal_reg >= Op_VecA && ideal_reg <= Op_VecZ, "not a vector: %d", ideal_reg); - return NULL; // not supported + return nullptr; // not supported } Node* fp = ret->in(TypeFunc::FramePtr); Node* mem = ret->in(TypeFunc::Memory); @@ -2575,25 +2575,25 @@ const RegMask* Matcher::regmask_for_ideal_register(uint ideal_reg, Node* ret) { Node* spill; switch (ideal_reg) { - case Op_RegN: spill = new LoadNNode(NULL, mem, fp, atp, t->is_narrowoop(), mo); break; - case Op_RegI: spill = new LoadINode(NULL, mem, fp, atp, t->is_int(), mo); break; - case Op_RegP: spill = new LoadPNode(NULL, mem, fp, atp, t->is_ptr(), mo); break; - case Op_RegF: spill = new LoadFNode(NULL, mem, fp, atp, t, mo); break; - case Op_RegD: spill = new LoadDNode(NULL, mem, fp, atp, t, mo); break; - case Op_RegL: spill = new LoadLNode(NULL, mem, fp, atp, t->is_long(), mo); break; + case Op_RegN: spill = new LoadNNode(nullptr, mem, fp, atp, t->is_narrowoop(), mo); break; + case Op_RegI: spill = new LoadINode(nullptr, mem, fp, atp, t->is_int(), mo); break; + case Op_RegP: spill = new LoadPNode(nullptr, mem, fp, atp, t->is_ptr(), mo); break; + case Op_RegF: spill = new LoadFNode(nullptr, mem, fp, atp, t, mo); break; + case Op_RegD: spill = new LoadDNode(nullptr, mem, fp, atp, t, mo); break; + case Op_RegL: spill = new LoadLNode(nullptr, mem, fp, atp, t->is_long(), mo); break; case Op_VecA: // fall-through case Op_VecS: // fall-through case Op_VecD: // fall-through case Op_VecX: // fall-through case Op_VecY: // fall-through - case Op_VecZ: spill = new LoadVectorNode(NULL, mem, fp, atp, t->is_vect()); break; + case Op_VecZ: spill = new LoadVectorNode(nullptr, mem, fp, atp, t->is_vect()); break; case Op_RegVectMask: return Matcher::predicate_reg_mask(); default: ShouldNotReachHere(); } MachNode* mspill = match_tree(spill); - assert(mspill != NULL, "matching failed: %d", ideal_reg); + assert(mspill != nullptr, "matching failed: %d", ideal_reg); // Handle generic vector operand case if (Matcher::supports_generic_vector_operands && t->isa_vect()) { specialize_mach_node(mspill); @@ -2629,7 +2629,7 @@ void Matcher::specialize_temp_node(MachTempNode* tmp, MachNode* use, uint idx) { // Compute concrete vector operand for a generic DEF/USE vector operand (of mach node m at index idx). MachOper* Matcher::specialize_vector_operand(MachNode* m, uint opnd_idx) { assert(Matcher::is_generic_vector(m->_opnds[opnd_idx]), "repeated updates"); - Node* def = NULL; + Node* def = nullptr; if (opnd_idx == 0) { // DEF def = m; // use mach node itself to compute vector operand type } else { @@ -2674,7 +2674,7 @@ void Matcher::specialize_generic_vector_operands() { while (live_nodes.size() > 0) { MachNode* m = live_nodes.pop()->isa_Mach(); - if (m != NULL) { + if (m != nullptr) { if (Matcher::is_generic_reg2reg_move(m)) { // Register allocator properly handles vec <=> leg moves using register masks. int opnd_idx = m->operand_index(1); @@ -2697,7 +2697,7 @@ bool Matcher::verify_after_postselect_cleanup() { C->identify_useful_nodes(useful); for (uint i = 0; i < useful.size(); i++) { MachNode* m = useful.at(i)->isa_Mach(); - if (m != NULL) { + if (m != nullptr) { assert(!Matcher::is_generic_reg2reg_move(m), "no MoveVec nodes allowed"); for (uint j = 0; j < m->num_opnds(); j++) { assert(!Matcher::is_generic_vector(m->_opnds[j]), "no generic vector operands allowed"); @@ -2720,7 +2720,7 @@ bool Matcher::post_store_load_barrier(const Node* vmb) { const MemBarNode* membar = vmb->as_MemBar(); // Get the Ideal Proj node, ctrl, that can be used to iterate forward - Node* ctrl = NULL; + Node* ctrl = nullptr; for (DUIterator_Fast imax, i = membar->fast_outs(imax); i < imax; i++) { Node* p = membar->fast_out(i); assert(p->is_Proj(), "only projections here"); @@ -2730,7 +2730,7 @@ bool Matcher::post_store_load_barrier(const Node* vmb) { break; } } - assert((ctrl != NULL), "missing control projection"); + assert((ctrl != nullptr), "missing control projection"); for (DUIterator_Fast jmax, j = ctrl->fast_outs(jmax); j < jmax; j++) { Node *x = ctrl->fast_out(j); @@ -2805,7 +2805,7 @@ bool Matcher::branches_to_uncommon_trap(const Node *n) { assert(n->is_If(), "You should only call this on if nodes."); IfNode *ifn = n->as_If(); - Node *ifFalse = NULL; + Node *ifFalse = nullptr; for (DUIterator_Fast imax, i = ifn->fast_outs(imax); i < imax; i++) { if (ifn->fast_out(i)->is_IfFalse()) { ifFalse = ifn->fast_out(i); @@ -2817,9 +2817,9 @@ bool Matcher::branches_to_uncommon_trap(const Node *n) { Node *reg = ifFalse; int cnt = 4; // We must protect against cycles. Limit to 4 iterations. // Alternatively use visited set? Seems too expensive. - while (reg != NULL && cnt > 0) { - CallNode *call = NULL; - RegionNode *nxt_reg = NULL; + while (reg != nullptr && cnt > 0) { + CallNode *call = nullptr; + RegionNode *nxt_reg = nullptr; for (DUIterator_Fast imax, i = reg->fast_outs(imax); i < imax; i++) { Node *o = reg->fast_out(i); if (o->is_Call()) { diff --git a/src/hotspot/share/opto/matcher.hpp b/src/hotspot/share/opto/matcher.hpp index 3582f4657ad40..3a1311cc03bad 100644 --- a/src/hotspot/share/opto/matcher.hpp +++ b/src/hotspot/share/opto/matcher.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -148,10 +148,10 @@ class Matcher : public PhaseTransform { // Accessors for the inherited field PhaseTransform::_nodes: void grow_new_node_array(uint idx_limit) { - _nodes.map(idx_limit-1, NULL); + _nodes.map(idx_limit-1, nullptr); } bool has_new_node(const Node* n) const { - return _nodes.at(n->_idx) != NULL; + return _nodes.at(n->_idx) != nullptr; } Node* new_node(const Node* n) const { assert(has_new_node(n), "set before get"); @@ -169,7 +169,7 @@ class Matcher : public PhaseTransform { Node* _mem_node; // Ideal memory node consumed by mach node #endif - // Mach node for ConP #NULL + // Mach node for ConP #null MachNode* _mach_null; void handle_precedence_edges(Node* n, MachNode *mach); diff --git a/src/hotspot/share/opto/mathexactnode.cpp b/src/hotspot/share/opto/mathexactnode.cpp index dddac5e581ea3..4c065e5cf52d3 100644 --- a/src/hotspot/share/opto/mathexactnode.cpp +++ b/src/hotspot/share/opto/mathexactnode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -192,8 +192,8 @@ struct IdealHelper { const Type* type1 = phase->type(arg1); const Type* type2 = phase->type(arg2); - if (type1 == NULL || type2 == NULL) { - return NULL; + if (type1 == nullptr || type2 == nullptr) { + return nullptr; } if (type1 != Type::TOP && type1->singleton() && @@ -204,9 +204,9 @@ struct IdealHelper { Node* con_result = ConINode::make(0); return con_result; } - return NULL; + return nullptr; } - return NULL; + return nullptr; } static const Type* Value(const OverflowOp* node, PhaseTransform* phase) { @@ -218,7 +218,7 @@ struct IdealHelper { const TypeClass* i1 = TypeClass::as_self(t1); const TypeClass* i2 = TypeClass::as_self(t2); - if (i1 == NULL || i2 == NULL) { + if (i1 == nullptr || i2 == nullptr) { return TypeInt::CC; } diff --git a/src/hotspot/share/opto/memnode.cpp b/src/hotspot/share/opto/memnode.cpp index 84efd9ff75d17..2ee1e5f52f766 100644 --- a/src/hotspot/share/opto/memnode.cpp +++ b/src/hotspot/share/opto/memnode.cpp @@ -65,14 +65,14 @@ uint MemNode::size_of() const { return sizeof(*this); } const TypePtr *MemNode::adr_type() const { Node* adr = in(Address); - if (adr == NULL) return NULL; // node is dead - const TypePtr* cross_check = NULL; + if (adr == nullptr) return nullptr; // node is dead + const TypePtr* cross_check = nullptr; DEBUG_ONLY(cross_check = _adr_type); return calculate_adr_type(adr->bottom_type(), cross_check); } bool MemNode::check_if_adr_maybe_raw(Node* adr) { - if (adr != NULL) { + if (adr != nullptr) { if (adr->bottom_type()->base() == Type::RawPtr || adr->bottom_type()->base() == Type::AnyPtr) { return true; } @@ -82,11 +82,11 @@ bool MemNode::check_if_adr_maybe_raw(Node* adr) { #ifndef PRODUCT void MemNode::dump_spec(outputStream *st) const { - if (in(Address) == NULL) return; // node is dead + if (in(Address) == nullptr) return; // node is dead #ifndef ASSERT // fake the missing field - const TypePtr* _adr_type = NULL; - if (in(Address) != NULL) + const TypePtr* _adr_type = nullptr; + if (in(Address) != nullptr) _adr_type = in(Address)->bottom_type()->isa_ptr(); #endif dump_adr_type(this, _adr_type, st); @@ -108,14 +108,14 @@ void MemNode::dump_spec(outputStream *st) const { void MemNode::dump_adr_type(const Node* mem, const TypePtr* adr_type, outputStream *st) { st->print(" @"); - if (adr_type == NULL) { - st->print("NULL"); + if (adr_type == nullptr) { + st->print("null"); } else { adr_type->dump_on(st); Compile* C = Compile::current(); - Compile::AliasType* atp = NULL; + Compile::AliasType* atp = nullptr; if (C->have_alias_type(adr_type)) atp = C->alias_type(adr_type); - if (atp == NULL) + if (atp == nullptr) st->print(", idx=?\?;"); else if (atp->index() == Compile::AliasIdxBot) st->print(", idx=Bot;"); @@ -139,16 +139,16 @@ extern void print_alias_types(); #endif Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase) { - assert((t_oop != NULL), "sanity"); + assert((t_oop != nullptr), "sanity"); bool is_instance = t_oop->is_known_instance_field(); bool is_boxed_value_load = t_oop->is_ptr_to_boxed_value() && - (load != NULL) && load->is_Load() && - (phase->is_IterGVN() != NULL); + (load != nullptr) && load->is_Load() && + (phase->is_IterGVN() != nullptr); if (!(is_instance || is_boxed_value_load)) return mchain; // don't try to optimize non-instance types uint instance_id = t_oop->instance_id(); Node *start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory); - Node *prev = NULL; + Node *prev = nullptr; Node *result = mchain; while (prev != result) { prev = result; @@ -169,7 +169,7 @@ Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oo AllocateNode* alloc = proj_in->as_Initialize()->allocation(); // Stop if this is the initialization for the object instance which // contains this memory slice, otherwise skip over it. - if ((alloc == NULL) || (alloc->_idx == instance_id)) { + if ((alloc == nullptr) || (alloc->_idx == instance_id)) { break; } if (is_instance) { @@ -182,7 +182,7 @@ Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oo } } } else if (proj_in->is_MemBar()) { - ArrayCopyNode* ac = NULL; + ArrayCopyNode* ac = nullptr; if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) { break; } @@ -198,7 +198,7 @@ Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oo } // Otherwise skip it (the call updated 'result' value). } else if (result->is_MergeMem()) { - result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, NULL, tty); + result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty); } } return result; @@ -206,12 +206,12 @@ Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oo Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) { const TypeOopPtr* t_oop = t_adr->isa_oopptr(); - if (t_oop == NULL) + if (t_oop == nullptr) return mchain; // don't try to optimize non-oop types Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase); bool is_instance = t_oop->is_known_instance_field(); PhaseIterGVN *igvn = phase->is_IterGVN(); - if (is_instance && igvn != NULL && result->is_Phi()) { + if (is_instance && igvn != nullptr && result->is_Phi()) { PhiNode *mphi = result->as_Phi(); assert(mphi->bottom_type() == Type::MEMORY, "memory phi required"); const TypePtr *t = mphi->adr_type(); @@ -236,10 +236,10 @@ static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const T { // Check that current type is consistent with the alias index used during graph construction assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx"); - bool consistent = adr_check == NULL || adr_check->empty() || + bool consistent = adr_check == nullptr || adr_check->empty() || phase->C->must_alias(adr_check, alias_idx ); // Sometimes dead array references collapse to a[-1], a[-2], or a[-3] - if( !consistent && adr_check != NULL && !adr_check->empty() && + if( !consistent && adr_check != nullptr && !adr_check->empty() && tp->isa_aryptr() && tp->offset() == Type::OffsetBot && adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot && ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() || @@ -250,8 +250,8 @@ static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const T } if( !consistent ) { st->print("alias_idx==%d, adr_check==", alias_idx); - if( adr_check == NULL ) { - st->print("NULL"); + if( adr_check == nullptr ) { + st->print("null"); } else { adr_check->dump(); } @@ -296,9 +296,9 @@ Node *MemNode::Ideal_common(PhaseGVN *phase, bool can_reshape) { PhaseIterGVN *igvn = phase->is_IterGVN(); // Wait if control on the worklist. - if (ctl && can_reshape && igvn != NULL) { - Node* bol = NULL; - Node* cmp = NULL; + if (ctl && can_reshape && igvn != nullptr) { + Node* bol = nullptr; + Node* cmp = nullptr; if (ctl->in(0)->is_If()) { assert(ctl->is_IfTrue() || ctl->is_IfFalse(), "sanity"); bol = ctl->in(0)->in(1); @@ -306,34 +306,34 @@ Node *MemNode::Ideal_common(PhaseGVN *phase, bool can_reshape) { cmp = ctl->in(0)->in(1)->in(1); } if (igvn->_worklist.member(ctl) || - (bol != NULL && igvn->_worklist.member(bol)) || - (cmp != NULL && igvn->_worklist.member(cmp)) ) { + (bol != nullptr && igvn->_worklist.member(bol)) || + (cmp != nullptr && igvn->_worklist.member(cmp)) ) { // This control path may be dead. // Delay this memory node transformation until the control is processed. igvn->_worklist.push(this); - return NodeSentinel; // caller will return NULL + return NodeSentinel; // caller will return null } } // Ignore if memory is dead, or self-loop Node *mem = in(MemNode::Memory); - if (phase->type( mem ) == Type::TOP) return NodeSentinel; // caller will return NULL + if (phase->type( mem ) == Type::TOP) return NodeSentinel; // caller will return null assert(mem != this, "dead loop in MemNode::Ideal"); - if (can_reshape && igvn != NULL && igvn->_worklist.member(mem)) { + if (can_reshape && igvn != nullptr && igvn->_worklist.member(mem)) { // This memory slice may be dead. // Delay this mem node transformation until the memory is processed. igvn->_worklist.push(this); - return NodeSentinel; // caller will return NULL + return NodeSentinel; // caller will return null } Node *address = in(MemNode::Address); const Type *t_adr = phase->type(address); - if (t_adr == Type::TOP) return NodeSentinel; // caller will return NULL + if (t_adr == Type::TOP) return NodeSentinel; // caller will return null if (can_reshape && is_unsafe_access() && (t_adr == TypePtr::NULL_PTR)) { // Unsafe off-heap access with zero address. Remove access and other control users // to not confuse optimizations and add a HaltNode to fail if this is ever executed. - assert(ctl != NULL, "unsafe accesses should be control dependent"); + assert(ctl != nullptr, "unsafe accesses should be control dependent"); for (DUIterator_Fast imax, i = ctl->fast_outs(imax); i < imax; i++) { Node* u = ctl->fast_out(i); if (u != ctl) { @@ -348,13 +348,13 @@ Node *MemNode::Ideal_common(PhaseGVN *phase, bool can_reshape) { return this; } - if (can_reshape && igvn != NULL && + if (can_reshape && igvn != nullptr && (igvn->_worklist.member(address) || (igvn->_worklist.size() > 0 && t_adr != adr_type())) ) { // The address's base and type may change when the address is processed. // Delay this mem node transformation until the address is processed. igvn->_worklist.push(this); - return NodeSentinel; // caller will return NULL + return NodeSentinel; // caller will return null } // Do NOT remove or optimize the next lines: ensure a new alias index @@ -364,15 +364,15 @@ Node *MemNode::Ideal_common(PhaseGVN *phase, bool can_reshape) { int alias_idx = phase->C->get_alias_index(t_adr->is_ptr()); } - Node* base = NULL; + Node* base = nullptr; if (address->is_AddP()) { base = address->in(AddPNode::Base); } - if (base != NULL && phase->type(base)->higher_equal(TypePtr::NULL_PTR) && + if (base != nullptr && phase->type(base)->higher_equal(TypePtr::NULL_PTR) && !t_adr->isa_rawptr()) { // Note: raw address has TOP base and top->higher_equal(TypePtr::NULL_PTR) is true. // Skip this node optimization if its address has TOP base. - return NodeSentinel; // caller will return NULL + return NodeSentinel; // caller will return null } // Avoid independent memory operations @@ -403,7 +403,7 @@ Node *MemNode::Ideal_common(PhaseGVN *phase, bool can_reshape) { } // let the subclass continue analyzing... - return NULL; + return nullptr; } // Helper function for proving some simple control dominations. @@ -414,12 +414,12 @@ Node *MemNode::Ideal_common(PhaseGVN *phase, bool can_reshape) { // control input of a memory operation predates (dominates) // an allocation it wants to look past. bool MemNode::all_controls_dominate(Node* dom, Node* sub) { - if (dom == NULL || dom->is_top() || sub == NULL || sub->is_top()) + if (dom == nullptr || dom->is_top() || sub == nullptr || sub->is_top()) return false; // Conservative answer for dead code // Check 'dom'. Skip Proj and CatchProj nodes. dom = dom->find_exact_control(dom); - if (dom == NULL || dom->is_top()) + if (dom == nullptr || dom->is_top()) return false; // Conservative answer for dead code if (dom == sub) { @@ -436,14 +436,14 @@ bool MemNode::all_controls_dominate(Node* dom, Node* sub) { // Currently 'sub' is either Allocate, Initialize or Start nodes. // Or Region for the check in LoadNode::Ideal(); - // 'sub' should have sub->in(0) != NULL. + // 'sub' should have sub->in(0) != nullptr. assert(sub->is_Allocate() || sub->is_Initialize() || sub->is_Start() || sub->is_Region() || sub->is_Call(), "expecting only these nodes"); // Get control edge of 'sub'. Node* orig_sub = sub; sub = sub->find_exact_control(sub->in(0)); - if (sub == NULL || sub->is_top()) + if (sub == nullptr || sub->is_top()) return false; // Conservative answer for dead code assert(sub->is_CFG(), "expecting control"); @@ -471,7 +471,7 @@ bool MemNode::all_controls_dominate(Node* dom, Node* sub) { if (!n->is_CFG() && n->pinned()) { // Check only own control edge for pinned non-control nodes. n = n->find_exact_control(n->in(0)); - if (n == NULL || n->is_top()) + if (n == nullptr || n->is_top()) return false; // Conservative answer for dead code assert(n->is_CFG(), "expecting control"); dom_list.push(n); @@ -485,7 +485,7 @@ bool MemNode::all_controls_dominate(Node* dom, Node* sub) { } else { // First, own control edge. Node* m = n->find_exact_control(n->in(0)); - if (m != NULL) { + if (m != nullptr) { if (m->is_top()) return false; // Conservative answer for dead code dom_list.push(m); @@ -494,7 +494,7 @@ bool MemNode::all_controls_dominate(Node* dom, Node* sub) { uint cnt = n->req(); for (uint i = 1; i < cnt; i++) { m = n->find_exact_control(n->in(i)); - if (m == NULL || m->is_top()) + if (m == nullptr || m->is_top()) continue; dom_list.push(m); } @@ -516,14 +516,14 @@ bool MemNode::detect_ptr_independence(Node* p1, AllocateNode* a1, // They may both manifestly be allocations, and they should differ. // Or, if they are not both allocations, they can be distinct constants. // Otherwise, one is an allocation and the other a pre-existing value. - if (a1 == NULL && a2 == NULL) { // neither an allocation + if (a1 == nullptr && a2 == nullptr) { // neither an allocation return (p1 != p2) && p1->is_Con() && p2->is_Con(); - } else if (a1 != NULL && a2 != NULL) { // both allocations + } else if (a1 != nullptr && a2 != nullptr) { // both allocations return (a1 != a2); - } else if (a1 != NULL) { // one allocation a1 + } else if (a1 != nullptr) { // one allocation a1 // (Note: p2->is_Con implies p2->in(0)->is_Root, which dominates.) return all_controls_dominate(p2, a1); - } else { //(a2 != NULL) // one allocation a2 + } else { //(a2 != null) // one allocation a2 return all_controls_dominate(p1, a2); } return false; @@ -537,10 +537,10 @@ bool MemNode::detect_ptr_independence(Node* p1, AllocateNode* a1, // (c) can_see_stored_value=false and ac cannot have set the value for this load. // In case (c) change the parameter mem to the memory input of ac to skip it // when searching stored value. -// Otherwise return NULL. +// Otherwise return null. Node* LoadNode::find_previous_arraycopy(PhaseTransform* phase, Node* ld_alloc, Node*& mem, bool can_see_stored_value) const { ArrayCopyNode* ac = find_array_copy_clone(phase, ld_alloc, mem); - if (ac != NULL) { + if (ac != nullptr) { Node* ld_addp = in(MemNode::Address); Node* src = ac->in(ArrayCopyNode::Src); const TypeAryPtr* ary_t = phase->type(src)->isa_aryptr(); @@ -548,7 +548,7 @@ Node* LoadNode::find_previous_arraycopy(PhaseTransform* phase, Node* ld_alloc, N // This is a load from a cloned array. The corresponding arraycopy ac must // have set the value for the load and we can return ac but only if the load // is known to be within bounds. This is checked below. - if (ary_t != NULL && ld_addp->is_AddP()) { + if (ary_t != nullptr && ld_addp->is_AddP()) { Node* ld_offs = ld_addp->in(AddPNode::Offset); BasicType ary_elem = ary_t->klass()->as_array_klass()->element_type()->basic_type(); jlong header = arrayOopDesc::base_offset_in_bytes(ary_elem); @@ -564,8 +564,8 @@ Node* LoadNode::find_previous_arraycopy(PhaseTransform* phase, Node* ld_alloc, N // The load is known to be out-of-bounds. } // The load could be out-of-bounds. It must not be hoisted but must remain - // dependent on the runtime range check. This is achieved by returning NULL. - } else if (mem->is_Proj() && mem->in(0) != NULL && mem->in(0)->is_ArrayCopy()) { + // dependent on the runtime range check. This is achieved by returning null. + } else if (mem->is_Proj() && mem->in(0) != nullptr && mem->in(0)->is_ArrayCopy()) { ArrayCopyNode* ac = mem->in(0)->as_ArrayCopy(); if (ac->is_arraycopy_validated() || @@ -591,18 +591,18 @@ Node* LoadNode::find_previous_arraycopy(PhaseTransform* phase, Node* ld_alloc, N } } } - return NULL; + return nullptr; } ArrayCopyNode* MemNode::find_array_copy_clone(PhaseTransform* phase, Node* ld_alloc, Node* mem) const { - if (mem->is_Proj() && mem->in(0) != NULL && (mem->in(0)->Opcode() == Op_MemBarStoreStore || + if (mem->is_Proj() && mem->in(0) != nullptr && (mem->in(0)->Opcode() == Op_MemBarStoreStore || mem->in(0)->Opcode() == Op_MemBarCPUOrder)) { - if (ld_alloc != NULL) { + if (ld_alloc != nullptr) { // Check if there is an array copy for a clone Node* mb = mem->in(0); - ArrayCopyNode* ac = NULL; - if (mb->in(0) != NULL && mb->in(0)->is_Proj() && - mb->in(0)->in(0) != NULL && mb->in(0)->in(0)->is_ArrayCopy()) { + ArrayCopyNode* ac = nullptr; + if (mb->in(0) != nullptr && mb->in(0)->is_Proj() && + mb->in(0)->in(0) != nullptr && mb->in(0)->in(0)->is_ArrayCopy()) { ac = mb->in(0)->in(0)->as_ArrayCopy(); } else { // Step over GC barrier when ReduceInitialCardMarks is disabled @@ -614,15 +614,15 @@ ArrayCopyNode* MemNode::find_array_copy_clone(PhaseTransform* phase, Node* ld_al } } - if (ac != NULL && ac->is_clonebasic()) { + if (ac != nullptr && ac->is_clonebasic()) { AllocateNode* alloc = AllocateNode::Ideal_allocation(ac->in(ArrayCopyNode::Dest), phase); - if (alloc != NULL && alloc == ld_alloc) { + if (alloc != nullptr && alloc == ld_alloc) { return ac; } } } } - return NULL; + return nullptr; } // The logic for reordering loads and stores uses four steps: @@ -646,7 +646,7 @@ Node* MemNode::find_previous_store(PhaseTransform* phase) { AllocateNode* alloc = AllocateNode::Ideal_allocation(base, phase); if (offset == Type::OffsetBot) - return NULL; // cannot unalias unless there are precise offsets + return nullptr; // cannot unalias unless there are precise offsets const bool adr_maybe_raw = check_if_adr_maybe_raw(adr); const TypeOopPtr *addr_t = adr->bottom_type()->isa_oopptr(); @@ -664,7 +664,7 @@ Node* MemNode::find_previous_store(PhaseTransform* phase) { Node* st_adr = mem->in(MemNode::Address); intptr_t st_offset = 0; Node* st_base = AddPNode::Ideal_base_and_offset(st_adr, phase, st_offset); - if (st_base == NULL) + if (st_base == nullptr) break; // inscrutable pointer // For raw accesses it's not enough to prove that constant offsets don't intersect. @@ -707,13 +707,13 @@ Node* MemNode::find_previous_store(PhaseTransform* phase) { } else if (mem->is_Proj() && mem->in(0)->is_Initialize()) { InitializeNode* st_init = mem->in(0)->as_Initialize(); AllocateNode* st_alloc = st_init->allocation(); - if (st_alloc == NULL) + if (st_alloc == nullptr) break; // something degenerated bool known_identical = false; bool known_independent = false; if (alloc == st_alloc) known_identical = true; - else if (alloc != NULL) + else if (alloc != nullptr) known_independent = true; else if (all_controls_dominate(this, st_alloc)) known_independent = true; @@ -738,14 +738,14 @@ Node* MemNode::find_previous_store(PhaseTransform* phase) { return mem; // let caller handle steps (c), (d) } - } else if (find_previous_arraycopy(phase, alloc, mem, false) != NULL) { + } else if (find_previous_arraycopy(phase, alloc, mem, false) != nullptr) { if (prev != mem) { // Found an arraycopy but it doesn't affect that load continue; } // Found an arraycopy that may affect that load return mem; - } else if (addr_t != NULL && addr_t->is_known_instance_field()) { + } else if (addr_t != nullptr && addr_t->is_known_instance_field()) { // Can't use optimize_simple_memory_chain() since it needs PhaseGVN. if (mem->is_Proj() && mem->in(0)->is_Call()) { // ArrayCopyNodes processed here as well. @@ -755,7 +755,7 @@ Node* MemNode::find_previous_store(PhaseTransform* phase) { continue; // (a) advance through independent call memory } } else if (mem->is_Proj() && mem->in(0)->is_MemBar()) { - ArrayCopyNode* ac = NULL; + ArrayCopyNode* ac = nullptr; if (ArrayCopyNode::may_modify(addr_t, mem->in(0)->as_MemBar(), phase, ac)) { break; } @@ -782,26 +782,26 @@ Node* MemNode::find_previous_store(PhaseTransform* phase) { break; } - return NULL; // bail out + return nullptr; // bail out } //----------------------calculate_adr_type------------------------------------- // Helper function. Notices when the given type of address hits top or bottom. // Also, asserts a cross-check of the type against the expected address type. const TypePtr* MemNode::calculate_adr_type(const Type* t, const TypePtr* cross_check) { - if (t == Type::TOP) return NULL; // does not touch memory any more? + if (t == Type::TOP) return nullptr; // does not touch memory any more? #ifdef ASSERT - if (!VerifyAliases || VMError::is_error_reported() || Node::in_dump()) cross_check = NULL; + if (!VerifyAliases || VMError::is_error_reported() || Node::in_dump()) cross_check = nullptr; #endif const TypePtr* tp = t->isa_ptr(); - if (tp == NULL) { - assert(cross_check == NULL || cross_check == TypePtr::BOTTOM, "expected memory type must be wide"); + if (tp == nullptr) { + assert(cross_check == nullptr || cross_check == TypePtr::BOTTOM, "expected memory type must be wide"); return TypePtr::BOTTOM; // touches lots of memory } else { #ifdef ASSERT // %%%% [phh] We don't check the alias index if cross_check is // TypeRawPtr::BOTTOM. Needs to be investigated. - if (cross_check != NULL && + if (cross_check != nullptr && cross_check != TypePtr::BOTTOM && cross_check != TypeRawPtr::BOTTOM) { // Recheck the alias index, to see if it has changed (due to a bug). @@ -883,11 +883,11 @@ Node* LoadNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypeP adr_type->offset() == arrayOopDesc::length_offset_in_bytes()), "use LoadRangeNode instead"); // Check control edge of raw loads - assert( ctl != NULL || C->get_alias_index(adr_type) != Compile::AliasIdxRaw || + assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw || // oop will be recorded in oop map if load crosses safepoint rt->isa_oopptr() || is_immutable_value(adr), "raw memory operations should have control edge"); - LoadNode* load = NULL; + LoadNode* load = nullptr; switch (bt) { case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break; case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break; @@ -913,7 +913,7 @@ Node* LoadNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypeP ShouldNotReachHere(); break; } - assert(load != NULL, "LoadNode should have been created"); + assert(load != nullptr, "LoadNode should have been created"); if (unaligned) { load->set_unaligned_access(); } @@ -939,10 +939,10 @@ uint LoadNode::hash() const { } static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) { - if ((atp != NULL) && (atp->index() >= Compile::AliasIdxRaw)) { - bool non_volatile = (atp->field() != NULL) && !atp->field()->is_volatile(); + if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) { + bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile(); bool is_stable_ary = FoldStableValues && - (tp != NULL) && (tp->isa_aryptr() != NULL) && + (tp != nullptr) && (tp->isa_aryptr() != nullptr) && tp->isa_aryptr()->is_stable(); return (eliminate_boxing && non_volatile) || is_stable_ary; @@ -959,7 +959,7 @@ Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const { intptr_t ld_off = 0; AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off); Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true); - if (ac != NULL) { + if (ac != nullptr) { assert(ac->is_ArrayCopy(), "what kind of node can this be?"); Node* mem = ac->in(TypeFunc::Memory); @@ -967,13 +967,13 @@ Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const { Node* src = ac->in(ArrayCopyNode::Src); if (!ac->as_ArrayCopy()->is_clonebasic() && !phase->type(src)->isa_aryptr()) { - return NULL; + return nullptr; } LoadNode* ld = clone()->as_Load(); Node* addp = in(MemNode::Address)->clone(); if (ac->as_ArrayCopy()->is_clonebasic()) { - assert(ld_alloc != NULL, "need an alloc"); + assert(ld_alloc != nullptr, "need an alloc"); assert(addp->is_AddP(), "address must be addp"); BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern"); @@ -1014,7 +1014,7 @@ Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const { ld->_control_dependency = UnknownControl; return ld; } - return NULL; + return nullptr; } @@ -1031,11 +1031,11 @@ Node* MemNode::can_see_stored_value(Node* st, PhaseTransform* phase) const { Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off); Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base, phase); const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr(); - Compile::AliasType* atp = (tp != NULL) ? phase->C->alias_type(tp) : NULL; + Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr; // This is more general than load from boxing objects. if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) { uint alias_idx = atp->index(); - Node* result = NULL; + Node* result = nullptr; Node* current = st; // Skip through chains of MemBarNodes checking the MergeMems for // new states for the slice of this load. Stop once any other @@ -1054,7 +1054,8 @@ Node* MemNode::can_see_stored_value(Node* st, PhaseTransform* phase) const { opc == Op_MemBarRelease || opc == Op_StoreFence || opc == Op_MemBarReleaseLock || - opc == Op_MemBarStoreStore) { + opc == Op_MemBarStoreStore || + opc == Op_StoreStoreFence) { Node* mem = current->in(0)->in(TypeFunc::Memory); if (mem->is_MergeMem()) { MergeMemNode* merge = mem->as_MergeMem(); @@ -1071,7 +1072,7 @@ Node* MemNode::can_see_stored_value(Node* st, PhaseTransform* phase) const { } break; } - if (result != NULL) { + if (result != nullptr) { st = result; } } @@ -1086,11 +1087,11 @@ Node* MemNode::can_see_stored_value(Node* st, PhaseTransform* phase) const { // Try harder before giving up. Unify base pointers with casts (e.g., raw/non-raw pointers). intptr_t st_off = 0; Node* st_base = AddPNode::Ideal_base_and_offset(st_adr, phase, st_off); - if (ld_base == NULL) return NULL; - if (st_base == NULL) return NULL; - if (!ld_base->eqv_uncast(st_base, /*keep_deps=*/true)) return NULL; - if (ld_off != st_off) return NULL; - if (ld_off == Type::OffsetBot) return NULL; + if (ld_base == nullptr) return nullptr; + if (st_base == nullptr) return nullptr; + if (!ld_base->eqv_uncast(st_base, /*keep_deps=*/true)) return nullptr; + if (ld_off != st_off) return nullptr; + if (ld_off == Type::OffsetBot) return nullptr; // Same base, same offset. // Possible improvement for arrays: check index value instead of absolute offset. @@ -1106,14 +1107,14 @@ Node* MemNode::can_see_stored_value(Node* st, PhaseTransform* phase) const { } // Now prove that we have a LoadQ matched to a StoreQ, for some Q. if (store_Opcode() != st->Opcode()) { - return NULL; + return nullptr; } // LoadVector/StoreVector needs additional check to ensure the types match. if (store_Opcode() == Op_StoreVector) { const TypeVect* in_vt = st->as_StoreVector()->vect_type(); const TypeVect* out_vt = as_LoadVector()->vect_type(); if (in_vt != out_vt) { - return NULL; + return nullptr; } } return st->in(MemNode::ValueIn); @@ -1130,7 +1131,7 @@ Node* MemNode::can_see_stored_value(Node* st, PhaseTransform* phase) const { // can create new nodes. Think of it as lazily manifesting // virtually pre-existing constants.) if (memory_type() != T_VOID) { - if (ReduceBulkZeroing || find_array_copy_clone(phase, ld_alloc, in(MemNode::Memory)) == NULL) { + if (ReduceBulkZeroing || find_array_copy_clone(phase, ld_alloc, in(MemNode::Memory)) == nullptr) { // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done // by the ArrayCopyNode. @@ -1146,10 +1147,10 @@ Node* MemNode::can_see_stored_value(Node* st, PhaseTransform* phase) const { if (st->is_Proj() && st->in(0)->is_Initialize()) { InitializeNode* init = st->in(0)->as_Initialize(); AllocateNode* alloc = init->allocation(); - if ((alloc != NULL) && (alloc == ld_alloc)) { + if ((alloc != nullptr) && (alloc == ld_alloc)) { // examine a captured store value st = init->find_captured_store(ld_off, memory_size(), phase); - if (st != NULL) { + if (st != nullptr) { continue; // take one more trip around } } @@ -1157,12 +1158,12 @@ Node* MemNode::can_see_stored_value(Node* st, PhaseTransform* phase) const { // Load boxed value from result of valueOf() call is input parameter. if (this->is_Load() && ld_adr->is_AddP() && - (tp != NULL) && tp->is_ptr_to_boxed_value()) { + (tp != nullptr) && tp->is_ptr_to_boxed_value()) { intptr_t ignore = 0; Node* base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ignore); BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); base = bs->step_over_gc_barrier(base); - if (base != NULL && base->is_Proj() && + if (base != nullptr && base->is_Proj() && base->as_Proj()->_con == TypeFunc::Parms && base->in(0)->is_CallStaticJava() && base->in(0)->as_CallStaticJava()->is_boxing_method()) { @@ -1173,7 +1174,7 @@ Node* MemNode::can_see_stored_value(Node* st, PhaseTransform* phase) const { break; } - return NULL; + return nullptr; } //----------------------is_instance_field_load_with_local_phi------------------ @@ -1182,7 +1183,7 @@ bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) { in(Address)->is_AddP() ) { const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr(); // Only instances and boxed values. - if( t_oop != NULL && + if( t_oop != nullptr && (t_oop->is_ptr_to_boxed_value() || t_oop->is_known_instance_field()) && t_oop->offset() != Type::OffsetBot && @@ -1236,7 +1237,7 @@ Node* LoadNode::Identity(PhaseGVN* phase) { // Use _idx of address base (could be Phi node) for boxed values. intptr_t ignore = 0; Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore); - if (base == NULL) { + if (base == nullptr) { return this; } this_iid = base->_idx; @@ -1257,7 +1258,7 @@ Node* LoadNode::Identity(PhaseGVN* phase) { // Construct an equivalent unsigned load. Node* LoadNode::convert_to_unsigned_load(PhaseGVN& gvn) { BasicType bt = T_ILLEGAL; - const Type* rt = NULL; + const Type* rt = nullptr; switch (Opcode()) { case Op_LoadUB: return this; case Op_LoadUS: return this; @@ -1265,7 +1266,7 @@ Node* LoadNode::convert_to_unsigned_load(PhaseGVN& gvn) { case Op_LoadS: bt = T_CHAR; rt = TypeInt::CHAR; break; default: assert(false, "no unsigned variant: %s", Name()); - return NULL; + return nullptr; } return LoadNode::make(gvn, in(MemNode::Control), in(MemNode::Memory), in(MemNode::Address), raw_adr_type(), rt, bt, _mo, _control_dependency, @@ -1275,7 +1276,7 @@ Node* LoadNode::convert_to_unsigned_load(PhaseGVN& gvn) { // Construct an equivalent signed load. Node* LoadNode::convert_to_signed_load(PhaseGVN& gvn) { BasicType bt = T_ILLEGAL; - const Type* rt = NULL; + const Type* rt = nullptr; switch (Opcode()) { case Op_LoadUB: bt = T_BYTE; rt = TypeInt::BYTE; break; case Op_LoadUS: bt = T_SHORT; rt = TypeInt::SHORT; break; @@ -1285,7 +1286,7 @@ Node* LoadNode::convert_to_signed_load(PhaseGVN& gvn) { case Op_LoadL: return this; default: assert(false, "no signed variant: %s", Name()); - return NULL; + return nullptr; } return LoadNode::make(gvn, in(MemNode::Control), in(MemNode::Memory), in(MemNode::Address), raw_adr_type(), rt, bt, _mo, _control_dependency, @@ -1309,7 +1310,7 @@ Node* LoadNode::convert_to_reinterpret_load(PhaseGVN& gvn, const Type* rt) { assert(has_reinterpret_variant(rt), "no reinterpret variant: %s %s", Name(), type2name(bt)); bool is_mismatched = is_mismatched_access(); const TypeRawPtr* raw_type = gvn.type(in(MemNode::Memory))->isa_rawptr(); - if (raw_type == NULL) { + if (raw_type == nullptr) { is_mismatched = true; // conservatively match all non-raw accesses as mismatched } const int op = Opcode(); @@ -1343,7 +1344,7 @@ Node* StoreNode::convert_to_reinterpret_store(PhaseGVN& gvn, Node* val, const Ty bool is_mismatched = is_mismatched_access(); const TypeRawPtr* raw_type = gvn.type(in(MemNode::Memory))->isa_rawptr(); - if (raw_type == NULL) { + if (raw_type == nullptr) { is_mismatched = true; // conservatively match all non-raw accesses as mismatched } if (is_mismatched) { @@ -1361,11 +1362,11 @@ Node* LoadNode::eliminate_autobox(PhaseIterGVN* igvn) { assert(igvn->C->eliminate_boxing(), "sanity"); intptr_t ignore = 0; Node* base = AddPNode::Ideal_base_and_offset(in(Address), igvn, ignore); - if ((base == NULL) || base->is_Phi()) { + if ((base == nullptr) || base->is_Phi()) { // Push the loads from the phi that comes from valueOf up // through it to allow elimination of the loads and the recovery // of the original value. It is done in split_through_phi(). - return NULL; + return nullptr; } else if (base->is_Load() || (base->is_DecodeN() && base->in(1)->is_Load())) { // Eliminate the load of boxed value for integer types from the cache @@ -1377,17 +1378,17 @@ Node* LoadNode::eliminate_autobox(PhaseIterGVN* igvn) { base = base->in(1); } if (!base->in(Address)->is_AddP()) { - return NULL; // Complex address + return nullptr; // Complex address } AddPNode* address = base->in(Address)->as_AddP(); Node* cache_base = address->in(AddPNode::Base); - if ((cache_base != NULL) && cache_base->is_DecodeN()) { + if ((cache_base != nullptr) && cache_base->is_DecodeN()) { // Get ConP node which is static 'cache' field. cache_base = cache_base->in(1); } - if ((cache_base != NULL) && cache_base->is_Con()) { + if ((cache_base != nullptr) && cache_base->is_Con()) { const TypeAryPtr* base_type = cache_base->bottom_type()->isa_aryptr(); - if ((base_type != NULL) && base_type->is_autobox_cache()) { + if ((base_type != nullptr) && base_type->is_autobox_cache()) { Node* elements[4]; int shift = exact_log2(type2aelembytes(T_OBJECT)); int count = address->unpack_offsets(elements, ARRAY_SIZE(elements)); @@ -1412,11 +1413,11 @@ Node* LoadNode::eliminate_autobox(PhaseIterGVN* igvn) { bt == T_INT || bt == T_LONG, "wrong type = %s", type2name(bt)); jlong cache_low = (bt == T_LONG) ? c.as_long() : c.as_int(); if (cache_low != (int)cache_low) { - return NULL; // should not happen since cache is array indexed by value + return nullptr; // should not happen since cache is array indexed by value } jlong offset = arrayOopDesc::base_offset_in_bytes(T_OBJECT) - (cache_low << shift); if (offset != (int)offset) { - return NULL; // should not happen since cache is array indexed by value + return nullptr; // should not happen since cache is array indexed by value } // Add up all the offsets making of the address of the load Node* result = elements[0]; @@ -1468,21 +1469,21 @@ Node* LoadNode::eliminate_autobox(PhaseIterGVN* igvn) { } } } - return NULL; + return nullptr; } static bool stable_phi(PhiNode* phi, PhaseGVN *phase) { Node* region = phi->in(0); - if (region == NULL) { + if (region == nullptr) { return false; // Wait stable graph } uint cnt = phi->req(); for (uint i = 1; i < cnt; i++) { Node* rc = region->in(i); - if (rc == NULL || phase->type(rc) == Type::TOP) + if (rc == nullptr || phase->type(rc) == Type::TOP) return false; // Wait stable graph Node* in = phi->in(i); - if (in == NULL || phase->type(in) == Type::TOP) + if (in == nullptr || phase->type(in) == Type::TOP) return false; // Wait stable graph } return true; @@ -1493,32 +1494,32 @@ Node* LoadNode::split_through_phi(PhaseGVN* phase) { if (req() > 3) { assert(is_LoadVector() && Opcode() != Op_LoadVector, "load has too many inputs"); // LoadVector subclasses such as LoadVectorMasked have extra inputs that the logic below doesn't take into account - return NULL; + return nullptr; } Node* mem = in(Memory); Node* address = in(Address); const TypeOopPtr *t_oop = phase->type(address)->isa_oopptr(); - assert((t_oop != NULL) && + assert((t_oop != nullptr) && (t_oop->is_known_instance_field() || t_oop->is_ptr_to_boxed_value()), "invalide conditions"); Compile* C = phase->C; intptr_t ignore = 0; Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore); - bool base_is_phi = (base != NULL) && base->is_Phi(); + bool base_is_phi = (base != nullptr) && base->is_Phi(); bool load_boxed_values = t_oop->is_ptr_to_boxed_value() && C->aggressive_unboxing() && - (base != NULL) && (base == address->in(AddPNode::Base)) && + (base != nullptr) && (base == address->in(AddPNode::Base)) && phase->type(base)->higher_equal(TypePtr::NOTNULL); if (!((mem->is_Phi() || base_is_phi) && (load_boxed_values || t_oop->is_known_instance_field()))) { - return NULL; // memory is not Phi + return nullptr; // memory is not Phi } if (mem->is_Phi()) { if (!stable_phi(mem->as_Phi(), phase)) { - return NULL; // Wait stable graph + return nullptr; // Wait stable graph } uint cnt = mem->req(); // Check for loop invariant memory. @@ -1543,14 +1544,14 @@ Node* LoadNode::split_through_phi(PhaseGVN* phase) { } if (base_is_phi) { if (!stable_phi(base->as_Phi(), phase)) { - return NULL; // Wait stable graph + return nullptr; // Wait stable graph } uint cnt = base->req(); // Check for loop invariant memory. if (cnt == 3) { for (uint i = 1; i < cnt; i++) { if (base->in(i) == base) { - return NULL; // Wait stable graph + return nullptr; // Wait stable graph } } } @@ -1562,7 +1563,7 @@ Node* LoadNode::split_through_phi(PhaseGVN* phase) { // Do nothing here if Identity will find a value // (to avoid infinite chain of value phis generation). if (this != Identity(phase)) { - return NULL; + return nullptr; } // Select Region to split through. @@ -1572,13 +1573,13 @@ Node* LoadNode::split_through_phi(PhaseGVN* phase) { region = mem->in(0); // Skip if the region dominates some control edge of the address. if (!MemNode::all_controls_dominate(address, region)) - return NULL; + return nullptr; } else if (!mem->is_Phi()) { assert(base_is_phi, "sanity"); region = base->in(0); // Skip if the region dominates some control edge of the memory. if (!MemNode::all_controls_dominate(mem, region)) - return NULL; + return nullptr; } else if (base->in(0) != mem->in(0)) { assert(base_is_phi && mem->is_Phi(), "sanity"); if (MemNode::all_controls_dominate(mem, base->in(0))) { @@ -1586,7 +1587,7 @@ Node* LoadNode::split_through_phi(PhaseGVN* phase) { } else if (MemNode::all_controls_dominate(address, mem->in(0))) { region = mem->in(0); } else { - return NULL; // complex graph + return nullptr; // complex graph } } else { assert(base->in(0) == mem->in(0), "sanity"); @@ -1602,17 +1603,17 @@ Node* LoadNode::split_through_phi(PhaseGVN* phase) { this_iid = base->_idx; } PhaseIterGVN* igvn = phase->is_IterGVN(); - Node* phi = new PhiNode(region, this_type, NULL, mem->_idx, this_iid, this_index, this_offset); + Node* phi = new PhiNode(region, this_type, nullptr, mem->_idx, this_iid, this_index, this_offset); for (uint i = 1; i < region->req(); i++) { Node* x; - Node* the_clone = NULL; + Node* the_clone = nullptr; Node* in = region->in(i); if (region->is_CountedLoop() && region->as_Loop()->is_strip_mined() && i == LoopNode::EntryControl && - in != NULL && in->is_OuterStripMinedLoop()) { + in != nullptr && in->is_OuterStripMinedLoop()) { // No node should go in the outer strip mined loop in = in->in(LoopNode::EntryControl); } - if (in == NULL || in == C->top()) { + if (in == nullptr || in == C->top()) { x = C->top(); // Dead path? Use a dead data op } else { x = this->clone(); // Else clone up the data op @@ -1621,7 +1622,7 @@ Node* LoadNode::split_through_phi(PhaseGVN* phase) { if (this->in(0) == region) { x->set_req(0, in); } else { - x->set_req(0, NULL); + x->set_req(0, nullptr); } if (mem->is_Phi() && (mem->in(0) == region)) { x->set_req(Memory, mem->in(i)); // Use pre-Phi input for the clone. @@ -1673,7 +1674,7 @@ Node* LoadNode::split_through_phi(PhaseGVN* phase) { } } } - if (x != the_clone && the_clone != NULL) { + if (x != the_clone && the_clone != nullptr) { igvn->remove_dead_node(the_clone); } phi->set_req(i, x); @@ -1688,14 +1689,14 @@ AllocateNode* LoadNode::is_new_object_mark_load(PhaseGVN *phase) const { Node* address = in(MemNode::Address); AllocateNode* alloc = AllocateNode::Ideal_allocation(address, phase); Node* mem = in(MemNode::Memory); - if (alloc != NULL && mem->is_Proj() && - mem->in(0) != NULL && + if (alloc != nullptr && mem->is_Proj() && + mem->in(0) != nullptr && mem->in(0) == alloc->initialization() && - alloc->initialization()->proj_out_or_null(0) != NULL) { + alloc->initialization()->proj_out_or_null(0) != nullptr) { return alloc; } } - return NULL; + return nullptr; } @@ -1706,10 +1707,10 @@ AllocateNode* LoadNode::is_new_object_mark_load(PhaseGVN *phase) const { // try to hook me up to the exact initializing store. Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (has_pinned_control_dependency()) { - return NULL; + return nullptr; } Node* p = MemNode::Ideal_common(phase, can_reshape); - if (p) return (p == NodeSentinel) ? NULL : p; + if (p) return (p == NodeSentinel) ? nullptr : p; Node* ctrl = in(MemNode::Control); Node* address = in(MemNode::Address); @@ -1720,7 +1721,7 @@ Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Skip up past a SafePoint control. Cannot do this for Stores because // pointer stores & cardmarks must stay on the same side of a SafePoint. - if( ctrl != NULL && ctrl->Opcode() == Op_SafePoint && + if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw && !addr_mark && (depends_only_on_test() || has_unknown_control_dependency())) { @@ -1731,15 +1732,15 @@ Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) { intptr_t ignore = 0; Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore); - if (base != NULL + if (base != nullptr && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) { // Check for useless control edge in some common special cases - if (in(MemNode::Control) != NULL + if (in(MemNode::Control) != nullptr && can_remove_control() && phase->type(base)->higher_equal(TypePtr::NOTNULL) && all_controls_dominate(base, phase->C->start())) { // A method-invariant, non-null address (constant or 'this' argument). - set_req(MemNode::Control, NULL); + set_req(MemNode::Control, nullptr); progress = true; } } @@ -1747,32 +1748,32 @@ Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) { Node* mem = in(MemNode::Memory); const TypePtr *addr_t = phase->type(address)->isa_ptr(); - if (can_reshape && (addr_t != NULL)) { + if (can_reshape && (addr_t != nullptr)) { // try to optimize our memory input Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase); if (opt_mem != mem) { set_req_X(MemNode::Memory, opt_mem, phase); - if (phase->type( opt_mem ) == Type::TOP) return NULL; + if (phase->type( opt_mem ) == Type::TOP) return nullptr; return this; } const TypeOopPtr *t_oop = addr_t->isa_oopptr(); - if ((t_oop != NULL) && + if ((t_oop != nullptr) && (t_oop->is_known_instance_field() || t_oop->is_ptr_to_boxed_value())) { PhaseIterGVN *igvn = phase->is_IterGVN(); - assert(igvn != NULL, "must be PhaseIterGVN when can_reshape is true"); + assert(igvn != nullptr, "must be PhaseIterGVN when can_reshape is true"); if (igvn->_worklist.member(opt_mem)) { // Delay this transformation until memory Phi is processed. igvn->_worklist.push(this); - return NULL; + return nullptr; } // Split instance field load through Phi. Node* result = split_through_phi(phase); - if (result != NULL) return result; + if (result != nullptr) return result; if (t_oop->is_ptr_to_boxed_value()) { Node* result = eliminate_autobox(igvn); - if (result != NULL) return result; + if (result != nullptr) return result; } } } @@ -1780,16 +1781,16 @@ Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Is there a dominating load that loads the same value? Leave // anything that is not a load of a field/array element (like // barriers etc.) alone - if (in(0) != NULL && !adr_type()->isa_rawptr() && can_reshape) { + if (in(0) != nullptr && !adr_type()->isa_rawptr() && can_reshape) { for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) { Node *use = mem->fast_out(i); if (use != this && use->Opcode() == Opcode() && - use->in(0) != NULL && + use->in(0) != nullptr && use->in(0) != in(0) && use->in(Address) == in(Address)) { Node* ctl = in(0); - for (int i = 0; i < 10 && ctl != NULL; i++) { + for (int i = 0; i < 10 && ctl != nullptr; i++) { ctl = IfNode::up_one_dom(ctl); if (ctl == use->in(0)) { set_req(0, use->in(0)); @@ -1813,14 +1814,14 @@ Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) { // the alias index stuff. So instead, peek through Stores and IFF we can // fold up, do so. Node* prev_mem = find_previous_store(phase); - if (prev_mem != NULL) { + if (prev_mem != nullptr) { Node* value = can_see_arraycopy_value(prev_mem, phase); - if (value != NULL) { + if (value != nullptr) { return value; } } // Steps (a), (b): Walk past independent stores to find an exact match. - if (prev_mem != NULL && prev_mem != in(MemNode::Memory)) { + if (prev_mem != nullptr && prev_mem != in(MemNode::Memory)) { // (c) See if we can fold up on the spot, but don't fold up here. // Fold-up might require truncation (for LoadB/LoadS/LoadUS) or // just return a prior value, which is done by Identity calls. @@ -1838,7 +1839,7 @@ Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) { return alloc->make_ideal_mark(phase, address, control, mem); } - return progress ? this : NULL; + return progress ? this : nullptr; } // Helper to recognize certain Klass fields which are invariant across @@ -1865,7 +1866,7 @@ LoadNode::load_array_final_field(const TypeKlassPtr *tkls, } // No match. - return NULL; + return nullptr; } //------------------------------Value----------------------------------------- @@ -1876,7 +1877,7 @@ const Type* LoadNode::Value(PhaseGVN* phase) const { if (t1 == Type::TOP) return Type::TOP; Node* adr = in(MemNode::Address); const TypePtr* tp = phase->type(adr)->isa_ptr(); - if (tp == NULL || tp->empty()) return Type::TOP; + if (tp == nullptr || tp->empty()) return Type::TOP; int off = tp->offset(); assert(off != Type::OffsetTop, "case covered by TypePtr::empty"); Compile* C = phase->C; @@ -1899,12 +1900,12 @@ const Type* LoadNode::Value(PhaseGVN* phase) const { if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) { // Make sure the reference is not into the header and the offset is constant ciObject* aobj = ary->const_oop(); - if (aobj != NULL && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) { + if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) { int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0); const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off, stable_dimension, memory_type(), is_unsigned()); - if (con_type != NULL) { + if (con_type != nullptr) { return con_type; } } @@ -1925,8 +1926,8 @@ const Type* LoadNode::Value(PhaseGVN* phase) const { // In fact, that could have been the original type of p1, and p1 could have // had an original form like p1:(AddP x x (LShiftL quux 3)), where the // expression (LShiftL quux 3) independently optimized to the constant 8. - if ((t->isa_int() == NULL) && (t->isa_long() == NULL) - && (_type->isa_vect() == NULL) + if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr) + && (_type->isa_vect() == nullptr) && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) { // t might actually be lower than _type, if _type is a unique // concrete subclass of abstract class t. @@ -1941,13 +1942,13 @@ const Type* LoadNode::Value(PhaseGVN* phase) const { if (phase->C->eliminate_boxing() && adr->is_AddP()) { // The pointers in the autobox arrays are always non-null Node* base = adr->in(AddPNode::Base); - if ((base != NULL) && base->is_DecodeN()) { + if ((base != nullptr) && base->is_DecodeN()) { // Get LoadN node which loads IntegerCache.cache field base = base->in(1); } - if ((base != NULL) && base->is_Con()) { + if ((base != nullptr) && base->is_Con()) { const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr(); - if ((base_type != NULL) && base_type->is_autobox_cache()) { + if ((base_type != nullptr) && base_type->is_autobox_cache()) { // It could be narrow oop assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity"); } @@ -1969,9 +1970,9 @@ const Type* LoadNode::Value(PhaseGVN* phase) const { // Optimize loads from constant fields. const TypeInstPtr* tinst = tp->is_instptr(); ciObject* const_oop = tinst->const_oop(); - if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != NULL && const_oop->is_instance()) { + if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) { const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), memory_type()); - if (con_type != NULL) { + if (con_type != nullptr) { return con_type; } } @@ -1993,7 +1994,7 @@ const Type* LoadNode::Value(PhaseGVN* phase) const { */ Node* adr2 = adr->in(MemNode::Address); const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr(); - if (tkls != NULL && !StressReflectiveCode) { + if (tkls != nullptr && !StressReflectiveCode) { ciKlass* klass = tkls->klass(); if (klass->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) { assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror"); @@ -2004,7 +2005,7 @@ const Type* LoadNode::Value(PhaseGVN* phase) const { } const TypeKlassPtr *tkls = tp->isa_klassptr(); - if (tkls != NULL && !StressReflectiveCode) { + if (tkls != nullptr && !StressReflectiveCode) { ciKlass* klass = tkls->klass(); if (klass->is_loaded() && tkls->klass_is_exact()) { // We are loading a field from a Klass metaobject whose identity @@ -2027,7 +2028,7 @@ const Type* LoadNode::Value(PhaseGVN* phase) const { return ss ? TypeKlassPtr::make(ss) : TypePtr::NULL_PTR; } const Type* aift = load_array_final_field(tkls, klass); - if (aift != NULL) return aift; + if (aift != nullptr) return aift; } // We can still check if we are loading from the primary_supers array at a @@ -2076,17 +2077,17 @@ const Type* LoadNode::Value(PhaseGVN* phase) const { // if the load is provably beyond the header of the object. // (Also allow a variable load from a fresh array to produce zero.) const TypeOopPtr *tinst = tp->isa_oopptr(); - bool is_instance = (tinst != NULL) && tinst->is_known_instance_field(); - bool is_boxed_value = (tinst != NULL) && tinst->is_ptr_to_boxed_value(); + bool is_instance = (tinst != nullptr) && tinst->is_known_instance_field(); + bool is_boxed_value = (tinst != nullptr) && tinst->is_ptr_to_boxed_value(); if (ReduceFieldZeroing || is_instance || is_boxed_value) { Node* value = can_see_stored_value(mem,phase); - if (value != NULL && value->is_Con()) { + if (value != nullptr && value->is_Con()) { assert(value->bottom_type()->higher_equal(_type),"sanity"); return value->bottom_type(); } } - bool is_vect = (_type->isa_vect() != NULL); + bool is_vect = (_type->isa_vect() != nullptr); if (is_instance && !is_vect) { // If we have an instance type and our memory input is the // programs's initial memory state, there is no matching store, @@ -2100,7 +2101,7 @@ const Type* LoadNode::Value(PhaseGVN* phase) const { } Node* alloc = is_new_object_mark_load(phase); - if (alloc != NULL && !(alloc->Opcode() == Op_Allocate && UseBiasedLocking)) { + if (alloc != nullptr && !(alloc->Opcode() == Op_Allocate && UseBiasedLocking)) { return TypeX::make(markWord::prototype().value()); } @@ -2123,7 +2124,7 @@ uint LoadNode::match_edge(uint idx) const { Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) { Node* mem = in(MemNode::Memory); Node* value = can_see_stored_value(mem,phase); - if (value != NULL) { + if (value != nullptr) { Node* narrow = Compile::narrow_value(T_BYTE, value, _type, phase, false); if (narrow != value) { return narrow; @@ -2136,7 +2137,7 @@ Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) { const Type* LoadBNode::Value(PhaseGVN* phase) const { Node* mem = in(MemNode::Memory); Node* value = can_see_stored_value(mem,phase); - if (value != NULL && value->is_Con() && + if (value != nullptr && value->is_Con() && !value->bottom_type()->higher_equal(_type)) { // If the input to the store does not fit with the load's result type, // it must be truncated. We can't delay until Ideal call since @@ -2157,7 +2158,7 @@ const Type* LoadBNode::Value(PhaseGVN* phase) const { Node* LoadUBNode::Ideal(PhaseGVN* phase, bool can_reshape) { Node* mem = in(MemNode::Memory); Node* value = can_see_stored_value(mem, phase); - if (value != NULL) { + if (value != nullptr) { Node* narrow = Compile::narrow_value(T_BOOLEAN, value, _type, phase, false); if (narrow != value) { return narrow; @@ -2170,7 +2171,7 @@ Node* LoadUBNode::Ideal(PhaseGVN* phase, bool can_reshape) { const Type* LoadUBNode::Value(PhaseGVN* phase) const { Node* mem = in(MemNode::Memory); Node* value = can_see_stored_value(mem,phase); - if (value != NULL && value->is_Con() && + if (value != nullptr && value->is_Con() && !value->bottom_type()->higher_equal(_type)) { // If the input to the store does not fit with the load's result type, // it must be truncated. We can't delay until Ideal call since @@ -2191,7 +2192,7 @@ const Type* LoadUBNode::Value(PhaseGVN* phase) const { Node* LoadUSNode::Ideal(PhaseGVN* phase, bool can_reshape) { Node* mem = in(MemNode::Memory); Node* value = can_see_stored_value(mem,phase); - if (value != NULL) { + if (value != nullptr) { Node* narrow = Compile::narrow_value(T_CHAR, value, _type, phase, false); if (narrow != value) { return narrow; @@ -2204,7 +2205,7 @@ Node* LoadUSNode::Ideal(PhaseGVN* phase, bool can_reshape) { const Type* LoadUSNode::Value(PhaseGVN* phase) const { Node* mem = in(MemNode::Memory); Node* value = can_see_stored_value(mem,phase); - if (value != NULL && value->is_Con() && + if (value != nullptr && value->is_Con() && !value->bottom_type()->higher_equal(_type)) { // If the input to the store does not fit with the load's result type, // it must be truncated. We can't delay until Ideal call since @@ -2225,7 +2226,7 @@ const Type* LoadUSNode::Value(PhaseGVN* phase) const { Node* LoadSNode::Ideal(PhaseGVN* phase, bool can_reshape) { Node* mem = in(MemNode::Memory); Node* value = can_see_stored_value(mem,phase); - if (value != NULL) { + if (value != nullptr) { Node* narrow = Compile::narrow_value(T_SHORT, value, _type, phase, false); if (narrow != value) { return narrow; @@ -2238,7 +2239,7 @@ Node* LoadSNode::Ideal(PhaseGVN* phase, bool can_reshape) { const Type* LoadSNode::Value(PhaseGVN* phase) const { Node* mem = in(MemNode::Memory); Node* value = can_see_stored_value(mem,phase); - if (value != NULL && value->is_Con() && + if (value != nullptr && value->is_Con() && !value->bottom_type()->higher_equal(_type)) { // If the input to the store does not fit with the load's result type, // it must be truncated. We can't delay until Ideal call since @@ -2255,7 +2256,7 @@ const Type* LoadSNode::Value(PhaseGVN* phase) const { Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) { // sanity check the alias category against the created node type const TypePtr *adr_type = adr->bottom_type()->isa_ptr(); - assert(adr_type != NULL, "expecting TypeKlassPtr"); + assert(adr_type != nullptr, "expecting TypeKlassPtr"); #ifdef _LP64 if (adr_type->is_ptr_to_narrowklass()) { assert(UseCompressedClassPointers, "no compressed klasses"); @@ -2291,7 +2292,7 @@ const Type* LoadNode::klass_value_common(PhaseGVN* phase) const { // Return a more precise klass, if possible const TypeInstPtr *tinst = tp->isa_instptr(); - if (tinst != NULL) { + if (tinst != nullptr) { ciInstanceKlass* ik = tinst->klass()->as_instance_klass(); int offset = tinst->offset(); if (ik == phase->C->env()->Class_klass() @@ -2301,7 +2302,7 @@ const Type* LoadNode::klass_value_common(PhaseGVN* phase) const { // the field which points to the VM's Klass metaobject. ciType* t = tinst->java_mirror_type(); // java_mirror_type returns non-null for compile-time Class constants. - if (t != NULL) { + if (t != nullptr) { // constant oop => constant klass if (offset == java_lang_Class::array_klass_offset()) { if (t->is_void()) { @@ -2312,7 +2313,7 @@ const Type* LoadNode::klass_value_common(PhaseGVN* phase) const { return TypeKlassPtr::make(ciArrayKlass::make(t)); } if (!t->is_klass()) { - // a primitive Class (e.g., int.class) has NULL for a klass field + // a primitive Class (e.g., int.class) has null for a klass field return TypePtr::NULL_PTR; } // (Folds up the 1st indirection in aClassConstant.getModifiers().) @@ -2345,9 +2346,9 @@ const Type* LoadNode::klass_value_common(PhaseGVN* phase) const { // Check for loading klass from an array const TypeAryPtr *tary = tp->isa_aryptr(); - if( tary != NULL ) { + if( tary != nullptr ) { ciKlass *tary_klass = tary->klass(); - if (tary_klass != NULL // can be NULL when at BOTTOM or TOP + if (tary_klass != nullptr // can be null when at BOTTOM or TOP && tary->offset() == oopDesc::klass_offset_in_bytes()) { if (tary->klass_is_exact()) { return TypeKlassPtr::make(tary_klass); @@ -2380,7 +2381,7 @@ const Type* LoadNode::klass_value_common(PhaseGVN* phase) const { // Check for loading klass from an array klass const TypeKlassPtr *tkls = tp->isa_klassptr(); - if (tkls != NULL && !StressReflectiveCode) { + if (tkls != nullptr && !StressReflectiveCode) { ciKlass* klass = tkls->klass(); if( !klass->is_loaded() ) return _type; // Bail out if not loaded @@ -2424,9 +2425,9 @@ Node* LoadNode::klass_identity_common(PhaseGVN* phase) { Node* adr = in(MemNode::Address); intptr_t offset = 0; Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset); - if (base == NULL) return this; + if (base == nullptr) return this; const TypeOopPtr* toop = phase->type(adr)->isa_oopptr(); - if (toop == NULL) return this; + if (toop == nullptr) return this; // Step over potential GC barrier for OopHandle resolve BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); @@ -2438,7 +2439,7 @@ Node* LoadNode::klass_identity_common(PhaseGVN* phase) { // This works even if the klass is not constant (clone or newArray). if (offset == oopDesc::klass_offset_in_bytes()) { Node* allocated_klass = AllocateNode::Ideal_klass(base, phase); - if (allocated_klass != NULL) { + if (allocated_klass != nullptr) { return allocated_klass; } } @@ -2459,7 +2460,7 @@ Node* LoadNode::klass_identity_common(PhaseGVN* phase) { if (base2->is_Load()) { /* direct load of a load which is the OopHandle */ Node* adr2 = base2->in(MemNode::Address); const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr(); - if (tkls != NULL && !tkls->empty() + if (tkls != nullptr && !tkls->empty() && (tkls->klass()->is_instance_klass() || tkls->klass()->is_array_klass()) && adr2->is_AddP() @@ -2519,22 +2520,22 @@ const Type* LoadRangeNode::Value(PhaseGVN* phase) const { // Feed through the length in AllocateArray(...length...)._length. Node *LoadRangeNode::Ideal(PhaseGVN *phase, bool can_reshape) { Node* p = MemNode::Ideal_common(phase, can_reshape); - if (p) return (p == NodeSentinel) ? NULL : p; + if (p) return (p == NodeSentinel) ? nullptr : p; // Take apart the address into an oop and and offset. // Return 'this' if we cannot. Node* adr = in(MemNode::Address); intptr_t offset = 0; Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset); - if (base == NULL) return NULL; + if (base == nullptr) return nullptr; const TypeAryPtr* tary = phase->type(adr)->isa_aryptr(); - if (tary == NULL) return NULL; + if (tary == nullptr) return nullptr; // We can fetch the length directly through an AllocateArrayNode. // This works even if the length is not constant (clone or newArray). if (offset == arrayOopDesc::length_offset_in_bytes()) { AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(base, phase); - if (alloc != NULL) { + if (alloc != nullptr) { Node* allocated_length = alloc->Ideal_length(); Node* len = alloc->make_ideal_length(tary, phase); if (allocated_length != len) { @@ -2544,7 +2545,7 @@ Node *LoadRangeNode::Ideal(PhaseGVN *phase, bool can_reshape) { } } - return NULL; + return nullptr; } //------------------------------Identity--------------------------------------- @@ -2558,15 +2559,15 @@ Node* LoadRangeNode::Identity(PhaseGVN* phase) { Node* adr = in(MemNode::Address); intptr_t offset = 0; Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset); - if (base == NULL) return this; + if (base == nullptr) return this; const TypeAryPtr* tary = phase->type(adr)->isa_aryptr(); - if (tary == NULL) return this; + if (tary == nullptr) return this; // We can fetch the length directly through an AllocateArrayNode. // This works even if the length is not constant (clone or newArray). if (offset == arrayOopDesc::length_offset_in_bytes()) { AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(base, phase); - if (alloc != NULL) { + if (alloc != nullptr) { Node* allocated_length = alloc->Ideal_length(); // Do not allow make_ideal_length to allocate a CastII node. Node* len = alloc->make_ideal_length(tary, phase, false); @@ -2588,7 +2589,7 @@ StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const assert((mo == unordered || mo == release), "unexpected"); Compile* C = gvn.C; assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw || - ctl != NULL, "raw memory operations should have control edge"); + ctl != nullptr, "raw memory operations should have control edge"); switch (bt) { case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case @@ -2618,7 +2619,7 @@ StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const } default: ShouldNotReachHere(); - return (StoreNode*)NULL; + return (StoreNode*)nullptr; } } @@ -2642,7 +2643,7 @@ uint StoreNode::hash() const { // try to capture it into the initialization, or hoist it above. Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) { Node* p = MemNode::Ideal_common(phase, can_reshape); - if (p) return (p == NodeSentinel) ? NULL : p; + if (p) return (p == NodeSentinel) ? nullptr : p; Node* mem = in(MemNode::Memory); Node* address = in(MemNode::Address); @@ -2700,7 +2701,7 @@ Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) { Node* moved = init->capture_store(this, offset, phase, can_reshape); // If the InitializeNode captured me, it made a raw copy of me, // and I need to disappear. - if (moved != NULL) { + if (moved != nullptr) { // %%% hack to ensure that Ideal returns a new node: mem = MergeMemNode::make(mem); return mem; // fold me away @@ -2721,7 +2722,7 @@ Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) { } } - return NULL; // No further progress + return nullptr; // No further progress } //------------------------------Value----------------------------------------- @@ -2778,9 +2779,9 @@ Node* StoreNode::Identity(PhaseGVN* phase) { // the store may also apply to zero-bits in an earlier object Node* prev_mem = find_previous_store(phase); // Steps (a), (b): Walk past independent stores to find an exact match. - if (prev_mem != NULL) { + if (prev_mem != nullptr) { Node* prev_val = can_see_stored_value(prev_mem, phase); - if (prev_val != NULL && prev_val == val) { + if (prev_val != nullptr && prev_val == val) { // prev_val and val might differ by a cast; it would be good // to keep the more informative of the two. result = mem; @@ -2790,12 +2791,12 @@ Node* StoreNode::Identity(PhaseGVN* phase) { } PhaseIterGVN* igvn = phase->is_IterGVN(); - if (result != this && igvn != NULL) { + if (result != this && igvn != nullptr) { MemBarNode* trailing = trailing_membar(); - if (trailing != NULL) { + if (trailing != nullptr) { #ifdef ASSERT const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr(); - assert(t_oop == NULL || t_oop->is_known_instance_field(), "only for non escaping objects"); + assert(t_oop == nullptr || t_oop->is_known_instance_field(), "only for non escaping objects"); #endif trailing->remove(igvn); } @@ -2831,7 +2832,7 @@ Node *StoreNode::Ideal_masked_input(PhaseGVN *phase, uint mask) { return this; } } - return NULL; + return nullptr; } @@ -2855,7 +2856,7 @@ Node *StoreNode::Ideal_sign_extended_input(PhaseGVN *phase, int num_bits) { } } } - return NULL; + return nullptr; } //------------------------------value_never_loaded----------------------------------- @@ -2866,7 +2867,7 @@ Node *StoreNode::Ideal_sign_extended_input(PhaseGVN *phase, int num_bits) { bool StoreNode::value_never_loaded( PhaseTransform *phase) const { Node *adr = in(Address); const TypeOopPtr *adr_oop = phase->type(adr)->isa_oopptr(); - if (adr_oop == NULL) + if (adr_oop == nullptr) return false; if (!adr_oop->is_known_instance_field()) return false; // if not a distinct instance, there may be aliases of the address @@ -2881,13 +2882,13 @@ bool StoreNode::value_never_loaded( PhaseTransform *phase) const { MemBarNode* StoreNode::trailing_membar() const { if (is_release()) { - MemBarNode* trailing_mb = NULL; + MemBarNode* trailing_mb = nullptr; for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) { Node* u = fast_out(i); if (u->is_MemBar()) { if (u->as_MemBar()->trailing_store()) { assert(u->Opcode() == Op_MemBarVolatile, ""); - assert(trailing_mb == NULL, "only one"); + assert(trailing_mb == nullptr, "only one"); trailing_mb = u->as_MemBar(); #ifdef ASSERT Node* leading = u->as_MemBar()->leading_membar(); @@ -2902,7 +2903,7 @@ MemBarNode* StoreNode::trailing_membar() const { } return trailing_mb; } - return NULL; + return nullptr; } @@ -2913,10 +2914,10 @@ MemBarNode* StoreNode::trailing_membar() const { // (a left shift, then right shift) we can skip both. Node *StoreBNode::Ideal(PhaseGVN *phase, bool can_reshape){ Node *progress = StoreNode::Ideal_masked_input(phase, 0xFF); - if( progress != NULL ) return progress; + if( progress != nullptr ) return progress; progress = StoreNode::Ideal_sign_extended_input(phase, 24); - if( progress != NULL ) return progress; + if( progress != nullptr ) return progress; // Finally check the default case return StoreNode::Ideal(phase, can_reshape); @@ -2928,10 +2929,10 @@ Node *StoreBNode::Ideal(PhaseGVN *phase, bool can_reshape){ // we can skip the AND operation Node *StoreCNode::Ideal(PhaseGVN *phase, bool can_reshape){ Node *progress = StoreNode::Ideal_masked_input(phase, 0xFFFF); - if( progress != NULL ) return progress; + if( progress != nullptr ) return progress; progress = StoreNode::Ideal_sign_extended_input(phase, 16); - if( progress != NULL ) return progress; + if( progress != nullptr ) return progress; // Finally check the default case return StoreNode::Ideal(phase, can_reshape); @@ -2955,7 +2956,7 @@ Node* StoreCMNode::Identity(PhaseGVN* phase) { //------------------------------Ideal--------------------------------------- Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){ Node* progress = StoreNode::Ideal(phase, can_reshape); - if (progress != NULL) return progress; + if (progress != nullptr) return progress; Node* my_store = in(MemNode::OopStore); if (my_store->is_MergeMem()) { @@ -2964,7 +2965,7 @@ Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){ return this; } - return NULL; + return nullptr; } //------------------------------Value----------------------------------------- @@ -2983,7 +2984,7 @@ const Type* StoreCMNode::Value(PhaseGVN* phase) const { //----------------------------------SCMemProjNode------------------------------ const Type* SCMemProjNode::Value(PhaseGVN* phase) const { - if (in(0) == NULL || phase->type(in(0)) == Type::TOP) { + if (in(0) == nullptr || phase->type(in(0)) == Type::TOP) { return Type::TOP; } return bottom_type(); @@ -3039,13 +3040,13 @@ bool LoadStoreNode::result_not_used() const { } MemBarNode* LoadStoreNode::trailing_membar() const { - MemBarNode* trailing = NULL; + MemBarNode* trailing = nullptr; for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) { Node* u = fast_out(i); if (u->is_MemBar()) { if (u->as_MemBar()->trailing_load_store()) { assert(u->Opcode() == Op_MemBarAcquire, ""); - assert(trailing == NULL, "only one"); + assert(trailing == nullptr, "only one"); trailing = u->as_MemBar(); #ifdef ASSERT Node* leading = trailing->leading_membar(); @@ -3066,7 +3067,7 @@ uint LoadStoreNode::size_of() const { return sizeof(*this); } //============================================================================= //----------------------------------LoadStoreConditionalNode-------------------- -LoadStoreConditionalNode::LoadStoreConditionalNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex ) : LoadStoreNode(c, mem, adr, val, NULL, TypeInt::BOOL, 5) { +LoadStoreConditionalNode::LoadStoreConditionalNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex ) : LoadStoreNode(c, mem, adr, val, nullptr, TypeInt::BOOL, 5) { init_req(ExpectedIn, ex ); } @@ -3083,7 +3084,7 @@ const Type* LoadStoreConditionalNode::Value(PhaseGVN* phase) const { //-------------------------------adr_type-------------------------------------- const TypePtr* ClearArrayNode::adr_type() const { Node *adr = in(3); - if (adr == NULL) return NULL; // node is dead + if (adr == nullptr) return nullptr; // node is dead return MemNode::calculate_adr_type(adr->bottom_type()); } @@ -3103,36 +3104,36 @@ Node* ClearArrayNode::Identity(PhaseGVN* phase) { // Clearing a short array is faster with stores Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Already know this is a large node, do not try to ideal it - if (_is_large) return NULL; + if (_is_large) return nullptr; const int unit = BytesPerLong; const TypeX* t = phase->type(in(2))->isa_intptr_t(); - if (!t) return NULL; - if (!t->is_con()) return NULL; + if (!t) return nullptr; + if (!t->is_con()) return nullptr; intptr_t raw_count = t->get_con(); intptr_t size = raw_count; if (!Matcher::init_array_count_is_in_bytes) size *= unit; // Clearing nothing uses the Identity call. // Negative clears are possible on dead ClearArrays // (see jck test stmt114.stmt11402.val). - if (size <= 0 || size % unit != 0) return NULL; + if (size <= 0 || size % unit != 0) return nullptr; intptr_t count = size / unit; // Length too long; communicate this to matchers and assemblers. // Assemblers are responsible to produce fast hardware clears for it. if (size > InitArrayShortSize) { return new ClearArrayNode(in(0), in(1), in(2), in(3), true); } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) { - return NULL; + return nullptr; } - if (!IdealizeClearArrayNode) return NULL; + if (!IdealizeClearArrayNode) return nullptr; Node *mem = in(1); - if( phase->type(mem)==Type::TOP ) return NULL; + if( phase->type(mem)==Type::TOP ) return nullptr; Node *adr = in(3); const Type* at = phase->type(adr); - if( at==Type::TOP ) return NULL; + if( at==Type::TOP ) return nullptr; const TypePtr* atp = at->isa_ptr(); // adjust atp to be the correct array element address type - if (atp == NULL) atp = TypePtr::BOTTOM; + if (atp == nullptr) atp = TypePtr::BOTTOM; else atp = atp->add_offset(Type::OffsetBot); // Get base for derived pointer purposes if( adr->Opcode() != Op_AddP ) Unimplemented(); @@ -3162,14 +3163,14 @@ bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseTransform* p // during macro nodes expansion. Before that ClearArray nodes are // only generated in PhaseMacroExpand::generate_arraycopy() (before // Allocate nodes are expanded) which follows allocations. - assert(alloc != NULL, "should have allocation"); + assert(alloc != nullptr, "should have allocation"); if (alloc->_idx == instance_id) { // Can not bypass initialization of the instance we are looking for. return false; } // Otherwise skip it. InitializeNode* init = alloc->initialization(); - if (init != NULL) + if (init != nullptr) *np = init->in(TypeFunc::Memory); else *np = alloc->in(TypeFunc::Memory); @@ -3258,7 +3259,7 @@ Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest, //============================================================================= MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent) - : MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)), + : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)), _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone) #ifdef ASSERT , _pair_idx(0) @@ -3269,7 +3270,7 @@ MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent) init_req(TypeFunc::I_O,top); init_req(TypeFunc::FramePtr,top); init_req(TypeFunc::ReturnAdr,top); - if (precedent != NULL) + if (precedent != nullptr) init_req(TypeFunc::Parms, precedent); } @@ -3286,15 +3287,15 @@ MemBarNode* MemBarNode::make(Compile* C, int opcode, int atp, Node* pn) { case Op_LoadFence: return new LoadFenceNode(C, atp, pn); case Op_MemBarRelease: return new MemBarReleaseNode(C, atp, pn); case Op_StoreFence: return new StoreFenceNode(C, atp, pn); + case Op_MemBarStoreStore: return new MemBarStoreStoreNode(C, atp, pn); + case Op_StoreStoreFence: return new StoreStoreFenceNode(C, atp, pn); case Op_MemBarAcquireLock: return new MemBarAcquireLockNode(C, atp, pn); case Op_MemBarReleaseLock: return new MemBarReleaseLockNode(C, atp, pn); case Op_MemBarVolatile: return new MemBarVolatileNode(C, atp, pn); case Op_MemBarCPUOrder: return new MemBarCPUOrderNode(C, atp, pn); case Op_OnSpinWait: return new OnSpinWaitNode(C, atp, pn); case Op_Initialize: return new InitializeNode(C, atp, pn); - case Op_MemBarStoreStore: return new MemBarStoreStoreNode(C, atp, pn); - case Op_Blackhole: return new BlackholeNode(C, atp, pn); - default: ShouldNotReachHere(); return NULL; + default: ShouldNotReachHere(); return nullptr; } } @@ -3305,15 +3306,15 @@ void MemBarNode::remove(PhaseIterGVN *igvn) { } if (trailing_store() || trailing_load_store()) { MemBarNode* leading = leading_membar(); - if (leading != NULL) { + if (leading != nullptr) { assert(leading->trailing_membar() == this, "inconsistent leading/trailing membars"); leading->remove(igvn); } } - if (proj_out_or_null(TypeFunc::Memory) != NULL) { + if (proj_out_or_null(TypeFunc::Memory) != nullptr) { igvn->replace_node(proj_out(TypeFunc::Memory), in(TypeFunc::Memory)); } - if (proj_out_or_null(TypeFunc::Control) != NULL) { + if (proj_out_or_null(TypeFunc::Control) != nullptr) { igvn->replace_node(proj_out(TypeFunc::Control), in(TypeFunc::Control)); } } @@ -3325,7 +3326,7 @@ Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (remove_dead_region(phase, can_reshape)) return this; // Don't bother trying to transform a dead node if (in(0) && in(0)->is_top()) { - return NULL; + return nullptr; } bool progress = false; @@ -3337,7 +3338,7 @@ Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Volatile field loads and stores. Node* my_mem = in(MemBarNode::Precedent); // The MembarAquire may keep an unused LoadNode alive through the Precedent edge - if ((my_mem != NULL) && (opc == Op_MemBarAcquire) && (my_mem->outcnt() == 1)) { + if ((my_mem != nullptr) && (opc == Op_MemBarAcquire) && (my_mem->outcnt() == 1)) { // if the Precedent is a decodeN and its input (a Load) is used at more than one place, // replace this Precedent (decodeN) with the Load instead. if ((my_mem->Opcode() == Op_DecodeN) && (my_mem->in(1)->outcnt() > 1)) { @@ -3349,14 +3350,14 @@ Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) { assert(my_mem->unique_out() == this, "sanity"); del_req(Precedent); phase->is_IterGVN()->_worklist.push(my_mem); // remove dead node later - my_mem = NULL; + my_mem = nullptr; } progress = true; } - if (my_mem != NULL && my_mem->is_Mem()) { + if (my_mem != nullptr && my_mem->is_Mem()) { const TypeOopPtr* t_oop = my_mem->in(MemNode::Address)->bottom_type()->isa_oopptr(); // Check for scalar replaced object reference. - if( t_oop != NULL && t_oop->is_known_instance_field() && + if( t_oop != nullptr && t_oop->is_known_instance_field() && t_oop->offset() != Type::OffsetBot && t_oop->offset() != Type::OffsetTop) { eliminate = true; @@ -3365,7 +3366,7 @@ Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) { } else if (opc == Op_MemBarRelease) { // Final field stores. Node* alloc = AllocateNode::Ideal_allocation(in(MemBarNode::Precedent), phase); - if ((alloc != NULL) && alloc->is_Allocate() && + if ((alloc != nullptr) && alloc->is_Allocate() && alloc->as_Allocate()->does_not_escape_thread()) { // The allocated object does not escape. eliminate = true; @@ -3380,7 +3381,7 @@ Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) { return new ConINode(TypeInt::ZERO); } } - return progress ? this : NULL; + return progress ? this : nullptr; } //------------------------------Value------------------------------------------ @@ -3400,7 +3401,7 @@ Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) { return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj); } ShouldNotReachHere(); - return NULL; + return nullptr; } void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) { @@ -3430,7 +3431,7 @@ MemBarNode* MemBarNode::trailing_membar() const { Node* c = trailing; uint i = 0; do { - trailing = NULL; + trailing = nullptr; for (; i < c->outcnt(); i++) { Node* next = c->raw_out(i); if (next != c && next->is_CFG()) { @@ -3445,7 +3446,7 @@ MemBarNode* MemBarNode::trailing_membar() const { break; } } - if (trailing != NULL && !seen.test_set(trailing->_idx)) { + if (trailing != nullptr && !seen.test_set(trailing->_idx)) { break; } while (multis.size() > 0) { @@ -3471,10 +3472,10 @@ MemBarNode* MemBarNode::leading_membar() const { VectorSet seen; Node_Stack regions(0); Node* leading = in(0); - while (leading != NULL && (!leading->is_MemBar() || !leading->as_MemBar()->leading())) { - while (leading == NULL || leading->is_top() || seen.test_set(leading->_idx)) { - leading = NULL; - while (regions.size() > 0 && leading == NULL) { + while (leading != nullptr && (!leading->is_MemBar() || !leading->as_MemBar()->leading())) { + while (leading == nullptr || leading->is_top() || seen.test_set(leading->_idx)) { + leading = nullptr; + while (regions.size() > 0 && leading == nullptr) { Node* r = regions.node(); uint i = regions.index(); if (i < r->req()) { @@ -3484,9 +3485,9 @@ MemBarNode* MemBarNode::leading_membar() const { regions.pop(); } } - if (leading == NULL) { + if (leading == nullptr) { assert(regions.size() == 0, "all paths should have been tried"); - return NULL; + return nullptr; } } if (leading->is_Region()) { @@ -3505,7 +3506,7 @@ MemBarNode* MemBarNode::leading_membar() const { if (n->is_Region()) { for (uint j = 1; j < n->req(); j++) { Node* in = n->in(j); - if (in != NULL && !in->is_top()) { + if (in != nullptr && !in->is_top()) { wq.push(in); } } @@ -3515,16 +3516,16 @@ MemBarNode* MemBarNode::leading_membar() const { found++; } else { Node* in = n->in(0); - if (in != NULL && !in->is_top()) { + if (in != nullptr && !in->is_top()) { wq.push(in); } } } } - assert(found == 1 || (found == 0 && leading == NULL), "consistency check failed"); + assert(found == 1 || (found == 0 && leading == nullptr), "consistency check failed"); #endif - if (leading == NULL) { - return NULL; + if (leading == nullptr) { + return nullptr; } MemBarNode* mb = leading->as_MemBar(); assert((mb->_kind == LeadingStore && _kind == TrailingStore) || @@ -3533,26 +3534,6 @@ MemBarNode* MemBarNode::leading_membar() const { return mb; } -#ifndef PRODUCT -void BlackholeNode::format(PhaseRegAlloc* ra, outputStream* st) const { - st->print("blackhole "); - bool first = true; - for (uint i = 0; i < req(); i++) { - Node* n = in(i); - if (n != NULL && OptoReg::is_valid(ra->get_reg_first(n))) { - if (first) { - first = false; - } else { - st->print(", "); - } - char buf[128]; - ra->dump_register(n, buf); - st->print("%s", buf); - } - } - st->cr(); -} -#endif //===========================InitializeNode==================================== // SUMMARY: @@ -3654,7 +3635,7 @@ InitializeNode::InitializeNode(Compile* C, int adr_type, Node* rawoop) assert(adr_type == Compile::AliasIdxRaw, "only valid atp"); assert(in(RawAddress) == rawoop, "proper init"); - // Note: allocation() can be NULL, for secondary initialization barriers + // Note: allocation() can be null, for secondary initialization barriers } // Since this node is not matched, it will be processed by the @@ -3700,7 +3681,7 @@ void InitializeNode::set_complete(PhaseGVN* phase) { // return false if the init contains any stores already bool AllocateNode::maybe_set_complete(PhaseGVN* phase) { InitializeNode* init = initialization(); - if (init == NULL || init->is_complete()) return false; + if (init == nullptr || init->is_complete()) return false; init->remove_extra_zeroes(); // for now, if this allocation has already collected any inits, bail: if (init->is_non_zero()) return false; @@ -3730,7 +3711,7 @@ intptr_t InitializeNode::get_store_offset(Node* st, PhaseTransform* phase) { intptr_t offset = -1; Node* base = AddPNode::Ideal_base_and_offset(st->in(MemNode::Address), phase, offset); - if (base == NULL) return -1; // something is dead, + if (base == nullptr) return -1; // something is dead, if (offset < 0) return -1; // dead, dead return offset; } @@ -3753,7 +3734,7 @@ bool InitializeNode::detect_init_independence(Node* value, PhaseGVN* phase) { } Node* n = worklist.at(j); - if (n == NULL) continue; // (can this really happen?) + if (n == nullptr) continue; // (can this really happen?) if (n->is_Proj()) n = n->in(0); if (n == this) return false; // found a cycle if (n->is_Con()) continue; @@ -3766,7 +3747,7 @@ bool InitializeNode::detect_init_independence(Node* value, PhaseGVN* phase) { } Node* ctl = n->in(0); - if (ctl != NULL && !ctl->is_top()) { + if (ctl != nullptr && !ctl->is_top()) { if (ctl->is_Proj()) ctl = ctl->in(0); if (ctl == this) return false; @@ -3782,7 +3763,7 @@ bool InitializeNode::detect_init_independence(Node* value, PhaseGVN* phase) { // Check data edges for possible dependencies on 'this'. for (uint i = 1; i < n->req(); i++) { Node* m = n->in(i); - if (m == NULL || m == n || m->is_top()) continue; + if (m == nullptr || m == n || m->is_top()) continue; // Only process data inputs once worklist.push(m); @@ -3801,7 +3782,7 @@ intptr_t InitializeNode::can_capture_store(StoreNode* st, PhaseGVN* phase, bool if (st->req() != MemNode::ValueIn + 1) return FAIL; // an inscrutable StoreNode (card mark?) Node* ctl = st->in(MemNode::Control); - if (!(ctl != NULL && ctl->is_Proj() && ctl->in(0) == this)) + if (!(ctl != nullptr && ctl->is_Proj() && ctl->in(0) == this)) return FAIL; // must be unconditional after the initialization Node* mem = st->in(MemNode::Memory); if (!(mem->is_Proj() && mem->in(0) == this)) @@ -3809,7 +3790,7 @@ intptr_t InitializeNode::can_capture_store(StoreNode* st, PhaseGVN* phase, bool Node* adr = st->in(MemNode::Address); intptr_t offset; AllocateNode* alloc = AllocateNode::Ideal_allocation(adr, phase, offset); - if (alloc == NULL) + if (alloc == nullptr) return FAIL; // inscrutable address if (alloc != allocation()) return FAIL; // wrong allocation! (store needs to float up) @@ -3837,7 +3818,7 @@ intptr_t InitializeNode::can_capture_store(StoreNode* st, PhaseGVN* phase, bool ResourceMark rm; Unique_Node_List mems; mems.push(mem); - Node* unique_merge = NULL; + Node* unique_merge = nullptr; for (uint next = 0; next < mems.size(); ++next) { Node *m = mems.at(next); for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax; j++) { @@ -3847,7 +3828,7 @@ intptr_t InitializeNode::can_capture_store(StoreNode* st, PhaseGVN* phase, bool } if (n == st) { continue; - } else if (n->in(0) != NULL && n->in(0) != ctl) { + } else if (n->in(0) != nullptr && n->in(0) != ctl) { // If the control of this use is different from the control // of the Store which is right after the InitializeNode then // this node cannot be between the InitializeNode and the @@ -3869,7 +3850,7 @@ intptr_t InitializeNode::can_capture_store(StoreNode* st, PhaseGVN* phase, bool break; } else { const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr(); - if (other_t_adr != NULL) { + if (other_t_adr != nullptr) { int other_alias_idx = phase->C->get_alias_index(other_t_adr); if (other_alias_idx == alias_idx) { // A load from the same memory slice as the store right @@ -3880,7 +3861,7 @@ intptr_t InitializeNode::can_capture_store(StoreNode* st, PhaseGVN* phase, bool Node* base = other_adr; assert(base->is_AddP(), "should be addp but is %s", base->Name()); base = base->in(AddPNode::Base); - if (base != NULL) { + if (base != nullptr) { base = base->uncast(); if (base->is_Proj() && base->in(0) == alloc) { failed = true; @@ -3925,7 +3906,7 @@ int InitializeNode::captured_store_insertion_point(intptr_t start, if (is_complete()) return FAIL; // arraycopy got here first; punt - assert(allocation() != NULL, "must be present"); + assert(allocation() != nullptr, "must be present"); // no negatives, no header fields: if (start < (intptr_t) allocation()->minimum_header_size()) return FAIL; @@ -3977,7 +3958,7 @@ Node* InitializeNode::find_captured_store(intptr_t start, int size_in_bytes, assert(stores_are_sane(phase), ""); int i = captured_store_insertion_point(start, size_in_bytes, phase); if (i == 0) { - return NULL; // something is dead + return nullptr; // something is dead } else if (i < 0) { return zero_memory(); // just primordial zero bits here } else { @@ -4021,14 +4002,14 @@ Node* InitializeNode::capture_store(StoreNode* st, intptr_t start, PhaseGVN* phase, bool can_reshape) { assert(stores_are_sane(phase), ""); - if (start < 0) return NULL; + if (start < 0) return nullptr; assert(can_capture_store(st, phase, can_reshape) == start, "sanity"); Compile* C = phase->C; int size_in_bytes = st->memory_size(); int i = captured_store_insertion_point(start, size_in_bytes, phase); - if (i == 0) return NULL; // bail out - Node* prev_mem = NULL; // raw memory for the captured store + if (i == 0) return nullptr; // bail out + Node* prev_mem = nullptr; // raw memory for the captured store if (i > 0) { prev_mem = in(i); // there is a pre-existing store under this one set_req(i, C->top()); // temporarily disconnect it @@ -4060,7 +4041,7 @@ Node* InitializeNode::capture_store(StoreNode* st, intptr_t start, // The caller may now kill the old guy. DEBUG_ONLY(Node* check_st = find_captured_store(start, size_in_bytes, phase)); - assert(check_st == new_st || check_st == NULL, "must be findable"); + assert(check_st == new_st || check_st == nullptr, "must be findable"); assert(!is_complete(), ""); return new_st; } @@ -4188,7 +4169,7 @@ InitializeNode::coalesce_subword_stores(intptr_t header_size, st = nodes[j]; st_off -= BytesPerInt; con = intcon[0]; - if (con != 0 && st != NULL && st->Opcode() == Op_StoreI) { + if (con != 0 && st != nullptr && st->Opcode() == Op_StoreI) { assert(st_off >= header_size, "still ignoring header"); assert(get_store_offset(st, phase) == st_off, "must be"); assert(in(i-1) == zmem, "must be"); @@ -4197,7 +4178,7 @@ InitializeNode::coalesce_subword_stores(intptr_t header_size, // Undo the effects of the previous loop trip, which swallowed st: intcon[0] = 0; // undo store_constant() set_req(i-1, st); // undo set_req(i, zmem) - nodes[j] = NULL; // undo nodes[j] = st + nodes[j] = nullptr; // undo nodes[j] = st --old_subword; // undo ++old_subword } continue; // This StoreI is already optimal. @@ -4234,7 +4215,7 @@ InitializeNode::coalesce_subword_stores(intptr_t header_size, } Node* old = nodes[j]; - assert(old != NULL, "need the prior store"); + assert(old != nullptr, "need the prior store"); intptr_t offset = (j * BytesPerLong); bool split = !Matcher::isSimpleConstant64(con); @@ -4311,7 +4292,7 @@ InitializeNode::coalesce_subword_stores(intptr_t header_size, if (PrintCompilation && WizardMode) tty->print_cr("Changed %d/%d subword/long constants into %d/%d int/long", old_subword, old_long, new_int, new_long); - if (C->log() != NULL) + if (C->log() != nullptr) C->log()->elem("comment that='%d/%d subword/long to %d/%d int/long'", old_subword, old_long, new_int, new_long); @@ -4382,7 +4363,7 @@ Node* InitializeNode::complete_stores(Node* rawctl, Node* rawmem, Node* rawptr, PhaseIterGVN* phase) { assert(!is_complete(), "not already complete"); assert(stores_are_sane(phase), ""); - assert(allocation() != NULL, "must be present"); + assert(allocation() != nullptr, "must be present"); remove_extra_zeroes(); @@ -4485,7 +4466,7 @@ Node* InitializeNode::complete_stores(Node* rawctl, Node* rawmem, Node* rawptr, // a large constant tile can be filled in by smaller non-constant stores. assert(st_off >= last_init_off, "inits do not reverse"); last_init_off = st_off; - const Type* val = NULL; + const Type* val = nullptr; if (st_size >= BytesPerInt && (val = phase->type(st->in(MemNode::ValueIn)))->singleton() && (int)val->basic_type() < (int)T_OBJECT) { @@ -4511,8 +4492,8 @@ Node* InitializeNode::complete_stores(Node* rawctl, Node* rawmem, Node* rawptr, intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint); if (zeroes_done + BytesPerLong >= size_limit) { AllocateNode* alloc = allocation(); - assert(alloc != NULL, "must be present"); - if (alloc != NULL && alloc->Opcode() == Op_Allocate) { + assert(alloc != nullptr, "must be present"); + if (alloc != nullptr && alloc->Opcode() == Op_Allocate) { Node* klass_node = alloc->in(AllocateNode::KlassNode); ciKlass* k = phase->type(klass_node)->is_klassptr()->klass(); if (zeroes_done == k->layout_helper()) @@ -4534,7 +4515,7 @@ Node* InitializeNode::complete_stores(Node* rawctl, Node* rawmem, Node* rawptr, bool InitializeNode::stores_are_sane(PhaseTransform* phase) { if (is_complete()) return true; // stores could be anything at this point - assert(allocation() != NULL, "must be present"); + assert(allocation() != nullptr, "must be present"); intptr_t last_off = allocation()->minimum_header_size(); for (uint i = InitializeNode::RawStores; i < req(); i++) { Node* st = in(i); @@ -4610,7 +4591,7 @@ bool InitializeNode::stores_are_sane(PhaseTransform* phase) { // memory state has an edge in(AliasIdxBot) which is a "wide" memory state, // containing all alias categories. // -// MergeMem nodes never (?) have control inputs, so in(0) is NULL. +// MergeMem nodes never (?) have control inputs, so in(0) is null. // // All other edges in(N) (including in(AliasIdxRaw), which is in(3)) are either // a memory state for the alias type , or else the top node, meaning that @@ -4663,7 +4644,7 @@ Node* MergeMemNode::make_empty_memory() { MergeMemNode::MergeMemNode(Node *new_base) : Node(1+Compile::AliasIdxRaw) { init_class_id(Class_MergeMem); // all inputs are nullified in Node::Node(int) - // set_input(0, NULL); // no control input + // set_input(0, nullptr); // no control input // Initialize the edges uniformly to top, for starters. Node* empty_mem = make_empty_memory(); @@ -4672,7 +4653,7 @@ MergeMemNode::MergeMemNode(Node *new_base) : Node(1+Compile::AliasIdxRaw) { } assert(empty_memory() == empty_mem, ""); - if( new_base != NULL && new_base->is_MergeMem() ) { + if( new_base != nullptr && new_base->is_MergeMem() ) { MergeMemNode* mdef = new_base->as_MergeMem(); assert(mdef->empty_memory() == empty_mem, "consistent sentinels"); for (MergeMemStream mms(this, mdef); mms.next_non_empty2(); ) { @@ -4722,19 +4703,19 @@ Node *MergeMemNode::Ideal(PhaseGVN *phase, bool can_reshape) { // relative to the "in(Bot)". Since we are patching both at the same time, // we have to be careful to read each "in(i)" relative to the old "in(Bot)", // but rewrite each "in(i)" relative to the new "in(Bot)". - Node *progress = NULL; + Node *progress = nullptr; Node* old_base = base_memory(); Node* empty_mem = empty_memory(); if (old_base == empty_mem) - return NULL; // Dead memory path. + return nullptr; // Dead memory path. MergeMemNode* old_mbase; - if (old_base != NULL && old_base->is_MergeMem()) + if (old_base != nullptr && old_base->is_MergeMem()) old_mbase = old_base->as_MergeMem(); else - old_mbase = NULL; + old_mbase = nullptr; Node* new_base = old_base; // simplify stacked MergeMems in base memory @@ -4785,10 +4766,10 @@ Node *MergeMemNode::Ideal(PhaseGVN *phase, bool can_reshape) { // simplify stacked MergeMems Node* new_mem = old_mem; MergeMemNode* old_mmem; - if (old_mem != NULL && old_mem->is_MergeMem()) + if (old_mem != nullptr && old_mem->is_MergeMem()) old_mmem = old_mem->as_MergeMem(); else - old_mmem = NULL; + old_mmem = nullptr; if (old_mmem == this) { // This can happen if loops break up and safepoints disappear. // A merge of BotPtr (default) with a RawPtr memory derived from a @@ -4803,7 +4784,7 @@ Node *MergeMemNode::Ideal(PhaseGVN *phase, bool can_reshape) { // from start. Update the input to TOP. new_mem = (new_base == this || new_base == empty_mem)? empty_mem : new_base; } - else if (old_mmem != NULL) { + else if (old_mmem != nullptr) { new_mem = old_mmem->memory_at(i); } // else preceding memory was not a MergeMem @@ -4838,7 +4819,7 @@ Node *MergeMemNode::Ideal(PhaseGVN *phase, bool can_reshape) { if( base_memory()->is_MergeMem() ) { MergeMemNode *new_mbase = base_memory()->as_MergeMem(); Node *m = phase->transform(new_mbase); // Rollup any cycles - if( m != NULL && + if( m != nullptr && (m->is_top() || (m->is_MergeMem() && m->as_MergeMem()->base_memory() == empty_mem)) ) { // propagate rollup of dead cycle to self @@ -4900,7 +4881,7 @@ void MergeMemNode::dump_spec(outputStream *st) const { st->print(" {"); Node* base_mem = base_memory(); for( uint i = Compile::AliasIdxRaw; i < req(); i++ ) { - Node* mem = (in(i) != NULL) ? memory_at(i) : base_mem; + Node* mem = (in(i) != nullptr) ? memory_at(i) : base_mem; if (mem == base_mem) { st->print(" -"); continue; } st->print( " N%d:", mem->_idx ); Compile::current()->get_adr_type(i)->dump_on(st); @@ -4924,7 +4905,7 @@ static void verify_memory_slice(const MergeMemNode* m, int alias_idx, Node* n) { if (VMError::is_error_reported()) return; // muzzle asserts when debugging an error if (Node::in_dump()) return; // muzzle asserts when printing assert(alias_idx >= Compile::AliasIdxRaw, "must not disturb base_memory or sentinel"); - assert(n != NULL, ""); + assert(n != nullptr, ""); // Elide intervening MergeMem's while (n->is_MergeMem()) { n = n->as_MergeMem()->memory_at(alias_idx); @@ -4934,7 +4915,7 @@ static void verify_memory_slice(const MergeMemNode* m, int alias_idx, Node* n) { if (n == m->empty_memory()) { // Implicit copy of base_memory() } else if (n_adr_type != TypePtr::BOTTOM) { - assert(n_adr_type != NULL, "new memory must have a well-defined adr_type"); + assert(n_adr_type != nullptr, "new memory must have a well-defined adr_type"); assert(C->must_alias(n_adr_type, alias_idx), "new memory must match selected slice"); } else { // A few places like make_runtime_call "know" that VM calls are narrow, @@ -4971,8 +4952,8 @@ Node* MergeMemNode::memory_at(uint alias_idx) const { // the array is sparse; empty slots are the "top" node n = base_memory(); assert(Node::in_dump() - || n == NULL || n->bottom_type() == Type::TOP - || n->adr_type() == NULL // address is TOP + || n == nullptr || n->bottom_type() == Type::TOP + || n->adr_type() == nullptr // address is TOP || n->adr_type() == TypePtr::BOTTOM || n->adr_type() == TypeRawPtr::BOTTOM || Compile::current()->AliasLevel() == 0, @@ -5014,7 +4995,7 @@ void MergeMemNode::set_memory_at(uint alias_idx, Node *n) { //--------------------------iteration_setup------------------------------------ void MergeMemNode::iteration_setup(const MergeMemNode* other) { - if (other != NULL) { + if (other != nullptr) { grow_to_match(other); // invariant: the finite support of mm2 is within mm->req() #ifdef ASSERT @@ -5025,7 +5006,7 @@ void MergeMemNode::iteration_setup(const MergeMemNode* other) { } // Replace spurious copies of base_memory by top. Node* base_mem = base_memory(); - if (base_mem != NULL && !base_mem->is_top()) { + if (base_mem != nullptr && !base_mem->is_top()) { for (uint i = Compile::AliasIdxBot+1, imax = req(); i < imax; i++) { if (in(i) == base_mem) set_req(i, empty_memory()); @@ -5055,7 +5036,7 @@ bool MergeMemNode::verify_sparse() const { // The following can happen in degenerate cases, since empty==top. if (is_empty_memory(base_mem)) return true; for (uint i = Compile::AliasIdxRaw; i < req(); i++) { - assert(in(i) != NULL, "sane slice"); + assert(in(i) != nullptr, "sane slice"); if (in(i) == base_mem) return false; // should have been the sentinel value! } return true; diff --git a/src/hotspot/share/opto/memnode.hpp b/src/hotspot/share/opto/memnode.hpp index 04d24c4cd9a81..99a3048627445 100644 --- a/src/hotspot/share/opto/memnode.hpp +++ b/src/hotspot/share/opto/memnode.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -37,7 +37,7 @@ class PhaseCCP; class PhaseTransform; //------------------------------MemNode---------------------------------------- -// Load or Store, possibly throwing a NULL pointer exception +// Load or Store, possibly throwing a null pointer exception class MemNode : public Node { private: bool _unaligned_access; // Unaligned access from unsafe @@ -92,7 +92,7 @@ class MemNode : public Node { debug_only(_adr_type=at; adr_type();) } - virtual Node* find_previous_arraycopy(PhaseTransform* phase, Node* ld_alloc, Node*& mem, bool can_see_stored_value) const { return NULL; } + virtual Node* find_previous_arraycopy(PhaseTransform* phase, Node* ld_alloc, Node*& mem, bool can_see_stored_value) const { return nullptr; } ArrayCopyNode* find_array_copy_clone(PhaseTransform* phase, Node* ld_alloc, Node* mem) const; static bool check_if_adr_maybe_raw(Node* adr); @@ -111,10 +111,10 @@ class MemNode : public Node { virtual const class TypePtr *adr_type() const; // returns bottom_type of address // Shared code for Ideal methods: - Node *Ideal_common(PhaseGVN *phase, bool can_reshape); // Return -1 for short-circuit NULL. + Node *Ideal_common(PhaseGVN *phase, bool can_reshape); // Return -1 for short-circuit null. // Helper function for adr_type() implementations. - static const TypePtr* calculate_adr_type(const Type* t, const TypePtr* cross_check = NULL); + static const TypePtr* calculate_adr_type(const Type* t, const TypePtr* cross_check = nullptr); // Raw access function, to allow copying of adr_type efficiently in // product builds and retain the debug info for debug builds. @@ -262,13 +262,13 @@ class LoadNode : public MemNode { virtual const Type *bottom_type() const; // Following method is copied from TypeNode: void set_type(const Type* t) { - assert(t != NULL, "sanity"); + assert(t != nullptr, "sanity"); debug_only(uint check_hash = (VerifyHashTableKeys && _hash_lock) ? hash() : NO_HASH); *(const Type**)&_type = t; // cast away const-ness // If this node is in the hash table, make sure it doesn't need a rehash. assert(check_hash == NO_HASH || check_hash == hash(), "type change must preserve hash code"); } - const Type* type() const { assert(_type != NULL, "sanity"); return _type; }; + const Type* type() const { assert(_type != nullptr, "sanity"); return _type; }; // Do not match memory edge virtual uint match_edge(uint idx) const; @@ -822,7 +822,7 @@ class SCMemProjNode : public ProjNode { virtual const Type *bottom_type() const {return Type::MEMORY;} virtual const TypePtr *adr_type() const { Node* ctrl = in(0); - if (ctrl == NULL) return NULL; // node is dead + if (ctrl == nullptr) return nullptr; // node is dead return ctrl->in(MemNode::Memory)->adr_type(); } virtual uint ideal_reg() const { return 0;} // memory projections don't have a register @@ -1215,7 +1215,7 @@ class MemBarNode: public MultiNode { // Optional 'precedent' becomes an extra edge if not null. static MemBarNode* make(Compile* C, int opcode, int alias_idx = Compile::AliasIdxBot, - Node* precedent = NULL); + Node* precedent = nullptr); MemBarNode* trailing_membar() const; MemBarNode* leading_membar() const; @@ -1309,6 +1309,13 @@ class MemBarStoreStoreNode: public MemBarNode { virtual int Opcode() const; }; +class StoreStoreFenceNode: public MemBarNode { +public: + StoreStoreFenceNode(Compile* C, int alias_idx, Node* precedent) + : MemBarNode(C, alias_idx, precedent) {} + virtual int Opcode() const; +}; + // Ordering between a volatile store and a following volatile load. // Requires multi-CPU visibility? class MemBarVolatileNode: public MemBarNode { @@ -1336,26 +1343,6 @@ class OnSpinWaitNode: public MemBarNode { virtual int Opcode() const; }; -//------------------------------BlackholeNode---------------------------- -// Blackhole all arguments. This node would survive through the compiler -// the effects on its arguments, and would be finally matched to nothing. -class BlackholeNode : public MemBarNode { -public: - BlackholeNode(Compile* C, int alias_idx, Node* precedent) - : MemBarNode(C, alias_idx, precedent) {} - virtual int Opcode() const; - virtual uint ideal_reg() const { return 0; } // not matched in the AD file - const RegMask &in_RegMask(uint idx) const { - // Fake the incoming arguments mask for blackholes: accept all registers - // and all stack slots. This would avoid any redundant register moves - // for blackhole inputs. - return RegMask::All; - } -#ifndef PRODUCT - virtual void format(PhaseRegAlloc* ra, outputStream* st) const; -#endif -}; - // Isolation of object setup after an AllocateNode and before next safepoint. // (See comment in memnode.cpp near InitializeNode::InitializeNode for semantics.) class InitializeNode: public MemBarNode { @@ -1422,12 +1409,12 @@ class InitializeNode: public MemBarNode { intptr_t can_capture_store(StoreNode* st, PhaseGVN* phase, bool can_reshape); // Capture another store; reformat it to write my internal raw memory. - // Return the captured copy, else NULL if there is some sort of problem. + // Return the captured copy, else null if there is some sort of problem. Node* capture_store(StoreNode* st, intptr_t start, PhaseGVN* phase, bool can_reshape); // Find captured store which corresponds to the range [start..start+size). // Return my own memory projection (meaning the initial zero bits) - // if there is no such store. Return NULL if there is a problem. + // if there is no such store. Return null if there is a problem. Node* find_captured_store(intptr_t start, int size_in_bytes, PhaseTransform* phase); // Called when the associated AllocateNode is expanded into CFG. @@ -1491,7 +1478,7 @@ class MergeMemNode: public Node { static Node* make_empty_memory(); // where the sentinel comes from bool is_empty_memory(Node* n) const { assert((n == empty_memory()) == n->is_top(), "sanity"); return n->is_top(); } // hook for the iterator, to perform any necessary setup - void iteration_setup(const MergeMemNode* other = NULL); + void iteration_setup(const MergeMemNode* other = nullptr); // push sentinels until I am at least as long as the other (semantic no-op) void grow_to_match(const MergeMemNode* other); bool verify_sparse() const PRODUCT_RETURN0; @@ -1511,7 +1498,7 @@ class MergeMemStream : public StackObj { Node* _mem2; int _cnt2; - void init(MergeMemNode* mm, const MergeMemNode* mm2 = NULL) { + void init(MergeMemNode* mm, const MergeMemNode* mm2 = nullptr) { // subsume_node will break sparseness at times, whenever a memory slice // folds down to a copy of the base ("fat") memory. In such a case, // the raw edge will update to base, although it should be top. @@ -1525,15 +1512,15 @@ class MergeMemStream : public StackObj { // // Also, iteration_setup repairs sparseness. assert(mm->verify_sparse(), "please, no dups of base"); - assert(mm2==NULL || mm2->verify_sparse(), "please, no dups of base"); + assert(mm2==nullptr || mm2->verify_sparse(), "please, no dups of base"); _mm = mm; _mm_base = mm->base_memory(); _mm2 = mm2; _cnt = mm->req(); _idx = Compile::AliasIdxBot-1; // start at the base memory - _mem = NULL; - _mem2 = NULL; + _mem = nullptr; + _mem2 = nullptr; } #ifdef ASSERT @@ -1591,7 +1578,7 @@ class MergeMemStream : public StackObj { return _mm_base; } const MergeMemNode* all_memory2() const { - assert(_mm2 != NULL, ""); + assert(_mm2 != nullptr, ""); return _mm2; } bool at_base_memory() const { @@ -1662,7 +1649,7 @@ class MergeMemStream : public StackObj { private: // find the next item, which might be empty bool next(bool have_mm2) { - assert((_mm2 != NULL) == have_mm2, "use other next"); + assert((_mm2 != nullptr) == have_mm2, "use other next"); assert_synch(); if (++_idx < _cnt) { // Note: This iterator allows _mm to be non-sparse. diff --git a/src/hotspot/share/opto/movenode.cpp b/src/hotspot/share/opto/movenode.cpp index 4e22a35aaec52..6f23eb10c1447 100644 --- a/src/hotspot/share/opto/movenode.cpp +++ b/src/hotspot/share/opto/movenode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -75,12 +75,12 @@ // Return a node which is more "ideal" than the current node. // Move constants to the right. Node *CMoveNode::Ideal(PhaseGVN *phase, bool can_reshape) { - if (in(0) != NULL && remove_dead_region(phase, can_reshape)) { + if (in(0) != nullptr && remove_dead_region(phase, can_reshape)) { return this; } // Don't bother trying to transform a dead node - if (in(0) != NULL && in(0)->is_top()) { - return NULL; + if (in(0) != nullptr && in(0)->is_top()) { + return nullptr; } assert(in(Condition) != this && in(IfFalse) != this && @@ -88,14 +88,14 @@ Node *CMoveNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (phase->type(in(Condition)) == Type::TOP || phase->type(in(IfFalse)) == Type::TOP || phase->type(in(IfTrue)) == Type::TOP) { - return NULL; + return nullptr; } // Canonicalize the node by moving constants to the right input. if (in(Condition)->is_Bool() && phase->type(in(IfFalse))->singleton() && !phase->type(in(IfTrue))->singleton()) { BoolNode* b = in(Condition)->as_Bool()->negate(phase); return make(in(Control), phase->transform(b), in(IfTrue), in(IfFalse), _type); } - return NULL; + return nullptr; } //------------------------------is_cmove_id------------------------------------ @@ -108,7 +108,7 @@ Node *CMoveNode::is_cmove_id( PhaseTransform *phase, Node *cmp, Node *t, Node *f // Give up this identity check for floating points because it may choose incorrect // value around 0.0 and -0.0 if ( cmp->Opcode()==Op_CmpF || cmp->Opcode()==Op_CmpD ) - return NULL; + return nullptr; // Check for "(t==f)?t:f;" and replace with "f" if( b->_test._test == BoolTest::eq ) return f; @@ -117,7 +117,7 @@ Node *CMoveNode::is_cmove_id( PhaseTransform *phase, Node *cmp, Node *t, Node *f if( b->_test._test == BoolTest::ne ) return t; } - return NULL; + return nullptr; } //------------------------------Identity--------------------------------------- @@ -180,7 +180,7 @@ CMoveNode *CMoveNode::make(Node *c, Node *bol, Node *left, Node *right, const Ty case T_NARROWOOP: return new CMoveNNode( c, bol, left, right, t ); default: ShouldNotReachHere(); - return NULL; + return nullptr; } } @@ -214,26 +214,26 @@ Node *CMoveINode::Ideal(PhaseGVN *phase, bool can_reshape) { if( phase->type(in(IfFalse)) == TypeInt::ZERO && phase->type(in(IfTrue)) == TypeInt::ONE ) { flip = 1 - flip; } else if( phase->type(in(IfFalse)) == TypeInt::ONE && phase->type(in(IfTrue)) == TypeInt::ZERO ) { - } else return NULL; + } else return nullptr; // Check for eq/ne test - if( !in(1)->is_Bool() ) return NULL; + if( !in(1)->is_Bool() ) return nullptr; BoolNode *bol = in(1)->as_Bool(); if( bol->_test._test == BoolTest::eq ) { } else if( bol->_test._test == BoolTest::ne ) { flip = 1-flip; - } else return NULL; + } else return nullptr; // Check for vs 0 or 1 - if( !bol->in(1)->is_Cmp() ) return NULL; + if( !bol->in(1)->is_Cmp() ) return nullptr; const CmpNode *cmp = bol->in(1)->as_Cmp(); if( phase->type(cmp->in(2)) == TypeInt::ZERO ) { } else if( phase->type(cmp->in(2)) == TypeInt::ONE ) { // Allow cmp-vs-1 if the other input is bounded by 0-1 if( phase->type(cmp->in(1)) != TypeInt::BOOL ) - return NULL; + return nullptr; flip = 1 - flip; - } else return NULL; + } else return nullptr; // Convert to a bool (flipped) // Build int->bool conversion @@ -258,7 +258,7 @@ Node *CMoveFNode::Ideal(PhaseGVN *phase, bool can_reshape) { int phi_x_idx = 0; // Index of phi input where to find naked x // Find the Bool - if( !in(1)->is_Bool() ) return NULL; + if( !in(1)->is_Bool() ) return nullptr; BoolNode *bol = in(1)->as_Bool(); // Check bool sense switch( bol->_test._test ) { @@ -266,13 +266,13 @@ Node *CMoveFNode::Ideal(PhaseGVN *phase, bool can_reshape) { case BoolTest::le: cmp_zero_idx = 2; phi_x_idx = IfFalse; break; case BoolTest::gt: cmp_zero_idx = 2; phi_x_idx = IfTrue; break; case BoolTest::ge: cmp_zero_idx = 1; phi_x_idx = IfFalse; break; - default: return NULL; break; + default: return nullptr; break; } // Find zero input of CmpF; the other input is being abs'd Node *cmpf = bol->in(1); - if( cmpf->Opcode() != Op_CmpF ) return NULL; - Node *X = NULL; + if( cmpf->Opcode() != Op_CmpF ) return nullptr; + Node *X = nullptr; bool flip = false; if( phase->type(cmpf->in(cmp_zero_idx)) == TypeF::ZERO ) { X = cmpf->in(3 - cmp_zero_idx); @@ -281,18 +281,18 @@ Node *CMoveFNode::Ideal(PhaseGVN *phase, bool can_reshape) { X = cmpf->in(cmp_zero_idx); flip = true; } else { - return NULL; + return nullptr; } // If X is found on the appropriate phi input, find the subtract on the other - if( X != in(phi_x_idx) ) return NULL; + if( X != in(phi_x_idx) ) return nullptr; int phi_sub_idx = phi_x_idx == IfTrue ? IfFalse : IfTrue; Node *sub = in(phi_sub_idx); // Allow only SubF(0,X) and fail out for all others; NegF is not OK if( sub->Opcode() != Op_SubF || sub->in(2) != X || - phase->type(sub->in(1)) != TypeF::ZERO ) return NULL; + phase->type(sub->in(1)) != TypeF::ZERO ) return nullptr; Node *abs = new AbsFNode( X ); if( flip ) @@ -314,7 +314,7 @@ Node *CMoveDNode::Ideal(PhaseGVN *phase, bool can_reshape) { int phi_x_idx = 0; // Index of phi input where to find naked x // Find the Bool - if( !in(1)->is_Bool() ) return NULL; + if( !in(1)->is_Bool() ) return nullptr; BoolNode *bol = in(1)->as_Bool(); // Check bool sense switch( bol->_test._test ) { @@ -322,13 +322,13 @@ Node *CMoveDNode::Ideal(PhaseGVN *phase, bool can_reshape) { case BoolTest::le: cmp_zero_idx = 2; phi_x_idx = IfFalse; break; case BoolTest::gt: cmp_zero_idx = 2; phi_x_idx = IfTrue; break; case BoolTest::ge: cmp_zero_idx = 1; phi_x_idx = IfFalse; break; - default: return NULL; break; + default: return nullptr; break; } // Find zero input of CmpD; the other input is being abs'd Node *cmpd = bol->in(1); - if( cmpd->Opcode() != Op_CmpD ) return NULL; - Node *X = NULL; + if( cmpd->Opcode() != Op_CmpD ) return nullptr; + Node *X = nullptr; bool flip = false; if( phase->type(cmpd->in(cmp_zero_idx)) == TypeD::ZERO ) { X = cmpd->in(3 - cmp_zero_idx); @@ -337,18 +337,18 @@ Node *CMoveDNode::Ideal(PhaseGVN *phase, bool can_reshape) { X = cmpd->in(cmp_zero_idx); flip = true; } else { - return NULL; + return nullptr; } // If X is found on the appropriate phi input, find the subtract on the other - if( X != in(phi_x_idx) ) return NULL; + if( X != in(phi_x_idx) ) return nullptr; int phi_sub_idx = phi_x_idx == IfTrue ? IfFalse : IfTrue; Node *sub = in(phi_sub_idx); // Allow only SubD(0,X) and fail out for all others; NegD is not OK if( sub->Opcode() != Op_SubD || sub->in(2) != X || - phase->type(sub->in(1)) != TypeD::ZERO ) return NULL; + phase->type(sub->in(1)) != TypeD::ZERO ) return nullptr; Node *abs = new AbsDNode( X ); if( flip ) @@ -364,7 +364,7 @@ Node* MoveNode::Ideal(PhaseGVN* phase, bool can_reshape) { // Fold reinterpret cast into memory operation: // MoveX2Y (LoadX mem) => LoadY mem LoadNode* ld = in(1)->isa_Load(); - if (ld != NULL && (ld->outcnt() == 1)) { // replace only + if (ld != nullptr && (ld->outcnt() == 1)) { // replace only const Type* rt = bottom_type(); if (ld->has_reinterpret_variant(rt)) { if (phase->C->post_loop_opts_phase()) { @@ -375,7 +375,7 @@ Node* MoveNode::Ideal(PhaseGVN* phase, bool can_reshape) { } } } - return NULL; + return nullptr; } Node* MoveNode::Identity(PhaseGVN* phase) { diff --git a/src/hotspot/share/opto/movenode.hpp b/src/hotspot/share/opto/movenode.hpp index 1e8b1b2489bc6..c6d292da14629 100644 --- a/src/hotspot/share/opto/movenode.hpp +++ b/src/hotspot/share/opto/movenode.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -39,7 +39,7 @@ class CMoveNode : public TypeNode { { init_class_id(Class_CMove); // all inputs are nullified in Node::Node(int) - // init_req(Control,NULL); + // init_req(Control,nullptr); init_req(Condition,bol); init_req(IfFalse,left); init_req(IfTrue,right); @@ -100,7 +100,7 @@ class CMoveNNode : public CMoveNode { // class MoveNode : public Node { protected: - MoveNode(Node* value) : Node(NULL, value) { + MoveNode(Node* value) : Node(nullptr, value) { init_class_id(Class_Move); } diff --git a/src/hotspot/share/opto/mulnode.cpp b/src/hotspot/share/opto/mulnode.cpp index 6e6026213062e..cc42318d0cb6c 100644 --- a/src/hotspot/share/opto/mulnode.cpp +++ b/src/hotspot/share/opto/mulnode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -61,7 +61,7 @@ Node* MulNode::Identity(PhaseGVN* phase) { Node *MulNode::Ideal(PhaseGVN *phase, bool can_reshape) { Node* in1 = in(1); Node* in2 = in(2); - Node* progress = NULL; // Progress flag + Node* progress = nullptr; // Progress flag // convert "max(a,b) * min(a,b)" into "a*b". if ((in(1)->Opcode() == max_opcode() && in(2)->Opcode() == min_opcode()) @@ -111,7 +111,7 @@ Node *MulNode::Ideal(PhaseGVN *phase, bool can_reshape) { if( t2->singleton() && // Right input is a constant? op != Op_MulF && // Float & double cannot reassociate op != Op_MulD ) { - if( t2 == Type::TOP ) return NULL; + if( t2 == Type::TOP ) return nullptr; Node *mul1 = in(1); #ifdef ASSERT // Check for dead loop @@ -214,8 +214,8 @@ Node *MulINode::Ideal(PhaseGVN *phase, bool can_reshape) { } // Now we have a constant Node on the right and the constant in con - if (con == 0) return NULL; // By zero is handled by Value call - if (con == 1) return NULL; // By one is handled by Identity call + if (con == 0) return nullptr; // By zero is handled by Value call + if (con == 1) return nullptr; // By one is handled by Identity call // Check for negative constant; if so negate the final result bool sign_flip = false; @@ -226,7 +226,7 @@ Node *MulINode::Ideal(PhaseGVN *phase, bool can_reshape) { } // Get low bit; check for being the only bit - Node *res = NULL; + Node *res = nullptr; unsigned int bit1 = abs_con & (0-abs_con); // Extract low bit if (bit1 == abs_con) { // Found a power of 2? res = new LShiftINode(in(1), phase->intcon(log2i_exact(bit1))); @@ -320,7 +320,7 @@ Node *MulLNode::Ideal(PhaseGVN *phase, bool can_reshape) { } // Get low bit; check for being the only bit - Node *res = NULL; + Node *res = nullptr; julong bit1 = abs_con & (0-abs_con); // Extract low bit if (bit1 == abs_con) { // Found a power of 2? res = new LShiftLNode(in(1), phase->intcon(log2i_exact(bit1))); @@ -474,7 +474,7 @@ Node* AndINode::Identity(PhaseGVN* phase) { int con = t2->get_con(); // Masking off high bits which are always zero is useless. const TypeInt* t1 = phase->type(in(1))->isa_int(); - if (t1 != NULL && t1->_lo >= 0) { + if (t1 != nullptr && t1->_lo >= 0) { jint t1_support = right_n_bits(1 + log2i_graceful(t1->_hi)); if ((t1_support & con) == t1_support) return in1; @@ -597,7 +597,7 @@ Node* AndLNode::Identity(PhaseGVN* phase) { jlong con = t2->get_con(); // Masking off high bits which are always zero is useless. const TypeLong* t1 = phase->type( in(1) )->isa_long(); - if (t1 != NULL && t1->_lo >= 0) { + if (t1 != nullptr && t1->_lo >= 0) { int bit_count = log2i_graceful(t1->_hi) + 1; jlong t1_support = jlong(max_julong >> (BitsPerJavaLong - bit_count)); if ((t1_support & con) == t1_support) @@ -665,7 +665,7 @@ Node *AndLNode::Ideal(PhaseGVN *phase, bool can_reshape) { static bool const_shift_count(PhaseGVN* phase, Node* shiftNode, int* count) { const TypeInt* tcount = phase->type(shiftNode->in(2))->isa_int(); - if (tcount != NULL && tcount->is_con()) { + if (tcount != nullptr && tcount->is_con()) { *count = tcount->get_con(); return true; } @@ -709,7 +709,7 @@ Node* LShiftINode::Identity(PhaseGVN* phase) { Node *LShiftINode::Ideal(PhaseGVN *phase, bool can_reshape) { int con = maskShiftAmount(phase, this, BitsPerJavaInteger); if (con == 0) { - return NULL; + return nullptr; } // Left input is an add of a constant? @@ -755,7 +755,7 @@ Node *LShiftINode::Ideal(PhaseGVN *phase, bool can_reshape) { phase->type(add1->in(2)) == TypeInt::make( bits_mask ) ) return new LShiftINode( add1->in(1), in(2) ); - return NULL; + return nullptr; } //------------------------------Value------------------------------------------ @@ -822,7 +822,7 @@ Node* LShiftLNode::Identity(PhaseGVN* phase) { Node *LShiftLNode::Ideal(PhaseGVN *phase, bool can_reshape) { int con = maskShiftAmount(phase, this, BitsPerJavaLong); if (con == 0) { - return NULL; + return nullptr; } // Left input is an add of a constant? @@ -865,7 +865,7 @@ Node *LShiftLNode::Ideal(PhaseGVN *phase, bool can_reshape) { phase->type(add1->in(2)) == TypeLong::make( bits_mask ) ) return new LShiftLNode( add1->in(1), in(2) ); - return NULL; + return nullptr; } //------------------------------Value------------------------------------------ @@ -933,7 +933,7 @@ Node* RShiftINode::Identity(PhaseGVN* phase) { int lo = (-1 << (BitsPerJavaInteger - ((uint)count)-1)); // FFFF8000 int hi = ~lo; // 00007FFF const TypeInt* t11 = phase->type(in(1)->in(1))->isa_int(); - if (t11 == NULL) { + if (t11 == nullptr) { return this; } // Does actual value fit inside of mask? @@ -949,11 +949,11 @@ Node* RShiftINode::Identity(PhaseGVN* phase) { Node *RShiftINode::Ideal(PhaseGVN *phase, bool can_reshape) { // Inputs may be TOP if they are dead. const TypeInt *t1 = phase->type(in(1))->isa_int(); - if (!t1) return NULL; // Left input is an integer + if (!t1) return nullptr; // Left input is an integer const TypeInt *t3; // type of in(1).in(2) int shift = maskShiftAmount(phase, this, BitsPerJavaInteger); if (shift == 0) { - return NULL; + return nullptr; } // Check for (x & 0xFF000000) >> 24, whose mask can be made smaller. @@ -971,7 +971,7 @@ Node *RShiftINode::Ideal(PhaseGVN *phase, bool can_reshape) { // Check for "(short[i] <<16)>>16" which simply sign-extends const Node *shl = in(1); - if( shl->Opcode() != Op_LShiftI ) return NULL; + if( shl->Opcode() != Op_LShiftI ) return nullptr; if( shift == 16 && (t3 = phase->type(shl->in(2))->isa_int()) && @@ -1007,7 +1007,7 @@ Node *RShiftINode::Ideal(PhaseGVN *phase, bool can_reshape) { } } - return NULL; + return nullptr; } //------------------------------Value------------------------------------------ @@ -1147,7 +1147,7 @@ Node* URShiftINode::Identity(PhaseGVN* phase) { t_lshift_count == phase->type(in(2))) { Node *x = add->in(1)->in(1); const TypeInt *t_x = phase->type(x)->isa_int(); - if (t_x != NULL && 0 <= t_x->_lo && t_x->_hi <= (max_jint>>LogBytesPerWord)) { + if (t_x != nullptr && 0 <= t_x->_lo && t_x->_hi <= (max_jint>>LogBytesPerWord)) { return x; } } @@ -1161,7 +1161,7 @@ Node* URShiftINode::Identity(PhaseGVN* phase) { Node *URShiftINode::Ideal(PhaseGVN *phase, bool can_reshape) { int con = maskShiftAmount(phase, this, BitsPerJavaInteger); if (con == 0) { - return NULL; + return nullptr; } // We'll be wanting the right-shift amount as a mask of that many bits @@ -1233,7 +1233,7 @@ Node *URShiftINode::Ideal(PhaseGVN *phase, bool can_reshape) { } } - return NULL; + return nullptr; } //------------------------------Value------------------------------------------ @@ -1325,7 +1325,7 @@ Node* URShiftLNode::Identity(PhaseGVN* phase) { Node *URShiftLNode::Ideal(PhaseGVN *phase, bool can_reshape) { int con = maskShiftAmount(phase, this, BitsPerJavaLong); if (con == 0) { - return NULL; + return nullptr; } // We'll be wanting the right-shift amount as a mask of that many bits @@ -1378,7 +1378,7 @@ Node *URShiftLNode::Ideal(PhaseGVN *phase, bool can_reshape) { return new URShiftLNode(in11, phase->intcon(63)); } } - return NULL; + return nullptr; } //------------------------------Value------------------------------------------ @@ -1572,7 +1572,7 @@ Node* RotateLeftNode::Ideal(PhaseGVN *phase, bool can_reshape) { return new RotateRightNode(in(1), phase->intcon(64 - (lshift & 63)), TypeLong::LONG); } } - return NULL; + return nullptr; } Node* RotateRightNode::Identity(PhaseGVN* phase) { diff --git a/src/hotspot/share/opto/mulnode.hpp b/src/hotspot/share/opto/mulnode.hpp index d399af5a990d1..d94b303f7a3ae 100644 --- a/src/hotspot/share/opto/mulnode.hpp +++ b/src/hotspot/share/opto/mulnode.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/opto/multnode.cpp b/src/hotspot/share/opto/multnode.cpp index e9881eb8b746c..e8c1e3840f404 100644 --- a/src/hotspot/share/opto/multnode.cpp +++ b/src/hotspot/share/opto/multnode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -59,24 +59,24 @@ ProjNode* MultiNode::proj_out_or_null(uint which_proj) const { continue; } } - return NULL; + return nullptr; } ProjNode* MultiNode::proj_out_or_null(uint which_proj, bool is_io_use) const { for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) { ProjNode* proj = fast_out(i)->isa_Proj(); - if (proj != NULL && (proj->_con == which_proj) && (proj->_is_io_use == is_io_use)) { + if (proj != nullptr && (proj->_con == which_proj) && (proj->_is_io_use == is_io_use)) { return proj; } } - return NULL; + return nullptr; } // Get a named projection ProjNode* MultiNode::proj_out(uint which_proj) const { assert((Opcode() != Op_If && Opcode() != Op_RangeCheck) || outcnt() == 2, "bad if #1"); ProjNode* p = proj_out_or_null(which_proj); - assert(p != NULL, "named projection %u not found", which_proj); + assert(p != nullptr, "named projection %u not found", which_proj); return p; } @@ -113,7 +113,7 @@ const Type* ProjNode::proj_type(const Type* t) const { } const Type *ProjNode::bottom_type() const { - if (in(0) == NULL) return Type::TOP; + if (in(0) == nullptr) return Type::TOP; return proj_type(in(0)->bottom_type()); } @@ -121,16 +121,16 @@ const TypePtr *ProjNode::adr_type() const { if (bottom_type() == Type::MEMORY) { // in(0) might be a narrow MemBar; otherwise we will report TypePtr::BOTTOM Node* ctrl = in(0); - if (ctrl == NULL) return NULL; // node is dead + if (ctrl == nullptr) return nullptr; // node is dead const TypePtr* adr_type = ctrl->adr_type(); #ifdef ASSERT if (!VMError::is_error_reported() && !Node::in_dump()) - assert(adr_type != NULL, "source must have adr_type"); + assert(adr_type != nullptr, "source must have adr_type"); #endif return adr_type; } assert(bottom_type()->base() != Type::Memory, "no other memories?"); - return NULL; + return nullptr; } bool ProjNode::pinned() const { return in(0)->pinned(); } @@ -142,7 +142,7 @@ void ProjNode::dump_compact_spec(outputStream *st) const { Node* o = this->out(i); if (not_a_node(o)) { st->print("[?]"); - } else if (o == NULL) { + } else if (o == nullptr) { st->print("[_]"); } else { st->print("[%d]", o->_idx); @@ -155,7 +155,7 @@ void ProjNode::dump_compact_spec(outputStream *st) const { //----------------------------check_con---------------------------------------- void ProjNode::check_con() const { Node* n = in(0); - if (n == NULL) return; // should be assert, but NodeHash makes bogons + if (n == nullptr) return; // should be assert, but NodeHash makes bogons if (n->is_Mach()) return; // mach. projs. are not type-safe if (n->is_Start()) return; // alas, starts can have mach. projs. also if (_con == SCMemProjNode::SCMEMPROJCON ) return; @@ -166,7 +166,7 @@ void ProjNode::check_con() const { //------------------------------Value------------------------------------------ const Type* ProjNode::Value(PhaseGVN* phase) const { - if (in(0) == NULL) return Type::TOP; + if (in(0) == nullptr) return Type::TOP; return proj_type(phase->type(in(0))); } @@ -183,14 +183,14 @@ uint ProjNode::ideal_reg() const { //-------------------------------is_uncommon_trap_proj---------------------------- // Return uncommon trap call node if proj is for "proj->[region->..]call_uct" -// NULL otherwise +// null otherwise CallStaticJavaNode* ProjNode::is_uncommon_trap_proj(Deoptimization::DeoptReason reason) { int path_limit = 10; Node* out = this; for (int ct = 0; ct < path_limit; ct++) { out = out->unique_ctrl_out(); - if (out == NULL) - return NULL; + if (out == nullptr) + return nullptr; if (out->is_CallStaticJava()) { CallStaticJavaNode* call = out->as_CallStaticJava(); int req = call->uncommon_trap_request(); @@ -200,12 +200,12 @@ CallStaticJavaNode* ProjNode::is_uncommon_trap_proj(Deoptimization::DeoptReason return call; } } - return NULL; // don't do further after call + return nullptr; // don't do further after call } if (out->Opcode() != Op_Region) - return NULL; + return nullptr; } - return NULL; + return nullptr; } //-------------------------------is_uncommon_trap_if_pattern------------------------- @@ -213,31 +213,31 @@ CallStaticJavaNode* ProjNode::is_uncommon_trap_proj(Deoptimization::DeoptReason // | // V // other_proj->[region->..]call_uct" -// NULL otherwise +// null otherwise // "must_reason_predicate" means the uct reason must be Reason_predicate CallStaticJavaNode* ProjNode::is_uncommon_trap_if_pattern(Deoptimization::DeoptReason reason) { Node *in0 = in(0); - if (!in0->is_If()) return NULL; + if (!in0->is_If()) return nullptr; // Variation of a dead If node. - if (in0->outcnt() < 2) return NULL; + if (in0->outcnt() < 2) return nullptr; IfNode* iff = in0->as_If(); // we need "If(Conv2B(Opaque1(...)))" pattern for reason_predicate if (reason != Deoptimization::Reason_none) { if (iff->in(1)->Opcode() != Op_Conv2B || iff->in(1)->in(1)->Opcode() != Op_Opaque1) { - return NULL; + return nullptr; } } ProjNode* other_proj = iff->proj_out(1-_con); CallStaticJavaNode* call = other_proj->is_uncommon_trap_proj(reason); - if (call != NULL) { + if (call != nullptr) { assert(reason == Deoptimization::Reason_none || Compile::current()->is_predicate_opaq(iff->in(1)->in(1)), "should be on the list"); return call; } - return NULL; + return nullptr; } ProjNode* ProjNode::other_if_proj() const { diff --git a/src/hotspot/share/opto/multnode.hpp b/src/hotspot/share/opto/multnode.hpp index efd43bc03f263..09552508aa34f 100644 --- a/src/hotspot/share/opto/multnode.hpp +++ b/src/hotspot/share/opto/multnode.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -92,13 +92,13 @@ class ProjNode : public Node { #endif // Return uncommon trap call node if proj is for "proj->[region->..]call_uct" - // NULL otherwise + // null otherwise CallStaticJavaNode* is_uncommon_trap_proj(Deoptimization::DeoptReason reason); // Return uncommon trap call node for "if(test)-> proj -> ... // | // V // other_proj->[region->..]call_uct" - // NULL otherwise + // null otherwise CallStaticJavaNode* is_uncommon_trap_if_pattern(Deoptimization::DeoptReason reason); // Return other proj node when this is a If proj node diff --git a/src/hotspot/share/opto/narrowptrnode.hpp b/src/hotspot/share/opto/narrowptrnode.hpp index 91bf76140db47..e7cd19cb42441 100644 --- a/src/hotspot/share/opto/narrowptrnode.hpp +++ b/src/hotspot/share/opto/narrowptrnode.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -34,7 +34,7 @@ class EncodeNarrowPtrNode : public TypeNode { EncodeNarrowPtrNode(Node* value, const Type* type): TypeNode(type, 2) { init_class_id(Class_EncodeNarrowPtr); - init_req(0, NULL); + init_req(0, nullptr); init_req(1, value); } public: @@ -77,7 +77,7 @@ class DecodeNarrowPtrNode : public TypeNode { DecodeNarrowPtrNode(Node* value, const Type* type): TypeNode(type, 2) { init_class_id(Class_DecodeNarrowPtr); - init_req(0, NULL); + init_req(0, nullptr); init_req(1, value); } public: diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp index 2dad491977a69..8575470987acd 100644 --- a/src/hotspot/share/opto/node.cpp +++ b/src/hotspot/share/opto/node.cpp @@ -66,7 +66,7 @@ extern int nodes_created; //-------------------------- construct_node------------------------------------ // Set a breakpoint here to identify where a particular node index is built. void Node::verify_construction() { - _debug_orig = NULL; + _debug_orig = nullptr; int old_debug_idx = Compile::debug_idx(); int new_debug_idx = old_debug_idx + 1; if (new_debug_idx > 0) { @@ -94,7 +94,7 @@ void Node::verify_construction() { BREAKPOINT; } #if OPTO_DU_ITERATOR_ASSERT - _last_del = NULL; + _last_del = nullptr; _del_tick = 0; #endif _hash_lock = 0; @@ -109,7 +109,7 @@ void DUIterator_Common::sample(const Node* node) { _node = node; _outcnt = node->_outcnt; _del_tick = node->_del_tick; - _last = NULL; + _last = nullptr; } void DUIterator_Common::verify(const Node* node, bool at_end_ok) { @@ -290,7 +290,7 @@ void DUIterator_Last::verify_step(uint num_edges) { // This constant used to initialize _out may be any non-null value. -// The value NULL is reserved for the top node only. +// The value null is reserved for the top node only. #define NO_OUT_ARRAY ((Node**)-1) // Out-of-line code from node constructors. @@ -312,7 +312,7 @@ inline int Node::Init(int req) { } // If there are default notes floating around, capture them: Node_Notes* nn = C->default_node_notes(); - if (nn != NULL) init_node_notes(C, idx, nn); + if (nn != nullptr) init_node_notes(C, idx, nn); // Note: At this point, C is dead, // and we begin to initialize the new Node. @@ -338,11 +338,11 @@ Node::Node(uint req) debug_only( verify_construction() ); NOT_PRODUCT(nodes_created++); if (req == 0) { - _in = NULL; + _in = nullptr; } else { Node** to = _in; for(uint i = 0; i < req; i++) { - to[i] = NULL; + to[i] = nullptr; } } } @@ -358,7 +358,7 @@ Node::Node(Node *n0) debug_only( verify_construction() ); NOT_PRODUCT(nodes_created++); assert( is_not_dead(n0), "can not use dead node"); - _in[0] = n0; if (n0 != NULL) n0->add_out((Node *)this); + _in[0] = n0; if (n0 != nullptr) n0->add_out((Node *)this); } //------------------------------Node------------------------------------------- @@ -373,8 +373,8 @@ Node::Node(Node *n0, Node *n1) NOT_PRODUCT(nodes_created++); assert( is_not_dead(n0), "can not use dead node"); assert( is_not_dead(n1), "can not use dead node"); - _in[0] = n0; if (n0 != NULL) n0->add_out((Node *)this); - _in[1] = n1; if (n1 != NULL) n1->add_out((Node *)this); + _in[0] = n0; if (n0 != nullptr) n0->add_out((Node *)this); + _in[1] = n1; if (n1 != nullptr) n1->add_out((Node *)this); } //------------------------------Node------------------------------------------- @@ -390,9 +390,9 @@ Node::Node(Node *n0, Node *n1, Node *n2) assert( is_not_dead(n0), "can not use dead node"); assert( is_not_dead(n1), "can not use dead node"); assert( is_not_dead(n2), "can not use dead node"); - _in[0] = n0; if (n0 != NULL) n0->add_out((Node *)this); - _in[1] = n1; if (n1 != NULL) n1->add_out((Node *)this); - _in[2] = n2; if (n2 != NULL) n2->add_out((Node *)this); + _in[0] = n0; if (n0 != nullptr) n0->add_out((Node *)this); + _in[1] = n1; if (n1 != nullptr) n1->add_out((Node *)this); + _in[2] = n2; if (n2 != nullptr) n2->add_out((Node *)this); } //------------------------------Node------------------------------------------- @@ -409,10 +409,10 @@ Node::Node(Node *n0, Node *n1, Node *n2, Node *n3) assert( is_not_dead(n1), "can not use dead node"); assert( is_not_dead(n2), "can not use dead node"); assert( is_not_dead(n3), "can not use dead node"); - _in[0] = n0; if (n0 != NULL) n0->add_out((Node *)this); - _in[1] = n1; if (n1 != NULL) n1->add_out((Node *)this); - _in[2] = n2; if (n2 != NULL) n2->add_out((Node *)this); - _in[3] = n3; if (n3 != NULL) n3->add_out((Node *)this); + _in[0] = n0; if (n0 != nullptr) n0->add_out((Node *)this); + _in[1] = n1; if (n1 != nullptr) n1->add_out((Node *)this); + _in[2] = n2; if (n2 != nullptr) n2->add_out((Node *)this); + _in[3] = n3; if (n3 != nullptr) n3->add_out((Node *)this); } //------------------------------Node------------------------------------------- @@ -430,11 +430,11 @@ Node::Node(Node *n0, Node *n1, Node *n2, Node *n3, Node *n4) assert( is_not_dead(n2), "can not use dead node"); assert( is_not_dead(n3), "can not use dead node"); assert( is_not_dead(n4), "can not use dead node"); - _in[0] = n0; if (n0 != NULL) n0->add_out((Node *)this); - _in[1] = n1; if (n1 != NULL) n1->add_out((Node *)this); - _in[2] = n2; if (n2 != NULL) n2->add_out((Node *)this); - _in[3] = n3; if (n3 != NULL) n3->add_out((Node *)this); - _in[4] = n4; if (n4 != NULL) n4->add_out((Node *)this); + _in[0] = n0; if (n0 != nullptr) n0->add_out((Node *)this); + _in[1] = n1; if (n1 != nullptr) n1->add_out((Node *)this); + _in[2] = n2; if (n2 != nullptr) n2->add_out((Node *)this); + _in[3] = n3; if (n3 != nullptr) n3->add_out((Node *)this); + _in[4] = n4; if (n4 != nullptr) n4->add_out((Node *)this); } //------------------------------Node------------------------------------------- @@ -454,12 +454,12 @@ Node::Node(Node *n0, Node *n1, Node *n2, Node *n3, assert( is_not_dead(n3), "can not use dead node"); assert( is_not_dead(n4), "can not use dead node"); assert( is_not_dead(n5), "can not use dead node"); - _in[0] = n0; if (n0 != NULL) n0->add_out((Node *)this); - _in[1] = n1; if (n1 != NULL) n1->add_out((Node *)this); - _in[2] = n2; if (n2 != NULL) n2->add_out((Node *)this); - _in[3] = n3; if (n3 != NULL) n3->add_out((Node *)this); - _in[4] = n4; if (n4 != NULL) n4->add_out((Node *)this); - _in[5] = n5; if (n5 != NULL) n5->add_out((Node *)this); + _in[0] = n0; if (n0 != nullptr) n0->add_out((Node *)this); + _in[1] = n1; if (n1 != nullptr) n1->add_out((Node *)this); + _in[2] = n2; if (n2 != nullptr) n2->add_out((Node *)this); + _in[3] = n3; if (n3 != nullptr) n3->add_out((Node *)this); + _in[4] = n4; if (n4 != nullptr) n4->add_out((Node *)this); + _in[5] = n5; if (n5 != nullptr) n5->add_out((Node *)this); } //------------------------------Node------------------------------------------- @@ -480,13 +480,13 @@ Node::Node(Node *n0, Node *n1, Node *n2, Node *n3, assert( is_not_dead(n4), "can not use dead node"); assert( is_not_dead(n5), "can not use dead node"); assert( is_not_dead(n6), "can not use dead node"); - _in[0] = n0; if (n0 != NULL) n0->add_out((Node *)this); - _in[1] = n1; if (n1 != NULL) n1->add_out((Node *)this); - _in[2] = n2; if (n2 != NULL) n2->add_out((Node *)this); - _in[3] = n3; if (n3 != NULL) n3->add_out((Node *)this); - _in[4] = n4; if (n4 != NULL) n4->add_out((Node *)this); - _in[5] = n5; if (n5 != NULL) n5->add_out((Node *)this); - _in[6] = n6; if (n6 != NULL) n6->add_out((Node *)this); + _in[0] = n0; if (n0 != nullptr) n0->add_out((Node *)this); + _in[1] = n1; if (n1 != nullptr) n1->add_out((Node *)this); + _in[2] = n2; if (n2 != nullptr) n2->add_out((Node *)this); + _in[3] = n3; if (n3 != nullptr) n3->add_out((Node *)this); + _in[4] = n4; if (n4 != nullptr) n4->add_out((Node *)this); + _in[5] = n5; if (n5 != nullptr) n5->add_out((Node *)this); + _in[6] = n6; if (n6 != nullptr) n6->add_out((Node *)this); } #ifdef __clang__ @@ -515,7 +515,7 @@ Node *Node::clone() const { for( i = 0; i < len(); i++ ) { Node *x = in(i); n->_in[i] = x; - if (x != NULL) x->add_out(n); + if (x != nullptr) x->add_out(n); } if (is_macro()) { C->add_macro_node(n); @@ -564,7 +564,7 @@ Node *Node::clone() const { if (n->is_Call()) { // CallGenerator is linked to the original node. CallGenerator* cg = n->as_Call()->generator(); - if (cg != NULL) { + if (cg != nullptr) { CallGenerator* cloned_cg = cg->with_call_node(n->as_Call()); n->as_Call()->set_generator(cloned_cg); @@ -589,10 +589,10 @@ void Node::setup_is_top() { if (this == (Node*)Compile::current()->top()) { // This node has just become top. Kill its out array. _outcnt = _outmax = 0; - _out = NULL; // marker value for top + _out = nullptr; // marker value for top assert(is_top(), "must be top"); } else { - if (_out == NULL) _out = NO_OUT_ARRAY; + if (_out == nullptr) _out = NO_OUT_ARRAY; assert(!is_top(), "must not be top"); } } @@ -600,8 +600,8 @@ void Node::setup_is_top() { //------------------------------~Node------------------------------------------ // Fancy destructor; eagerly attempt to reclaim Node numberings and storage void Node::destruct(PhaseValues* phase) { - Compile* compile = (phase != NULL) ? phase->C : Compile::current(); - if (phase != NULL && phase->is_IterGVN()) { + Compile* compile = (phase != nullptr) ? phase->C : Compile::current(); + if (phase != nullptr && phase->is_IterGVN()) { phase->is_IterGVN()->_worklist.remove(this); } // If this is the most recently created node, reclaim its index. Otherwise, @@ -613,42 +613,16 @@ void Node::destruct(PhaseValues* phase) { } // Clear debug info: Node_Notes* nn = compile->node_notes_at(_idx); - if (nn != NULL) nn->clear(); + if (nn != nullptr) nn->clear(); // Walk the input array, freeing the corresponding output edges _cnt = _max; // forget req/prec distinction uint i; for( i = 0; i < _max; i++ ) { - set_req(i, NULL); + set_req(i, nullptr); //assert(def->out(def->outcnt()-1) == (Node *)this,"bad def-use hacking in reclaim"); } assert(outcnt() == 0, "deleting a node must not leave a dangling use"); - // See if the input array was allocated just prior to the object - int edge_size = _max*sizeof(void*); - int out_edge_size = _outmax*sizeof(void*); - char *edge_end = ((char*)_in) + edge_size; - char *out_array = (char*)(_out == NO_OUT_ARRAY? NULL: _out); - int node_size = size_of(); - - // Free the output edge array - if (out_edge_size > 0) { - compile->node_arena()->Afree(out_array, out_edge_size); - } - - // Free the input edge array and the node itself - if( edge_end == (char*)this ) { - // It was; free the input array and object all in one hit -#ifndef ASSERT - compile->node_arena()->Afree(_in,edge_size+node_size); -#endif - } else { - // Free just the input array - compile->node_arena()->Afree(_in,edge_size); - // Free just the object -#ifndef ASSERT - compile->node_arena()->Afree(this,node_size); -#endif - } if (is_macro()) { compile->remove_macro_node(this); } @@ -667,13 +641,43 @@ void Node::destruct(PhaseValues* phase) { } BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); bs->unregister_potential_barrier_node(this); + + // See if the input array was allocated just prior to the object + int edge_size = _max*sizeof(void*); + int out_edge_size = _outmax*sizeof(void*); + char *in_array = ((char*)_in); + char *edge_end = in_array + edge_size; + char *out_array = (char*)(_out == NO_OUT_ARRAY? nullptr: _out); + int node_size = size_of(); + #ifdef ASSERT // We will not actually delete the storage, but we'll make the node unusable. + compile->remove_modified_node(this); *(address*)this = badAddress; // smash the C++ vtbl, probably _in = _out = (Node**) badAddress; _max = _cnt = _outmax = _outcnt = 0; - compile->remove_modified_node(this); #endif + + // Free the output edge array + if (out_edge_size > 0) { + compile->node_arena()->Afree(out_array, out_edge_size); + } + + // Free the input edge array and the node itself + if( edge_end == (char*)this ) { + // It was; free the input array and object all in one hit +#ifndef ASSERT + compile->node_arena()->Afree(in_array, edge_size+node_size); +#endif + } else { + // Free just the input array + compile->node_arena()->Afree(in_array, edge_size); + + // Free just the object +#ifndef ASSERT + compile->node_arena()->Afree(this, node_size); +#endif + } } //------------------------------grow------------------------------------------- @@ -685,10 +689,10 @@ void Node::grow(uint len) { _max = 4; _in = (Node**)arena->Amalloc(4*sizeof(Node*)); Node** to = _in; - to[0] = NULL; - to[1] = NULL; - to[2] = NULL; - to[3] = NULL; + to[0] = nullptr; + to[1] = nullptr; + to[2] = nullptr; + to[3] = nullptr; return; } new_max = next_power_of_2(len); @@ -696,7 +700,7 @@ void Node::grow(uint len) { // Previously I was using only powers-of-2 which peaked at 128 edges. //if( new_max >= limit ) new_max = limit-1; _in = (Node**)arena->Arealloc(_in, _max*sizeof(Node*), new_max*sizeof(Node*)); - Copy::zero_to_bytes(&_in[_max], (new_max-_max)*sizeof(Node*)); // NULL all new space + Copy::zero_to_bytes(&_in[_max], (new_max-_max)*sizeof(Node*)); // null all new space _max = new_max; // Record new max length // This assertion makes sure that Node::_max is wide enough to // represent the numerical value of new_max. @@ -718,9 +722,9 @@ void Node::out_grow( uint len ) { // Trimming to limit allows a uint8 to handle up to 255 edges. // Previously I was using only powers-of-2 which peaked at 128 edges. //if( new_max >= limit ) new_max = limit-1; - assert(_out != NULL && _out != NO_OUT_ARRAY, "out must have sensible value"); + assert(_out != nullptr && _out != NO_OUT_ARRAY, "out must have sensible value"); _out = (Node**)arena->Arealloc(_out,_outmax*sizeof(Node*),new_max*sizeof(Node*)); - //Copy::zero_to_bytes(&_out[_outmax], (new_max-_outmax)*sizeof(Node*)); // NULL all new space + //Copy::zero_to_bytes(&_out[_outmax], (new_max-_outmax)*sizeof(Node*)); // null all new space _outmax = new_max; // Record new max length // This assertion makes sure that Node::_max is wide enough to // represent the numerical value of new_max. @@ -734,7 +738,7 @@ bool Node::is_dead() const { if( is_top() || is_Mach() || (Opcode() == Op_Node && _outcnt > 0) ) return false; for( uint i = 0; i < _max; i++ ) - if( _in[i] != NULL ) + if( _in[i] != nullptr ) return false; dump(); return true; @@ -762,7 +766,7 @@ bool Node::is_reachable_from_root() const { //------------------------------is_unreachable--------------------------------- bool Node::is_unreachable(PhaseIterGVN &igvn) const { assert(!is_Mach(), "doesn't work with MachNodes"); - return outcnt() == 0 || igvn.type(this) == Type::TOP || (in(0) != NULL && in(0)->is_top()); + return outcnt() == 0 || igvn.type(this) == Type::TOP || (in(0) != nullptr && in(0)->is_top()); } //------------------------------add_req---------------------------------------- @@ -771,19 +775,19 @@ void Node::add_req( Node *n ) { assert( is_not_dead(n), "can not use dead node"); // Look to see if I can move precedence down one without reallocating - if( (_cnt >= _max) || (in(_max-1) != NULL) ) + if( (_cnt >= _max) || (in(_max-1) != nullptr) ) grow( _max+1 ); // Find a precedence edge to move - if( in(_cnt) != NULL ) { // Next precedence edge is busy? + if( in(_cnt) != nullptr ) { // Next precedence edge is busy? uint i; for( i=_cnt; i<_max; i++ ) - if( in(i) == NULL ) // Find the NULL at end of prec edge list + if( in(i) == nullptr ) // Find the null at end of prec edge list break; // There must be one, since we grew the array _in[i] = in(_cnt); // Move prec over, making space for req edge } _in[_cnt++] = n; // Stuff over old prec edge - if (n != NULL) n->add_out((Node *)this); + if (n != nullptr) n->add_out((Node *)this); } //---------------------------add_req_batch------------------------------------- @@ -802,10 +806,10 @@ void Node::add_req_batch( Node *n, uint m ) { grow( _max+m ); // Find a precedence edge to move - if( _in[_cnt] != NULL ) { // Next precedence edge is busy? + if( _in[_cnt] != nullptr ) { // Next precedence edge is busy? uint i; for( i=_cnt; i<_max; i++ ) - if( _in[i] == NULL ) // Find the NULL at end of prec edge list + if( _in[i] == nullptr ) // Find the null at end of prec edge list break; // There must be one, since we grew the array // Slide all the precs over by m positions (assume #prec << m). Copy::conjoint_words_to_higher((HeapWord*)&_in[_cnt], (HeapWord*)&_in[_cnt+m], ((i-_cnt)*sizeof(Node*))); @@ -817,7 +821,7 @@ void Node::add_req_batch( Node *n, uint m ) { } // Insert multiple out edges on the node. - if (n != NULL && !n->is_top()) { + if (n != nullptr && !n->is_top()) { for(uint i=0; iadd_out((Node *)this); } @@ -832,7 +836,7 @@ void Node::del_req( uint idx ) { "remove node from hash table before modifying it"); // First remove corresponding def-use edge Node *n = in(idx); - if (n != NULL) n->del_out((Node *)this); + if (n != nullptr) n->del_out((Node *)this); _in[idx] = in(--_cnt); // Compact the array // Avoid spec violation: Gap in prec edges. close_prec_gap_at(_cnt); @@ -847,7 +851,7 @@ void Node::del_req_ordered( uint idx ) { "remove node from hash table before modifying it"); // First remove corresponding def-use edge Node *n = in(idx); - if (n != NULL) n->del_out((Node *)this); + if (n != nullptr) n->del_out((Node *)this); if (idx < --_cnt) { // Not last edge ? Copy::conjoint_words_to_lower((HeapWord*)&_in[idx+1], (HeapWord*)&_in[idx], ((_cnt-idx)*sizeof(Node*))); } @@ -860,14 +864,14 @@ void Node::del_req_ordered( uint idx ) { // Insert a new required input at the end void Node::ins_req( uint idx, Node *n ) { assert( is_not_dead(n), "can not use dead node"); - add_req(NULL); // Make space + add_req(nullptr); // Make space assert( idx < _max, "Must have allocated enough space"); // Slide over if(_cnt-idx-1 > 0) { Copy::conjoint_words_to_higher((HeapWord*)&_in[idx], (HeapWord*)&_in[idx+1], ((_cnt-idx-1)*sizeof(Node*))); } _in[idx] = n; // Stuff over old required edge - if (n != NULL) n->add_out((Node *)this); // Add reciprocal def-use edge + if (n != nullptr) n->add_out((Node *)this); // Add reciprocal def-use edge } //-----------------------------find_edge--------------------------------------- @@ -885,13 +889,13 @@ int Node::replace_edge(Node* old, Node* neww, PhaseGVN* gvn) { for (uint i = 0; i < len(); i++) { if (in(i) == old) { if (i < req()) { - if (gvn != NULL) { + if (gvn != nullptr) { set_req_X(i, neww, gvn); } else { set_req(i, neww); } } else { - assert(gvn == NULL || gvn->is_IterGVN() == NULL, "no support for igvn here"); + assert(gvn == nullptr || gvn->is_IterGVN() == nullptr, "no support for igvn here"); assert(find_prec_edge(neww) == -1, "spec violation: duplicated prec edge (node %d -> %d)", _idx, neww->_idx); set_prec(i, neww); } @@ -917,7 +921,7 @@ int Node::replace_edges_in_range(Node* old, Node* neww, int start, int end, Phas } //-------------------------disconnect_inputs----------------------------------- -// NULL out all inputs to eliminate incoming Def-Use edges. +// null out all inputs to eliminate incoming Def-Use edges. void Node::disconnect_inputs(Compile* C) { // the layout of Node::_in // r: a required input, null is allowed @@ -936,7 +940,7 @@ void Node::disconnect_inputs(Compile* C) { // Remove precedence edges if any exist // Note: Safepoints may have precedence edges, even during parsing for (uint i = len(); i > req(); ) { - rm_prec(--i); // no-op if _in[i] is nullptr + rm_prec(--i); // no-op if _in[i] is null } #ifdef ASSERT @@ -973,12 +977,12 @@ Node* Node::find_out_with(int opcode) { return use; } } - return NULL; + return nullptr; } // Return true if the current node has an out that matches opcode. bool Node::has_out_with(int opcode) { - return (find_out_with(opcode) != NULL); + return (find_out_with(opcode) != nullptr); } // Return true if the current node has an out that matches any of the opcodes. @@ -1009,7 +1013,7 @@ Node* Node::uncast_helper(const Node* p, bool keep_deps) { } assert(depth_count++ < K, "infinite loop in Node::uncast_helper"); #endif - if (p == NULL || p->req() != 2) { + if (p == nullptr || p->req() != 2) { break; } else if (p->is_ConstraintCast()) { if (keep_deps && p->as_ConstraintCast()->carry_dependency()) { @@ -1025,35 +1029,35 @@ Node* Node::uncast_helper(const Node* p, bool keep_deps) { //------------------------------add_prec--------------------------------------- // Add a new precedence input. Precedence inputs are unordered, with -// duplicates removed and NULLs packed down at the end. +// duplicates removed and nulls packed down at the end. void Node::add_prec( Node *n ) { assert( is_not_dead(n), "can not use dead node"); - // Check for NULL at end + // Check for null at end if( _cnt >= _max || in(_max-1) ) grow( _max+1 ); // Find a precedence edge to move uint i = _cnt; - while( in(i) != NULL ) { + while( in(i) != nullptr ) { if (in(i) == n) return; // Avoid spec violation: duplicated prec edge. i++; } - _in[i] = n; // Stuff prec edge over NULL - if ( n != NULL) n->add_out((Node *)this); // Add mirror edge + _in[i] = n; // Stuff prec edge over null + if ( n != nullptr) n->add_out((Node *)this); // Add mirror edge #ifdef ASSERT - while ((++i)<_max) { assert(_in[i] == NULL, "spec violation: Gap in prec edges (node %d)", _idx); } + while ((++i)<_max) { assert(_in[i] == nullptr, "spec violation: Gap in prec edges (node %d)", _idx); } #endif } //------------------------------rm_prec---------------------------------------- // Remove a precedence input. Precedence inputs are unordered, with -// duplicates removed and NULLs packed down at the end. +// duplicates removed and nulls packed down at the end. void Node::rm_prec( uint j ) { assert(j < _max, "oob: i=%d, _max=%d", j, _max); assert(j >= _cnt, "not a precedence edge"); - if (_in[j] == NULL) return; // Avoid spec violation: Gap in prec edges. + if (_in[j] == nullptr) return; // Avoid spec violation: Gap in prec edges. _in[j]->del_out((Node *)this); close_prec_gap_at(j); } @@ -1065,12 +1069,12 @@ uint Node::size_of() const { return sizeof(*this); } uint Node::ideal_reg() const { return 0; } //------------------------------jvms------------------------------------------- -JVMState* Node::jvms() const { return NULL; } +JVMState* Node::jvms() const { return nullptr; } #ifdef ASSERT //------------------------------jvms------------------------------------------- bool Node::verify_jvms(const JVMState* using_jvms) const { - for (JVMState* jvms = this->jvms(); jvms != NULL; jvms = jvms->caller()) { + for (JVMState* jvms = this->jvms(); jvms != nullptr; jvms = jvms->caller()) { if (jvms == using_jvms) return true; } return false; @@ -1150,13 +1154,13 @@ const Type* Node::Value(PhaseGVN* phase) const { // pointer. If ANY change is made, it must return the root of the reshaped // graph - even if the root is the same Node. Example: swapping the inputs // to an AddINode gives the same answer and same root, but you still have to -// return the 'this' pointer instead of NULL. +// return the 'this' pointer instead of null. // // You cannot return an OLD Node, except for the 'this' pointer. Use the // Identity call to return an old Node; basically if Identity can find -// another Node have the Ideal call make no change and return NULL. +// another Node have the Ideal call make no change and return null. // Example: AddINode::Ideal must check for add of zero; in this case it -// returns NULL instead of doing any graph reshaping. +// returns null instead of doing any graph reshaping. // // You cannot modify any old Nodes except for the 'this' pointer. Due to // sharing there may be other users of the old Nodes relying on their current @@ -1191,7 +1195,7 @@ const Type* Node::Value(PhaseGVN* phase) const { // the same Opcode as the 'this' pointer use 'clone'. // Node *Node::Ideal(PhaseGVN *phase, bool can_reshape) { - return NULL; // Default to being Ideal already + return nullptr; // Default to being Ideal already } // Some nodes have specific Ideal subgraph transformations only if they are @@ -1224,17 +1228,17 @@ bool Node::has_special_unique_user() const { //--------------------------find_exact_control--------------------------------- // Skip Proj and CatchProj nodes chains. Check for Null and Top. Node* Node::find_exact_control(Node* ctrl) { - if (ctrl == NULL && this->is_Region()) + if (ctrl == nullptr && this->is_Region()) ctrl = this->as_Region()->is_copy(); - if (ctrl != NULL && ctrl->is_CatchProj()) { + if (ctrl != nullptr && ctrl->is_CatchProj()) { if (ctrl->as_CatchProj()->_con == CatchProjNode::fall_through_index) ctrl = ctrl->in(0); - if (ctrl != NULL && !ctrl->is_top()) + if (ctrl != nullptr && !ctrl->is_top()) ctrl = ctrl->in(0); } - if (ctrl != NULL && ctrl->is_Proj()) + if (ctrl != nullptr && ctrl->is_Proj()) ctrl = ctrl->in(0); return ctrl; @@ -1248,7 +1252,7 @@ Node* Node::find_exact_control(Node* ctrl) { // not an exhaustive search for a counterexample. bool Node::dominates(Node* sub, Node_List &nlist) { assert(this->is_CFG(), "expecting control"); - assert(sub != NULL && sub->is_CFG(), "expecting control"); + assert(sub != nullptr && sub->is_CFG(), "expecting control"); // detect dead cycle without regions int iterations_without_region_limit = DominatorSearchLimit; @@ -1265,7 +1269,7 @@ bool Node::dominates(Node* sub, Node_List &nlist) { // same region again, go through a different input. Eventually we // will either exit through the loop head, or give up. // (If we get confused, break out and return a conservative 'false'.) - while (sub != NULL) { + while (sub != nullptr) { if (sub->is_top()) break; // Conservative answer for dead code. if (sub == dom) { if (nlist.size() == 0) { @@ -1297,7 +1301,7 @@ bool Node::dominates(Node* sub, Node_List &nlist) { } else if (sub == up && sub->is_Region() && sub->req() == 2) { // Take in(1) path on the way up to 'dom' for regions with only one input up = sub->in(1); - } else if (sub == up && sub->is_Region() && sub->req() == 3) { + } else if (sub == up && sub->is_Region()) { // Try both paths for Regions with 2 input paths (it may be a loop head). // It could give conservative 'false' answer without information // which region's input is the entry path. @@ -1332,7 +1336,7 @@ bool Node::dominates(Node* sub, Node_List &nlist) { uint skip = region_was_visited_before ? 1 : 0; for (uint i = 1; i < sub->req(); i++) { Node* in = sub->in(i); - if (in != NULL && !in->is_top() && in != sub) { + if (in != nullptr && !in->is_top() && in != sub) { if (skip == 0) { up = in; break; @@ -1408,7 +1412,7 @@ static void kill_dead_code( Node *dead, PhaseIterGVN *igvn ) { Node* in = use->in(j); if (in == dead) { // Turn all dead inputs into TOP use->set_req(j, top); - } else if (in != NULL && !in->is_top()) { + } else if (in != nullptr && !in->is_top()) { dead_use = false; } } @@ -1432,7 +1436,7 @@ static void kill_dead_code( Node *dead, PhaseIterGVN *igvn ) { // Kill all inputs to the dead guy for (uint i=0; i < dead->req(); i++) { Node *n = dead->in(i); // Get input to dead guy - if (n != NULL && !n->is_top()) { // Input is valid? + if (n != nullptr && !n->is_top()) { // Input is valid? dead->set_req(i, top); // Smash input away if (n->outcnt() == 0) { // Input also goes dead? if (!n->is_Con()) @@ -1481,7 +1485,7 @@ bool Node::remove_dead_region(PhaseGVN *phase, bool can_reshape) { uint Node::hash() const { uint sum = 0; for( uint i=0; i<_cnt; i++ ) // Add in all inputs - sum = (sum<<1)-(uintptr_t)in(i); // Ignore embedded NULLs + sum = (sum<<1)-(uintptr_t)in(i); // Ignore embedded nulls return (sum>>2) + _cnt + Opcode(); } @@ -1518,7 +1522,7 @@ const TypeInt* Node::find_int_type() const { assert(is_Mach(), "should be ConNode(TypeNode) or else a MachNode"); return this->bottom_type()->isa_int(); } - return NULL; + return nullptr; } const TypeInteger* Node::find_integer_type(BasicType bt) const { @@ -1528,7 +1532,7 @@ const TypeInteger* Node::find_integer_type(BasicType bt) const { assert(is_Mach(), "should be ConNode(TypeNode) or else a MachNode"); return this->bottom_type()->isa_integer(bt); } - return NULL; + return nullptr; } // Get a pointer constant from a ConstNode. @@ -1553,7 +1557,7 @@ const TypeLong* Node::find_long_type() const { assert(is_Mach(), "should be ConNode(TypeNode) or else a MachNode"); return this->bottom_type()->isa_long(); } - return NULL; + return nullptr; } @@ -1563,9 +1567,9 @@ const TypeLong* Node::find_long_type() const { const TypePtr* Node::get_ptr_type() const { const TypePtr* tp = this->bottom_type()->make_ptr(); #ifdef ASSERT - if (tp == NULL) { + if (tp == nullptr) { this->dump(1); - assert((tp != NULL), "unexpected node type"); + assert((tp != nullptr), "unexpected node type"); } #endif return tp; @@ -1615,8 +1619,8 @@ Node* Node::find_ctrl(int idx) { //------------------------------find------------------------------------------- // Tries to find the node with the index |idx| starting from this node. If idx is negative, -// the search also includes forward (out) edges. Returns NULL if not found. -// If only_ctrl is set, the search will only be done on control nodes. Returns NULL if +// the search also includes forward (out) edges. Returns null if not found. +// If only_ctrl is set, the search will only be done on control nodes. Returns null if // not found or if the node to be found is not a control node (search will not find it). Node* Node::find(const int idx, bool only_ctrl) { ResourceMark rm; @@ -1625,14 +1629,14 @@ Node* Node::find(const int idx, bool only_ctrl) { Node_List worklist; Arena* old_arena = Compile::current()->old_arena(); add_to_worklist(this, &worklist, old_arena, &old_space, &new_space); - Node* result = NULL; + Node* result = nullptr; int node_idx = (idx >= 0) ? idx : -idx; for (uint list_index = 0; list_index < worklist.size(); list_index++) { Node* n = worklist[list_index]; if ((int)n->_idx == node_idx debug_only(|| n->debug_idx() == node_idx)) { - if (result != NULL) { + if (result != nullptr) { tty->print("find: " INTPTR_FORMAT " and " INTPTR_FORMAT " both have idx==%d\n", (uintptr_t)result, (uintptr_t)n, node_idx); } @@ -1655,7 +1659,7 @@ Node* Node::find(const int idx, bool only_ctrl) { #ifdef ASSERT // Search along debug_orig edges last Node* orig = n->debug_orig(); - while (orig != NULL && add_to_worklist(orig, &worklist, old_arena, &old_space, &new_space)) { + while (orig != nullptr && add_to_worklist(orig, &worklist, old_arena, &old_space, &new_space)) { orig = orig->debug_orig(); } #endif // ASSERT @@ -1665,7 +1669,7 @@ Node* Node::find(const int idx, bool only_ctrl) { bool Node::add_to_worklist(Node* n, Node_List* worklist, Arena* old_arena, VectorSet* old_space, VectorSet* new_space) { if (not_a_node(n)) { - return false; // Gracefully handle NULL, -1, 0xabababab, etc. + return false; // Gracefully handle null, -1, 0xabababab, etc. } // Contained in new_space or old_space? Check old_arena first since it's mostly empty. @@ -1683,7 +1687,7 @@ const char *Node::Name() const { return NodeClassNames[Opcode()]; } static bool is_disconnected(const Node* n) { for (uint i = 0; i < n->req(); i++) { - if (n->in(i) != NULL) return false; + if (n->in(i) != nullptr) return false; } return true; } @@ -1692,15 +1696,15 @@ static bool is_disconnected(const Node* n) { void Node::dump_orig(outputStream *st, bool print_key) const { Compile* C = Compile::current(); Node* orig = _debug_orig; - if (not_a_node(orig)) orig = NULL; - if (orig != NULL && !C->node_arena()->contains(orig)) orig = NULL; - if (orig == NULL) return; + if (not_a_node(orig)) orig = nullptr; + if (orig != nullptr && !C->node_arena()->contains(orig)) orig = nullptr; + if (orig == nullptr) return; if (print_key) { st->print(" !orig="); } Node* fast = orig->debug_orig(); // tortoise & hare algorithm to detect loops - if (not_a_node(fast)) fast = NULL; - while (orig != NULL) { + if (not_a_node(fast)) fast = nullptr; + while (orig != nullptr) { bool discon = is_disconnected(orig); // if discon, print [123] else 123 if (discon) st->print("["); if (!Compile::current()->node_arena()->contains(orig)) @@ -1708,16 +1712,16 @@ void Node::dump_orig(outputStream *st, bool print_key) const { st->print("%d", orig->_idx); if (discon) st->print("]"); orig = orig->debug_orig(); - if (not_a_node(orig)) orig = NULL; - if (orig != NULL && !C->node_arena()->contains(orig)) orig = NULL; - if (orig != NULL) st->print(","); - if (fast != NULL) { + if (not_a_node(orig)) orig = nullptr; + if (orig != nullptr && !C->node_arena()->contains(orig)) orig = nullptr; + if (orig != nullptr) st->print(","); + if (fast != nullptr) { // Step fast twice for each single step of orig: fast = fast->debug_orig(); - if (not_a_node(fast)) fast = NULL; - if (fast != NULL && fast != orig) { + if (not_a_node(fast)) fast = nullptr; + if (fast != nullptr && fast != orig) { fast = fast->debug_orig(); - if (not_a_node(fast)) fast = NULL; + if (not_a_node(fast)) fast = nullptr; } if (fast == orig) { st->print("..."); @@ -1730,16 +1734,16 @@ void Node::dump_orig(outputStream *st, bool print_key) const { void Node::set_debug_orig(Node* orig) { _debug_orig = orig; if (BreakAtNode == 0) return; - if (not_a_node(orig)) orig = NULL; + if (not_a_node(orig)) orig = nullptr; int trip = 10; - while (orig != NULL) { + while (orig != nullptr) { if (orig->debug_idx() == BreakAtNode || (int)orig->_idx == BreakAtNode) { tty->print_cr("BreakAtNode: _idx=%d _debug_idx=%d orig._idx=%d orig._debug_idx=%d", this->_idx, this->debug_idx(), orig->_idx, orig->debug_idx()); BREAKPOINT; } orig = orig->debug_orig(); - if (not_a_node(orig)) orig = NULL; + if (not_a_node(orig)) orig = nullptr; if (trip-- <= 0) break; } } @@ -1788,7 +1792,7 @@ void Node::dump(const char* suffix, bool mark, outputStream *st) const { const Type *t = bottom_type(); - if (t != NULL && (t->isa_instptr() || t->isa_klassptr())) { + if (t != nullptr && (t->isa_instptr() || t->isa_klassptr())) { const TypeInstPtr *toop = t->isa_instptr(); const TypeKlassPtr *tkls = t->isa_klassptr(); ciKlass* klass = toop ? toop->klass() : (tkls ? tkls->klass() : NULL ); @@ -1817,8 +1821,8 @@ void Node::dump(const char* suffix, bool mark, outputStream *st) const { if (is_new) { DEBUG_ONLY(dump_orig(st)); Node_Notes* nn = C->node_notes_at(_idx); - if (nn != NULL && !nn->is_clear()) { - if (nn->jvms() != NULL) { + if (nn != nullptr && !nn->is_clear()) { + if (nn->jvms() != nullptr) { st->print(" !jvms:"); nn->jvms()->dump_spec(st); } @@ -1833,7 +1837,7 @@ void Node::dump_req(outputStream *st) const { // Dump the required input edges for (uint i = 0; i < req(); i++) { // For all required inputs Node* d = in(i); - if (d == NULL) { + if (d == nullptr) { st->print("_ "); } else if (not_a_node(d)) { st->print("not_a_node "); // uninitialized, sentinel, garbage, etc. @@ -1850,7 +1854,7 @@ void Node::dump_prec(outputStream *st) const { int any_prec = 0; for (uint i = req(); i < len(); i++) { // For all precedence inputs Node* p = in(i); - if (p != NULL) { + if (p != nullptr) { if (!any_prec++) st->print(" |"); if (not_a_node(p)) { st->print("not_a_node "); continue; } st->print("%c%d ", Compile::current()->node_arena()->contains(in(i)) ? ' ' : 'o', in(i)->_idx); @@ -1865,7 +1869,7 @@ void Node::dump_out(outputStream *st) const { // Dump the output edges for (uint i = 0; i < _outcnt; i++) { // For all outputs Node* u = _out[i]; - if (u == NULL) { + if (u == nullptr) { st->print("_ "); } else if (not_a_node(u)) { st->print("not_a_node "); @@ -2228,7 +2232,7 @@ void Node::verify(int verify_depth, VectorSet& visited, Node_List& worklist) { Node* n = worklist[list_index]; if (n->is_Con() && n->bottom_type() == Type::TOP) { - if (C->cached_top_node() == NULL) { + if (C->cached_top_node() == nullptr) { C->set_cached_top_node((Node*)n); } assert(C->cached_top_node() == n, "TOP node must be unique"); @@ -2324,14 +2328,14 @@ void Node_Array::insert(uint i, Node* n) { void Node_Array::remove(uint i) { Copy::conjoint_words_to_lower((HeapWord*)&_nodes[i + 1], (HeapWord*)&_nodes[i], ((_max - i - 1) * sizeof(Node*))); - _nodes[_max - 1] = NULL; + _nodes[_max - 1] = nullptr; } void Node_Array::dump() const { #ifndef PRODUCT for (uint i = 0; i < _max; i++) { Node* nn = _nodes[i]; - if (nn != NULL) { + if (nn != nullptr) { tty->print("%5d--> ",i); nn->dump(); } } @@ -2346,7 +2350,7 @@ bool Node::is_iteratively_computed() { if (ideal_reg()) { // does operation have a result register? for (uint i = 1; i < req(); i++) { Node* n = in(i); - if (n != NULL && n->is_Phi()) { + if (n != nullptr && n->is_Phi()) { for (uint j = 1; j < n->req(); j++) { if (n->in(j) == this) { return true; @@ -2360,7 +2364,7 @@ bool Node::is_iteratively_computed() { //--------------------------find_similar------------------------------ // Return a node with opcode "opc" and same inputs as "this" if one can -// be found; Otherwise return NULL; +// be found; Otherwise return null; Node* Node::find_similar(int opc) { if (req() >= 2) { Node* def = in(1); @@ -2383,19 +2387,19 @@ Node* Node::find_similar(int opc) { } } } - return NULL; + return nullptr; } //--------------------------unique_ctrl_out------------------------------ // Return the unique control out if only one. Null if none or more than one. Node* Node::unique_ctrl_out() const { - Node* found = NULL; + Node* found = nullptr; for (uint i = 0; i < outcnt(); i++) { Node* use = raw_out(i); if (use->is_CFG() && use != this) { - if (found != NULL) { - return NULL; + if (found != nullptr) { + return nullptr; } found = use; } @@ -2404,7 +2408,7 @@ Node* Node::unique_ctrl_out() const { } void Node::ensure_control_or_add_prec(Node* c) { - if (in(0) == NULL) { + if (in(0) == nullptr) { set_req(0, c); } else if (in(0) != c) { add_prec(c); @@ -2415,7 +2419,7 @@ bool Node::is_dead_loop_safe() const { if (is_Phi()) { return true; } - if (is_Proj() && in(0) == NULL) { + if (is_Proj() && in(0) == nullptr) { return true; } if ((_flags & (Flag_is_dead_loop_safe | Flag_is_Con)) != 0) { @@ -2468,7 +2472,7 @@ void Node_List::dump_simple() const { if( _nodes[i] ) { tty->print(" %d", _nodes[i]->_idx); } else { - tty->print(" NULL"); + tty->print(" null"); } } #endif @@ -2494,7 +2498,7 @@ void Unique_Node_List::remove(Node* n) { void Unique_Node_List::remove_useless_nodes(VectorSet &useful) { for (uint i = 0; i < size(); ++i) { Node *n = at(i); - assert( n != NULL, "Did not expect null entries in worklist"); + assert( n != nullptr, "Did not expect null entries in worklist"); if (!useful.test(n->_idx)) { _in_worklist.remove(n->_idx); map(i, Node_List::pop()); @@ -2522,7 +2526,7 @@ Node* Node_Stack::find(uint idx) const { return node_at(i); } } - return NULL; + return nullptr; } //============================================================================= diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp index 2a78e259d3e73..890e8db077d74 100644 --- a/src/hotspot/share/opto/node.hpp +++ b/src/hotspot/share/opto/node.hpp @@ -70,6 +70,7 @@ class CmpNode; class CodeBuffer; class ConstraintCastNode; class ConNode; +class ConINode; class CompareAndSwapNode; class CompareAndExchangeNode; class CountedLoopNode; @@ -254,7 +255,7 @@ class Node { // Create a new Node with given input edges. // This version requires use of the "edge-count" new. - // E.g. new (C,3) FooNode( C, NULL, left, right ); + // E.g. new (C,3) FooNode( C, nullptr, left, right ); Node( Node *n0 ); Node( Node *n0, Node *n1 ); Node( Node *n0, Node *n1, Node *n2 ); @@ -270,10 +271,10 @@ class Node { // Clone a Node, immediately supplying one or two new edges. // The first and second arguments, if non-null, replace in(1) and in(2), // respectively. - Node* clone_with_data_edge(Node* in1, Node* in2 = NULL) const { + Node* clone_with_data_edge(Node* in1, Node* in2 = nullptr) const { Node* nn = clone(); - if (in1 != NULL) nn->set_req(1, in1); - if (in2 != NULL) nn->set_req(2, in2); + if (in1 != nullptr) nn->set_req(1, in1); + if (in2 != nullptr) nn->set_req(2, in2); return nn; } @@ -292,10 +293,10 @@ class Node { Node **_out; // Array of def-use references to Nodes // Input edges are split into two categories. Required edges are required - // for semantic correctness; order is important and NULLs are allowed. + // for semantic correctness; order is important and nulls are allowed. // Precedence edges are used to help determine execution order and are // added, e.g., for scheduling purposes. They are unordered and not - // duplicated; they have no embedded NULLs. Edges from 0 to _cnt-1 + // duplicated; they have no embedded nulls. Edges from 0 to _cnt-1 // are required, from _cnt to _max-1 are precedence edges. node_idx_t _cnt; // Total number of required Node inputs. @@ -390,8 +391,8 @@ class Node { // Reference to the i'th input Node. Error if out of bounds. Node* in(uint i) const { assert(i < _max, "oob: i=%d, _max=%d", i, _max); return _in[i]; } - // Reference to the i'th input Node. NULL if out of bounds. - Node* lookup(uint i) const { return ((i < _max) ? _in[i] : NULL); } + // Reference to the i'th input Node. null if out of bounds. + Node* lookup(uint i) const { return ((i < _max) ? _in[i] : nullptr); } // Reference to the i'th output Node. Error if out of bounds. // Use this accessor sparingly. We are going trying to use iterators instead. Node* raw_out(uint i) const { assert(i < _outcnt,"oob"); return _out[i]; } @@ -434,9 +435,9 @@ class Node { assert( !VerifyHashTableKeys || _hash_lock == 0, "remove node from hash table before modifying it"); Node** p = &_in[i]; // cache this._in, across the del_out call - if (*p != NULL) (*p)->del_out((Node *)this); + if (*p != nullptr) (*p)->del_out((Node *)this); (*p) = n; - if (n != NULL) n->add_out((Node *)this); + if (n != nullptr) n->add_out((Node *)this); Compile::current()->record_modified_node(this); } // Light version of set_req() to init inputs after node creation. @@ -446,9 +447,9 @@ class Node { assert( i < _cnt, "oob"); assert( !VerifyHashTableKeys || _hash_lock == 0, "remove node from hash table before modifying it"); - assert( _in[i] == NULL, "sanity"); + assert( _in[i] == nullptr, "sanity"); _in[i] = n; - if (n != NULL) n->add_out((Node *)this); + if (n != nullptr) n->add_out((Node *)this); Compile::current()->record_modified_node(this); } // Find first occurrence of n among my edges: @@ -456,22 +457,22 @@ class Node { int find_prec_edge(Node* n) { for (uint i = req(); i < len(); i++) { if (_in[i] == n) return i; - if (_in[i] == NULL) { - DEBUG_ONLY( while ((++i) < len()) assert(_in[i] == NULL, "Gap in prec edges!"); ) + if (_in[i] == nullptr) { + DEBUG_ONLY( while ((++i) < len()) assert(_in[i] == nullptr, "Gap in prec edges!"); ) break; } } return -1; } - int replace_edge(Node* old, Node* neww, PhaseGVN* gvn = NULL); + int replace_edge(Node* old, Node* neww, PhaseGVN* gvn = nullptr); int replace_edges_in_range(Node* old, Node* neww, int start, int end, PhaseGVN* gvn); - // NULL out all inputs to eliminate incoming Def-Use edges. + // null out all inputs to eliminate incoming Def-Use edges. void disconnect_inputs(Compile* C); // Quickly, return true if and only if I am Compile::current()->top(). bool is_top() const { - assert((this == (Node*) Compile::current()->top()) == (_out == NULL), ""); - return (_out == NULL); + assert((this == (Node*) Compile::current()->top()) == (_out == nullptr), ""); + return (_out == nullptr); } // Reaffirm invariants for is_top. (Only from Compile::set_cached_top_node.) void setup_is_top(); @@ -519,14 +520,14 @@ class Node { void close_prec_gap_at(uint gap) { assert(_cnt <= gap && gap < _max, "no valid prec edge"); uint i = gap; - Node *last = NULL; + Node *last = nullptr; for (; i < _max-1; ++i) { Node *next = _in[i+1]; - if (next == NULL) break; + if (next == nullptr) break; last = next; } - _in[gap] = last; // Move last slot to empty one. - _in[i] = NULL; // NULL out last slot. + _in[gap] = last; // Move last slot to empty one. + _in[i] = nullptr; // null out last slot. } public: @@ -553,11 +554,11 @@ class Node { assert(i >= _cnt, "not a precedence edge"); // Avoid spec violation: duplicated prec edge. if (_in[i] == n) return; - if (n == NULL || find_prec_edge(n) != -1) { + if (n == nullptr || find_prec_edge(n) != -1) { rm_prec(i); return; } - if (_in[i] != NULL) _in[i]->del_out((Node *)this); + if (_in[i] != nullptr) _in[i]->del_out((Node *)this); _in[i] = n; n->add_out((Node *)this); } @@ -581,7 +582,7 @@ class Node { // Iterators over input Nodes for a Node X are written as: // for( i = 0; i < X.req(); i++ ) ... X[i] ... - // NOTE: Required edges can contain embedded NULL pointers. + // NOTE: Required edges can contain embedded null pointers. //----------------- Other Node Properties @@ -707,6 +708,8 @@ class Node { DEFINE_CLASS_ID(EncodePKlass, EncodeNarrowPtr, 1) DEFINE_CLASS_ID(Vector, Type, 7) DEFINE_CLASS_ID(VectorMaskCmp, Vector, 0) + DEFINE_CLASS_ID(Con, Type, 8) + DEFINE_CLASS_ID(ConI, Con, 0) DEFINE_CLASS_ID(Proj, Node, 3) DEFINE_CLASS_ID(CatchProj, Proj, 0) @@ -824,11 +827,11 @@ class Node { return ((_class_id & ClassMask_##type) == Class_##type); \ } \ type##Node *as_##type() const { \ - assert(is_##type(), "invalid node class: %s", Name()); \ + assert(is_##type(), "invalid node class: %s", Name()); \ return (type##Node*)this; \ } \ type##Node* isa_##type() const { \ - return (is_##type()) ? as_##type() : NULL; \ + return (is_##type()) ? as_##type() : nullptr; \ } DEFINE_CLASS_QUERY(AbstractLock) @@ -854,6 +857,7 @@ class Node { DEFINE_CLASS_QUERY(CheckCastPP) DEFINE_CLASS_QUERY(CastII) DEFINE_CLASS_QUERY(CastLL) + DEFINE_CLASS_QUERY(ConI) DEFINE_CLASS_QUERY(ConstraintCast) DEFINE_CLASS_QUERY(ClearArray) DEFINE_CLASS_QUERY(CMove) @@ -981,7 +985,7 @@ class Node { // The node is a "macro" node which needs to be expanded before matching bool is_macro() const { return (_flags & Flag_is_macro) != 0; } // The node is expensive: the best control is set during loop opts - bool is_expensive() const { return (_flags & Flag_is_expensive) != 0 && in(0) != NULL; } + bool is_expensive() const { return (_flags & Flag_is_expensive) != 0 && in(0) != nullptr; } // An arithmetic node which accumulates a data in a loop. // It must have the loop's phi as input and provide a def to the phi. @@ -1006,10 +1010,10 @@ class Node { void raise_bottom_type(const Type* new_type); // Get the address type with which this node uses and/or defs memory, - // or NULL if none. The address type is conservatively wide. + // or null if none. The address type is conservatively wide. // Returns non-null for calls, membars, loads, stores, etc. // Returns TypePtr::BOTTOM if the node touches memory "broadly". - virtual const class TypePtr *adr_type() const { return NULL; } + virtual const class TypePtr *adr_type() const { return nullptr; } // Return an existing node which computes the same function as this node. // The optimistic combined algorithm requires this to return a Node which @@ -1067,7 +1071,7 @@ class Node { bool is_cloop_ind_var() const; // Return a node with opcode "opc" and same inputs as "this" if one can - // be found; Otherwise return NULL; + // be found; Otherwise return null; Node* find_similar(int opc); // Return the unique control out if only one. Null if none or more than one. @@ -1097,7 +1101,7 @@ class Node { // Should we clone rather than spill this instruction? bool rematerialize() const; - // Return JVM State Object if this Node carries debug info, or NULL otherwise + // Return JVM State Object if this Node carries debug info, or null otherwise virtual JVMState* jvms() const; // Print as assembly @@ -1113,12 +1117,12 @@ class Node { // return value_if_unknown. jint find_int_con(jint value_if_unknown) const { const TypeInt* t = find_int_type(); - return (t != NULL && t->is_con()) ? t->get_con() : value_if_unknown; + return (t != nullptr && t->is_con()) ? t->get_con() : value_if_unknown; } // Return the constant, knowing it is an integer constant already jint get_int() const { const TypeInt* t = find_int_type(); - guarantee(t != NULL, "must be con"); + guarantee(t != nullptr, "must be con"); return t->get_con(); } // Here's where the work is done. Can produce non-constant int types too. @@ -1128,18 +1132,18 @@ class Node { // Same thing for long (and intptr_t, via type.hpp): jlong get_long() const { const TypeLong* t = find_long_type(); - guarantee(t != NULL, "must be con"); + guarantee(t != nullptr, "must be con"); return t->get_con(); } jlong find_long_con(jint value_if_unknown) const { const TypeLong* t = find_long_type(); - return (t != NULL && t->is_con()) ? t->get_con() : value_if_unknown; + return (t != nullptr && t->is_con()) ? t->get_con() : value_if_unknown; } const TypeLong* find_long_type() const; jlong get_integer_as_long(BasicType bt) const { const TypeInteger* t = find_integer_type(bt); - guarantee(t != NULL, "must be con"); + guarantee(t != nullptr, "must be con"); return t->get_con_as_long(bt); } const TypePtr* get_ptr_type() const; @@ -1263,7 +1267,7 @@ class Node { }; inline bool not_a_node(const Node* n) { - if (n == NULL) return true; + if (n == nullptr) return true; if (((intptr_t)n & 1) != 0) return true; // uninitialized, etc. if (*(address*)n == badAddress) return true; // kill by Node::destruct return false; @@ -1523,7 +1527,7 @@ class SimpleDUIterator : public StackObj { //----------------------------------------------------------------------------- // Map dense integer indices to Nodes. Uses classic doubling-array trick. -// Abstractly provides an infinite array of Node*'s, initialized to NULL. +// Abstractly provides an infinite array of Node*'s, initialized to null. // Note that the constructor just zeros things, and since I use Arena // allocation I do not need a destructor to reclaim storage. class Node_Array : public ResourceObj { @@ -1540,15 +1544,15 @@ class Node_Array : public ResourceObj { } Node_Array(Node_Array* na) : _a(na->_a), _max(na->_max), _nodes(na->_nodes) {} - Node *operator[] ( uint i ) const // Lookup, or NULL for not mapped - { return (i<_max) ? _nodes[i] : (Node*)NULL; } + Node *operator[] ( uint i ) const // Lookup, or null for not mapped + { return (i<_max) ? _nodes[i] : (Node*)nullptr; } Node* at(uint i) const { assert(i<_max,"oob"); return _nodes[i]; } Node** adr() { return _nodes; } // Extend the mapping: index i maps to Node *n. void map( uint i, Node *n ) { if( i>=_max ) grow(i); _nodes[i] = n; } void insert( uint i, Node *n ); void remove( uint i ); // Remove, preserving order - // Clear all entries in _nodes to NULL but keep storage + // Clear all entries in _nodes to null but keep storage void clear() { Copy::zero_to_bytes(_nodes, _max * sizeof(Node*)); } @@ -1647,6 +1651,11 @@ inline void Compile::record_for_igvn(Node* n) { _for_igvn->push(n); } +// Inline definition of Compile::remove_for_igvn must be deferred to this point. +inline void Compile::remove_for_igvn(Node* n) { + _for_igvn->remove(n); +} + //------------------------------Node_Stack------------------------------------- class Node_Stack { friend class VMStructs; @@ -1726,7 +1735,7 @@ class Node_Notes { JVMState* _jvms; public: - Node_Notes(JVMState* jvms = NULL) { + Node_Notes(JVMState* jvms = nullptr) { _jvms = jvms; } @@ -1735,12 +1744,12 @@ class Node_Notes { // True if there is nothing here. bool is_clear() { - return (_jvms == NULL); + return (_jvms == nullptr); } // Make there be nothing here. void clear() { - _jvms = NULL; + _jvms = nullptr; } // Make a new, clean node notes. @@ -1759,8 +1768,8 @@ class Node_Notes { // Absorb any information from source. bool update_from(Node_Notes* source) { bool changed = false; - if (source != NULL) { - if (source->jvms() != NULL) { + if (source != nullptr) { + if (source->jvms() != nullptr) { set_jvms(source->jvms()); changed = true; } @@ -1775,22 +1784,22 @@ Compile::locate_node_notes(GrowableArray* arr, int idx, bool can_grow) { assert(idx >= 0, "oob"); int block_idx = (idx >> _log2_node_notes_block_size); - int grow_by = (block_idx - (arr == NULL? 0: arr->length())); + int grow_by = (block_idx - (arr == nullptr? 0: arr->length())); if (grow_by >= 0) { - if (!can_grow) return NULL; + if (!can_grow) return nullptr; grow_node_notes(arr, grow_by + 1); } - if (arr == NULL) return NULL; + if (arr == nullptr) return nullptr; // (Every element of arr is a sub-array of length _node_notes_block_size.) return arr->at(block_idx) + (idx & (_node_notes_block_size-1)); } inline bool Compile::set_node_notes_at(int idx, Node_Notes* value) { - if (value == NULL || value->is_clear()) + if (value == nullptr || value->is_clear()) return false; // nothing to write => write nothing Node_Notes* loc = locate_node_notes(_node_note_array, idx, true); - assert(loc != NULL, ""); + assert(loc != nullptr, ""); return loc->update_from(value); } @@ -1805,13 +1814,13 @@ class TypeNode : public Node { const Type* const _type; public: void set_type(const Type* t) { - assert(t != NULL, "sanity"); + assert(t != nullptr, "sanity"); debug_only(uint check_hash = (VerifyHashTableKeys && _hash_lock) ? hash() : NO_HASH); *(const Type**)&_type = t; // cast away const-ness // If this node is in the hash table, make sure it doesn't need a rehash. assert(check_hash == NO_HASH || check_hash == hash(), "type change must preserve hash code"); } - const Type* type() const { assert(_type != NULL, "sanity"); return _type; }; + const Type* type() const { assert(_type != nullptr, "sanity"); return _type; }; TypeNode( const Type *t, uint required ) : Node(required), _type(t) { init_class_id(Class_Type); } diff --git a/src/hotspot/share/opto/opaquenode.cpp b/src/hotspot/share/opto/opaquenode.cpp index c66df16b2d039..2593db31c9ac2 100644 --- a/src/hotspot/share/opto/opaquenode.cpp +++ b/src/hotspot/share/opto/opaquenode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -76,7 +76,7 @@ Node *ProfileBooleanNode::Ideal(PhaseGVN *phase, bool can_reshape) { _delay_removal = false; return this; } else { - return NULL; + return nullptr; } } diff --git a/src/hotspot/share/opto/opaquenode.hpp b/src/hotspot/share/opto/opaquenode.hpp index cb7ff23764bfd..e375cfee0a8e4 100644 --- a/src/hotspot/share/opto/opaquenode.hpp +++ b/src/hotspot/share/opto/opaquenode.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -35,7 +35,7 @@ class Opaque1Node : public Node { virtual uint hash() const ; // { return NO_HASH; } virtual bool cmp( const Node &n ) const; public: - Opaque1Node(Compile* C, Node *n) : Node(NULL, n) { + Opaque1Node(Compile* C, Node *n) : Node(nullptr, n) { // Put it on the Macro nodes list to removed during macro nodes expansion. init_flags(Flag_is_macro); init_class_id(Class_Opaque1); @@ -43,13 +43,13 @@ class Opaque1Node : public Node { } // Special version for the pre-loop to hold the original loop limit // which is consumed by range check elimination. - Opaque1Node(Compile* C, Node *n, Node* orig_limit) : Node(NULL, n, orig_limit) { + Opaque1Node(Compile* C, Node *n, Node* orig_limit) : Node(nullptr, n, orig_limit) { // Put it on the Macro nodes list to removed during macro nodes expansion. init_flags(Flag_is_macro); init_class_id(Class_Opaque1); C->add_macro_node(this); } - Node* original_loop_limit() { return req()==3 ? in(2) : NULL; } + Node* original_loop_limit() { return req()==3 ? in(2) : nullptr; } virtual int Opcode() const; virtual const Type *bottom_type() const { return TypeInt::INT; } virtual Node* Identity(PhaseGVN* phase); @@ -114,7 +114,7 @@ class Opaque3Node : public Opaque2Node { // GraphKit::must_be_not_null(). class Opaque4Node : public Node { public: - Opaque4Node(Compile* C, Node *tst, Node* final_tst) : Node(NULL, tst, final_tst) { + Opaque4Node(Compile* C, Node *tst, Node* final_tst) : Node(nullptr, tst, final_tst) { init_flags(Flag_is_macro); C->add_macro_node(this); } diff --git a/src/hotspot/share/opto/output.cpp b/src/hotspot/share/opto/output.cpp index 57d2fe05481e4..8a1ed0d316089 100644 --- a/src/hotspot/share/opto/output.cpp +++ b/src/hotspot/share/opto/output.cpp @@ -298,29 +298,29 @@ PhaseOutput::PhaseOutput() _first_block_size(0), _handler_table(), _inc_table(), - _oop_map_set(NULL), - _scratch_buffer_blob(NULL), - _scratch_locs_memory(NULL), + _oop_map_set(nullptr), + _scratch_buffer_blob(nullptr), + _scratch_locs_memory(nullptr), _scratch_const_size(-1), _in_scratch_emit_size(false), _frame_slots(0), _code_offsets(), _node_bundling_limit(0), - _node_bundling_base(NULL), + _node_bundling_base(nullptr), _orig_pc_slot(0), _orig_pc_slot_offset_in_bytes(0), _buf_sizes(), - _block(NULL), + _block(nullptr), _index(0) { C->set_output(this); - if (C->stub_name() == NULL) { + if (C->stub_name() == nullptr) { _orig_pc_slot = C->fixed_slots() - (sizeof(address) / VMRegImpl::stack_slot_size); } } PhaseOutput::~PhaseOutput() { - C->set_output(NULL); - if (_scratch_buffer_blob != NULL) { + C->set_output(nullptr); + if (_scratch_buffer_blob != nullptr) { BufferBlob::free(_scratch_buffer_blob); } } @@ -419,7 +419,7 @@ void PhaseOutput::Output() { // Complete sizing of codebuffer CodeBuffer* cb = init_buffer(); - if (cb == NULL || C->failing()) { + if (cb == nullptr || C->failing()) { return; } @@ -440,7 +440,7 @@ bool PhaseOutput::need_stack_bang(int frame_size_in_bytes) const { // unexpected stack overflow (compiled method stack banging should // guarantee it doesn't happen) so we always need the stack bang in // a debug VM. - return (C->stub_function() == NULL && + return (C->stub_function() == nullptr && (C->has_java_calls() || frame_size_in_bytes > os::vm_page_size()>>3 DEBUG_ONLY(|| true))); } @@ -450,7 +450,7 @@ bool PhaseOutput::need_register_stack_bang() const { // This is only used on architectures which have split register // and memory stacks (ie. IA64). // Bang if the method is not a stub function and has java calls - return (C->stub_function() == NULL && C->has_java_calls()); + return (C->stub_function() == nullptr && C->has_java_calls()); } @@ -656,8 +656,8 @@ void PhaseOutput::shorten_branches(uint* blk_starts) { for (uint i = 0; i < nblocks; i++) { Block* block = C->cfg()->get_block(i); int idx = jmp_nidx[i]; - MachNode* mach = (idx == -1) ? NULL: block->get_node(idx)->as_Mach(); - if (mach != NULL && mach->may_be_short_branch()) { + MachNode* mach = (idx == -1) ? nullptr: block->get_node(idx)->as_Mach(); + if (mach != nullptr && mach->may_be_short_branch()) { #ifdef ASSERT assert(jmp_size[i] > 0 && mach->is_MachBranch(), "sanity"); int j; @@ -721,7 +721,7 @@ void PhaseOutput::shorten_branches(uint* blk_starts) { has_short_branch_candidate = true; } } // (mach->may_be_short_branch()) - if (mach != NULL && (mach->may_be_short_branch() || + if (mach != nullptr && (mach->may_be_short_branch() || mach->avoid_back_to_back(MachNode::AVOID_AFTER))) { last_may_be_short_branch_adr = blk_starts[i] + jmp_offset[i] + jmp_size[i]; } @@ -787,12 +787,12 @@ PhaseOutput::sv_for_node_id(GrowableArray *objs, int id) { } } // Otherwise.. - return NULL; + return nullptr; } void PhaseOutput::set_sv_for_object_node(GrowableArray *objs, ObjectValue* sv ) { - assert(sv_for_node_id(objs, sv->id()) == NULL, "Precondition"); + assert(sv_for_node_id(objs, sv->id()) == nullptr, "Precondition"); objs->append(sv); } @@ -821,7 +821,7 @@ void PhaseOutput::FillLocArray( int idx, MachSafePointNode* sfpt, Node *local, SafePointScalarObjectNode* spobj = local->as_SafePointScalarObject(); ObjectValue* sv = sv_for_node_id(objs, spobj->_idx); - if (sv == NULL) { + if (sv == nullptr) { ciKlass* cik = t->is_oopptr()->klass(); assert(cik->is_instance_klass() || cik->is_array_klass(), "Not supported allocation."); @@ -909,7 +909,7 @@ void PhaseOutput::FillLocArray( int idx, MachSafePointNode* sfpt, Node *local, ShouldNotReachHere(); // Caller should skip 2nd halves break; case Type::AnyPtr: - array->append(new ConstantOopWriteValue(NULL)); + array->append(new ConstantOopWriteValue(nullptr)); break; case Type::AryPtr: case Type::InstPtr: // fall through @@ -917,7 +917,7 @@ void PhaseOutput::FillLocArray( int idx, MachSafePointNode* sfpt, Node *local, break; case Type::NarrowOop: if (t == TypeNarrowOop::NULL_PTR) { - array->append(new ConstantOopWriteValue(NULL)); + array->append(new ConstantOopWriteValue(nullptr)); } else { array->append(new ConstantOopWriteValue(t->make_ptr()->isa_oopptr()->const_oop()->constant_encoding())); } @@ -1010,7 +1010,7 @@ void PhaseOutput::Process_OopMap_Node(MachNode *mach, int current_offset) { // Add the safepoint in the DebugInfoRecorder if( !mach->is_MachCall() ) { - mcall = NULL; + mcall = nullptr; C->debug_info()->add_safepoint(safepoint_pc_offset, sfn->_oop_map); } else { mcall = mach->as_MachCall(); @@ -1035,7 +1035,7 @@ void PhaseOutput::Process_OopMap_Node(MachNode *mach, int current_offset) { } // Loop over the JVMState list to add scope information - // Do not skip safepoints with a NULL method, they need monitor info + // Do not skip safepoints with a null method, they need monitor info JVMState* youngest_jvms = sfn->jvms(); int max_depth = youngest_jvms->depth(); @@ -1048,13 +1048,13 @@ void PhaseOutput::Process_OopMap_Node(MachNode *mach, int current_offset) { for (int depth = 1; depth <= max_depth; depth++) { JVMState* jvms = youngest_jvms->of_depth(depth); int idx; - ciMethod* method = jvms->has_method() ? jvms->method() : NULL; + ciMethod* method = jvms->has_method() ? jvms->method() : nullptr; // Safepoints that do not have method() set only provide oop-map and monitor info // to support GC; these do not support deoptimization. - int num_locs = (method == NULL) ? 0 : jvms->loc_size(); - int num_exps = (method == NULL) ? 0 : jvms->stk_size(); + int num_locs = (method == nullptr) ? 0 : jvms->loc_size(); + int num_exps = (method == nullptr) ? 0 : jvms->stk_size(); int num_mon = jvms->nof_monitors(); - assert(method == NULL || jvms->bci() < 0 || num_locs == method->max_locals(), + assert(method == nullptr || jvms->bci() < 0 || num_locs == method->max_locals(), "JVMS local count must match that of the method"); // Add Local and Expression Stack Information @@ -1089,12 +1089,12 @@ void PhaseOutput::Process_OopMap_Node(MachNode *mach, int current_offset) { Node* obj_node = sfn->monitor_obj(jvms, idx); // Create ScopeValue for object - ScopeValue *scval = NULL; + ScopeValue *scval = nullptr; if (obj_node->is_SafePointScalarObject()) { SafePointScalarObjectNode* spobj = obj_node->as_SafePointScalarObject(); scval = PhaseOutput::sv_for_node_id(objs, spobj->_idx); - if (scval == NULL) { + if (scval == nullptr) { const Type *t = spobj->bottom_type(); ciKlass* cik = t->is_oopptr()->klass(); assert(cik->is_instance_klass() || @@ -1180,7 +1180,7 @@ class NonSafepointEmitter { public: NonSafepointEmitter(Compile* compile) { this->C = compile; - _pending_jvms = NULL; + _pending_jvms = nullptr; _pending_offset = 0; } @@ -1188,19 +1188,19 @@ class NonSafepointEmitter { if (!C->debug_info()->recording_non_safepoints()) return; Node_Notes* nn = C->node_notes_at(n->_idx); - if (nn == NULL || nn->jvms() == NULL) return; - if (_pending_jvms != NULL && + if (nn == nullptr || nn->jvms() == nullptr) return; + if (_pending_jvms != nullptr && _pending_jvms->same_calls_as(nn->jvms())) { // Repeated JVMS? Stretch it up here. _pending_offset = pc_offset; } else { - if (_pending_jvms != NULL && + if (_pending_jvms != nullptr && _pending_offset < pc_offset) { emit_non_safepoint(); } - _pending_jvms = NULL; + _pending_jvms = nullptr; if (pc_offset > C->debug_info()->last_pc_offset()) { - // This is the only way _pending_jvms can become non-NULL: + // This is the only way _pending_jvms can become non-null: _pending_jvms = nn->jvms(); _pending_offset = pc_offset; } @@ -1209,19 +1209,19 @@ class NonSafepointEmitter { // Stay out of the way of real safepoints: void observe_safepoint(JVMState* jvms, int pc_offset) { - if (_pending_jvms != NULL && + if (_pending_jvms != nullptr && !_pending_jvms->same_calls_as(jvms) && _pending_offset < pc_offset) { emit_non_safepoint(); } - _pending_jvms = NULL; + _pending_jvms = nullptr; } void flush_at_end() { - if (_pending_jvms != NULL) { + if (_pending_jvms != nullptr) { emit_non_safepoint(); } - _pending_jvms = NULL; + _pending_jvms = nullptr; } }; @@ -1230,7 +1230,7 @@ void NonSafepointEmitter::emit_non_safepoint() { int pc_offset = _pending_offset; // Clear it now: - _pending_jvms = NULL; + _pending_jvms = nullptr; DebugInformationRecorder* debug_info = C->debug_info(); assert(debug_info->recording_non_safepoints(), "sanity"); @@ -1241,7 +1241,7 @@ void NonSafepointEmitter::emit_non_safepoint() { // Visit scopes from oldest to youngest. for (int depth = 1; depth <= max_depth; depth++) { JVMState* jvms = youngest_jvms->of_depth(depth); - ciMethod* method = jvms->has_method() ? jvms->method() : NULL; + ciMethod* method = jvms->has_method() ? jvms->method() : nullptr; assert(!jvms->should_reexecute() || depth==max_depth, "reexecute allowed only for the youngest"); methodHandle null_mh; debug_info->describe_scope(pc_offset, null_mh, method, jvms->bci(), jvms->should_reexecute()); @@ -1340,9 +1340,9 @@ CodeBuffer* PhaseOutput::init_buffer() { cb->initialize(total_req, _buf_sizes._reloc); // Have we run out of code space? - if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) { + if ((cb->blob() == nullptr) || (!CompileBroker::should_compile_new_jobs())) { C->record_failure("CodeCache is full"); - return NULL; + return nullptr; } // Configure the code buffer. cb->initialize_consts_size(const_req); @@ -1397,13 +1397,13 @@ void PhaseOutput::fill_buffer(CodeBuffer* cb, uint* blk_starts) { // Create an array of unused labels, one for each basic block, if printing is enabled #if defined(SUPPORT_OPTO_ASSEMBLY) - int* node_offsets = NULL; + int* node_offsets = nullptr; uint node_offset_limit = C->unique(); if (C->print_assembly()) { node_offsets = NEW_RESOURCE_ARRAY(int, node_offset_limit); } - if (node_offsets != NULL) { + if (node_offsets != nullptr) { // We need to initialize. Unused array elements may contain garbage and mess up PrintOptoAssembly. memset(node_offsets, 0, node_offset_limit*sizeof(int)); } @@ -1426,7 +1426,7 @@ void PhaseOutput::fill_buffer(CodeBuffer* cb, uint* blk_starts) { } // Now fill in the code buffer - Node* delay_slot = NULL; + Node* delay_slot = nullptr; for (uint i = 0; i < nblocks; i++) { Block* block = C->cfg()->get_block(i); _block = block; @@ -1467,7 +1467,7 @@ void PhaseOutput::fill_buffer(CodeBuffer* cb, uint* blk_starts) { // See if delay slots are supported if (valid_bundle_info(n) && node_bundling(n)->used_in_unconditional_delay()) { - assert(delay_slot == NULL, "no use of delay slot node"); + assert(delay_slot == nullptr, "no use of delay slot node"); assert(n->size(C->regalloc()) == Pipeline::instr_unit_size(), "delay slot instruction wrong size"); delay_slot = n; @@ -1517,7 +1517,7 @@ void PhaseOutput::fill_buffer(CodeBuffer* cb, uint* blk_starts) { C->cfg()->map_node_to_block(nop, block); // Ensure enough space. cb->insts()->maybe_expand_to_ensure_remaining(MAX_inst_size); - if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) { + if ((cb->blob() == nullptr) || (!CompileBroker::should_compile_new_jobs())) { C->record_failure("CodeCache is full"); return; } @@ -1546,7 +1546,7 @@ void PhaseOutput::fill_buffer(CodeBuffer* cb, uint* blk_starts) { if (!is_mcall) { MachSafePointNode *sfn = mach->as_MachSafePoint(); // !!!!! Stubs only need an oopmap right now, so bail out - if (sfn->jvms()->method() == NULL) { + if (sfn->jvms()->method() == nullptr) { // Write the oopmap directly to the code blob??!! continue; } @@ -1573,7 +1573,7 @@ void PhaseOutput::fill_buffer(CodeBuffer* cb, uint* blk_starts) { bool delay_slot_is_used = valid_bundle_info(n) && C->output()->node_bundling(n)->use_unconditional_delay(); if (!delay_slot_is_used && mach->may_be_short_branch()) { - assert(delay_slot == NULL, "not expecting delay slot node"); + assert(delay_slot == nullptr, "not expecting delay slot node"); int br_size = n->size(C->regalloc()); int offset = blk_starts[block_num] - current_offset; if (block_num >= i) { @@ -1638,7 +1638,7 @@ void PhaseOutput::fill_buffer(CodeBuffer* cb, uint* blk_starts) { int count = 0; for (uint prec = mach->req(); prec < mach->len(); prec++) { Node *oop_store = mach->in(prec); // Precedence edge - if (oop_store == NULL) continue; + if (oop_store == nullptr) continue; count++; uint i4; for (i4 = 0; i4 < last_inst; ++i4) { @@ -1669,14 +1669,14 @@ void PhaseOutput::fill_buffer(CodeBuffer* cb, uint* blk_starts) { // Verify that there is sufficient space remaining cb->insts()->maybe_expand_to_ensure_remaining(MAX_inst_size); - if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) { + if ((cb->blob() == nullptr) || (!CompileBroker::should_compile_new_jobs())) { C->record_failure("CodeCache is full"); return; } // Save the offset for the listing #if defined(SUPPORT_OPTO_ASSEMBLY) - if ((node_offsets != NULL) && (n->_idx < node_offset_limit)) { + if ((node_offsets != nullptr) && (n->_idx < node_offset_limit)) { node_offsets[n->_idx] = cb->insts_size(); } #endif @@ -1732,14 +1732,14 @@ void PhaseOutput::fill_buffer(CodeBuffer* cb, uint* blk_starts) { // See if this instruction has a delay slot if (valid_bundle_info(n) && node_bundling(n)->use_unconditional_delay()) { - guarantee(delay_slot != NULL, "expecting delay slot node"); + guarantee(delay_slot != nullptr, "expecting delay slot node"); // Back up 1 instruction cb->set_insts_end(cb->insts_end() - Pipeline::instr_unit_size()); // Save the offset for the listing #if defined(SUPPORT_OPTO_ASSEMBLY) - if ((node_offsets != NULL) && (delay_slot->_idx < node_offset_limit)) { + if ((node_offsets != nullptr) && (delay_slot->_idx < node_offset_limit)) { node_offsets[delay_slot->_idx] = cb->insts_size(); } #endif @@ -1748,9 +1748,9 @@ void PhaseOutput::fill_buffer(CodeBuffer* cb, uint* blk_starts) { if (delay_slot->is_MachSafePoint()) { MachNode *mach = delay_slot->as_Mach(); // !!!!! Stubs only need an oopmap right now, so bail out - if (!mach->is_MachCall() && mach->as_MachSafePoint()->jvms()->method() == NULL) { + if (!mach->is_MachCall() && mach->as_MachSafePoint()->jvms()->method() == nullptr) { // Write the oopmap directly to the code blob??!! - delay_slot = NULL; + delay_slot = nullptr; continue; } @@ -1765,7 +1765,7 @@ void PhaseOutput::fill_buffer(CodeBuffer* cb, uint* blk_starts) { delay_slot->emit(*cb, C->regalloc()); // Don't reuse it - delay_slot = NULL; + delay_slot = nullptr; } } // End for all instructions in block @@ -1853,7 +1853,7 @@ void PhaseOutput::fill_buffer(CodeBuffer* cb, uint* blk_starts) { } // One last check for failed CodeBuffer::expand: - if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) { + if ((cb->blob() == nullptr) || (!CompileBroker::should_compile_new_jobs())) { C->record_failure("CodeCache is full"); return; } @@ -1873,21 +1873,21 @@ void PhaseOutput::fill_buffer(CodeBuffer* cb, uint* blk_starts) { // This output goes directly to the tty, not the compiler log. // To enable tools to match it up with the compilation activity, // be sure to tag this tty output with the compile ID. - if (xtty != NULL) { + if (xtty != nullptr) { xtty->head("opto_assembly compile_id='%d'%s", C->compile_id(), C->is_osr_compilation() ? " compile_kind='osr'" : ""); } - if (C->method() != NULL) { + if (C->method() != nullptr) { tty->print_cr("----------------------- MetaData before Compile_id = %d ------------------------", C->compile_id()); C->method()->print_metadata(); - } else if (C->stub_name() != NULL) { + } else if (C->stub_name() != nullptr) { tty->print_cr("----------------------------- RuntimeStub %s -------------------------------", C->stub_name()); } tty->cr(); tty->print_cr("------------------------ OptoAssembly for Compile_id = %d -----------------------", C->compile_id()); dump_asm(node_offsets, node_offset_limit); tty->print_cr("--------------------------------------------------------------------------------"); - if (xtty != NULL) { + if (xtty != nullptr) { // print_metadata and dump_asm above may safepoint which makes us loose the ttylock. // Retake lock too make sure the end tag is coherent, and that xmlStream->pop_tag is done // thread safe @@ -1905,7 +1905,7 @@ void PhaseOutput::FillExceptionTables(uint cnt, uint *call_returns, uint *inct_s uint inct_cnt = 0; for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) { Block* block = C->cfg()->get_block(i); - Node *n = NULL; + Node *n = nullptr; int j; // Find the branch; ignore trailing NOPs. @@ -1966,7 +1966,7 @@ void PhaseOutput::FillExceptionTables(uint cnt, uint *call_returns, uint *inct_s // Set the offset of the return from the call assert(handler_bcis.find(-1) != -1, "must have default handler"); - _handler_table.add_subtable(call_return, &handler_bcis, NULL, &handler_pcos); + _handler_table.add_subtable(call_return, &handler_bcis, nullptr, &handler_pcos); continue; } @@ -2004,7 +2004,7 @@ Scheduling::Scheduling(Arena *arena, Compile &compile) _available(arena), _reg_node(arena), _pinch_free_list(arena), - _next_node(NULL), + _next_node(nullptr), _bundle_instr_count(0), _bundle_cycle_number(0), _bundle_use(0, 0, resource_count, &_bundle_use_elements[0]) @@ -2255,9 +2255,9 @@ Node * Scheduling::ChooseNodeToBundle() { #ifndef PRODUCT if (_cfg->C->trace_opto_output()) - tty->print("# ChooseNodeToBundle: NULL\n"); + tty->print("# ChooseNodeToBundle: null\n"); #endif - return (NULL); + return (nullptr); } // Fast path, if only 1 instruction in the bundle @@ -2606,7 +2606,7 @@ void Scheduling::ComputeUseCount(const Block *bb) { _scheduled.clear(); // No delay slot specified - _unconditional_delay_slot = NULL; + _unconditional_delay_slot = nullptr; #ifdef ASSERT for( uint i=0; i < bb->number_of_nodes(); i++ ) @@ -2668,7 +2668,7 @@ void Scheduling::DoScheduling() { tty->print("# -> DoScheduling\n"); #endif - Block *succ_bb = NULL; + Block *succ_bb = nullptr; Block *bb; Compile* C = Compile::current(); @@ -2770,7 +2770,7 @@ void Scheduling::DoScheduling() { // Schedule the remaining instructions in the block while ( _available.size() > 0 ) { Node *n = ChooseNodeToBundle(); - guarantee(n != NULL, "no nodes available"); + guarantee(n != nullptr, "no nodes available"); AddNodeToBundle(n,bb); } @@ -2845,7 +2845,7 @@ void Scheduling::verify_do_def( Node *n, OptoReg::Name def, const char *msg ) { prior_use->dump(); assert(edge_from_to(prior_use,n), "%s", msg); } - _reg_node.map(def,NULL); // Kill live USEs + _reg_node.map(def,nullptr); // Kill live USEs } } @@ -2922,7 +2922,7 @@ void Scheduling::anti_do_def( Block *b, Node *def, OptoReg::Name def_reg, int is } Node *pinch = _reg_node[def_reg]; // Get pinch point - if ((pinch == NULL) || _cfg->get_block_for_node(pinch) != b || // No pinch-point yet? + if ((pinch == nullptr) || _cfg->get_block_for_node(pinch) != b || // No pinch-point yet? is_def ) { // Check for a true def (not a kill) _reg_node.map(def_reg,def); // Record def/kill as the optimistic pinch-point return; @@ -2932,7 +2932,7 @@ void Scheduling::anti_do_def( Block *b, Node *def, OptoReg::Name def_reg, int is debug_only( def = (Node*)((intptr_t)0xdeadbeef); ) // After some number of kills there _may_ be a later def - Node *later_def = NULL; + Node *later_def = nullptr; Compile* C = Compile::current(); @@ -2954,9 +2954,9 @@ void Scheduling::anti_do_def( Block *b, Node *def, OptoReg::Name def_reg, int is _reg_node.map(def_reg,pinch); // Record pinch-point //regalloc()->set_bad(pinch->_idx); // Already initialized this way. if( later_def->outcnt() == 0 || later_def->ideal_reg() == MachProjNode::fat_proj ) { // Distinguish def from kill - pinch->init_req(0, C->top()); // set not NULL for the next call + pinch->init_req(0, C->top()); // set not null for the next call add_prec_edge_from_to(later_def,pinch); // Add edge from kill to pinch - later_def = NULL; // and no later def + later_def = nullptr; // and no later def } pinch->set_req(0,later_def); // Hook later def so we can find it } else { // Else have valid pinch point @@ -2975,7 +2975,7 @@ void Scheduling::anti_do_def( Block *b, Node *def, OptoReg::Name def_reg, int is if( _regalloc->get_reg_first(uses->in(i)) == def_reg || _regalloc->get_reg_second(uses->in(i)) == def_reg ) { // Yes, found a use/kill pinch-point - pinch->set_req(0,NULL); // + pinch->set_req(0,nullptr); // pinch->replace_by(kill); // Move anti-dep edges up pinch = kill; _reg_node.map(def_reg,pinch); @@ -2993,7 +2993,7 @@ void Scheduling::anti_do_use( Block *b, Node *use, OptoReg::Name use_reg ) { return; Node *pinch = _reg_node[use_reg]; // Get pinch point // Check for no later def_reg/kill in block - if ((pinch != NULL) && _cfg->get_block_for_node(pinch) == b && + if ((pinch != nullptr) && _cfg->get_block_for_node(pinch) == b && // Use has to be block-local as well _cfg->get_block_for_node(use) == b) { if( pinch->Opcode() == Op_Node && // Real pinch-point (not optimistic?) @@ -3045,14 +3045,14 @@ void Scheduling::ComputeRegisterAntidependencies(Block *b) { // put an edge from the pinch point to the USE. // To be expedient, the _reg_node array is pre-allocated for the whole - // compilation. _reg_node is lazily initialized; it either contains a NULL, + // compilation. _reg_node is lazily initialized; it either contains a null, // or a valid def/kill/pinch-point, or a leftover node from some prior - // block. Leftover node from some prior block is treated like a NULL (no + // block. Leftover node from some prior block is treated like a null (no // prior def, so no anti-dependence needed). Valid def is distinguished by // it being in the current block. bool fat_proj_seen = false; uint last_safept = _bb_end-1; - Node* end_node = (_bb_end-1 >= _bb_start) ? b->get_node(last_safept) : NULL; + Node* end_node = (_bb_end-1 >= _bb_start) ? b->get_node(last_safept) : nullptr; Node* last_safept_node = end_node; for( uint i = _bb_end-1; i >= _bb_start; i-- ) { Node *n = b->get_node(i); @@ -3170,12 +3170,12 @@ void Scheduling::garbage_collect_pinch_nodes() { int trace_cnt = 0; for (uint k = 0; k < _reg_node.Size(); k++) { Node* pinch = _reg_node[k]; - if ((pinch != NULL) && pinch->Opcode() == Op_Node && + if ((pinch != nullptr) && pinch->Opcode() == Op_Node && // no predecence input edges - (pinch->req() == pinch->len() || pinch->in(pinch->req()) == NULL) ) { + (pinch->req() == pinch->len() || pinch->in(pinch->req()) == nullptr) ) { cleanup_pinch(pinch); _pinch_free_list.push(pinch); - _reg_node.map(k, NULL); + _reg_node.map(k, nullptr); #ifndef PRODUCT if (_cfg->C->trace_opto_output()) { trace_cnt++; @@ -3210,7 +3210,7 @@ void Scheduling::cleanup_pinch( Node *pinch ) { i -= uses_found; // we deleted 1 or more copies of this edge } // May have a later_def entry - pinch->set_req(0, NULL); + pinch->set_req(0, nullptr); } #ifndef PRODUCT @@ -3263,10 +3263,10 @@ void PhaseOutput::init_scratch_buffer_blob(int const_size) { // constant section is big enough, use it. Otherwise free the // current and allocate a new one. BufferBlob* blob = scratch_buffer_blob(); - if ((blob != NULL) && (const_size <= _scratch_const_size)) { + if ((blob != nullptr) && (const_size <= _scratch_const_size)) { // Use the current blob. } else { - if (blob != NULL) { + if (blob != nullptr) { BufferBlob::free(blob); } @@ -3277,7 +3277,7 @@ void PhaseOutput::init_scratch_buffer_blob(int const_size) { // Record the buffer blob for next time. set_scratch_buffer_blob(blob); // Have we run out of code space? - if (scratch_buffer_blob() == NULL) { + if (scratch_buffer_blob() == nullptr) { // Let CompilerBroker disable further compilations. C->record_failure("Not enough space for scratch buffer in CodeCache"); return; @@ -3307,7 +3307,7 @@ uint PhaseOutput::scratch_emit_size(const Node* n) { // The allocation of the scratch buffer blob is particularly // expensive, since it has to grab the code cache lock. BufferBlob* blob = this->scratch_buffer_blob(); - assert(blob != NULL, "Initialize BufferBlob at start"); + assert(blob != nullptr, "Initialize BufferBlob at start"); assert(blob->size() > MAX_inst_size, "sanity"); relocInfo* locs_buf = scratch_locs_memory(); address blob_begin = blob->content_begin(); @@ -3316,7 +3316,7 @@ uint PhaseOutput::scratch_emit_size(const Node* n) { CodeBuffer buf(blob_begin, blob_end - blob_begin); buf.initialize_consts_size(_scratch_const_size); buf.initialize_stubs_size(MAX_stubs_size); - assert(locs_buf != NULL, "sanity"); + assert(locs_buf != nullptr, "sanity"); int lsize = MAX_locs_size / 3; buf.consts()->initialize_shared_locs(&locs_buf[lsize * 0], lsize); buf.insts()->initialize_shared_locs( &locs_buf[lsize * 1], lsize); @@ -3329,7 +3329,7 @@ uint PhaseOutput::scratch_emit_size(const Node* n) { // Do the emission. Label fakeL; // Fake label for branch instructions. - Label* saveL = NULL; + Label* saveL = nullptr; uint save_bnum = 0; bool is_branch = n->is_MachBranch(); if (is_branch) { @@ -3355,7 +3355,7 @@ uint PhaseOutput::scratch_emit_size(const Node* n) { void PhaseOutput::install() { if (!C->should_install_code()) { return; - } else if (C->stub_function() != NULL) { + } else if (C->stub_function() != nullptr) { install_stub(C->stub_name()); } else { install_code(C->method(), @@ -3406,14 +3406,14 @@ void PhaseOutput::install_code(ciMethod* target, C->rtm_state(), C->native_invokers()); - if (C->log() != NULL) { // Print code cache state into compiler log + if (C->log() != nullptr) { // Print code cache state into compiler log C->log()->code_cache_state(); } } } void PhaseOutput::install_stub(const char* stub_name) { // Entry point will be accessed using stub_entry_point(); - if (code_buffer() == NULL) { + if (code_buffer() == nullptr) { Matcher::soft_match_failure(); } else { if (PrintAssembly && (WizardMode || Verbose)) @@ -3431,7 +3431,7 @@ void PhaseOutput::install_stub(const char* stub_name) { frame_size_in_words(), oop_map_set(), false); - assert(rs != NULL && rs->is_runtime_stub(), "sanity check"); + assert(rs != nullptr && rs->is_runtime_stub(), "sanity check"); C->set_stub_entry_point(rs->entry_point()); } @@ -3474,7 +3474,7 @@ void PhaseOutput::dump_asm_on(outputStream* st, int* pcs, uint pc_limit) { int pc_digits = 3; // #chars required for pc int sb_chars = 3; // #chars for "start bundle" indicator int tab_size = 8; - if (pcs != NULL) { + if (pcs != nullptr) { int max_pc = 0; for (uint i = 0; i < pc_limit; i++) { max_pc = (max_pc < pcs[i]) ? pcs[i] : max_pc; @@ -3493,7 +3493,7 @@ void PhaseOutput::dump_asm_on(outputStream* st, int* pcs, uint pc_limit) { char starts_bundle = ' '; C->regalloc()->dump_frame(); - Node *n = NULL; + Node *n = nullptr; for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) { if (VMThread::should_terminate()) { cut_short = true; @@ -3504,7 +3504,7 @@ void PhaseOutput::dump_asm_on(outputStream* st, int* pcs, uint pc_limit) { continue; } n = block->head(); - if ((pcs != NULL) && (n->_idx < pc_limit)) { + if ((pcs != nullptr) && (n->_idx < pc_limit)) { pc = pcs[n->_idx]; st->print("%*.*x", pc_digits, pc_digits, pc); } @@ -3519,7 +3519,7 @@ void PhaseOutput::dump_asm_on(outputStream* st, int* pcs, uint pc_limit) { } // For all instructions - Node *delay = NULL; + Node *delay = nullptr; for (uint j = 0; j < block->number_of_nodes(); j++) { if (VMThread::should_terminate()) { cut_short = true; @@ -3551,7 +3551,7 @@ void PhaseOutput::dump_asm_on(outputStream* st, int* pcs, uint pc_limit) { !n->is_top() && // Debug info table constants !(n->is_Con() && !n->is_Mach())// Debug info table constants ) { - if ((pcs != NULL) && (n->_idx < pc_limit)) { + if ((pcs != nullptr) && (n->_idx < pc_limit)) { pc = pcs[n->_idx]; st->print("%*.*x", pc_digits, pc_digits, pc); } else { @@ -3568,12 +3568,12 @@ void PhaseOutput::dump_asm_on(outputStream* st, int* pcs, uint pc_limit) { // then back up and print it if (valid_bundle_info(n) && node_bundling(n)->use_unconditional_delay()) { // Coverity finding - Explicit null dereferenced. - guarantee(delay != NULL, "no unconditional delay instruction"); + guarantee(delay != nullptr, "no unconditional delay instruction"); if (WizardMode) delay->dump(); if (node_bundling(delay)->starts_bundle()) starts_bundle = '+'; - if ((pcs != NULL) && (n->_idx < pc_limit)) { + if ((pcs != nullptr) && (n->_idx < pc_limit)) { pc = pcs[n->_idx]; st->print("%*.*x", pc_digits, pc_digits, pc); } else { @@ -3584,7 +3584,7 @@ void PhaseOutput::dump_asm_on(outputStream* st, int* pcs, uint pc_limit) { st->fill_to(prefix_len); delay->format(C->regalloc(), st); st->cr(); - delay = NULL; + delay = nullptr; } // Dump the exception table as well @@ -3595,7 +3595,7 @@ void PhaseOutput::dump_asm_on(outputStream* st, int* pcs, uint pc_limit) { st->bol(); // Make sure we start on a new line } st->cr(); // one empty line between blocks - assert(cut_short || delay == NULL, "no unconditional delay branch"); + assert(cut_short || delay == nullptr, "no unconditional delay branch"); } // End of per-block dump if (cut_short) st->print_cr("*** disassembly is cut short ***"); diff --git a/src/hotspot/share/opto/output.hpp b/src/hotspot/share/opto/output.hpp index 4f296c64b8efb..84298750feafd 100644 --- a/src/hotspot/share/opto/output.hpp +++ b/src/hotspot/share/opto/output.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -179,7 +179,7 @@ class PhaseOutput : public Phase { // The architecture description provides short branch variants for some long // branch instructions. Replace eligible long branches with short branches. void shorten_branches(uint* blk_starts); - // If "objs" contains an ObjectValue whose id is "id", returns it, else NULL. + // If "objs" contains an ObjectValue whose id is "id", returns it, else null. static ObjectValue* sv_for_node_id(GrowableArray *objs, int id); static void set_sv_for_object_node(GrowableArray *objs, ObjectValue* sv); void FillLocArray( int idx, MachSafePointNode* sfpt, Node *local, diff --git a/src/hotspot/share/opto/parse.hpp b/src/hotspot/share/opto/parse.hpp index 580de65092843..1e7de6e56090c 100644 --- a/src/hotspot/share/opto/parse.hpp +++ b/src/hotspot/share/opto/parse.hpp @@ -180,14 +180,14 @@ class Parse : public GraphKit { void set_start_map(SafePointNode* m) { assert(!is_merged(), ""); _start_map = m; } // True after any predecessor flows control into this block - bool is_merged() const { return _start_map != NULL; } + bool is_merged() const { return _start_map != nullptr; } #ifdef ASSERT // True after backedge predecessor flows control into this block bool has_merged_backedge() const { return _has_merged_backedge; } void mark_merged_backedge(Block* pred) { assert(is_SEL_head(), "should be loop head"); - if (pred != NULL && is_SEL_backedge(pred)) { + if (pred != nullptr && is_SEL_backedge(pred)) { assert(is_parsed(), "block should be parsed before merging backedges"); _has_merged_backedge = true; } @@ -254,7 +254,7 @@ class Parse : public GraphKit { // path number ("pnum"). int add_new_path(); - // Initialize me by recording the parser's map. My own map must be NULL. + // Initialize me by recording the parser's map. My own map must be null. void record_state(Parse* outer); }; @@ -374,7 +374,7 @@ class Parse : public GraphKit { void set_wrote_fields(bool z) { _wrote_fields = z; } Node* alloc_with_final() const { return _alloc_with_final; } void set_alloc_with_final(Node* n) { - assert((_alloc_with_final == NULL) || (_alloc_with_final == n), "different init objects?"); + assert((_alloc_with_final == nullptr) || (_alloc_with_final == n), "different init objects?"); _alloc_with_final = n; } @@ -401,7 +401,7 @@ class Parse : public GraphKit { Block* start_block() { return rpo_at(flow()->start_block()->rpo()); } - // Can return NULL if the flow pass did not complete a block. + // Can return null if the flow pass did not complete a block. Block* successor_for_bci(int bci) { return block()->successor_for_bci(bci); } diff --git a/src/hotspot/share/opto/parse1.cpp b/src/hotspot/share/opto/parse1.cpp index 7819812e1fee0..7cb74b3574d3e 100644 --- a/src/hotspot/share/opto/parse1.cpp +++ b/src/hotspot/share/opto/parse1.cpp @@ -78,7 +78,7 @@ void Parse::print_statistics() { tty->print_cr("Blocks parsed: %d Blocks seen: %d", blocks_parsed, blocks_seen); if (explicit_null_checks_inserted) { - tty->print_cr("%d original NULL checks - %d elided (%2d%%); optimizer leaves %d,", + tty->print_cr("%d original null checks - %d elided (%2d%%); optimizer leaves %d,", explicit_null_checks_inserted, explicit_null_checks_elided, (100*explicit_null_checks_elided)/explicit_null_checks_inserted, all_null_checks_found); @@ -112,7 +112,7 @@ Node *Parse::fetch_interpreter_state(int index, // Very similar to LoadNode::make, except we handle un-aligned longs and // doubles on Sparc. Intel can handle them just fine directly. - Node *l = NULL; + Node *l = nullptr; switch (bt) { // Signature is flattened case T_INT: l = new LoadINode(ctl, mem, adr, TypeRawPtr::BOTTOM, TypeInt::INT, MemNode::unordered); break; case T_FLOAT: l = new LoadFNode(ctl, mem, adr, TypeRawPtr::BOTTOM, Type::FLOAT, MemNode::unordered); break; @@ -152,7 +152,7 @@ Node* Parse::check_interpreter_type(Node* l, const Type* type, // TypeFlow may assert null-ness if a type appears unloaded. if (type == TypePtr::NULL_PTR || - (tp != NULL && !tp->klass()->is_loaded())) { + (tp != nullptr && !tp->klass()->is_loaded())) { // Value must be null, not a real oop. Node* chk = _gvn.transform( new CmpPNode(l, null()) ); Node* tst = _gvn.transform( new BoolNode(chk, BoolTest::eq) ); @@ -168,9 +168,9 @@ Node* Parse::check_interpreter_type(Node* l, const Type* type, // When paths are cut off, values at later merge points can rise // toward more specific classes. Make sure these specific classes // are still in effect. - if (tp != NULL && tp->klass() != C->env()->Object_klass()) { + if (tp != nullptr && tp->klass() != C->env()->Object_klass()) { // TypeFlow asserted a specific object type. Value must have that type. - Node* bad_type_ctrl = NULL; + Node* bad_type_ctrl = nullptr; l = gen_checkcast(l, makecon(TypeKlassPtr::make(tp->klass())), &bad_type_ctrl); bad_type_exit->control()->add_req(bad_type_ctrl); } @@ -272,7 +272,7 @@ void Parse::load_interpreter_state(Node* osr_buf) { const Type *type = osr_block->local_type_at(index); - if (type->isa_oopptr() != NULL) { + if (type->isa_oopptr() != nullptr) { // 6403625: Verify that the interpreter oopMap thinks that the oop is live // else we might load a stale oop if the MethodLiveness disagrees with the @@ -281,7 +281,7 @@ void Parse::load_interpreter_state(Node* osr_buf) { // if (!live_oops.at(index)) { - if (C->log() != NULL) { + if (C->log() != nullptr) { C->log()->elem("OSR_mismatch local_index='%d'",index); } set_local(index, null()); @@ -303,7 +303,7 @@ void Parse::load_interpreter_state(Node* osr_buf) { // Construct code to access the appropriate local. BasicType bt = type->basic_type(); if (type == TypePtr::NULL_PTR) { - // Ptr types are mixed together with T_ADDRESS but NULL is + // Ptr types are mixed together with T_ADDRESS but null is // really for T_OBJECT types so correct it. bt = T_OBJECT; } @@ -338,7 +338,7 @@ void Parse::load_interpreter_state(Node* osr_buf) { Node* l = local(index); if (l->is_top()) continue; // nothing here const Type *type = osr_block->local_type_at(index); - if (type->isa_oopptr() != NULL) { + if (type->isa_oopptr() != nullptr) { if (!live_oops.at(index)) { // skip type check for dead oops continue; @@ -400,10 +400,10 @@ Parse::Parse(JVMState* caller, ciMethod* parse_method, float expected_uses) _wrote_volatile = false; _wrote_stable = false; _wrote_fields = false; - _alloc_with_final = NULL; + _alloc_with_final = nullptr; _entry_bci = InvocationEntryBci; - _tf = NULL; - _block = NULL; + _tf = nullptr; + _block = nullptr; _first_return = true; _replaced_nodes_for_exceptions = false; _new_idx = C->unique(); @@ -449,7 +449,7 @@ Parse::Parse(JVMState* caller, ciMethod* parse_method, float expected_uses) } CompileLog* log = C->log(); - if (log != NULL) { + if (log != nullptr) { log->begin_head("parse method='%d' uses='%f'", log->identify(parse_method), expected_uses); if (depth() == 1 && C->is_osr_compilation()) { @@ -474,7 +474,7 @@ Parse::Parse(JVMState* caller, ciMethod* parse_method, float expected_uses) if (total_count < old_count || total_count < md_count) total_count = (uint)-1; C->set_trap_count(reason, total_count); - if (log != NULL) + if (log != nullptr) log->elem("observe trap='%s' count='%d' total='%d'", Deoptimization::trap_reason_name(reason), md_count, total_count); @@ -483,11 +483,11 @@ Parse::Parse(JVMState* caller, ciMethod* parse_method, float expected_uses) // Accumulate total sum of decompilations, also. C->set_decompile_count(C->decompile_count() + md->decompile_count()); - if (log != NULL && method()->has_exception_handlers()) { + if (log != nullptr && method()->has_exception_handlers()) { log->elem("observe that='has_exception_handlers'"); } - assert(InlineTree::check_can_parse(method()) == NULL, "Can not parse this method, cutout earlier"); + assert(InlineTree::check_can_parse(method()) == nullptr, "Can not parse this method, cutout earlier"); assert(method()->has_balanced_monitors(), "Can not parse unbalanced monitors, cutout earlier"); // Always register dependence if JVMTI is enabled, because @@ -554,7 +554,7 @@ Parse::Parse(JVMState* caller, ciMethod* parse_method, float expected_uses) SafePointNode* entry_map = create_entry_map(); // Check for bailouts during map initialization - if (failing() || entry_map == NULL) { + if (failing() || entry_map == nullptr) { if (log) log->done("parse"); return; } @@ -613,8 +613,6 @@ Parse::Parse(JVMState* caller, ciMethod* parse_method, float expected_uses) // Parse all the basic blocks. do_all_blocks(); - C->set_default_node_notes(caller_nn); - // Check for bailouts during conversion to graph if (failing()) { if (log) log->done("parse"); @@ -625,6 +623,10 @@ Parse::Parse(JVMState* caller, ciMethod* parse_method, float expected_uses) set_map(entry_map); do_exits(); + // Only reset this now, to make sure that debug information emitted + // for exiting control flow still refers to the inlined method. + C->set_default_node_notes(caller_nn); + if (log) log->done("parse nodes='%d' live='%d' memory='" SIZE_FORMAT "'", C->unique(), C->live_nodes(), C->node_arena()->used()); } @@ -810,7 +812,7 @@ void Parse::build_exits() { //----------------------------build_start_state------------------------------- // Construct a state which contains only the incoming arguments from an -// unknown caller. The method & bci will be NULL & InvocationEntryBci. +// unknown caller. The method & bci will be null & InvocationEntryBci. JVMState* Compile::build_start_state(StartNode* start, const TypeFunc* tf) { int arg_size = tf->domain()->cnt(); int max_size = MAX2(arg_size, (int)tf->range()->cnt()); @@ -819,7 +821,7 @@ JVMState* Compile::build_start_state(StartNode* start, const TypeFunc* tf) { record_for_igvn(map); assert(arg_size == TypeFunc::Parms + (is_osr_compilation() ? 1 : method()->arg_size()), "correct arg_size"); Node_Notes* old_nn = default_node_notes(); - if (old_nn != NULL && has_method()) { + if (old_nn != nullptr && has_method()) { Node_Notes* entry_nn = old_nn->clone(this); JVMState* entry_jvms = new(this) JVMState(method(), old_nn->jvms()); entry_jvms->set_offsets(0); @@ -845,7 +847,7 @@ JVMState* Compile::build_start_state(StartNode* start, const TypeFunc* tf) { //-----------------------------make_node_notes--------------------------------- Node_Notes* Parse::make_node_notes(Node_Notes* caller_nn) { - if (caller_nn == NULL) return NULL; + if (caller_nn == nullptr) return nullptr; Node_Notes* nn = caller_nn->clone(C); JVMState* caller_jvms = nn->jvms(); JVMState* jvms = new (C) JVMState(method(), caller_jvms); @@ -907,14 +909,14 @@ void Parse::do_exceptions() { if (failing()) { // Pop them all off and throw them away. - while (pop_exception_state() != NULL) ; + while (pop_exception_state() != nullptr) ; return; } PreserveJVMState pjvms(this, false); SafePointNode* ex_map; - while ((ex_map = pop_exception_state()) != NULL) { + while ((ex_map = pop_exception_state()) != nullptr) { if (!method()->has_exception_handlers()) { // Common case: Transfer control outward. // Doing it this early allows the exceptions to common up @@ -1067,7 +1069,7 @@ void Parse::do_exits() { SafePointNode* normal_map = kit.map(); // keep this guy safe // Now re-collect the exceptions into _exits: SafePointNode* ex_map; - while ((ex_map = kit.pop_exception_state()) != NULL) { + while ((ex_map = kit.pop_exception_state()) != nullptr) { Node* ex_oop = kit.use_exception_state(ex_map); // Force the exiting JVM state to have this method at InvocationEntryBci. // The exiting JVM state is otherwise a copy of the calling JVMS. @@ -1102,7 +1104,7 @@ void Parse::do_exits() { // Capture very early exceptions (receiver null checks) from caller JVMS GraphKit caller(_caller); SafePointNode* ex_map; - while ((ex_map = caller.pop_exception_state()) != NULL) { + while ((ex_map = caller.pop_exception_state()) != nullptr) { _exits.add_exception_state(ex_map); } } @@ -1118,7 +1120,7 @@ SafePointNode* Parse::create_entry_map() { uint len = TypeFunc::Parms + method()->max_locals() + method()->max_stack(); if (len >= 32760) { C->record_method_not_compilable("too many local variables"); - return NULL; + return nullptr; } // clear current replaced nodes that are of no use from here on (map was cloned in build_exits). @@ -1132,21 +1134,21 @@ SafePointNode* Parse::create_entry_map() { if (kit.stopped()) { _exits.add_exception_states_from(_caller); _exits.set_jvms(_caller); - return NULL; + return nullptr; } } - assert(method() != NULL, "parser must have a method"); + assert(method() != nullptr, "parser must have a method"); // Create an initial safepoint to hold JVM state during parsing - JVMState* jvms = new (C) JVMState(method(), _caller->has_method() ? _caller : NULL); + JVMState* jvms = new (C) JVMState(method(), _caller->has_method() ? _caller : nullptr); set_map(new SafePointNode(len, jvms)); jvms->set_map(map()); record_for_igvn(map()); assert(jvms->endoff() == len, "correct jvms sizing"); SafePointNode* inmap = _caller->map(); - assert(inmap != NULL, "must have inmap"); + assert(inmap != nullptr, "must have inmap"); // In case of null check on receiver above map()->transfer_replaced_nodes_from(inmap, _new_idx); @@ -1202,7 +1204,7 @@ void Parse::do_method_entry() { Node* receiver_obj = local(0); const TypeInstPtr* receiver_type = _gvn.type(receiver_obj)->isa_instptr(); - if (receiver_type != NULL && !receiver_type->higher_equal(holder_type)) { + if (receiver_type != nullptr && !receiver_type->higher_equal(holder_type)) { // Receiver should always be a subtype of callee holder. // But, since C2 type system doesn't properly track interfaces, // the invariant can't be expressed in the type system for default methods. @@ -1232,7 +1234,7 @@ void Parse::do_method_entry() { // FastLockNode becomes the new control parent to pin it to the start. // Setup Object Pointer - Node *lock_obj = NULL; + Node *lock_obj = nullptr; if (method()->is_static()) { ciInstance* mirror = _method->holder()->java_mirror(); const TypeInstPtr *t_lock = TypeInstPtr::make(mirror); @@ -1280,11 +1282,11 @@ Parse::Block::Block(Parse* outer, int rpo) : _live_locals() { _is_parsed = false; _is_handler = false; _has_merged_backedge = false; - _start_map = NULL; + _start_map = nullptr; _has_predicates = false; _num_successors = 0; _all_successors = 0; - _successors = NULL; + _successors = nullptr; assert(pred_count() == 0 && preds_parsed() == 0, "sanity"); assert(!(is_merged() || is_parsed() || is_handler() || has_merged_backedge()), "sanity"); assert(_live_locals.size() == 0, "sanity"); @@ -1303,7 +1305,7 @@ void Parse::Block::init_graph(Parse* outer) { int ne = tfe->length(); _num_successors = ns; _all_successors = ns+ne; - _successors = (ns+ne == 0) ? NULL : NEW_RESOURCE_ARRAY(Block*, ns+ne); + _successors = (ns+ne == 0) ? nullptr : NEW_RESOURCE_ARRAY(Block*, ns+ne); int p = 0; for (int i = 0; i < ns+ne; i++) { ciTypeFlow::Block* tf2 = (i < ns) ? tfs->at(i) : tfe->at(i-ns); @@ -1346,7 +1348,7 @@ Parse::Block* Parse::Block::successor_for_bci(int bci) { // of bytecodes. For example, "obj.field = null" is executable even // if the field's type is an unloaded class; the flow pass used to // make a trap for such code. - return NULL; + return nullptr; } @@ -1526,7 +1528,7 @@ void Parse::do_one_block() { } assert(bci() < block()->limit(), "bci still in block"); - if (log != NULL) { + if (log != nullptr) { // Output an optional context marker, to help place actions // that occur during parsing of this BC. If there is no log // output until the next context string, this context string @@ -1561,7 +1563,7 @@ void Parse::do_one_block() { NOT_PRODUCT( parse_histogram()->record_change(); ); - if (log != NULL) + if (log != nullptr) log->clear_context(); // skip marker if nothing was printed // Fall into next bytecode. Each bytecode normally has 1 sequential @@ -1576,7 +1578,7 @@ void Parse::do_one_block() { void Parse::set_parse_bci(int bci) { set_bci(bci); Node_Notes* nn = C->default_node_notes(); - if (nn == NULL) return; + if (nn == nullptr) return; // Collect debug info for inlined calls unless -XX:-DebugInlinedCalls. if (!DebugInlinedCalls && depth() > 1) { @@ -1585,7 +1587,7 @@ void Parse::set_parse_bci(int bci) { // Update the JVMS annotation, if present. JVMState* jvms = nn->jvms(); - if (jvms != NULL && jvms->bci() != bci) { + if (jvms != nullptr && jvms->bci() != bci) { // Update the JVMS. jvms = jvms->clone_shallow(C); jvms->set_bci(bci); @@ -1597,7 +1599,7 @@ void Parse::set_parse_bci(int bci) { // Merge the current mapping into the basic block starting at bci void Parse::merge(int target_bci) { Block* target = successor_for_bci(target_bci); - if (target == NULL) { handle_missing_successor(target_bci); return; } + if (target == nullptr) { handle_missing_successor(target_bci); return; } assert(!target->is_ready(), "our arrival must be expected"); int pnum = target->next_path_num(); merge_common(target, pnum); @@ -1607,7 +1609,7 @@ void Parse::merge(int target_bci) { // Merge the current mapping into the basic block, using a new path void Parse::merge_new_path(int target_bci) { Block* target = successor_for_bci(target_bci); - if (target == NULL) { handle_missing_successor(target_bci); return; } + if (target == nullptr) { handle_missing_successor(target_bci); return; } assert(!target->is_ready(), "new path into frozen graph"); int pnum = target->add_new_path(); merge_common(target, pnum); @@ -1624,7 +1626,7 @@ void Parse::merge_exception(int target_bci) { #endif assert(sp() == 1, "must have only the throw exception on the stack"); Block* target = successor_for_bci(target_bci); - if (target == NULL) { handle_missing_successor(target_bci); return; } + if (target == nullptr) { handle_missing_successor(target_bci); return; } assert(target->is_handler(), "exceptions are handled by special blocks"); int pnum = target->add_new_path(); merge_common(target, pnum); @@ -1685,8 +1687,8 @@ void Parse::merge_common(Parse::Block* target, int pnum) { RegionNode *r = new RegionNode(edges+1); gvn().set_type(r, Type::CONTROL); record_for_igvn(r); - // zap all inputs to NULL for debugging (done in Node(uint) constructor) - // for (int j = 1; j < edges+1; j++) { r->init_req(j, NULL); } + // zap all inputs to null for debugging (done in Node(uint) constructor) + // for (int j = 1; j < edges+1; j++) { r->init_req(j, nullptr); } r->init_req(pnum, control()); set_control(r); set_parse_bci(current_bci); // Restore bci @@ -1745,7 +1747,7 @@ void Parse::merge_common(Parse::Block* target, int pnum) { if (m->is_Phi() && m->as_Phi()->region() == r) phi = m->as_Phi(); else - phi = NULL; + phi = nullptr; if (m != n) { // Different; must merge switch (j) { // Frame pointer and Return Address never changes @@ -1753,11 +1755,11 @@ void Parse::merge_common(Parse::Block* target, int pnum) { case TypeFunc::ReturnAdr: break; case TypeFunc::Memory: // Merge inputs to the MergeMem node - assert(phi == NULL, "the merge contains phis, not vice versa"); + assert(phi == nullptr, "the merge contains phis, not vice versa"); merge_memory_edges(n->as_MergeMem(), pnum, nophi); continue; default: // All normal stuff - if (phi == NULL) { + if (phi == nullptr) { const JVMState* jvms = map()->jvms(); if (EliminateNestedLocks && jvms->is_mon(j) && jvms->is_monitor_box(j)) { @@ -1779,7 +1781,7 @@ void Parse::merge_common(Parse::Block* target, int pnum) { // - the corresponding control edges is top (a dead incoming path) // It is a bug if we create a phi which sees a garbage value on a live path. - if (phi != NULL) { + if (phi != nullptr) { assert(n != top() || r->in(pnum) == top(), "live value must not be garbage"); assert(phi->region() == r, ""); phi->set_req(pnum, n); // Then add 'n' to the merge @@ -1825,15 +1827,15 @@ void Parse::merge_common(Parse::Block* target, int pnum) { //--------------------------merge_memory_edges--------------------------------- void Parse::merge_memory_edges(MergeMemNode* n, int pnum, bool nophi) { // (nophi means we must not create phis, because we already parsed here) - assert(n != NULL, ""); + assert(n != nullptr, ""); // Merge the inputs to the MergeMems MergeMemNode* m = merged_memory(); assert(control()->is_Region(), "must be merging to a region"); RegionNode* r = control()->as_Region(); - PhiNode* base = NULL; - MergeMemNode* remerge = NULL; + PhiNode* base = nullptr; + MergeMemNode* remerge = nullptr; for (MergeMemStream mms(m, n); mms.next_non_empty2(); ) { Node *p = mms.force_memory(); Node *q = mms.memory2(); @@ -1841,9 +1843,9 @@ void Parse::merge_memory_edges(MergeMemNode* n, int pnum, bool nophi) { // Trouble: No new splits allowed after a loop body is parsed. // Instead, wire the new split into a MergeMem on the backedge. // The optimizer will sort it out, slicing the phi. - if (remerge == NULL) { - guarantee(base != NULL, ""); - assert(base->in(0) != NULL, "should not be xformed away"); + if (remerge == nullptr) { + guarantee(base != nullptr, ""); + assert(base->in(0) != nullptr, "should not be xformed away"); remerge = MergeMemNode::make(base->in(pnum)); gvn().set_type(remerge, Type::MEMORY); base->set_req(pnum, remerge); @@ -1859,10 +1861,10 @@ void Parse::merge_memory_edges(MergeMemNode* n, int pnum, bool nophi) { if (p->is_Phi() && p->as_Phi()->region() == r) phi = p->as_Phi(); else - phi = NULL; + phi = nullptr; } // Insert q into local phi - if (phi != NULL) { + if (phi != nullptr) { assert(phi->region() == r, ""); p = phi; phi->set_req(pnum, q); @@ -1876,7 +1878,7 @@ void Parse::merge_memory_edges(MergeMemNode* n, int pnum, bool nophi) { } } // Transform base last, in case we must fiddle with remerging. - if (base != NULL && pnum == 1) { + if (base != nullptr && pnum == 1) { record_for_igvn(base); m->set_base_memory( _gvn.transform_no_reclaim(base) ); } @@ -1935,7 +1937,7 @@ int Parse::Block::add_new_path() { // Add new path to the region. uint pnum = r->req(); - r->add_req(NULL); + r->add_req(nullptr); for (uint i = 1; i < map->req(); i++) { Node* n = map->in(i); @@ -1945,13 +1947,13 @@ int Parse::Block::add_new_path() { Node* phi = mms.memory(); if (phi->is_Phi() && phi->as_Phi()->region() == r) { assert(phi->req() == pnum, "must be same size as region"); - phi->add_req(NULL); + phi->add_req(nullptr); } } } else { if (n->is_Phi() && n->as_Phi()->region() == r) { assert(n->req() == pnum, "must be same size as region"); - n->add_req(NULL); + n->add_req(nullptr); } } } @@ -1967,9 +1969,9 @@ PhiNode *Parse::ensure_phi(int idx, bool nocreate) { assert(region->is_Region(), ""); Node* o = map->in(idx); - assert(o != NULL, ""); + assert(o != nullptr, ""); - if (o == top()) return NULL; // TOP always merges into TOP + if (o == top()) return nullptr; // TOP always merges into TOP if (o->is_Phi() && o->as_Phi()->region() == region) { return o->as_Phi(); @@ -1978,7 +1980,7 @@ PhiNode *Parse::ensure_phi(int idx, bool nocreate) { // Now use a Phi here for merging assert(!nocreate, "Cannot build a phi for a block already parsed."); const JVMState* jvms = map->jvms(); - const Type* t = NULL; + const Type* t = nullptr; if (jvms->is_loc(idx)) { t = block()->local_type_at(idx - jvms->locoff()); } else if (jvms->is_stk(idx)) { @@ -1997,14 +1999,14 @@ PhiNode *Parse::ensure_phi(int idx, bool nocreate) { // makes it go dead. if (t == Type::BOTTOM) { map->set_req(idx, top()); - return NULL; + return nullptr; } // Do not create phis for top either. // A top on a non-null control flow must be an unused even after the.phi. if (t == Type::TOP || t == Type::HALF) { map->set_req(idx, top()); - return NULL; + return nullptr; } PhiNode* phi = PhiNode::make(region, o, t); @@ -2022,7 +2024,7 @@ PhiNode *Parse::ensure_memory_phi(int idx, bool nocreate) { assert(region->is_Region(), ""); Node *o = (idx == Compile::AliasIdxBot)? mem->base_memory(): mem->memory_at(idx); - assert(o != NULL && o != top(), ""); + assert(o != nullptr && o != top(), ""); PhiNode* phi; if (o->is_Phi() && o->as_Phi()->region() == region) { @@ -2056,11 +2058,11 @@ PhiNode *Parse::ensure_memory_phi(int idx, bool nocreate) { // class need finalization. void Parse::call_register_finalizer() { Node* receiver = local(0); - assert(receiver != NULL && receiver->bottom_type()->isa_instptr() != NULL, + assert(receiver != nullptr && receiver->bottom_type()->isa_instptr() != nullptr, "must have non-null instance type"); const TypeInstPtr *tinst = receiver->bottom_type()->isa_instptr(); - if (tinst != NULL && tinst->klass()->is_loaded() && !tinst->klass_is_exact()) { + if (tinst != nullptr && tinst->klass()->is_loaded() && !tinst->klass_is_exact()) { // The type isn't known exactly so see if CHA tells us anything. ciInstanceKlass* ik = tinst->klass()->as_instance_klass(); if (!Dependencies::has_finalizable_subclass(ik)) { @@ -2074,10 +2076,10 @@ void Parse::call_register_finalizer() { // finalization. In general this will fold up since the concrete // class is often visible so the access flags are constant. Node* klass_addr = basic_plus_adr( receiver, receiver, oopDesc::klass_offset_in_bytes() ); - Node* klass = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), klass_addr, TypeInstPtr::KLASS)); + Node* klass = _gvn.transform(LoadKlassNode::make(_gvn, nullptr, immutable_memory(), klass_addr, TypeInstPtr::KLASS)); Node* access_flags_addr = basic_plus_adr(klass, klass, in_bytes(Klass::access_flags_offset())); - Node* access_flags = make_load(NULL, access_flags_addr, TypeInt::INT, T_INT, MemNode::unordered); + Node* access_flags = make_load(nullptr, access_flags_addr, TypeInt::INT, T_INT, MemNode::unordered); Node* mask = _gvn.transform(new AndINode(access_flags, intcon(JVM_ACC_HAS_FINALIZER))); Node* check = _gvn.transform(new CmpINode(mask, intcon(0))); @@ -2100,7 +2102,7 @@ void Parse::call_register_finalizer() { Node *call = make_runtime_call(RC_NO_LEAF, OptoRuntime::register_finalizer_Type(), OptoRuntime::register_finalizer_Java(), - NULL, TypePtr::BOTTOM, + nullptr, TypePtr::BOTTOM, receiver); make_slow_call_ex(call, env()->Throwable_klass(), true); @@ -2229,7 +2231,7 @@ void Parse::return_current(Node* value) { } // frame pointer is always same, already captured - if (value != NULL) { + if (value != nullptr) { // If returning oops to an interface-return, there is a silent free // cast from oop to interface allowed by the Verifier. Make it explicit // here. @@ -2277,7 +2279,7 @@ void Parse::add_safepoint() { kill_dead_locals(); // Clone the JVM State - SafePointNode *sfpnt = new SafePointNode(parms, NULL); + SafePointNode *sfpnt = new SafePointNode(parms, nullptr); // Capture memory state BEFORE a SafePoint. Since we can block at a // SafePoint we need our GC state to be safe; i.e. we need all our current @@ -2319,7 +2321,7 @@ void Parse::add_safepoint() { // Provide an edge from root to safepoint. This makes the safepoint // appear useful until the parse has completed. if (transformed_sfpnt->is_SafePoint()) { - assert(C->root() != NULL, "Expect parse is still valid"); + assert(C->root() != nullptr, "Expect parse is still valid"); C->root()->add_prec(transformed_sfpnt); } } @@ -2327,8 +2329,8 @@ void Parse::add_safepoint() { #ifndef PRODUCT //------------------------show_parse_info-------------------------------------- void Parse::show_parse_info() { - InlineTree* ilt = NULL; - if (C->ilt() != NULL) { + InlineTree* ilt = nullptr; + if (C->ilt() != nullptr) { JVMState* caller_jvms = is_osr_parse() ? caller()->caller() : caller(); ilt = InlineTree::find_subtree_from_root(C->ilt(), caller_jvms, method()); } @@ -2393,7 +2395,7 @@ void Parse::show_parse_info() { //------------------------------dump------------------------------------------- // Dump information associated with the bytecodes of current _method void Parse::dump() { - if( method() != NULL ) { + if( method() != nullptr ) { // Iterate over bytecodes ciBytecodeStream iter(method()); for( Bytecodes::Code bc = iter.next(); bc != ciBytecodeStream::EOBC() ; bc = iter.next() ) { diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index 783f2a1c24301..aedf9485bae8b 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -140,11 +140,11 @@ Node* Parse::array_addressing(BasicType type, int vals, const Type*& elemtype) { bool need_range_check = true; if (idxtype->_hi < sizetype->_lo && idxtype->_lo >= 0) { need_range_check = false; - if (C->log() != NULL) C->log()->elem("observe that='!need_range_check'"); + if (C->log() != nullptr) C->log()->elem("observe that='!need_range_check'"); } ciKlass * arytype_klass = arytype->klass(); - if ((arytype_klass != NULL) && (!arytype_klass->is_loaded())) { + if ((arytype_klass != nullptr) && (!arytype_klass->is_loaded())) { // Only fails for some -Xcomp runs // The class is unloaded. We have to run this bytecode in the interpreter. uncommon_trap(Deoptimization::Reason_unloaded, @@ -189,7 +189,7 @@ Node* Parse::array_addressing(BasicType type, int vals, const Type*& elemtype) { // See IfNode::Ideal, is_range_check, adjust_check. uncommon_trap(Deoptimization::Reason_range_check, Deoptimization::Action_make_not_entrant, - NULL, "range_check"); + nullptr, "range_check"); } else { // If we have already recompiled with the range-check-widening // heroic optimization turned off, then we must really be throwing @@ -233,7 +233,7 @@ void Parse::jump_if_true_fork(IfNode *iff, int dest_bci_if_true, bool unc) { repush_if_args(); uncommon_trap(Deoptimization::Reason_unstable_if, Deoptimization::Action_reinterpret, - NULL, + nullptr, "taken always"); } else { assert(dest_bci_if_true != never_reached, "inconsistent dest"); @@ -255,7 +255,7 @@ void Parse::jump_if_false_fork(IfNode *iff, int dest_bci_if_true, bool unc) { repush_if_args(); uncommon_trap(Deoptimization::Reason_unstable_if, Deoptimization::Action_reinterpret, - NULL, + nullptr, "taken never"); } else { assert(dest_bci_if_true != never_reached, "inconsistent dest"); @@ -274,7 +274,7 @@ void Parse::jump_if_always_fork(int dest_bci, bool unc) { repush_if_args(); uncommon_trap(Deoptimization::Reason_unstable_if, Deoptimization::Action_reinterpret, - NULL, + nullptr, "taken never"); } else { assert(dest_bci != never_reached, "inconsistent dest"); @@ -423,10 +423,10 @@ void Parse::do_tableswitch() { } ciMethodData* methodData = method()->method_data(); - ciMultiBranchData* profile = NULL; + ciMultiBranchData* profile = nullptr; if (methodData->is_mature() && UseSwitchProfiling) { ciProfileData* data = methodData->bci_to_data(bci()); - if (data != NULL && data->is_MultiBranchData()) { + if (data != nullptr && data->is_MultiBranchData()) { profile = (ciMultiBranchData*)data; } } @@ -439,7 +439,7 @@ void Parse::do_tableswitch() { int rp = -1; if (lo_index != min_jint) { uint cnt = 1; - if (profile != NULL) { + if (profile != nullptr) { cnt = profile->default_count() / (hi_index != max_jint ? 2 : 1); } ranges[++rp].setRange(min_jint, lo_index-1, default_dest, cnt); @@ -449,7 +449,7 @@ void Parse::do_tableswitch() { int dest = iter().get_dest_table(j+3); makes_backward_branch |= (dest <= bci()); uint cnt = 1; - if (profile != NULL) { + if (profile != nullptr) { cnt = profile->count_at(j); } if (rp < 0 || !ranges[rp].adjoin(match_int, dest, cnt, trim_ranges)) { @@ -460,7 +460,7 @@ void Parse::do_tableswitch() { assert(ranges[rp].hi() == highest, ""); if (highest != max_jint) { uint cnt = 1; - if (profile != NULL) { + if (profile != nullptr) { cnt = profile->default_count() / (lo_index != min_jint ? 2 : 1); } if (!ranges[rp].adjoinRange(highest+1, max_jint, default_dest, cnt, trim_ranges)) { @@ -497,10 +497,10 @@ void Parse::do_lookupswitch() { } ciMethodData* methodData = method()->method_data(); - ciMultiBranchData* profile = NULL; + ciMultiBranchData* profile = nullptr; if (methodData->is_mature() && UseSwitchProfiling) { ciProfileData* data = methodData->bci_to_data(bci()); - if (data != NULL && data->is_MultiBranchData()) { + if (data != nullptr && data->is_MultiBranchData()) { profile = (ciMultiBranchData*)data; } } @@ -513,7 +513,7 @@ void Parse::do_lookupswitch() { table[3*j+0] = iter().get_int_table(2+2*j); table[3*j+1] = iter().get_dest_table(2+2*j+1); // Handle overflow when converting from uint to jint - table[3*j+2] = (profile == NULL) ? 1 : MIN2(max_jint, profile->count_at(j)); + table[3*j+2] = (profile == nullptr) ? 1 : MIN2(max_jint, profile->count_at(j)); } qsort(table, len, 3*sizeof(table[0]), jint_cmp); } @@ -531,7 +531,7 @@ void Parse::do_lookupswitch() { defaults += (float)max_jint - prev + 1; } float default_cnt = 1; - if (profile != NULL) { + if (profile != nullptr) { default_cnt = profile->default_count()/defaults; } @@ -615,12 +615,12 @@ class SwitchRanges : public ResourceObj { } _state; SwitchRanges(SwitchRange *lo, SwitchRange *hi) - : _lo(lo), _hi(hi), _mid(NULL), + : _lo(lo), _hi(hi), _mid(nullptr), _cost(0), _state(Start) { } SwitchRanges() - : _lo(NULL), _hi(NULL), _mid(NULL), + : _lo(nullptr), _hi(nullptr), _mid(nullptr), _cost(0), _state(Start) {} }; @@ -634,7 +634,7 @@ static float compute_tree_cost(SwitchRange *lo, SwitchRange *hi, float total_cnt do { SwitchRanges& r = *tree.adr_at(tree.length()-1); if (r._hi != r._lo) { - if (r._mid == NULL) { + if (r._mid == nullptr) { float r_cnt = sum_of_cnts(r._lo, r._hi); if (r_cnt == 0) { @@ -643,7 +643,7 @@ static float compute_tree_cost(SwitchRange *lo, SwitchRange *hi, float total_cnt continue; } - SwitchRange* mid = NULL; + SwitchRange* mid = nullptr; mid = r._lo; for (float cnt = 0; ; ) { assert(mid <= r._hi, "out of bounds"); @@ -692,7 +692,7 @@ void Parse::linear_search_switch_ranges(Node* key_val, SwitchRange*& lo, SwitchR SwitchRange* array1 = lo; SwitchRange* array2 = NEW_RESOURCE_ARRAY(SwitchRange, nr); - SwitchRange* ranges = NULL; + SwitchRange* ranges = nullptr; while (nr >= 2) { assert(lo == array1 || lo == array2, "one the 2 already allocated arrays"); @@ -888,15 +888,15 @@ bool Parse::create_jump_tables(Node* key_val, SwitchRange* lo, SwitchRange* hi) } ciMethodData* methodData = method()->method_data(); - ciMultiBranchData* profile = NULL; + ciMultiBranchData* profile = nullptr; if (methodData->is_mature()) { ciProfileData* data = methodData->bci_to_data(bci()); - if (data != NULL && data->is_MultiBranchData()) { + if (data != nullptr && data->is_MultiBranchData()) { profile = (ciMultiBranchData*)data; } } - Node* jtn = _gvn.transform(new JumpNode(control(), key_val, num_cases, probs, profile == NULL ? COUNT_UNKNOWN : total)); + Node* jtn = _gvn.transform(new JumpNode(control(), key_val, num_cases, probs, profile == nullptr ? COUNT_UNKNOWN : total)); // These are the switch destinations hanging off the jumpnode i = 0; @@ -950,7 +950,7 @@ void Parse::jump_switch_ranges(Node* key_val, SwitchRange *lo, SwitchRange *hi, jint min_val = min_jint; jint max_val = max_jint; const TypeInt* ti = key_val->bottom_type()->isa_int(); - if (ti != NULL) { + if (ti != nullptr) { min_val = ti->_lo; max_val = ti->_hi; assert(min_val <= max_val, "invalid int type"); @@ -987,7 +987,7 @@ void Parse::jump_switch_ranges(Node* key_val, SwitchRange *lo, SwitchRange *hi, if (create_jump_tables(key_val, lo, hi)) return; - SwitchRange* mid = NULL; + SwitchRange* mid = nullptr; float total_cnt = sum_of_cnts(lo, hi); int nr = hi - lo + 1; @@ -1111,7 +1111,7 @@ void Parse::modf() { Node *f1 = pop(); Node* c = make_runtime_call(RC_LEAF, OptoRuntime::modf_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::frem), - "frem", NULL, //no memory effects + "frem", nullptr, //no memory effects f1, f2); Node* res = _gvn.transform(new ProjNode(c, TypeFunc::Parms + 0)); @@ -1123,7 +1123,7 @@ void Parse::modd() { Node *d1 = pop_pair(); Node* c = make_runtime_call(RC_LEAF, OptoRuntime::Math_DD_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::drem), - "drem", NULL, //no memory effects + "drem", nullptr, //no memory effects d1, top(), d2, top()); Node* res_d = _gvn.transform(new ProjNode(c, TypeFunc::Parms + 0)); @@ -1140,7 +1140,7 @@ void Parse::l2f() { Node* f1 = pop(); Node* c = make_runtime_call(RC_LEAF, OptoRuntime::l2f_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::l2f), - "l2f", NULL, //no memory effects + "l2f", nullptr, //no memory effects f1, f2); Node* res = _gvn.transform(new ProjNode(c, TypeFunc::Parms + 0)); @@ -1222,7 +1222,7 @@ float Parse::dynamic_branch_prediction(float &cnt, BoolTest::mask btest, Node* t ciMethodData* methodData = method()->method_data(); if (!methodData->is_mature()) return PROB_UNKNOWN; ciProfileData* data = methodData->bci_to_data(bci()); - if (data == NULL) { + if (data == nullptr) { return PROB_UNKNOWN; } if (!data->is_JumpData()) return PROB_UNKNOWN; @@ -1242,7 +1242,7 @@ float Parse::dynamic_branch_prediction(float &cnt, BoolTest::mask btest, Node* t // Give up if too few (or too many, in which case the sum will overflow) counts to be meaningful. // We also check that individual counters are positive first, otherwise the sum can become positive. if (taken < 0 || not_taken < 0 || taken + not_taken < 40) { - if (C->log() != NULL) { + if (C->log() != nullptr) { C->log()->elem("branch target_bci='%d' taken='%d' not_taken='%d'", iter().get_dest(), taken, not_taken); } return PROB_UNKNOWN; @@ -1272,12 +1272,12 @@ float Parse::dynamic_branch_prediction(float &cnt, BoolTest::mask btest, Node* t assert((cnt > 0.0f) && (prob > 0.0f), "Bad frequency assignment in if"); - if (C->log() != NULL) { - const char* prob_str = NULL; + if (C->log() != nullptr) { + const char* prob_str = nullptr; if (prob >= PROB_MAX) prob_str = (prob == PROB_MAX) ? "max" : "always"; if (prob <= PROB_MIN) prob_str = (prob == PROB_MIN) ? "min" : "never"; char prob_str_buf[30]; - if (prob_str == NULL) { + if (prob_str == nullptr) { jio_snprintf(prob_str_buf, sizeof(prob_str_buf), "%20.2f", prob); prob_str = prob_str_buf; } @@ -1316,7 +1316,7 @@ float Parse::branch_prediction(float& cnt, // of the OSR-ed method, and we want to deopt to gather more stats. // If you have ANY counts, then this loop is simply 'cold' relative // to the OSR loop. - if (data == NULL || + if (data == nullptr || (data->as_BranchData()->taken() + data->as_BranchData()->not_taken() == 0)) { // This is the only way to return PROB_UNKNOWN: return PROB_UNKNOWN; @@ -1366,8 +1366,8 @@ inline int Parse::repush_if_args() { int bc_depth = - Bytecodes::depth(iter().cur_bc()); assert(bc_depth == 1 || bc_depth == 2, "only two kinds of branches"); DEBUG_ONLY(sync_jvms()); // argument(n) requires a synced jvms - assert(argument(0) != NULL, "must exist"); - assert(bc_depth == 1 || argument(1) != NULL, "two must exist"); + assert(argument(0) != nullptr, "must exist"); + assert(bc_depth == 1 || argument(1) != nullptr, "two must exist"); inc_sp(bc_depth); return bc_depth; } @@ -1389,7 +1389,7 @@ void Parse::do_ifnull(BoolTest::mask btest, Node *c) { repush_if_args(); // to gather stats on loop uncommon_trap(Deoptimization::Reason_unreached, Deoptimization::Action_reinterpret, - NULL, "cold"); + nullptr, "cold"); if (C->eliminate_boxing()) { // Mark the successor blocks as parsed branch_block->next_path_num(); @@ -1460,7 +1460,7 @@ void Parse::do_if(BoolTest::mask btest, Node* c) { repush_if_args(); // to gather stats on loop uncommon_trap(Deoptimization::Reason_unreached, Deoptimization::Action_reinterpret, - NULL, "cold"); + nullptr, "cold"); if (C->eliminate_boxing()) { // Mark the successor blocks as parsed branch_block->next_path_num(); @@ -1490,7 +1490,7 @@ void Parse::do_if(BoolTest::mask btest, Node* c) { if (tst->is_Bool()) { // Refresh c from the transformed bool node, since it may be // simpler than the original c. Also re-canonicalize btest. - // This wins when (Bool ne (Conv2B p) 0) => (Bool ne (CmpP p NULL)). + // This wins when (Bool ne (Conv2B p) 0) => (Bool ne (CmpP p null)). // That can arise from statements like: if (x instanceof C) ... if (tst != tst0) { // Canonicalize one more time since transform can change it. @@ -1595,7 +1595,7 @@ void Parse::adjust_map_after_if(BoolTest::mask btest, Node* c, float prob, Block repush_if_args(); uncommon_trap(Deoptimization::Reason_unstable_if, Deoptimization::Action_reinterpret, - NULL, + nullptr, (is_fallthrough ? "taken always" : "taken never")); return; } @@ -1633,25 +1633,25 @@ static Node* extract_obj_from_klass_load(PhaseGVN* gvn, Node* n) { Node* ldk; if (n->is_DecodeNKlass()) { if (n->in(1)->Opcode() != Op_LoadNKlass) { - return NULL; + return nullptr; } else { ldk = n->in(1); } } else if (n->Opcode() != Op_LoadKlass) { - return NULL; + return nullptr; } else { ldk = n; } - assert(ldk != NULL && ldk->is_Load(), "should have found a LoadKlass or LoadNKlass node"); + assert(ldk != nullptr && ldk->is_Load(), "should have found a LoadKlass or LoadNKlass node"); Node* adr = ldk->in(MemNode::Address); intptr_t off = 0; Node* obj = AddPNode::Ideal_base_and_offset(adr, gvn, off); - if (obj == NULL || off != oopDesc::klass_offset_in_bytes()) // loading oopDesc::_klass? - return NULL; + if (obj == nullptr || off != oopDesc::klass_offset_in_bytes()) // loading oopDesc::_klass? + return nullptr; const TypePtr* tp = gvn->type(obj)->is_ptr(); - if (tp == NULL || !(tp->isa_instptr() || tp->isa_aryptr())) // is obj a Java object ptr? - return NULL; + if (tp == nullptr || !(tp->isa_instptr() || tp->isa_aryptr())) // is obj a Java object ptr? + return nullptr; return obj; } @@ -1664,13 +1664,13 @@ void Parse::sharpen_type_after_if(BoolTest::mask btest, if (btest == BoolTest::eq && tcon->isa_klassptr()) { Node* obj = extract_obj_from_klass_load(&_gvn, val); const TypeOopPtr* con_type = tcon->isa_klassptr()->as_instance_type(); - if (obj != NULL && (con_type->isa_instptr() || con_type->isa_aryptr())) { + if (obj != nullptr && (con_type->isa_instptr() || con_type->isa_aryptr())) { // Found: // Bool(CmpP(LoadKlass(obj._klass), ConP(Foo.klass)), [eq]) // or the narrowOop equivalent. const Type* obj_type = _gvn.type(obj); const TypeOopPtr* tboth = obj_type->join_speculative(con_type)->isa_oopptr(); - if (tboth != NULL && tboth->klass_is_exact() && tboth != obj_type && + if (tboth != nullptr && tboth->klass_is_exact() && tboth != obj_type && tboth->higher_equal(obj_type)) { // obj has to be of the exact type Foo if the CmpP succeeds. int obj_in_map = map()->find_edge(obj); @@ -1703,8 +1703,8 @@ void Parse::sharpen_type_after_if(BoolTest::mask btest, // Check for a comparison to a constant, and "know" that the compared // value is constrained on this path. assert(tcon->singleton(), ""); - ConstraintCastNode* ccast = NULL; - Node* cast = NULL; + ConstraintCastNode* ccast = nullptr; + Node* cast = nullptr; switch (btest) { case BoolTest::eq: // Constant test? @@ -1741,7 +1741,7 @@ void Parse::sharpen_type_after_if(BoolTest::mask btest, break; } - if (ccast != NULL) { + if (ccast != nullptr) { const Type* tcc = ccast->as_Type()->type(); assert(tcc != tval && tcc->higher_equal(tval), "must improve"); // Delay transform() call to allow recovery of pre-cast value @@ -1752,7 +1752,7 @@ void Parse::sharpen_type_after_if(BoolTest::mask btest, cast = ccast; } - if (cast != NULL) { // Here's the payoff. + if (cast != nullptr) { // Here's the payoff. replace_in_map(val, cast); } } @@ -1772,8 +1772,8 @@ Node* Parse::optimize_cmp_with_klass(Node* c) { if (c->Opcode() == Op_CmpP && (c->in(1)->Opcode() == Op_LoadKlass || c->in(1)->Opcode() == Op_DecodeNKlass) && c->in(2)->is_Con()) { - Node* load_klass = NULL; - Node* decode = NULL; + Node* load_klass = nullptr; + Node* decode = nullptr; if (c->in(1)->Opcode() == Op_DecodeNKlass) { decode = c->in(1); load_klass = c->in(1)->in(1); @@ -1784,7 +1784,7 @@ Node* Parse::optimize_cmp_with_klass(Node* c) { Node* addp = load_klass->in(2); Node* obj = addp->in(AddPNode::Address); const TypeOopPtr* obj_type = _gvn.type(obj)->is_oopptr(); - if (obj_type->speculative_type_not_null() != NULL) { + if (obj_type->speculative_type_not_null() != nullptr) { ciKlass* k = obj_type->speculative_type(); inc_sp(2); obj = maybe_cast_profiled_obj(obj, k); @@ -1794,7 +1794,7 @@ Node* Parse::optimize_cmp_with_klass(Node* c) { load_klass = load_klass->clone(); load_klass->set_req(2, addp); load_klass = _gvn.transform(load_klass); - if (decode != NULL) { + if (decode != nullptr) { decode = decode->clone(); decode->set_req(1, load_klass); load_klass = _gvn.transform(decode); @@ -1882,7 +1882,7 @@ void Parse::do_one_bytecode() { assert(constant.basic_type() != T_OBJECT || constant.as_object()->is_instance(), "must be java_mirror of klass"); const Type* con_type = Type::make_from_constant(constant); - if (con_type != NULL) { + if (con_type != nullptr) { push_node(con_type->basic_type(), makecon(con_type)); } } else { @@ -1890,14 +1890,14 @@ void Parse::do_one_bytecode() { if (iter().is_in_error()) { uncommon_trap(Deoptimization::make_trap_request(Deoptimization::Reason_unhandled, Deoptimization::Action_none), - NULL, "constant in error state", true /* must_throw */); + nullptr, "constant in error state", true /* must_throw */); } else { int index = iter().get_constant_pool_index(); uncommon_trap(Deoptimization::make_trap_request(Deoptimization::Reason_unloaded, Deoptimization::Action_reinterpret, index), - NULL, "unresolved constant", false /* must_throw */); + nullptr, "unresolved constant", false /* must_throw */); } } break; @@ -2540,17 +2540,17 @@ void Parse::do_one_bytecode() { case Bytecodes::_i2b: // Sign extend a = pop(); - a = Compile::narrow_value(T_BYTE, a, NULL, &_gvn, true); + a = Compile::narrow_value(T_BYTE, a, nullptr, &_gvn, true); push(a); break; case Bytecodes::_i2s: a = pop(); - a = Compile::narrow_value(T_SHORT, a, NULL, &_gvn, true); + a = Compile::narrow_value(T_SHORT, a, nullptr, &_gvn, true); push(a); break; case Bytecodes::_i2c: a = pop(); - a = Compile::narrow_value(T_CHAR, a, NULL, &_gvn, true); + a = Compile::narrow_value(T_CHAR, a, nullptr, &_gvn, true); push(a); break; @@ -2574,7 +2574,7 @@ void Parse::do_one_bytecode() { // Exit points of synchronized methods must have an unlock node case Bytecodes::_return: - return_current(NULL); + return_current(nullptr); break; case Bytecodes::_ireturn: @@ -2590,7 +2590,7 @@ void Parse::do_one_bytecode() { break; case Bytecodes::_athrow: - // null exception oop throws NULL pointer exception + // null exception oop throws null pointer exception null_check(peek()); if (stopped()) return; // Hook the thrown exception directly to subsequent handlers. @@ -2624,7 +2624,7 @@ void Parse::do_one_bytecode() { ciMethodData* methodData = method()->method_data(); if (!methodData->is_mature()) break; ciProfileData* data = methodData->bci_to_data(bci()); - assert(data != NULL && data->is_JumpData(), "need JumpData for taken branch"); + assert(data != nullptr && data->is_JumpData(), "need JumpData for taken branch"); int taken = ((ciJumpData*)data)->taken(); taken = method()->scale_count(taken); target_block->set_count(taken); diff --git a/src/hotspot/share/opto/parse3.cpp b/src/hotspot/share/opto/parse3.cpp index a6c1b73ec051c..cb9133e0bb7c5 100644 --- a/src/hotspot/share/opto/parse3.cpp +++ b/src/hotspot/share/opto/parse3.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -60,7 +60,7 @@ void Parse::do_field_access(bool is_get, bool is_field) { !(method()->holder() == field_holder && method()->is_object_initializer())) { uncommon_trap(Deoptimization::Reason_unhandled, Deoptimization::Action_reinterpret, - NULL, "put to call site target field"); + nullptr, "put to call site target field"); return; } @@ -118,7 +118,7 @@ void Parse::do_get_xxx(Node* obj, ciField* field, bool is_field) { (bt != T_OBJECT || field->type()->is_loaded())) { // final or stable field Node* con = make_constant_from_field(field, obj); - if (con != NULL) { + if (con != nullptr) { push_node(field->layout_type(), con); return; } @@ -156,7 +156,7 @@ void Parse::do_get_xxx(Node* obj, ciField* field, bool is_field) { } else { type = TypeOopPtr::make_from_constant(con)->isa_oopptr(); } - assert(type != NULL, "field singleton type must be consistent"); + assert(type != nullptr, "field singleton type must be consistent"); } else { type = TypeOopPtr::make_from_klass(field_klass->as_klass()); } @@ -186,7 +186,7 @@ void Parse::do_get_xxx(Node* obj, ciField* field, bool is_field) { if (PrintOpto && (Verbose || WizardMode)) { method()->print_name(); tty->print_cr(" asserting nullness of field at bci: %d", bci()); } - if (C->log() != NULL) { + if (C->log() != nullptr) { C->log()->elem("assert_null reason='field' klass='%d'", C->log()->identify(field->type())); } @@ -242,7 +242,7 @@ void Parse::do_put_xxx(Node* obj, ciField* field, bool is_field) { // Any method can write a @Stable field; insert memory barriers after those also. if (field->is_final()) { set_wrote_final(true); - if (AllocateNode::Ideal_allocation(obj, &_gvn) != NULL) { + if (AllocateNode::Ideal_allocation(obj, &_gvn) != nullptr) { // Preserve allocation ptr to create precedent edge to it in membar // generated on exit from constructor. // Can't bind stable with its allocation, only record allocation for final field. @@ -298,7 +298,7 @@ void Parse::do_newarray(BasicType elem_type) { // Also handle the degenerate 1-dimensional case of anewarray. Node* Parse::expand_multianewarray(ciArrayKlass* array_klass, Node* *lengths, int ndimensions, int nargs) { Node* length = lengths[0]; - assert(length != NULL, ""); + assert(length != nullptr, ""); Node* array = new_array(makecon(TypeKlassPtr::make(array_klass)), length, nargs); if (ndimensions > 1) { jint length_con = find_int_con(length, -1); @@ -331,7 +331,7 @@ void Parse::do_multianewarray() { // get the lengths from the stack (first dimension is on top) Node** length = NEW_RESOURCE_ARRAY(Node*, ndimensions + 1); - length[ndimensions] = NULL; // terminating null for make_runtime_call + length[ndimensions] = nullptr; // terminating null for make_runtime_call int j; for (j = ndimensions-1; j >= 0 ; j--) length[j] = pop(); @@ -356,7 +356,7 @@ void Parse::do_multianewarray() { // Can use multianewarray instead of [a]newarray if only one dimension, // or if all non-final dimensions are small constants. if (ndimensions == 1 || (1 <= expand_count && expand_count <= expand_limit)) { - Node* obj = NULL; + Node* obj = nullptr; // Set the original stack and the reexecute bit for the interpreter // to reexecute the multianewarray bytecode if deoptimization happens. // Do it unconditionally even for one dimension multianewarray. @@ -371,7 +371,7 @@ void Parse::do_multianewarray() { return; } - address fun = NULL; + address fun = nullptr; switch (ndimensions) { case 1: ShouldNotReachHere(); break; case 2: fun = OptoRuntime::multianewarray2_Java(); break; @@ -379,19 +379,19 @@ void Parse::do_multianewarray() { case 4: fun = OptoRuntime::multianewarray4_Java(); break; case 5: fun = OptoRuntime::multianewarray5_Java(); break; }; - Node* c = NULL; + Node* c = nullptr; - if (fun != NULL) { + if (fun != nullptr) { c = make_runtime_call(RC_NO_LEAF | RC_NO_IO, OptoRuntime::multianewarray_Type(ndimensions), - fun, NULL, TypeRawPtr::BOTTOM, + fun, nullptr, TypeRawPtr::BOTTOM, makecon(TypeKlassPtr::make(array_klass)), length[0], length[1], length[2], - (ndimensions > 2) ? length[3] : NULL, - (ndimensions > 3) ? length[4] : NULL); + (ndimensions > 2) ? length[3] : nullptr, + (ndimensions > 3) ? length[4] : nullptr); } else { // Create a java array for dimension sizes - Node* dims = NULL; + Node* dims = nullptr; { PreserveReexecuteState preexecs(this); inc_sp(ndimensions); Node* dims_array_klass = makecon(TypeKlassPtr::make(ciArrayKlass::make(ciType::make(T_INT)))); @@ -406,7 +406,7 @@ void Parse::do_multianewarray() { c = make_runtime_call(RC_NO_LEAF | RC_NO_IO, OptoRuntime::multianewarrayN_Type(), - OptoRuntime::multianewarrayN_Java(), NULL, TypeRawPtr::BOTTOM, + OptoRuntime::multianewarrayN_Java(), nullptr, TypeRawPtr::BOTTOM, makecon(TypeKlassPtr::make(array_klass)), dims); } @@ -421,7 +421,7 @@ void Parse::do_multianewarray() { type = type->is_aryptr()->cast_to_exactness(true); const TypeInt* ltype = _gvn.find_int_type(length[0]); - if (ltype != NULL) + if (ltype != nullptr) type = type->is_aryptr()->cast_to_size(ltype); // We cannot sharpen the nested sub-arrays, since the top level is mutable. diff --git a/src/hotspot/share/opto/parseHelper.cpp b/src/hotspot/share/opto/parseHelper.cpp index 957a6f7ea46d1..cc232ae0f64f9 100644 --- a/src/hotspot/share/opto/parseHelper.cpp +++ b/src/hotspot/share/opto/parseHelper.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -69,11 +69,11 @@ void Parse::do_checkcast() { Node *obj = peek(); // Throw uncommon trap if class is not loaded or the value we are casting - // _from_ is not loaded, and value is not null. If the value _is_ NULL, + // _from_ is not loaded, and value is not null. If the value _is_ null, // then the checkcast does nothing. const TypeOopPtr *tp = _gvn.type(obj)->isa_oopptr(); if (!will_link || (tp && tp->klass() && !tp->klass()->is_loaded())) { - if (C->log() != NULL) { + if (C->log() != nullptr) { if (!will_link) { C->log()->elem("assert_null reason='checkcast' klass='%d'", C->log()->identify(klass)); @@ -112,7 +112,7 @@ void Parse::do_instanceof() { ciKlass* klass = iter().get_klass(will_link); if (!will_link) { - if (C->log() != NULL) { + if (C->log() != nullptr) { C->log()->elem("assert_null reason='instanceof' klass='%d'", C->log()->identify(klass)); } @@ -156,7 +156,7 @@ void Parse::array_store_check() { int klass_offset = oopDesc::klass_offset_in_bytes(); Node* p = basic_plus_adr( ary, ary, klass_offset ); // p's type is array-of-OOPS plus klass_offset - Node* array_klass = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, TypeInstPtr::KLASS)); + Node* array_klass = _gvn.transform(LoadKlassNode::make(_gvn, nullptr, immutable_memory(), p, TypeInstPtr::KLASS)); // Get the array klass const TypeKlassPtr *tak = _gvn.type(array_klass)->is_klassptr(); @@ -210,7 +210,7 @@ void Parse::array_store_check() { // Use the exact constant value we know it is. replace_in_map(array_klass,con); CompileLog* log = C->log(); - if (log != NULL) { + if (log != nullptr) { log->elem("cast_up reason='monomorphic_array' from='%d' to='(exact)'", log->identify(tak->klass())); } @@ -226,7 +226,7 @@ void Parse::array_store_check() { // We are allowed to use the constant type only if cast succeeded. If always_see_exact_class is true, // we must set a control edge from the IfTrue node created by the uncommon_trap above to the // LoadKlassNode. - Node* a_e_klass = _gvn.transform(LoadKlassNode::make(_gvn, always_see_exact_class ? control() : NULL, + Node* a_e_klass = _gvn.transform(LoadKlassNode::make(_gvn, always_see_exact_class ? control() : nullptr, immutable_memory(), p2, tak)); // Check (the hard way) and throw if not a subklass. @@ -283,8 +283,8 @@ void Parse::do_new() { // Debug dump of the mapping from address types to MergeMemNode indices. void Parse::dump_map_adr_mem() const { tty->print_cr("--- Mapping from address types to memory Nodes ---"); - MergeMemNode *mem = map() == NULL ? NULL : (map()->memory()->is_MergeMem() ? - map()->memory()->as_MergeMem() : NULL); + MergeMemNode *mem = map() == nullptr ? nullptr : (map()->memory()->is_MergeMem() ? + map()->memory()->as_MergeMem() : nullptr); for (uint i = 0; i < (uint)C->num_alias_types(); i++) { C->alias_type(i)->print_on(tty); tty->print("\t"); diff --git a/src/hotspot/share/opto/phase.cpp b/src/hotspot/share/opto/phase.cpp index 4b0920558a259..90b9a7d527ce5 100644 --- a/src/hotspot/share/opto/phase.cpp +++ b/src/hotspot/share/opto/phase.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -40,7 +40,7 @@ elapsedTimer Phase::_t_stubCompilation; elapsedTimer Phase::timers[max_phase_timers]; //------------------------------Phase------------------------------------------ -Phase::Phase( PhaseNumber pnum ) : _pnum(pnum), C( pnum == Compiler ? NULL : Compile::current()) { +Phase::Phase( PhaseNumber pnum ) : _pnum(pnum), C( pnum == Compiler ? nullptr : Compile::current()) { // Poll for requests from shutdown mechanism to quiesce compiler (4448539, 4448544). // This is an effective place to poll, since the compiler is full of phases. // In particular, every inlining site uses a recursively created Parse phase. diff --git a/src/hotspot/share/opto/phaseX.cpp b/src/hotspot/share/opto/phaseX.cpp index 1600e1f82d753..8386018b4debf 100644 --- a/src/hotspot/share/opto/phaseX.cpp +++ b/src/hotspot/share/opto/phaseX.cpp @@ -56,7 +56,7 @@ NodeHash::NodeHash(uint est_max_size) : #endif { // _sentinel must be in the current node space - _sentinel = new ProjNode(NULL, TypeFunc::Control); + _sentinel = new ProjNode(nullptr, TypeFunc::Control); memset(_table,0,sizeof(Node*)*_max); } @@ -73,7 +73,7 @@ NodeHash::NodeHash(Arena *arena, uint est_max_size) : #endif { // _sentinel must be in the current node space - _sentinel = new ProjNode(NULL, TypeFunc::Control); + _sentinel = new ProjNode(nullptr, TypeFunc::Control); memset(_table,0,sizeof(Node*)*_max); } @@ -99,7 +99,7 @@ Node *NodeHash::hash_find( const Node *n ) { uint hash = n->hash(); if (hash == Node::NO_HASH) { NOT_PRODUCT( _lookup_misses++ ); - return NULL; + return nullptr; } uint key = hash & (_max-1); uint stride = key | 0x01; @@ -107,7 +107,7 @@ Node *NodeHash::hash_find( const Node *n ) { Node *k = _table[key]; // Get hashed value if( !k ) { // ?Miss? NOT_PRODUCT( _lookup_misses++ ); - return NULL; // Miss! + return nullptr; // Miss! } int op = n->Opcode(); @@ -129,11 +129,11 @@ Node *NodeHash::hash_find( const Node *n ) { k = _table[key]; // Get hashed value if( !k ) { // ?Miss? NOT_PRODUCT( _lookup_misses++ ); - return NULL; // Miss! + return nullptr; // Miss! } } ShouldNotReachHere(); - return NULL; + return nullptr; } //------------------------------hash_find_insert------------------------------- @@ -144,7 +144,7 @@ Node *NodeHash::hash_find_insert( Node *n ) { uint hash = n->hash(); if (hash == Node::NO_HASH) { NOT_PRODUCT( _lookup_misses++ ); - return NULL; + return nullptr; } uint key = hash & (_max-1); uint stride = key | 0x01; // stride must be relatively prime to table siz @@ -156,7 +156,7 @@ Node *NodeHash::hash_find_insert( Node *n ) { _table[key] = n; // Insert into table! debug_only(n->enter_hash_lock()); // Lock down the node while in the table. check_grow(); // Grow table if insert hit limit - return NULL; // Miss! + return nullptr; // Miss! } else if( k == _sentinel ) { first_sentinel = key; // Can insert here @@ -185,7 +185,7 @@ Node *NodeHash::hash_find_insert( Node *n ) { _table[key] = n; // Insert into table! debug_only(n->enter_hash_lock()); // Lock down the node while in the table. check_grow(); // Grow table if insert hit limit - return NULL; // Miss! + return nullptr; // Miss! } else if( first_sentinel == 0 && k == _sentinel ) { first_sentinel = key; // Can insert here @@ -193,7 +193,7 @@ Node *NodeHash::hash_find_insert( Node *n ) { } ShouldNotReachHere(); - return NULL; + return nullptr; } //------------------------------hash_insert------------------------------------ @@ -235,7 +235,7 @@ bool NodeHash::hash_delete( const Node *n ) { uint key = hash & (_max-1); uint stride = key | 0x01; debug_only( uint counter = 0; ); - for( ; /* (k != NULL) && (k != _sentinel) */; ) { + for( ; /* (k != nullptr) && (k != _sentinel) */; ) { debug_only( counter++ ); NOT_PRODUCT( _delete_probes++ ); k = _table[key]; // Get hashed value @@ -294,7 +294,7 @@ void NodeHash::grow() { } //------------------------------clear------------------------------------------ -// Clear all entries in _table to NULL but keep storage +// Clear all entries in _table to null but keep storage void NodeHash::clear() { #ifdef ASSERT // Unlock all nodes upon removal from table. @@ -319,7 +319,7 @@ void NodeHash::remove_useless_nodes(VectorSet &useful) { Node *sentinel_node = sentinel(); for( uint i = 0; i < max; ++i ) { Node *n = at(i); - if(n != NULL && n != sentinel_node && !useful.test(n->_idx)) { + if(n != nullptr && n != sentinel_node && !useful.test(n->_idx)) { debug_only(n->exit_hash_lock()); // Unlock the node when removed _table[i] = sentinel_node; // Replace with placeholder } @@ -335,7 +335,7 @@ void NodeHash::check_no_speculative_types() { Node *sentinel_node = sentinel(); for (uint i = 0; i < max; ++i) { Node *n = at(i); - if (n != NULL && + if (n != nullptr && n != sentinel_node && n->is_Type() && live_nodes.member(n)) { @@ -379,7 +379,7 @@ Node *NodeHash::find_index(uint idx) { // For debugging if( !m || m == _sentinel ) continue; if( m->_idx == (uint)idx ) return m; } - return NULL; + return nullptr; } #endif @@ -469,6 +469,14 @@ PhaseRenumberLive::PhaseRenumberLive(PhaseGVN* gvn, uint worklist_size = worklist->size(); + GrowableArray* old_node_note_array = C->node_note_array(); + if (old_node_note_array != nullptr) { + int new_size = (_useful.size() >> 8) + 1; // The node note array uses blocks, see C->_log2_node_notes_block_size + new_size = MAX2(8, new_size); + C->set_node_note_array(new (C->comp_arena()) GrowableArray (C->comp_arena(), new_size, 0, nullptr)); + C->grow_node_notes(C->node_note_array(), new_size); + } + // Iterate over the set of live nodes. for (uint current_idx = 0; current_idx < _useful.size(); current_idx++) { Node* n = _useful.at(current_idx); @@ -484,6 +492,11 @@ PhaseRenumberLive::PhaseRenumberLive(PhaseGVN* gvn, assert(_old2new_map.at(n->_idx) == -1, "already seen"); _old2new_map.at_put(n->_idx, current_idx); + if (old_node_note_array != nullptr) { + Node_Notes* nn = C->locate_node_notes(old_node_note_array, n->_idx); + C->set_node_notes_at(current_idx, nn); + } + n->set_idx(current_idx); // Update node ID. if (in_worklist) { @@ -554,7 +567,7 @@ int PhaseRenumberLive::update_embedded_ids(Node* n) { } const Type* type = _new_type_array.fast_lookup(n->_idx); - if (type != NULL && type->isa_oopptr() && type->is_oopptr()->is_known_instance()) { + if (type != nullptr && type->isa_oopptr() && type->is_oopptr()->is_known_instance()) { if (!_is_pass_finished) { return -1; // delay } @@ -582,7 +595,7 @@ PhaseTransform::PhaseTransform( PhaseNumber pnum ) : Phase(pnum), set_allow_progress(true); #endif // Force allocation for currently existing nodes - _types.map(C->unique(), NULL); + _types.map(C->unique(), nullptr); } //------------------------------PhaseTransform--------------------------------- @@ -598,7 +611,7 @@ PhaseTransform::PhaseTransform( Arena *arena, PhaseNumber pnum ) : Phase(pnum), set_allow_progress(true); #endif // Force allocation for currently existing nodes - _types.map(C->unique(), NULL); + _types.map(C->unique(), nullptr); } //------------------------------PhaseTransform--------------------------------- @@ -625,22 +638,22 @@ void PhaseTransform::init_con_caches() { //--------------------------------find_int_type-------------------------------- const TypeInt* PhaseTransform::find_int_type(Node* n) { - if (n == NULL) return NULL; + if (n == nullptr) return nullptr; // Call type_or_null(n) to determine node's type since we might be in // parse phase and call n->Value() may return wrong type. // (For example, a phi node at the beginning of loop parsing is not ready.) const Type* t = type_or_null(n); - if (t == NULL) return NULL; + if (t == nullptr) return nullptr; return t->isa_int(); } //-------------------------------find_long_type-------------------------------- const TypeLong* PhaseTransform::find_long_type(Node* n) { - if (n == NULL) return NULL; + if (n == nullptr) return nullptr; // (See comment above on type_or_null.) const Type* t = type_or_null(n); - if (t == NULL) return NULL; + if (t == nullptr) return nullptr; return t->isa_long(); } @@ -682,7 +695,7 @@ void PhaseTransform::dump_nodes_and_types_recur( const Node *n, uint depth, bool dump_nodes_and_types_recur( n->in(i), depth-1, only_ctrl, visited ); } n->dump(); - if (type_or_null(n) != NULL) { + if (type_or_null(n) != nullptr) { tty->print(" "); type(n)->dump(); tty->cr(); } } @@ -745,10 +758,10 @@ ConNode* PhaseValues::uncached_makecon(const Type *t) { assert(t->singleton(), "must be a constant"); ConNode* x = ConNode::make(t); ConNode* k = (ConNode*)hash_find_insert(x); // Value numbering - if (k == NULL) { + if (k == nullptr) { set_type(x, t); // Missed, provide type mapping GrowableArray* nna = C->node_note_array(); - if (nna != NULL) { + if (nna != nullptr) { Node_Notes* loc = C->locate_node_notes(nna, x->_idx, true); loc->clear(); // do not put debug info on constants } @@ -765,7 +778,7 @@ ConINode* PhaseTransform::intcon(jint i) { // Small integer? Check cache! Check that cached node is not dead if (i >= _icon_min && i <= _icon_max) { ConINode* icon = _icons[i-_icon_min]; - if (icon != NULL && icon->in(TypeFunc::Control) != NULL) + if (icon != nullptr && icon->in(TypeFunc::Control) != nullptr) return icon; } ConINode* icon = (ConINode*) uncached_makecon(TypeInt::make(i)); @@ -781,7 +794,7 @@ ConLNode* PhaseTransform::longcon(jlong l) { // Small integer? Check cache! Check that cached node is not dead if (l >= _lcon_min && l <= _lcon_max) { ConLNode* lcon = _lcons[l-_lcon_min]; - if (lcon != NULL && lcon->in(TypeFunc::Control) != NULL) + if (lcon != nullptr && lcon->in(TypeFunc::Control) != nullptr) return lcon; } ConLNode* lcon = (ConLNode*) uncached_makecon(TypeLong::make(l)); @@ -806,7 +819,7 @@ ConNode* PhaseTransform::integercon(jlong l, BasicType bt) { ConNode* PhaseTransform::zerocon(BasicType bt) { assert((uint)bt <= _zcon_max, "domain check"); ConNode* zcon = _zcons[bt]; - if (zcon != NULL && zcon->in(TypeFunc::Control) != NULL) + if (zcon != nullptr && zcon->in(TypeFunc::Control) != nullptr) return zcon; zcon = (ConNode*) uncached_makecon(Type::get_zero_type(bt)); _zcons[bt] = zcon; @@ -818,7 +831,7 @@ ConNode* PhaseTransform::zerocon(BasicType bt) { //============================================================================= Node* PhaseGVN::apply_ideal(Node* k, bool can_reshape) { Node* i = BarrierSet::barrier_set()->barrier_set_c2()->ideal_node(this, k, can_reshape); - if (i == NULL) { + if (i == nullptr) { i = k->Ideal(this, can_reshape); } return i; @@ -841,7 +854,7 @@ Node *PhaseGVN::transform_no_reclaim(Node *n) { Node* k = n; Node* i = apply_ideal(k, /*can_reshape=*/false); NOT_PRODUCT(uint loop_count = 1;) - while (i != NULL) { + while (i != nullptr) { assert(i->_idx >= k->_idx, "Idealize should return new nodes, use Identity to return old nodes" ); k = i; #ifdef ASSERT @@ -862,11 +875,11 @@ Node *PhaseGVN::transform_no_reclaim(Node *n) { // cache Value. Later requests for the local phase->type of this Node can // use the cached Value instead of suffering with 'bottom_type'. const Type* t = k->Value(this); // Get runtime Value set - assert(t != NULL, "value sanity"); + assert(t != nullptr, "value sanity"); if (type_or_null(k) != t) { #ifndef PRODUCT // Do not count initial visit to node as a transformation - if (type_or_null(k) == NULL) { + if (type_or_null(k) == nullptr) { inc_new_values(); set_progress(); } @@ -912,7 +925,7 @@ bool PhaseGVN::is_dominator_helper(Node *d, Node *n, bool linear_only) { while (d != n) { n = IfNode::up_one_dom(n, linear_only); i++; - if (n == NULL || i >= 100) { + if (n == nullptr || i >= 100) { return false; } } @@ -925,7 +938,7 @@ bool PhaseGVN::is_dominator_helper(Node *d, Node *n, bool linear_only) { // or through an other data node excluding cons and phis. void PhaseGVN::dead_loop_check( Node *n ) { // Phi may reference itself in a loop - if (n != NULL && !n->is_dead_loop_safe() && !n->is_CFG()) { + if (n != nullptr && !n->is_dead_loop_safe() && !n->is_CFG()) { // Do 2 levels check and only data inputs. bool no_dead_loop = true; uint cnt = n->req(); @@ -933,7 +946,7 @@ void PhaseGVN::dead_loop_check( Node *n ) { Node *in = n->in(i); if (in == n) { no_dead_loop = false; - } else if (in != NULL && !in->is_dead_loop_safe()) { + } else if (in != nullptr && !in->is_dead_loop_safe()) { uint icnt = in->req(); for (uint j = 1; j < icnt && no_dead_loop; j++) { if (in->in(j) == n || in->in(j) == in) @@ -988,7 +1001,7 @@ PhaseIterGVN::PhaseIterGVN(PhaseGVN* gvn) : PhaseGVN(gvn), max = _table.size(); for( uint i = 0; i < max; ++i ) { Node *n = _table.at(i); - if(n != NULL && n != _table.sentinel() && n->outcnt() == 0) { + if(n != nullptr && n != _table.sentinel() && n->outcnt() == 0) { if( n->is_top() ) continue; // If remove_useless_nodes() has run, we expect no such nodes left. assert(false, "remove_useless_nodes missed this node"); @@ -1035,7 +1048,7 @@ void PhaseIterGVN::verify_step(Node* n) { } for (int i = 0; i < _verify_window_size; i++) { Node* n = _verify_window[i]; - if (n == NULL) { + if (n == nullptr) { continue; } if (n->in(0) == NodeSentinel) { // xform_idom @@ -1059,7 +1072,7 @@ void PhaseIterGVN::trace_PhaseIterGVN(Node* n, Node* nn, const Type* oldtype) { if (nn != n) { // print old node tty->print("< "); - if (oldtype != newtype && oldtype != NULL) { + if (oldtype != newtype && oldtype != nullptr) { oldtype->dump(); } do { tty->print("\t"); } while (tty->position() < 16); @@ -1068,14 +1081,14 @@ void PhaseIterGVN::trace_PhaseIterGVN(Node* n, Node* nn, const Type* oldtype) { } if (oldtype != newtype || nn != n) { // print new node and/or new type - if (oldtype == NULL) { + if (oldtype == nullptr) { tty->print("* "); } else if (nn != n) { tty->print("> "); } else { tty->print("= "); } - if (newtype == NULL) { + if (newtype == nullptr) { tty->print("null"); } else { newtype->dump(); @@ -1093,7 +1106,7 @@ void PhaseIterGVN::trace_PhaseIterGVN(Node* n, Node* nn, const Type* oldtype) { } if (nn != n) { // ignore n, it might be subsumed - verify_step((Node*) NULL); + verify_step((Node*) nullptr); } } } @@ -1102,12 +1115,12 @@ void PhaseIterGVN::init_verifyPhaseIterGVN() { _verify_counter = 0; _verify_full_passes = 0; for (int i = 0; i < _verify_window_size; i++) { - _verify_window[i] = NULL; + _verify_window[i] = nullptr; } #ifdef ASSERT // Verify that all modified nodes are on _worklist Unique_Node_List* modified_list = C->modified_nodes(); - while (modified_list != NULL && modified_list->size()) { + while (modified_list != nullptr && modified_list->size()) { Node* n = modified_list->pop(); if (!n->is_Con() && !_worklist.member(n)) { n->dump(); @@ -1121,7 +1134,7 @@ void PhaseIterGVN::verify_PhaseIterGVN() { #ifdef ASSERT // Verify nodes with changed inputs. Unique_Node_List* modified_list = C->modified_nodes(); - while (modified_list != NULL && modified_list->size()) { + while (modified_list != nullptr && modified_list->size()) { Node* n = modified_list->pop(); if (!n->is_Con()) { // skip Con nodes n->dump(); @@ -1142,7 +1155,7 @@ void PhaseIterGVN::verify_PhaseIterGVN() { } #ifdef ASSERT - if (modified_list != NULL) { + if (modified_list != nullptr) { while (modified_list->size() > 0) { Node* n = modified_list->pop(); n->dump(); @@ -1220,7 +1233,7 @@ void PhaseIterGVN::optimize() { Node* PhaseIterGVN::register_new_node_with_optimizer(Node* n, Node* orig) { set_type_bottom(n); _worklist.push(n); - if (orig != NULL) C->copy_node_notes_to(n, orig); + if (orig != nullptr) C->copy_node_notes_to(n, orig); return n; } @@ -1235,7 +1248,7 @@ Node *PhaseIterGVN::transform( Node *n ) { // If brand new node, make space in type array, and give it a type. ensure_type_or_null(n); - if (type_or_null(n) == NULL) { + if (type_or_null(n) == nullptr) { set_type_bottom(n); } @@ -1262,7 +1275,7 @@ Node *PhaseIterGVN::transform_old(Node* n) { #endif DEBUG_ONLY(uint loop_count = 1;) - while (i != NULL) { + while (i != nullptr) { #ifdef ASSERT if (loop_count >= K + C->live_nodes()) { dump_infinite_loop_info(i, "PhaseIterGVN::transform_old"); @@ -1294,7 +1307,7 @@ Node *PhaseIterGVN::transform_old(Node* n) { // See what kind of values 'k' takes on at runtime const Type* t = k->Value(this); - assert(t != NULL, "value sanity"); + assert(t != nullptr, "value sanity"); // Since I just called 'Value' to compute the set of run-time values // for this Node, and 'Value' is non-local (and therefore expensive) I'll @@ -1379,8 +1392,8 @@ void PhaseIterGVN::remove_globally_dead_node( Node *dead ) { // Smash all inputs to 'dead', isolating him completely for (uint i = 0; i < dead->req(); i++) { Node *in = dead->in(i); - if (in != NULL && in != C->top()) { // Points to something? - int nrep = dead->replace_edge(in, NULL, this); // Kill edges + if (in != nullptr && in != C->top()) { // Points to something? + int nrep = dead->replace_edge(in, nullptr, this); // Kill edges assert((nrep > 0), "sanity"); if (in->outcnt() == 0) { // Made input go dead? _stack.push(in, PROCESS_INPUTS); // Recursively remove @@ -1405,7 +1418,7 @@ void PhaseIterGVN::remove_globally_dead_node( Node *dead ) { BarrierSet::barrier_set()->barrier_set_c2()->enqueue_useful_gc_barrier(this, in); } if (ReduceFieldZeroing && dead->is_Load() && i == MemNode::Memory && - in->is_Proj() && in->in(0) != NULL && in->in(0)->is_Initialize()) { + in->is_Proj() && in->in(0) != nullptr && in->in(0)->is_Initialize()) { // A Load that directly follows an InitializeNode is // going away. The Stores that follow are candidates // again to be captured by the InitializeNode. @@ -1416,7 +1429,7 @@ void PhaseIterGVN::remove_globally_dead_node( Node *dead ) { } } } - } // if (in != NULL && in != C->top()) + } // if (in != nullptr && in != C->top()) } // for (uint i = 0; i < dead->req(); i++) if (recurse) { continue; @@ -1472,11 +1485,11 @@ void PhaseIterGVN::subsume_node( Node *old, Node *nn ) { // Search for instance field data PhiNodes in the same region pointing to the old // memory PhiNode and update their instance memory ids to point to the new node. - if (old->is_Phi() && old->as_Phi()->type()->has_memory() && old->in(0) != NULL) { + if (old->is_Phi() && old->as_Phi()->type()->has_memory() && old->in(0) != nullptr) { Node* region = old->in(0); for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) { PhiNode* phi = region->fast_out(i)->isa_Phi(); - if (phi != NULL && phi->inst_mem_id() == (int)old->_idx) { + if (phi != nullptr && phi->inst_mem_id() == (int)old->_idx) { phi->set_inst_mem_id((int)nn->_idx); } } @@ -1487,7 +1500,7 @@ void PhaseIterGVN::subsume_node( Node *old, Node *nn ) { temp->init_req(0,nn); // Add a use to nn to prevent him from dying remove_dead_node( old ); temp->del_req(0); // Yank bogus edge - if (nn != NULL && nn->outcnt() == 0) { + if (nn != nullptr && nn->outcnt() == 0) { _worklist.push(nn); } #ifndef PRODUCT @@ -1519,14 +1532,14 @@ static PhiNode* countedloop_phi_from_cmp(CmpNode* cmp, Node* n) { BaseCountedLoopEndNode* cle = iff->as_BaseCountedLoopEnd(); if (cle->limit() == n) { PhiNode* phi = cle->phi(); - if (phi != NULL) { + if (phi != nullptr) { return phi; } } } } } - return NULL; + return nullptr; } void PhaseIterGVN::add_users_to_worklist( Node *n ) { @@ -1541,12 +1554,12 @@ void PhaseIterGVN::add_users_to_worklist( Node *n ) { add_users_to_worklist0(use); // If we changed the receiver type to a call, we need to revisit - // the Catch following the call. It's looking for a non-NULL + // the Catch following the call. It's looking for a non-null // receiver to know when to enable the regular fall-through path // in addition to the NullPtrException path. if (use->is_CallDynamicJava() && n == use->in(TypeFunc::Parms)) { Node* p = use->as_CallDynamicJava()->proj_out_or_null(TypeFunc::Control); - if (p != NULL) { + if (p != nullptr) { add_users_to_worklist0(p); } } @@ -1574,7 +1587,7 @@ void PhaseIterGVN::add_users_to_worklist( Node *n ) { } if (use_op == Op_CmpI) { Node* phi = countedloop_phi_from_cmp((CmpINode*)use, n); - if (phi != NULL) { + if (phi != nullptr) { // If an opaque node feeds into the limit condition of a // CountedLoop, we need to process the Phi node for the // induction variable when the opaque node is removed: @@ -1637,14 +1650,24 @@ void PhaseIterGVN::add_users_to_worklist( Node *n ) { // If changed initialization activity, check dependent Stores if (use_op == Op_Allocate || use_op == Op_AllocateArray) { InitializeNode* init = use->as_Allocate()->initialization(); - if (init != NULL) { + if (init != nullptr) { Node* imem = init->proj_out_or_null(TypeFunc::Memory); - if (imem != NULL) add_users_to_worklist0(imem); + if (imem != nullptr) add_users_to_worklist0(imem); } } + // If the ValidLengthTest input changes then the fallthrough path out of the AllocateArray may have become dead. + // CatchNode::Value() is responsible for killing that path. The CatchNode has to be explicitly enqueued for igvn + // to guarantee the change is not missed. + if (use_op == Op_AllocateArray && n == use->in(AllocateNode::ValidLengthTest)) { + Node* p = use->as_AllocateArray()->proj_out_or_null(TypeFunc::Control); + if (p != nullptr) { + add_users_to_worklist0(p); + } + } + if (use_op == Op_Initialize) { Node* imem = use->as_Initialize()->proj_out_or_null(TypeFunc::Memory); - if (imem != NULL) add_users_to_worklist0(imem); + if (imem != nullptr) add_users_to_worklist0(imem); } // Loading the java mirror from a Klass requires two loads and the type // of the mirror load depends on the type of 'n'. See LoadNode::Value(). @@ -1680,7 +1703,7 @@ void PhaseIterGVN::remove_speculative_types() { assert(UseTypeSpeculation, "speculation is off"); for (uint i = 0; i < _types.Size(); i++) { const Type* t = _types.fast_lookup(i); - if (t != NULL) { + if (t != nullptr) { _types.map(i, t->remove_speculative()); } } @@ -1805,15 +1828,16 @@ void PhaseCCP::analyze() { } } // If we changed the receiver type to a call, we need to revisit - // the Catch following the call. It's looking for a non-NULL + // the Catch following the call. It's looking for a non-nullptr // receiver to know when to enable the regular fall-through path // in addition to the NullPtrException path + // Same is true if the type of a ValidLengthTest input to an AllocateArrayNode changes. if (m->is_Call()) { for (DUIterator_Fast i2max, i2 = m->fast_outs(i2max); i2 < i2max; i2++) { Node* p = m->fast_out(i2); // Propagate changes to uses if (p->is_Proj() && p->as_Proj()->_con == TypeFunc::Control) { Node* catch_node = p->find_out_with(Op_Catch); - if (catch_node != NULL) { + if (catch_node != nullptr) { worklist.push(catch_node); } } @@ -1844,7 +1868,7 @@ void PhaseCCP::analyze() { // PhiNode::Value(). if (m_op == Op_CmpI || m_op == Op_CmpL) { PhiNode* phi = countedloop_phi_from_cmp(m->as_Cmp(), n); - if (phi != NULL) { + if (phi != nullptr) { worklist.push(phi); } } @@ -1914,7 +1938,7 @@ void PhaseCCP::do_transform() { // Convert any of his old-space children into new-space children. Node *PhaseCCP::transform( Node *n ) { Node *new_node = _nodes[n->_idx]; // Check for transformed node - if( new_node != NULL ) + if( new_node != nullptr ) return new_node; // Been there, done that, return old answer assert(n->is_Root(), "traversal must start at root"); @@ -1935,7 +1959,7 @@ Node *PhaseCCP::transform( Node *n ) { for (uint i = 0; i < _root_and_safepoints.size(); ++i) { Node* nn = _root_and_safepoints.at(i); Node* new_node = _nodes[nn->_idx]; - assert(new_node == NULL, ""); + assert(new_node == nullptr, ""); new_node = transform_once(nn); // Check for constant _nodes.map(nn->_idx, new_node); // Flag as having been cloned transform_stack.push(new_node); // Process children of cloned node @@ -1947,9 +1971,9 @@ Node *PhaseCCP::transform( Node *n ) { uint cnt = clone->req(); for( uint i = 0; i < cnt; i++ ) { // For all inputs do Node *input = clone->in(i); - if( input != NULL ) { // Ignore NULLs + if( input != nullptr ) { // Ignore nulls Node *new_input = _nodes[input->_idx]; // Check for cloned input node - if( new_input == NULL ) { + if( new_input == nullptr ) { new_input = transform_once(input); // Check for constant _nodes.map( input->_idx, new_input );// Flag as having been cloned transform_stack.push(new_input); // Process children of cloned node @@ -1990,7 +2014,7 @@ Node *PhaseCCP::transform_once( Node *n ) { Node *nn = n; // Default is to return the original constant if( t == Type::TOP ) { // cache my top node on the Compile instance - if( C->cached_top_node() == NULL || C->cached_top_node()->in(0) == NULL ) { + if( C->cached_top_node() == nullptr || C->cached_top_node()->in(0) == nullptr ) { C->set_cached_top_node(ConNode::make(Type::TOP)); set_type(C->top(), Type::TOP); } @@ -2002,7 +2026,7 @@ Node *PhaseCCP::transform_once( Node *n ) { NOT_PRODUCT( inc_constants(); ) } else if( n->is_Region() ) { // Unreachable region // Note: nn == C->top() - n->set_req(0, NULL); // Cut selfreference + n->set_req(0, nullptr); // Cut selfreference bool progress = true; uint max = n->outcnt(); DUIterator i; @@ -2035,7 +2059,7 @@ Node *PhaseCCP::transform_once( Node *n ) { _worklist.push(n); // n re-enters the hash table via the worklist } - // TEMPORARY fix to ensure that 2nd GVN pass eliminates NULL checks + // TEMPORARY fix to ensure that 2nd GVN pass eliminates null checks switch( n->Opcode() ) { case Op_FastLock: // Revisit FastLocks for lock coarsening case Op_If: @@ -2095,7 +2119,7 @@ PhasePeephole::~PhasePeephole() { //------------------------------transform-------------------------------------- Node *PhasePeephole::transform( Node *n ) { ShouldNotCallThis(); - return NULL; + return nullptr; } //------------------------------do_transform----------------------------------- @@ -2210,7 +2234,7 @@ void Node::set_req_X( uint i, Node *n, PhaseIterGVN *igvn ) { void Node::set_req_X(uint i, Node *n, PhaseGVN *gvn) { PhaseIterGVN* igvn = gvn->is_IterGVN(); - if (igvn == NULL) { + if (igvn == nullptr) { set_req(i, n); return; } @@ -2243,7 +2267,7 @@ void Type_Array::grow( uint i ) { if( !_max ) { _max = 1; _types = (const Type**)_a->Amalloc( _max * sizeof(Type*) ); - _types[0] = NULL; + _types[0] = nullptr; } uint old = _max; _max = next_power_of_2(i); @@ -2256,7 +2280,7 @@ void Type_Array::grow( uint i ) { void Type_Array::dump() const { uint max = Size(); for( uint i = 0; i < max; i++ ) { - if( _types[i] != NULL ) { + if( _types[i] != nullptr ) { tty->print(" %d\t== ", i); _types[i]->dump(); tty->cr(); } } diff --git a/src/hotspot/share/opto/phaseX.hpp b/src/hotspot/share/opto/phaseX.hpp index 6d0d8ca46658f..3627774f97e8b 100644 --- a/src/hotspot/share/opto/phaseX.hpp +++ b/src/hotspot/share/opto/phaseX.hpp @@ -47,7 +47,7 @@ class PhaseRegAlloc; //----------------------------------------------------------------------------- -// Expandable closed hash-table of nodes, initialized to NULL. +// Expandable closed hash-table of nodes, initialized to null. // Note that the constructor just zeros things // Storage is reclaimed when the Arena's lifetime is over. class NodeHash : public StackObj { @@ -82,7 +82,7 @@ class NodeHash : public StackObj { // Return 75% of _max, rounded up. uint insert_limit() const { return _max - (_max>>2); } - void clear(); // Set all entries to NULL, keep storage. + void clear(); // Set all entries to null, keep storage. // Size of hash table uint size() const { return _max; } // Return Node* at index in table @@ -116,7 +116,7 @@ class NodeHash : public StackObj { //----------------------------------------------------------------------------- // Map dense integer indices to Types. Uses classic doubling-array trick. -// Abstractly provides an infinite array of Type*'s, initialized to NULL. +// Abstractly provides an infinite array of Type*'s, initialized to null. // Note that the constructor just zeros things, and since I use Arena // allocation I do not need a destructor to reclaim storage. // Despite the general name, this class is customized for use by PhaseTransform. @@ -125,8 +125,8 @@ class Type_Array : public StackObj { uint _max; const Type **_types; void grow( uint i ); // Grow array node to fit - const Type *operator[] ( uint i ) const // Lookup, or NULL for not mapped - { return (i<_max) ? _types[i] : (Type*)NULL; } + const Type *operator[] ( uint i ) const // Lookup, or null for not mapped + { return (i<_max) ? _types[i] : (Type*)nullptr; } friend class PhaseTransform; public: Type_Array(Arena *a) : _a(a), _max(0), _types(0) {} @@ -219,38 +219,43 @@ class PhaseTransform : public Phase { // Get a previously recorded type for the node n. // This type must already have been recorded. // If you want the type of a very new (untransformed) node, - // you must use type_or_null, and test the result for NULL. + // you must use type_or_null, and test the result for null. const Type* type(const Node* n) const { assert(_pnum != Ideal_Loop, "should not be used from PhaseIdealLoop"); - assert(n != NULL, "must not be null"); + assert(n != nullptr, "must not be null"); const Type* t = _types.fast_lookup(n->_idx); - assert(t != NULL, "must set before get"); + assert(t != nullptr, "must set before get"); return t; } // Get a previously recorded type for the node n, - // or else return NULL if there is none. + // or else return null if there is none. const Type* type_or_null(const Node* n) const { assert(_pnum != Ideal_Loop, "should not be used from PhaseIdealLoop"); return _types.fast_lookup(n->_idx); } // Record a type for a node. void set_type(const Node* n, const Type *t) { - assert(t != NULL, "type must not be null"); + assert(t != nullptr, "type must not be null"); _types.map(n->_idx, t); } + void clear_type(const Node* n) { + if (n->_idx < _types.Size()) { + _types.map(n->_idx, nullptr); + } + } // Record an initial type for a node, the node's bottom type. void set_type_bottom(const Node* n) { // Use this for initialization when bottom_type() (or better) is not handy. // Usually the initialization shoudl be to n->Value(this) instead, // or a hand-optimized value like Type::MEMORY or Type::CONTROL. - assert(_types[n->_idx] == NULL, "must set the initial type just once"); + assert(_types[n->_idx] == nullptr, "must set the initial type just once"); _types.map(n->_idx, n->bottom_type()); } // Make sure the types array is big enough to record a size for the node n. // (In product builds, we never want to do range checks on the types array!) void ensure_type_or_null(const Node* n) { if (n->_idx >= _types.Size()) - _types.map(n->_idx, NULL); // Grow the types array as needed. + _types.map(n->_idx, nullptr); // Grow the types array as needed. } // Utility functions: @@ -258,18 +263,18 @@ class PhaseTransform : public Phase { const TypeLong* find_long_type(Node* n); jint find_int_con( Node* n, jint value_if_unknown) { const TypeInt* t = find_int_type(n); - return (t != NULL && t->is_con()) ? t->get_con() : value_if_unknown; + return (t != nullptr && t->is_con()) ? t->get_con() : value_if_unknown; } jlong find_long_con(Node* n, jlong value_if_unknown) { const TypeLong* t = find_long_type(n); - return (t != NULL && t->is_con()) ? t->get_con() : value_if_unknown; + return (t != nullptr && t->is_con()) ? t->get_con() : value_if_unknown; } // Make an idealized constant, i.e., one of ConINode, ConPNode, ConFNode, etc. // Same as transform(ConNode::make(t)). ConNode* makecon(const Type* t); virtual ConNode* uncached_makecon(const Type* t) // override in PhaseValues - { ShouldNotCallThis(); return NULL; } + { ShouldNotCallThis(); return nullptr; } // Fast int or long constant. Same as TypeInt::make(i) or TypeLong::make(l). ConINode* intcon(jint i); @@ -333,7 +338,7 @@ class PhaseTransform : public Phase { // Caller guarantees that old_type and new_type are no higher than limit_type. virtual const Type* saturate(const Type* new_type, const Type* old_type, const Type* limit_type) const - { ShouldNotCallThis(); return NULL; } + { ShouldNotCallThis(); return nullptr; } // true if CFG node d dominates CFG node n virtual bool is_dominator(Node *d, Node *n) { fatal("unimplemented for this pass"); return false; }; @@ -371,7 +376,7 @@ class PhaseValues : public PhaseTransform { PhaseValues(Arena* arena, uint est_max_size); PhaseValues(PhaseValues* pt); NOT_PRODUCT(~PhaseValues();) - PhaseIterGVN* is_IterGVN() { return (_iterGVN) ? (PhaseIterGVN*)this : NULL; } + PhaseIterGVN* is_IterGVN() { return (_iterGVN) ? (PhaseIterGVN*)this : nullptr; } // Some Ideal and other transforms delete --> modify --> insert values bool hash_delete(Node* n) { return _table.hash_delete(n); } @@ -493,7 +498,7 @@ class PhaseIterGVN : public PhaseGVN { // transforms can be triggered on the region. // Optional 'orig' is an earlier version of this node. // It is significant only for debugging and profiling. - Node* register_new_node_with_optimizer(Node* n, Node* orig = NULL); + Node* register_new_node_with_optimizer(Node* n, Node* orig = nullptr); // Kill a globally dead Node. All uses are also globally dead and are // aggressively trimmed. diff --git a/src/hotspot/share/opto/phasetype.hpp b/src/hotspot/share/opto/phasetype.hpp index 2aa55a655537f..4e098f54987b3 100644 --- a/src/hotspot/share/opto/phasetype.hpp +++ b/src/hotspot/share/opto/phasetype.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -122,7 +122,7 @@ class CompilerPhaseTypeHelper { case PHASE_DEBUG: return "Debug"; default: ShouldNotReachHere(); - return NULL; + return nullptr; } } }; diff --git a/src/hotspot/share/opto/postaloc.cpp b/src/hotspot/share/opto/postaloc.cpp index 520505e46bbae..52d6f01973761 100644 --- a/src/hotspot/share/opto/postaloc.cpp +++ b/src/hotspot/share/opto/postaloc.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -65,7 +65,7 @@ bool PhaseChaitin::may_be_copy_of_callee( Node *def ) const { def = def->in(1); else break; - guarantee(def != NULL, "must not resurrect dead copy"); + guarantee(def != nullptr, "must not resurrect dead copy"); } // If we reached the end and didn't find a callee save proj // then this may be a callee save proj so we return true @@ -87,10 +87,10 @@ int PhaseChaitin::yank(Node *old, Block *current_block, Node_List *value, Node_L } _cfg.unmap_node_from_block(old); OptoReg::Name old_reg = lrgs(_lrg_map.live_range_id(old)).reg(); - assert(value != NULL || regnd == NULL, "sanity"); - if (value != NULL && regnd != NULL && regnd->at(old_reg) == old) { // Instruction is currently available? - value->map(old_reg, NULL); // Yank from value/regnd maps - regnd->map(old_reg, NULL); // This register's value is now unknown + assert(value != nullptr || regnd == nullptr, "sanity"); + if (value != nullptr && regnd != nullptr && regnd->at(old_reg) == old) { // Instruction is currently available? + value->map(old_reg, nullptr); // Yank from value/regnd maps + regnd->map(old_reg, nullptr); // This register's value is now unknown } return blk_adjust; } @@ -147,8 +147,8 @@ int PhaseChaitin::yank_if_dead_recurse(Node *old, Node *orig_old, Block *current for (uint i = 1; i < old->req(); i++) { Node* n = old->in(i); - if (n != NULL) { - old->set_req(i, NULL); + if (n != nullptr) { + old->set_req(i, nullptr); blk_adjust += yank_if_dead_recurse(n, orig_old, current_block, value, regnd); } } @@ -218,7 +218,7 @@ Node *PhaseChaitin::skip_copies( Node *c ) { int idx = c->is_Copy(); uint is_oop = lrgs(_lrg_map.live_range_id(c))._is_oop; while (idx != 0) { - guarantee(c->in(idx) != NULL, "must not resurrect dead copy"); + guarantee(c->in(idx) != nullptr, "must not resurrect dead copy"); if (lrgs(_lrg_map.live_range_id(c->in(idx)))._is_oop != is_oop) { break; // casting copy, not the same value } @@ -241,7 +241,7 @@ int PhaseChaitin::elide_copy( Node *n, int k, Block *current_block, Node_List *v int idx; while( (idx=x->is_Copy()) != 0 ) { Node *copy = x->in(idx); - guarantee(copy != NULL, "must not resurrect dead copy"); + guarantee(copy != nullptr, "must not resurrect dead copy"); if(lrgs(_lrg_map.live_range_id(copy)).reg() != nk_reg) { break; } @@ -258,8 +258,8 @@ int PhaseChaitin::elide_copy( Node *n, int k, Block *current_block, Node_List *v return blk_adjust; // Only check stupid copies! } // Loop backedges won't have a value-mapping yet - assert(regnd != NULL || value == NULL, "sanity"); - if (value == NULL || regnd == NULL) { + assert(regnd != nullptr || value == nullptr, "sanity"); + if (value == nullptr || regnd == nullptr) { return blk_adjust; } @@ -291,7 +291,7 @@ int PhaseChaitin::elide_copy( Node *n, int k, Block *current_block, Node_List *v // register. // Also handle duplicate copies here. - const Type *t = val->is_Con() ? val->bottom_type() : NULL; + const Type *t = val->is_Con() ? val->bottom_type() : nullptr; // Scan all registers to see if this value is around already for( uint reg = 0; reg < (uint)_max_reg; reg++ ) { @@ -361,7 +361,7 @@ bool PhaseChaitin::eliminate_copy_of_constant(Node* val, Node* n, Node_List& value, Node_List& regnd, OptoReg::Name nreg, OptoReg::Name nreg2) { if (value[nreg] != val && val->is_Con() && - value[nreg] != NULL && value[nreg]->is_Con() && + value[nreg] != nullptr && value[nreg]->is_Con() && (nreg2 == OptoReg::Bad || value[nreg] == value[nreg2]) && value[nreg]->bottom_type() == val->bottom_type() && value[nreg]->as_Mach()->rule() == val->as_Mach()->rule()) { @@ -441,7 +441,7 @@ int PhaseChaitin::possibly_merge_multidef(Node *n, uint k, Block *block, RegToDe OptoReg::Name reg = lrgs(lrg).reg(); Node* def = reg2defuse.at(reg).def(); - if (def != NULL && lrg == _lrg_map.live_range_id(def) && def != n->in(k)) { + if (def != nullptr && lrg == _lrg_map.live_range_id(def) && def != n->in(k)) { // Same lrg but different node, we have to merge. MachMergeNode* merge; if (def->is_MachMerge()) { // is it already a merge? @@ -465,7 +465,7 @@ int PhaseChaitin::possibly_merge_multidef(Node *n, uint k, Block *block, RegToDe if (use == n) { break; } - use->replace_edge(def, merge, NULL); + use->replace_edge(def, merge, nullptr); } } if (merge->find_edge(n->in(k)) == -1) { @@ -485,10 +485,10 @@ int PhaseChaitin::possibly_merge_multidef(Node *n, uint k, Block *block, RegToDe //------------------------------post_allocate_copy_removal--------------------- // Post-Allocation peephole copy removal. We do this in 1 pass over the // basic blocks. We maintain a mapping of registers to Nodes (an array of -// Nodes indexed by machine register or stack slot number). NULL means that a +// Nodes indexed by machine register or stack slot number). null means that a // register is not mapped to any Node. We can (want to have!) have several // registers map to the same Node. We walk forward over the instructions -// updating the mapping as we go. At merge points we force a NULL if we have +// updating the mapping as we go. At merge points we force a null if we have // to merge 2 different Nodes into the same register. Phi functions will give // us a new Node if there is a proper value merging. Since the blocks are // arranged in some RPO, we will visit all parent blocks before visiting any @@ -536,7 +536,7 @@ void PhaseChaitin::post_allocate_copy_removal() { // of registers at the start. Check for this, while updating copies // along Phi input edges bool missing_some_inputs = false; - Block *freed = NULL; + Block *freed = nullptr; for (j = 1; j < block->num_preds(); j++) { Block* pb = _cfg.get_block_for_node(block->pred(j)); // Remove copies along phi edges @@ -586,7 +586,7 @@ void PhaseChaitin::post_allocate_copy_removal() { value.copy(*blk2value[freed->_pre_order]); regnd.copy(*blk2regnd[freed->_pre_order]); } - // Merge all inputs together, setting to NULL any conflicts. + // Merge all inputs together, setting to null any conflicts. for (j = 1; j < block->num_preds(); j++) { Block* pb = _cfg.get_block_for_node(block->pred(j)); if (pb == freed) { @@ -595,8 +595,8 @@ void PhaseChaitin::post_allocate_copy_removal() { Node_List &p_regnd = *blk2regnd[pb->_pre_order]; for (uint k = 0; k < (uint)_max_reg; k++) { if (regnd[k] != p_regnd[k]) { // Conflict on reaching defs? - value.map(k, NULL); // Then no value handy - regnd.map(k, NULL); + value.map(k, nullptr); // Then no value handy + regnd.map(k, nullptr); } } } @@ -610,7 +610,7 @@ void PhaseChaitin::post_allocate_copy_removal() { OptoReg::Name preg = lrgs(pidx).reg(); // Remove copies remaining on edges. Check for junk phi. - Node *u = NULL; + Node *u = nullptr; for (k = 1; k < phi->req(); k++) { Node *x = phi->in(k); if( phi != x && u != x ) // Found a different input @@ -663,7 +663,7 @@ void PhaseChaitin::post_allocate_copy_removal() { uint k; for (k = 1; k < n->req(); k++) { Node *def = n->in(k); // n->in(k) is a USE; def is the DEF for this USE - guarantee(def != NULL, "no disconnected nodes at this point"); + guarantee(def != nullptr, "no disconnected nodes at this point"); uint useidx = _lrg_map.live_range_id(def); // useidx is the live range index for this USE if( useidx ) { @@ -671,7 +671,7 @@ void PhaseChaitin::post_allocate_copy_removal() { if( !value[ureg] ) { int idx; // Skip occasional useless copy while( (idx=def->is_Copy()) != 0 && - def->in(idx) != NULL && // NULL should not happen + def->in(idx) != nullptr && // null should not happen ureg == lrgs(_lrg_map.live_range_id(def->in(idx))).reg()) def = def->in(idx); Node *valdef = skip_copies(def); // tighten up val through non-useless copies @@ -717,9 +717,9 @@ void PhaseChaitin::post_allocate_copy_removal() { // definition could in fact be a kill projection with a count of // 0 which is safe but since those are uninteresting for copy // elimination just delete them as well. - if (regnd[nreg] != NULL && regnd[nreg]->outcnt() == 0) { - regnd.map(nreg, NULL); - value.map(nreg, NULL); + if (regnd[nreg] != nullptr && regnd[nreg]->outcnt() == 0) { + regnd.map(nreg, nullptr); + value.map(nreg, nullptr); } uint n_ideal_reg = n->ideal_reg(); diff --git a/src/hotspot/share/opto/reg_split.cpp b/src/hotspot/share/opto/reg_split.cpp index 1d2717e755603..d34f0b6634809 100644 --- a/src/hotspot/share/opto/reg_split.cpp +++ b/src/hotspot/share/opto/reg_split.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -66,10 +66,10 @@ Node *PhaseChaitin::get_spillcopy_wide(MachSpillCopyNode::SpillType spill_type, def->_idx, def->Name(), use->_idx, use->Name(), ireg, MachSpillCopyNode::spill_type(spill_type)); C->record_method_not_compilable("attempted to spill a non-spillable item"); - return NULL; + return nullptr; } if (C->check_node_count(NodeLimitFudgeFactor, out_of_nodes)) { - return NULL; + return nullptr; } const RegMask *i_mask = &def->out_RegMask(); const RegMask *w_mask = C->matcher()->idealreg2spillmask[ireg]; @@ -164,7 +164,7 @@ uint PhaseChaitin::split_DEF( Node *def, Block *b, int loc, uint maxlrg, Node ** assert( loc >= 0, "must insert past block head" ); // Get a def-side SpillCopy - Node *spill = get_spillcopy_wide(MachSpillCopyNode::Definition, def, NULL, 0); + Node *spill = get_spillcopy_wide(MachSpillCopyNode::Definition, def, nullptr, 0); // Did we fail to split?, then bail if (!spill) { return 0; @@ -363,7 +363,7 @@ Node *PhaseChaitin::split_Rematerialize(Node *def, Block *b, uint insidx, uint & } Node *spill = clone_node(def, b, C); - if (spill == NULL || C->check_node_count(NodeLimitFudgeFactor, out_of_nodes)) { + if (spill == nullptr || C->check_node_count(NodeLimitFudgeFactor, out_of_nodes)) { // Check when generating nodes return 0; } @@ -561,7 +561,7 @@ uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { bool *UPblock = UP[bidx]; for( slidx = 0; slidx < spill_cnt; slidx++ ) { UPblock[slidx] = true; // Assume they start in registers - Reachblock[slidx] = NULL; // Assume that no def is present + Reachblock[slidx] = nullptr; // Assume that no def is present } } @@ -652,8 +652,8 @@ uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { // Move n2/u2 to n1/u1 for next iteration n1 = n2; u1 = u2; - // Preserve a non-NULL predecessor for later type referencing - if( (n3 == NULL) && (n2 != NULL) ){ + // Preserve a non-null predecessor for later type referencing + if( (n3 == nullptr) && (n2 != nullptr) ){ n3 = n2; u3 = u2; } @@ -663,8 +663,8 @@ uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { for( insidx = 1; insidx <= b->end_idx(); insidx++ ) { n1 = b->get_node(insidx); // bail if this is not a phi - phi = n1->is_Phi() ? n1->as_Phi() : NULL; - if( phi == NULL ) { + phi = n1->is_Phi() ? n1->as_Phi() : nullptr; + if( phi == nullptr ) { // Keep track of index of first non-PhiNode instruction in block non_phi = insidx; // break out of the for loop as we have handled all phi nodes @@ -687,7 +687,7 @@ uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { if( needs_phi ) { // create a new phi node and insert it into the block // type is taken from left over pointer to a predecessor - guarantee(n3, "No non-NULL reaching DEF for a Phi"); + guarantee(n3, "No non-null reaching DEF for a Phi"); phi = new PhiNode(b->head(), n3->bottom_type()); // initialize the Reaches entry for this LRG Reachblock[slidx] = phi; @@ -700,7 +700,7 @@ uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { assert(_lrg_map.find_id(phi) == lidx, "Bad update on Union-Find mapping"); } // end if not found correct phi // Here you have either found or created the Phi, so record it - assert(phi != NULL,"Must have a Phi Node here"); + assert(phi != nullptr,"Must have a Phi Node here"); phis->push(phi); // PhiNodes should either force the LRG UP or DOWN depending // on its inputs and the register pressure in the Phi's block. @@ -753,7 +753,7 @@ uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { // Memoize any DOWN reaching definitions for use as DEBUG info for( insidx = 0; insidx < spill_cnt; insidx++ ) { - debug_defs[insidx] = (UPblock[insidx]) ? NULL : Reachblock[insidx]; + debug_defs[insidx] = (UPblock[insidx]) ? nullptr : Reachblock[insidx]; if( UPblock[insidx] ) // Memoize UP decision at block start UP_entry[insidx]->set( b->_pre_order ); } @@ -774,13 +774,13 @@ uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { // ranges; they are busy getting modifed in this pass. if( lrgs(defidx).reg() < LRG::SPILL_REG ) { uint i; - Node *u = NULL; + Node *u = nullptr; // Look for the Phi merging 2 unique inputs for( i = 1; i < cnt; i++ ) { // Ignore repeats and self if( n->in(i) != u && n->in(i) != n ) { // Found a unique input - if( u != NULL ) // If it's the 2nd, bail out + if( u != nullptr ) // If it's the 2nd, bail out break; u = n->in(i); // Else record it } @@ -816,7 +816,7 @@ uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { // Check for need to split at HRP boundary - split if UP n1 = Reachblock[slidx]; // bail out if no reaching DEF - if( n1 == NULL ) continue; + if( n1 == nullptr ) continue; // bail out if live range is 'isolated' around inner loop uint lidx = lidxs.at(slidx); // If live range is currently UP @@ -826,7 +826,7 @@ uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { if( is_high_pressure( b, &lrgs(lidx), insidx ) && !n1->rematerialize() ) { // If there is already a valid stack definition available, use it - if( debug_defs[slidx] != NULL ) { + if( debug_defs[slidx] != nullptr ) { Reachblock[slidx] = debug_defs[slidx]; } else { @@ -861,7 +861,7 @@ uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { if (!maxlrg) { return 0; } - // Spill of NULL check mem op goes into the following block. + // Spill of null check mem op goes into the following block. if (b->end_idx() > orig_eidx) { insidx++; } @@ -891,7 +891,7 @@ uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { // Remove coalesced copy from CFG if (copyidx && defidx == _lrg_map.live_range_id(n->in(copyidx))) { n->replace_by( n->in(copyidx) ); - n->set_req( copyidx, NULL ); + n->set_req( copyidx, nullptr ); b->remove_node(insidx--); b->_ihrp_index--; // Adjust the point where we go hi-pressure b->_fhrp_index--; @@ -925,7 +925,7 @@ uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { // Check for valid reaching DEF slidx = lrg2reach[useidx]; Node *def = Reachblock[slidx]; - assert( def != NULL, "Using Undefined Value in Split()\n"); + assert( def != nullptr, "Using Undefined Value in Split()\n"); // (+++) %%%% remove this in favor of pre-pass in matcher.cpp // monitor references do not care where they live, so just hook @@ -934,7 +934,7 @@ uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { // so that the allocator does not see it anymore, and therefore // does not attempt to assign it a register. def = clone_node(def, b, C); - if (def == NULL || C->check_node_count(NodeLimitFudgeFactor, out_of_nodes)) { + if (def == nullptr || C->check_node_count(NodeLimitFudgeFactor, out_of_nodes)) { return 0; } _lrg_map.extend(def->_idx, 0); @@ -952,7 +952,7 @@ uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { insidx += b->number_of_nodes()-old_size; } - MachNode *mach = n->is_Mach() ? n->as_Mach() : NULL; + MachNode *mach = n->is_Mach() ? n->as_Mach() : nullptr; // Base pointers and oopmap references do not care where they live. if ((inpidx >= oopoff) || (mach && mach->ideal_Opcode() == Op_AddP && inpidx == AddPNode::Base)) { @@ -981,7 +981,7 @@ uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { Node *derived_debug = debug_defs[slidx]; if( ((inpidx - oopoff) & 1) == DERIVED && // derived vs base? mach && mach->ideal_Opcode() != Op_Halt && - derived_debug != NULL && + derived_debug != nullptr && derived_debug != def ) { // Actual 2nd value appears // We have already set 'def' as a derived value. // Also set debug_defs[slidx] as a derived value. @@ -1009,7 +1009,7 @@ uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { if( jvms && b->_freq > BLOCK_FREQUENCY(0.5) ) { uint debug_start = jvms->debug_start(); // If this is debug info use & there is a reaching DOWN def - if ((debug_start <= inpidx) && (debug_defs[slidx] != NULL)) { + if ((debug_start <= inpidx) && (debug_defs[slidx] != nullptr)) { assert(inpidx < oopoff, "handle only debug info here"); // Just hook it in & move on n->set_req(inpidx, debug_defs[slidx]); @@ -1230,7 +1230,7 @@ uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { // UP should come from the outRegmask() of the DEF UPblock[slidx] = defup; // Update debug list of reaching down definitions, kill if DEF is UP - debug_defs[slidx] = defup ? NULL : n; + debug_defs[slidx] = defup ? nullptr : n; #ifndef PRODUCT // DEBUG if( trace_spilling() ) { @@ -1294,9 +1294,9 @@ uint PhaseChaitin::Split(uint maxlrg, ResourceArea* split_arena) { } } #endif - Reachblock[slidx] = NULL; + Reachblock[slidx] = nullptr; } else { - assert(Reachblock[slidx] != NULL,"No reaching definition for liveout value"); + assert(Reachblock[slidx] != nullptr,"No reaching definition for liveout value"); } } #ifndef PRODUCT diff --git a/src/hotspot/share/opto/regmask.hpp b/src/hotspot/share/opto/regmask.hpp index 1694367554d4b..233dd0da2d7f3 100644 --- a/src/hotspot/share/opto/regmask.hpp +++ b/src/hotspot/share/opto/regmask.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -99,7 +99,7 @@ class RegMask { // requirement is internal to the allocator, and independent of any // particular platform. enum { SlotsPerLong = 2, - SlotsPerVecA = 8, + SlotsPerVecA = RISCV_ONLY(4) NOT_RISCV(8), SlotsPerVecS = 1, SlotsPerVecD = 2, SlotsPerVecX = 4, diff --git a/src/hotspot/share/opto/replacednodes.cpp b/src/hotspot/share/opto/replacednodes.cpp index 0a89b06a7e7d0..6030ceafb6888 100644 --- a/src/hotspot/share/opto/replacednodes.cpp +++ b/src/hotspot/share/opto/replacednodes.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,13 +29,13 @@ #include "opto/replacednodes.hpp" void ReplacedNodes::allocate_if_necessary() { - if (_replaced_nodes == NULL) { + if (_replaced_nodes == nullptr) { _replaced_nodes = new GrowableArray(); } } bool ReplacedNodes::is_empty() const { - return _replaced_nodes == NULL || _replaced_nodes->length() == 0; + return _replaced_nodes == nullptr || _replaced_nodes->length() == 0; } bool ReplacedNodes::has_node(const ReplacedNode& r) const { @@ -78,7 +78,7 @@ void ReplacedNodes::transfer_from(const ReplacedNodes& other, uint idx) { } void ReplacedNodes::clone() { - if (_replaced_nodes != NULL) { + if (_replaced_nodes != nullptr) { GrowableArray* replaced_nodes_clone = new GrowableArray(); replaced_nodes_clone->appendAll(_replaced_nodes); _replaced_nodes = replaced_nodes_clone; @@ -86,7 +86,7 @@ void ReplacedNodes::clone() { } void ReplacedNodes::reset() { - if (_replaced_nodes != NULL) { + if (_replaced_nodes != nullptr) { _replaced_nodes->clear(); } } @@ -130,7 +130,7 @@ void ReplacedNodes::apply(Compile* C, Node* ctl) { ReplacedNode replaced = _replaced_nodes->at(i); Node* initial = replaced.initial(); Node* improved = replaced.improved(); - assert (ctl != NULL && !ctl->is_top(), "replaced node should have actual control"); + assert (ctl != nullptr && !ctl->is_top(), "replaced node should have actual control"); ResourceMark rm; Unique_Node_List work; @@ -150,7 +150,7 @@ void ReplacedNodes::apply(Compile* C, Node* ctl) { if (use->outcnt() == 0) { continue; } - if (n->is_CFG() || (n->in(0) != NULL && !n->in(0)->is_top())) { + if (n->is_CFG() || (n->in(0) != nullptr && !n->in(0)->is_top())) { // Skip projections, since some of the multi nodes aren't CFG (e.g., LoadStore and SCMemProj). if (n->is_Proj()) { n = n->in(0); @@ -164,7 +164,7 @@ void ReplacedNodes::apply(Compile* C, Node* ctl) { n = IfNode::up_one_dom(n); depth++; // limit search depth - if (depth >= 100 || n == NULL) { + if (depth >= 100 || n == nullptr) { replace = false; break; } diff --git a/src/hotspot/share/opto/replacednodes.hpp b/src/hotspot/share/opto/replacednodes.hpp index ebd3363fc58c8..c569e55ce5ff2 100644 --- a/src/hotspot/share/opto/replacednodes.hpp +++ b/src/hotspot/share/opto/replacednodes.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -48,7 +48,7 @@ class ReplacedNodes { Node* _initial; Node* _improved; public: - ReplacedNode() : _initial(NULL), _improved(NULL) {} + ReplacedNode() : _initial(nullptr), _improved(nullptr) {} ReplacedNode(Node* initial, Node* improved) : _initial(initial), _improved(improved) {} Node* initial() const { return _initial; } Node* improved() const { return _improved; } @@ -65,7 +65,7 @@ class ReplacedNodes { public: ReplacedNodes() - : _replaced_nodes(NULL) {} + : _replaced_nodes(nullptr) {} void clone(); void record(Node* initial, Node* improved); diff --git a/src/hotspot/share/opto/rootnode.cpp b/src/hotspot/share/opto/rootnode.cpp index b5f41ad385109..024ac13094da6 100644 --- a/src/hotspot/share/opto/rootnode.cpp +++ b/src/hotspot/share/opto/rootnode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -58,7 +58,7 @@ Node *RootNode::Ideal(PhaseGVN *phase, bool can_reshape) { // If we want to get the rest of the win later, we should pattern match // simple recursive call trees to closed-form solutions. - return modified ? this : NULL; + return modified ? this : nullptr; } //============================================================================= @@ -78,7 +78,7 @@ uint HaltNode::size_of() const { return sizeof(*this); } //------------------------------Ideal------------------------------------------ Node *HaltNode::Ideal(PhaseGVN *phase, bool can_reshape) { - return remove_dead_region(phase, can_reshape) ? this : NULL; + return remove_dead_region(phase, can_reshape) ? this : nullptr; } //------------------------------Value------------------------------------------ diff --git a/src/hotspot/share/opto/runtime.cpp b/src/hotspot/share/opto/runtime.cpp index b5cda23020ae5..c8e52153f4794 100644 --- a/src/hotspot/share/opto/runtime.cpp +++ b/src/hotspot/share/opto/runtime.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -92,22 +92,22 @@ // Compiled code entry points -address OptoRuntime::_new_instance_Java = NULL; -address OptoRuntime::_new_array_Java = NULL; -address OptoRuntime::_new_array_nozero_Java = NULL; -address OptoRuntime::_multianewarray2_Java = NULL; -address OptoRuntime::_multianewarray3_Java = NULL; -address OptoRuntime::_multianewarray4_Java = NULL; -address OptoRuntime::_multianewarray5_Java = NULL; -address OptoRuntime::_multianewarrayN_Java = NULL; -address OptoRuntime::_vtable_must_compile_Java = NULL; -address OptoRuntime::_complete_monitor_locking_Java = NULL; -address OptoRuntime::_monitor_notify_Java = NULL; -address OptoRuntime::_monitor_notifyAll_Java = NULL; -address OptoRuntime::_rethrow_Java = NULL; - -address OptoRuntime::_slow_arraycopy_Java = NULL; -address OptoRuntime::_register_finalizer_Java = NULL; +address OptoRuntime::_new_instance_Java = nullptr; +address OptoRuntime::_new_array_Java = nullptr; +address OptoRuntime::_new_array_nozero_Java = nullptr; +address OptoRuntime::_multianewarray2_Java = nullptr; +address OptoRuntime::_multianewarray3_Java = nullptr; +address OptoRuntime::_multianewarray4_Java = nullptr; +address OptoRuntime::_multianewarray5_Java = nullptr; +address OptoRuntime::_multianewarrayN_Java = nullptr; +address OptoRuntime::_vtable_must_compile_Java = nullptr; +address OptoRuntime::_complete_monitor_locking_Java = nullptr; +address OptoRuntime::_monitor_notify_Java = nullptr; +address OptoRuntime::_monitor_notifyAll_Java = nullptr; +address OptoRuntime::_rethrow_Java = nullptr; + +address OptoRuntime::_slow_arraycopy_Java = nullptr; +address OptoRuntime::_register_finalizer_Java = nullptr; ExceptionBlob* OptoRuntime::_exception_blob; @@ -126,7 +126,7 @@ static bool check_compiled_frame(JavaThread* thread) { #define gen(env, var, type_func_gen, c_func, fancy_jump, pass_tls, return_pc) \ var = generate_stub(env, type_func_gen, CAST_FROM_FN_PTR(address, c_func), #var, fancy_jump, pass_tls, return_pc); \ - if (var == NULL) { return false; } + if (var == nullptr) { return false; } bool OptoRuntime::generate(ciEnv* env) { @@ -177,7 +177,7 @@ const char* OptoRuntime::stub_name(address entry) { #ifndef PRODUCT CodeBlob* cb = CodeCache::find_blob(entry); RuntimeStub* rs =(RuntimeStub *)cb; - assert(rs != NULL && rs->is_runtime_stub(), "not a runtime stub"); + assert(rs != nullptr && rs->is_runtime_stub(), "not a runtime stub"); return rs->name(); #else // Fast implementation for product mode (maybe it should be inlined too) @@ -299,10 +299,10 @@ JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_nozero_C(Klass* array_type, int len SharedRuntime::on_slowpath_allocation_exit(current); oop result = current->vm_result(); - if ((len > 0) && (result != NULL) && + if ((len > 0) && (result != nullptr) && is_deoptimized_caller_frame(current)) { // Zero array here if the caller is deoptimized. - int size = ((typeArrayOop)result)->object_size(); + int size = TypeArrayKlass::cast(array_type)->oop_size(result); BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type(); const size_t hs = arrayOopDesc::header_size(elem_type); // Align to next 8 bytes to avoid trashing arrays's length. @@ -605,12 +605,12 @@ const TypeFunc *OptoRuntime::monitor_notify_Type() { const TypeFunc* OptoRuntime::flush_windows_Type() { // create input type (domain) const Type** fields = TypeTuple::fields(1); - fields[TypeFunc::Parms+0] = NULL; // void + fields[TypeFunc::Parms+0] = nullptr; // void const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms, fields); // create result type fields = TypeTuple::fields(1); - fields[TypeFunc::Parms+0] = NULL; // void + fields[TypeFunc::Parms+0] = nullptr; // void const TypeTuple *range = TypeTuple::make(TypeFunc::Parms, fields); return TypeFunc::make(domain, range); @@ -754,7 +754,7 @@ static const TypeFunc* make_arraycopy_Type(ArrayCopyType act) { int retcnt = (act == ac_checkcast || act == ac_generic ? 1 : 0); fields = TypeTuple::fields(1); if (retcnt == 0) - fields[TypeFunc::Parms+0] = NULL; // void + fields[TypeFunc::Parms+0] = nullptr; // void else fields[TypeFunc::Parms+0] = TypeInt::INT; // status result, if needed const TypeTuple* range = TypeTuple::make(TypeFunc::Parms+retcnt, fields); @@ -796,7 +796,7 @@ const TypeFunc* OptoRuntime::array_fill_Type() { // create result type fields = TypeTuple::fields(1); - fields[TypeFunc::Parms+0] = NULL; // void + fields[TypeFunc::Parms+0] = nullptr; // void const TypeTuple *range = TypeTuple::make(TypeFunc::Parms, fields); return TypeFunc::make(domain, range); @@ -817,7 +817,7 @@ const TypeFunc* OptoRuntime::aescrypt_block_Type() { // no result type needed fields = TypeTuple::fields(1); - fields[TypeFunc::Parms+0] = NULL; // void + fields[TypeFunc::Parms+0] = nullptr; // void const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields); return TypeFunc::make(domain, range); } @@ -972,7 +972,7 @@ const TypeFunc* OptoRuntime::digestBase_implCompress_Type(bool is_sha3) { // no result type needed fields = TypeTuple::fields(1); - fields[TypeFunc::Parms+0] = NULL; // void + fields[TypeFunc::Parms+0] = nullptr; // void const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields); return TypeFunc::make(domain, range); } @@ -1018,7 +1018,7 @@ const TypeFunc* OptoRuntime::multiplyToLen_Type() { // no result type needed fields = TypeTuple::fields(1); - fields[TypeFunc::Parms+0] = NULL; + fields[TypeFunc::Parms+0] = nullptr; const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields); return TypeFunc::make(domain, range); } @@ -1038,7 +1038,7 @@ const TypeFunc* OptoRuntime::squareToLen_Type() { // no result type needed fields = TypeTuple::fields(1); - fields[TypeFunc::Parms+0] = NULL; + fields[TypeFunc::Parms+0] = nullptr; const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields); return TypeFunc::make(domain, range); } @@ -1126,7 +1126,7 @@ const TypeFunc * OptoRuntime::bigIntegerShift_Type() { // no result type needed fields = TypeTuple::fields(1); - fields[TypeFunc::Parms + 0] = NULL; + fields[TypeFunc::Parms + 0] = nullptr; const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields); return TypeFunc::make(domain, range); } @@ -1166,7 +1166,7 @@ const TypeFunc* OptoRuntime::ghash_processBlocks_Type() { // result type needed fields = TypeTuple::fields(1); - fields[TypeFunc::Parms+0] = NULL; // void + fields[TypeFunc::Parms+0] = nullptr; // void const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields); return TypeFunc::make(domain, range); } @@ -1187,7 +1187,7 @@ const TypeFunc* OptoRuntime::base64_encodeBlock_Type() { // result type needed fields = TypeTuple::fields(1); - fields[TypeFunc::Parms + 0] = NULL; // void + fields[TypeFunc::Parms + 0] = nullptr; // void const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields); return TypeFunc::make(domain, range); } @@ -1224,7 +1224,7 @@ const TypeFunc* OptoRuntime::osr_end_Type() { // create result type fields = TypeTuple::fields(1); // fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // locked oop - fields[TypeFunc::Parms+0] = NULL; // void + fields[TypeFunc::Parms+0] = nullptr; // void const TypeTuple *range = TypeTuple::make(TypeFunc::Parms, fields); return TypeFunc::make(domain, range); } @@ -1258,8 +1258,8 @@ JRT_ENTRY_NO_ASYNC(address, OptoRuntime::handle_exception_C_helper(JavaThread* c // is only used to pass arguments into the method. Not for general // exception handling. DO NOT CHANGE IT to use pending_exception, since // the runtime stubs checks this on exit. - assert(current->exception_oop() != NULL, "exception oop is found"); - address handler_address = NULL; + assert(current->exception_oop() != nullptr, "exception oop is found"); + address handler_address = nullptr; Handle exception(current, current->exception_oop()); address pc = current->exception_pc(); @@ -1292,7 +1292,7 @@ JRT_ENTRY_NO_ASYNC(address, OptoRuntime::handle_exception_C_helper(JavaThread* c // using rethrow node nm = CodeCache::find_nmethod(pc); - assert(nm != NULL, "No NMethod found"); + assert(nm != nullptr, "No NMethod found"); if (nm->is_native_method()) { fatal("Native method should not have path to exception handling"); } else { @@ -1330,12 +1330,12 @@ JRT_ENTRY_NO_ASYNC(address, OptoRuntime::handle_exception_C_helper(JavaThread* c } else { handler_address = - force_unwind ? NULL : nm->handler_for_exception_and_pc(exception, pc); + force_unwind ? nullptr : nm->handler_for_exception_and_pc(exception, pc); - if (handler_address == NULL) { + if (handler_address == nullptr) { bool recursive_exception = false; handler_address = SharedRuntime::compute_compiled_exc_handler(nm, pc, exception, force_unwind, true, recursive_exception); - assert (handler_address != NULL, "must have compiled handler"); + assert (handler_address != nullptr, "must have compiled handler"); // Update the exception cache only when the unwind was not forced // and there didn't happen another exception during the computation of the // compiled exception handler. Checking for exception oop equality is not @@ -1381,8 +1381,8 @@ address OptoRuntime::handle_exception_C(JavaThread* current) { SharedRuntime::_find_handler_ctr++; // find exception handler #endif debug_only(NoHandleMark __hm;) - nmethod* nm = NULL; - address handler_address = NULL; + nmethod* nm = nullptr; + address handler_address = nullptr; { // Enter the VM @@ -1395,7 +1395,7 @@ address OptoRuntime::handle_exception_C(JavaThread* current) { // Now check to see if the handler we are returning is in a now // deoptimized frame - if (nm != NULL) { + if (nm != nullptr) { RegisterMap map(current, false); frame caller = current->last_frame().sender(&map); #ifdef ASSERT @@ -1431,10 +1431,6 @@ address OptoRuntime::handle_exception_C(JavaThread* current) { // *THIS IS NOT RECOMMENDED PROGRAMMING STYLE* // address OptoRuntime::rethrow_C(oopDesc* exception, JavaThread* thread, address ret_pc) { - - // Enable WXWrite: the function called directly by compiled code. - MACOS_AARCH64_ONLY(ThreadWXEnable wx(WXWrite, thread)); - // The frame we rethrow the exception to might not have been processed by the GC yet. // The stack watermark barrier takes care of detecting that and ensuring the frame // has updated oops. @@ -1443,7 +1439,7 @@ address OptoRuntime::rethrow_C(oopDesc* exception, JavaThread* thread, address r #ifndef PRODUCT SharedRuntime::_rethrow_ctr++; // count rethrows #endif - assert (exception != NULL, "should have thrown a NULLPointerException"); + assert (exception != nullptr, "should have thrown a NullPointerException"); #ifdef ASSERT if (!(exception->is_a(vmClasses::Throwable_klass()))) { // should throw an exception here @@ -1576,7 +1572,7 @@ JRT_END //----------------------------------------------------------------------------- -NamedCounter * volatile OptoRuntime::_named_counters = NULL; +NamedCounter * volatile OptoRuntime::_named_counters = nullptr; // // dump the collected NamedCounters. @@ -1638,7 +1634,7 @@ NamedCounter* OptoRuntime::new_named_counter(JVMState* youngest_jvms, NamedCount stringStream st; for (int depth = max_depth; depth >= 1; depth--) { JVMState* jvms = youngest_jvms->of_depth(depth); - ciMethod* m = jvms->has_method() ? jvms->method() : NULL; + ciMethod* m = jvms->has_method() ? jvms->method() : nullptr; if (!first) { st.print(" "); } else { @@ -1646,7 +1642,7 @@ NamedCounter* OptoRuntime::new_named_counter(JVMState* youngest_jvms, NamedCount } int bci = jvms->bci(); if (bci < 0) bci = 0; - if (m != NULL) { + if (m != nullptr) { st.print("%s.%s", m->holder()->name()->as_utf8(), m->name()->as_utf8()); } else { st.print("no method"); @@ -1667,7 +1663,7 @@ NamedCounter* OptoRuntime::new_named_counter(JVMState* youngest_jvms, NamedCount // add counters so this is safe. NamedCounter* head; do { - c->set_next(NULL); + c->set_next(nullptr); head = _named_counters; c->set_next(head); } while (Atomic::cmpxchg(&_named_counters, head, c) != head); diff --git a/src/hotspot/share/opto/runtime.hpp b/src/hotspot/share/opto/runtime.hpp index 26f80acac7632..a703da0c6d155 100644 --- a/src/hotspot/share/opto/runtime.hpp +++ b/src/hotspot/share/opto/runtime.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -75,13 +75,13 @@ class NamedCounter : public CHeapObj { public: NamedCounter(const char *n, CounterTag tag = NoTag): - _name(n == NULL ? NULL : os::strdup(n)), + _name(n == nullptr ? nullptr : os::strdup(n)), _count(0), _tag(tag), - _next(NULL) {} + _next(nullptr) {} ~NamedCounter() { - if (_name != NULL) { + if (_name != nullptr) { os::free((void*)_name); } } @@ -94,7 +94,7 @@ class NamedCounter : public CHeapObj { NamedCounter* next() const { return _next; } void set_next(NamedCounter* next) { - assert(_next == NULL || next == NULL, "already set"); + assert(_next == nullptr || next == nullptr, "already set"); _next = next; } diff --git a/src/hotspot/share/opto/split_if.cpp b/src/hotspot/share/opto/split_if.cpp index 012c1a69905e8..4427c859e3586 100644 --- a/src/hotspot/share/opto/split_if.cpp +++ b/src/hotspot/share/opto/split_if.cpp @@ -128,8 +128,8 @@ bool PhaseIdealLoop::split_up( Node *n, Node *blk1, Node *blk2 ) { } } else { // We might see an Opaque1 from a loop limit check here - assert(use->is_If() || use->is_CMove() || use->Opcode() == Op_Opaque1, "unexpected node type"); - Node *use_c = use->is_If() ? use->in(0) : get_ctrl(use); + assert(use->is_If() || use->is_CMove() || use->Opcode() == Op_Opaque1 || use->is_AllocateArray(), "unexpected node type"); + Node *use_c = (use->is_If() || use->is_AllocateArray()) ? use->in(0) : get_ctrl(use); if (use_c == blk1 || use_c == blk2) { assert(use->is_CMove(), "unexpected node type"); continue; @@ -166,14 +166,15 @@ bool PhaseIdealLoop::split_up( Node *n, Node *blk1, Node *blk2 ) { --j; } else { // We might see an Opaque1 from a loop limit check here - assert(u->is_If() || u->is_CMove() || u->Opcode() == Op_Opaque1, "unexpected node type"); - assert(u->in(1) == bol, ""); + assert(u->is_If() || u->is_CMove() || u->Opcode() == Op_Opaque1 || u->is_AllocateArray(), "unexpected node type"); + assert(u->is_AllocateArray() || u->in(1) == bol, ""); + assert(!u->is_AllocateArray() || u->in(AllocateNode::ValidLengthTest) == bol, "wrong input to AllocateArray"); // Get control block of either the CMove or the If input - Node *u_ctrl = u->is_If() ? u->in(0) : get_ctrl(u); + Node *u_ctrl = (u->is_If() || u->is_AllocateArray()) ? u->in(0) : get_ctrl(u); assert((u_ctrl != blk1 && u_ctrl != blk2) || u->is_CMove(), "won't converge"); Node *x = bol->clone(); register_new_node(x, u_ctrl); - _igvn.replace_input_of(u, 1, x); + _igvn.replace_input_of(u, u->is_AllocateArray() ? AllocateNode::ValidLengthTest : 1, x); --j; } } @@ -207,7 +208,7 @@ bool PhaseIdealLoop::split_up( Node *n, Node *blk1, Node *blk2 ) { Node* m = wq.at(i); if (m->is_If()) { assert(skeleton_predicate_has_opaque(m->as_If()), "opaque node not reachable from if?"); - Node* bol = clone_skeleton_predicate_bool(m, NULL, NULL, m->in(0)); + Node* bol = clone_skeleton_predicate_bool(m, nullptr, nullptr, m->in(0)); _igvn.replace_input_of(m, 1, bol); } else { assert(!m->is_CFG(), "not CFG expected"); @@ -251,7 +252,7 @@ bool PhaseIdealLoop::split_up( Node *n, Node *blk1, Node *blk2 ) { // ConvI2L may have type information on it which becomes invalid if // it moves up in the graph so change any clones so widen the type // to TypeLong::INT when pushing it up. - const Type* rtype = NULL; + const Type* rtype = nullptr; if (n->Opcode() == Op_ConvI2L && n->bottom_type() != TypeLong::INT) { rtype = TypeLong::INT; } @@ -261,7 +262,7 @@ bool PhaseIdealLoop::split_up( Node *n, Node *blk1, Node *blk2 ) { for( uint j = 1; j < blk1->req(); j++ ) { Node *x = n->clone(); // Widen the type of the ConvI2L when pushing up. - if (rtype != NULL) x->as_Type()->set_type(rtype); + if (rtype != nullptr) x->as_Type()->set_type(rtype); if( n->in(0) && n->in(0) == blk1 ) x->set_req( 0, blk1->in(j) ); for( uint i = 1; i < n->req(); i++ ) { @@ -411,7 +412,7 @@ Node *PhaseIdealLoop::find_use_block( Node *use, Node *def, Node *old_false, Nod set_ctrl(use, new_true); } - if (use_blk == NULL) { // He's dead, Jim + if (use_blk == nullptr) { // He's dead, Jim _igvn.replace_node(use, C->top()); } @@ -489,7 +490,7 @@ void PhaseIdealLoop::do_split_if( Node *iff ) { for (j = n->outs(); n->has_out(j); j++) { Node* m = n->out(j); // If m is dead, throw it away, and declare progress - if (_nodes[m->_idx] == NULL) { + if (_nodes[m->_idx] == nullptr) { _igvn.remove_dead_node(m); // fall through } @@ -513,8 +514,8 @@ void PhaseIdealLoop::do_split_if( Node *iff ) { // Replace both uses of 'new_iff' with Regions merging True/False // paths. This makes 'new_iff' go dead. - Node *old_false = NULL, *old_true = NULL; - Node *new_false = NULL, *new_true = NULL; + Node *old_false = nullptr, *old_true = nullptr; + Node *new_false = nullptr, *new_true = nullptr; for (DUIterator_Last j2min, j2 = iff->last_outs(j2min); j2 >= j2min; --j2) { Node *ifp = iff->last_out(j2); assert( ifp->Opcode() == Op_IfFalse || ifp->Opcode() == Op_IfTrue, "" ); @@ -550,7 +551,7 @@ void PhaseIdealLoop::do_split_if( Node *iff ) { // Lazy replace IDOM info with the region's dominator lazy_replace(iff, region_dom); lazy_update(region, region_dom); // idom must be update before handle_uses - region->set_req(0, NULL); // Break the self-cycle. Required for lazy_update to work on region + region->set_req(0, nullptr); // Break the self-cycle. Required for lazy_update to work on region // Now make the original merge point go dead, by handling all its uses. small_cache region_cache; diff --git a/src/hotspot/share/opto/stringopts.cpp b/src/hotspot/share/opto/stringopts.cpp index e75b2cb2da3d0..200e071db923e 100644 --- a/src/hotspot/share/opto/stringopts.cpp +++ b/src/hotspot/share/opto/stringopts.cpp @@ -72,7 +72,7 @@ class StringConcat : public ResourceObj { StringConcat(PhaseStringOpts* stringopts, CallStaticJavaNode* end): _stringopts(stringopts), _string_alloc(NULL), - _begin(NULL), + _begin(nullptr), _end(end), _multiple(false) { _arguments = new Node(1); @@ -144,7 +144,7 @@ class StringConcat : public ResourceObj { if (call->is_CallStaticJava()) { CallStaticJavaNode* csj = call->as_CallStaticJava(); ciMethod* m = csj->method(); - if (m != NULL && + if (m != nullptr && (m->intrinsic_id() == vmIntrinsics::_StringBuilder_toString || m->intrinsic_id() == vmIntrinsics::_StringBuffer_toString)) { return true; @@ -156,7 +156,7 @@ class StringConcat : public ResourceObj { static Node* skip_string_null_check(Node* value) { // Look for a diamond shaped Null check of toString() result // (could be code from String.valueOf()): - // (Proj == NULL) ? "null":"CastPP(Proj)#NotNULL + // (Proj == nullptr) ? "null":"CastPP(Proj)#Notnull if (value->is_Phi()) { int true_path = value->as_Phi()->is_diamond_phi(); if (true_path != 0) { @@ -217,13 +217,13 @@ class StringConcat : public ResourceObj { void maybe_log_transform() { CompileLog* log = _stringopts->C->log(); - if (log != NULL) { + if (log != nullptr) { log->head("replace_string_concat arguments='%d' string_alloc='%d' multiple='%d'", num_arguments(), _string_alloc != NULL, _multiple); JVMState* p = _begin->jvms(); - while (p != NULL) { + while (p != nullptr) { log->elem("jvms bci='%d' method='%d'", p->bci(), log->identify(p->method())); p = p->caller(); } @@ -238,7 +238,7 @@ class StringConcat : public ResourceObj { // Build a new call using the jvms state of the allocate address call_addr = SharedRuntime::uncommon_trap_blob()->entry_point(); const TypeFunc* call_type = OptoRuntime::uncommon_trap_Type(); - const TypePtr* no_memory_effects = NULL; + const TypePtr* no_memory_effects = nullptr; Compile* C = _stringopts->C; CallStaticJavaNode* call = new CallStaticJavaNode(call_type, call_addr, "uncommon_trap", no_memory_effects); @@ -350,22 +350,22 @@ void StringConcat::eliminate_call(CallNode* call) { Compile* C = _stringopts->C; CallProjections projs; call->extract_projections(&projs, false); - if (projs.fallthrough_catchproj != NULL) { + if (projs.fallthrough_catchproj != nullptr) { C->gvn_replace_by(projs.fallthrough_catchproj, call->in(TypeFunc::Control)); } - if (projs.fallthrough_memproj != NULL) { + if (projs.fallthrough_memproj != nullptr) { C->gvn_replace_by(projs.fallthrough_memproj, call->in(TypeFunc::Memory)); } - if (projs.catchall_memproj != NULL) { + if (projs.catchall_memproj != nullptr) { C->gvn_replace_by(projs.catchall_memproj, C->top()); } - if (projs.fallthrough_ioproj != NULL) { + if (projs.fallthrough_ioproj != nullptr) { C->gvn_replace_by(projs.fallthrough_ioproj, call->in(TypeFunc::I_O)); } - if (projs.catchall_ioproj != NULL) { + if (projs.catchall_ioproj != nullptr) { C->gvn_replace_by(projs.catchall_ioproj, C->top()); } - if (projs.catchall_catchproj != NULL) { + if (projs.catchall_catchproj != nullptr) { // EA can't cope with the partially collapsed graph this // creates so put it on the worklist to be collapsed later. for (SimpleDUIterator i(projs.catchall_catchproj); i.has_next(); i.next()) { @@ -377,7 +377,7 @@ void StringConcat::eliminate_call(CallNode* call) { } C->gvn_replace_by(projs.catchall_catchproj, C->top()); } - if (projs.resproj != NULL) { + if (projs.resproj != nullptr) { C->gvn_replace_by(projs.resproj, C->top()); } C->gvn_replace_by(call, C->top()); @@ -390,11 +390,11 @@ void StringConcat::eliminate_initialize(InitializeNode* init) { assert(init->outcnt() <= 2, "only a control and memory projection expected"); assert(init->req() <= InitializeNode::RawStores, "no pending inits"); Node *ctrl_proj = init->proj_out_or_null(TypeFunc::Control); - if (ctrl_proj != NULL) { + if (ctrl_proj != nullptr) { C->gvn_replace_by(ctrl_proj, init->in(TypeFunc::Control)); } Node *mem_proj = init->proj_out_or_null(TypeFunc::Memory); - if (mem_proj != NULL) { + if (mem_proj != nullptr) { Node *mem = init->in(TypeFunc::Memory); C->gvn_replace_by(mem_proj, mem); } @@ -411,7 +411,7 @@ Node_List PhaseStringOpts::collect_toString_calls() { // Prime the worklist for (uint i = 1; i < C->root()->len(); i++) { Node* n = C->root()->in(i); - if (n != NULL && !_visited.test_set(n->_idx)) { + if (n != nullptr && !_visited.test_set(n->_idx)) { worklist.push(n); } } @@ -422,12 +422,12 @@ Node_List PhaseStringOpts::collect_toString_calls() { CallStaticJavaNode* csj = ctrl->as_CallStaticJava(); string_calls.push(csj); } - if (ctrl->in(0) != NULL && !_visited.test_set(ctrl->in(0)->_idx)) { + if (ctrl->in(0) != nullptr && !_visited.test_set(ctrl->in(0)->_idx)) { worklist.push(ctrl->in(0)); } if (ctrl->is_Region()) { for (uint i = 1; i < ctrl->len(); i++) { - if (ctrl->in(i) != NULL && !_visited.test_set(ctrl->in(i)->_idx)) { + if (ctrl->in(i) != nullptr && !_visited.test_set(ctrl->in(i)->_idx)) { worklist.push(ctrl->in(i)); } } @@ -451,7 +451,7 @@ StringConcat* PhaseStringOpts::build_candidate(CallStaticJavaNode* call) { int_sig = ciSymbols::int_StringBuffer_signature(); char_sig = ciSymbols::char_StringBuffer_signature(); } else { - return NULL; + return nullptr; } #ifndef PRODUCT if (PrintOptimizeStringConcat) { @@ -462,7 +462,7 @@ StringConcat* PhaseStringOpts::build_candidate(CallStaticJavaNode* call) { StringConcat* sc = new StringConcat(this, call); - AllocateNode* alloc = NULL; + AllocateNode* alloc = nullptr; InitializeNode* init = NULL; // possible opportunity for StringBuilder fusion @@ -473,14 +473,14 @@ StringConcat* PhaseStringOpts::build_candidate(CallStaticJavaNode* call) { recv = recv->in(0); } cnode = recv->isa_CallStaticJava(); - if (cnode == NULL) { + if (cnode == nullptr) { alloc = recv->isa_Allocate(); - if (alloc == NULL) { + if (alloc == nullptr) { break; } // Find the constructor call Node* result = alloc->result_cast(); - if (result == NULL || !result->is_CheckCastPP() || alloc->in(TypeFunc::Memory)->is_top()) { + if (result == nullptr || !result->is_CheckCastPP() || alloc->in(TypeFunc::Memory)->is_top()) { // strange looking allocation #ifndef PRODUCT if (PrintOptimizeStringConcat) { @@ -490,11 +490,11 @@ StringConcat* PhaseStringOpts::build_candidate(CallStaticJavaNode* call) { #endif break; } - Node* constructor = NULL; + Node* constructor = nullptr; for (SimpleDUIterator i(result); i.has_next(); i.next()) { CallStaticJavaNode *use = i.get()->isa_CallStaticJava(); - if (use != NULL && - use->method() != NULL && + if (use != nullptr && + use->method() != nullptr && !use->method()->is_static() && use->method()->name() == ciSymbols::object_initializer_name() && use->method()->holder() == m->holder()) { @@ -505,7 +505,7 @@ StringConcat* PhaseStringOpts::build_candidate(CallStaticJavaNode* call) { sig == ciSymbols::string_void_signature()) { if (sig == ciSymbols::string_void_signature()) { // StringBuilder(String) so pick this up as the first argument - assert(use->in(TypeFunc::Parms + 1) != NULL, "what?"); + assert(use->in(TypeFunc::Parms + 1) != nullptr, "what?"); const Type* type = _gvn->type(use->in(TypeFunc::Parms + 1)); if (type == TypePtr::NULL_PTR) { // StringBuilder(null) throws exception. @@ -516,14 +516,14 @@ StringConcat* PhaseStringOpts::build_candidate(CallStaticJavaNode* call) { tty->cr(); } #endif - return NULL; + return nullptr; } // StringBuilder(str) argument needs null check. sc->push_string_null_check(use->in(TypeFunc::Parms + 1)); } else if (sig == ciSymbols::int_void_signature()) { // StringBuilder(int) case. Node* parm = use->in(TypeFunc::Parms + 1); - assert(parm != NULL, "must exist"); + assert(parm != nullptr, "must exist"); const TypeInt* type = _gvn->type(parm)->is_int(); if (type->_hi < 0) { // Initial capacity argument is always negative in which case StringBuilder(int) throws @@ -536,7 +536,7 @@ StringConcat* PhaseStringOpts::build_candidate(CallStaticJavaNode* call) { tty->cr(); } #endif - return NULL; + return nullptr; } else if (type->_lo < 0) { // Argument could be negative: We need a runtime check to throw NegativeArraySizeException in that case. sc->push_negative_int_check(parm); @@ -555,7 +555,7 @@ StringConcat* PhaseStringOpts::build_candidate(CallStaticJavaNode* call) { break; } } - if (constructor == NULL) { + if (constructor == nullptr) { // couldn't find constructor #ifndef PRODUCT if (PrintOptimizeStringConcat) { @@ -576,9 +576,9 @@ StringConcat* PhaseStringOpts::build_candidate(CallStaticJavaNode* call) { if (sc->validate_control_flow() && sc->validate_mem_flow()) { return sc; } else { - return NULL; + return nullptr; } - } else if (cnode->method() == NULL) { + } else if (cnode->method() == nullptr) { break; } else if (!cnode->method()->is_static() && cnode->method()->holder() == m->holder() && @@ -588,7 +588,7 @@ StringConcat* PhaseStringOpts::build_candidate(CallStaticJavaNode* call) { cnode->method()->signature()->as_symbol() == int_sig)) { sc->add_control(cnode); Node* arg = cnode->in(TypeFunc::Parms + 1); - if (arg == NULL || arg->is_top()) { + if (arg == nullptr || arg->is_top()) { #ifndef PRODUCT if (PrintOptimizeStringConcat) { tty->print("giving up because the call is effectively dead"); @@ -604,7 +604,7 @@ StringConcat* PhaseStringOpts::build_candidate(CallStaticJavaNode* call) { } else { if (arg->is_Proj() && arg->in(0)->is_CallStaticJava()) { CallStaticJavaNode* csj = arg->in(0)->as_CallStaticJava(); - if (csj->method() != NULL && + if (csj->method() != nullptr && csj->method()->intrinsic_id() == vmIntrinsics::_Integer_toString && arg->outcnt() == 1) { // _control is the list of StringBuilder calls nodes which @@ -636,7 +636,7 @@ StringConcat* PhaseStringOpts::build_candidate(CallStaticJavaNode* call) { break; } } - return NULL; + return nullptr; } @@ -667,7 +667,7 @@ PhaseStringOpts::PhaseStringOpts(PhaseGVN* gvn, Unique_Node_List*): Node_List toStrings = collect_toString_calls(); while (toStrings.size() > 0) { StringConcat* sc = build_candidate(toStrings.pop()->as_CallStaticJava()); - if (sc != NULL) { + if (sc != nullptr) { concats.push(sc); } } @@ -794,7 +794,7 @@ bool StringConcat::validate_mem_flow() { if (!_constructors.contains(curr)) { NOT_PRODUCT(path.push(curr);) Node* mem = curr->in(TypeFunc::Memory); - assert(mem != NULL, "calls should have memory edge"); + assert(mem != nullptr, "calls should have memory edge"); assert(!mem->is_Phi(), "should be handled by control flow validation"); NOT_PRODUCT(path.push(mem);) while (mem->is_MergeMem()) { @@ -851,8 +851,8 @@ bool StringConcat::validate_mem_flow() { assert(curr->is_Call(), "constructor should be a call"); // Go up the control starting from the constructor call Node* ctrl = curr->in(0); - IfNode* iff = NULL; - RegionNode* copy = NULL; + IfNode* iff = nullptr; + RegionNode* copy = nullptr; while (true) { // skip known check patterns @@ -862,10 +862,10 @@ bool StringConcat::validate_mem_flow() { ctrl = copy->is_copy(); } else { // a cast assert(ctrl->req() == 3 && - ctrl->in(1) != NULL && ctrl->in(1)->is_Proj() && - ctrl->in(2) != NULL && ctrl->in(2)->is_Proj() && + ctrl->in(1) != nullptr && ctrl->in(1)->is_Proj() && + ctrl->in(2) != nullptr && ctrl->in(2)->is_Proj() && ctrl->in(1)->in(0) == ctrl->in(2)->in(0) && - ctrl->in(1)->in(0) != NULL && ctrl->in(1)->in(0)->is_If(), + ctrl->in(1)->in(0) != nullptr && ctrl->in(1)->in(0)->is_If(), "must be a simple diamond"); Node* true_proj = ctrl->in(1)->is_IfTrue() ? ctrl->in(1) : ctrl->in(2); for (SimpleDUIterator i(true_proj); i.has_next(); i.next()) { @@ -949,7 +949,7 @@ bool StringConcat::validate_control_flow() { ctrl_path.push(cn->proj_out(0)); ctrl_path.push(cn->proj_out(0)->unique_out()); Node* catchproj = cn->proj_out(0)->unique_out()->as_Catch()->proj_out_or_null(0); - if (catchproj != NULL) { + if (catchproj != nullptr) { ctrl_path.push(catchproj); } } else { @@ -970,7 +970,7 @@ bool StringConcat::validate_control_flow() { IfNode* iff = ptr->in(0)->as_If(); BoolNode* b = iff->in(1)->isa_Bool(); - if (b == NULL) { + if (b == nullptr) { #ifndef PRODUCT if (PrintOptimizeStringConcat) { tty->print_cr("unexpected input to IfNode"); @@ -991,11 +991,11 @@ bool StringConcat::validate_control_flow() { if (b->_test._test == BoolTest::ne && v2->bottom_type() == TypePtr::NULL_PTR && v1->is_Proj() && ctrl_path.member(v1->in(0))) { - // NULL check of the return value of the append + // null check of the return value of the append null_check_count++; if (otherproj->outcnt() == 1) { CallStaticJavaNode* call = otherproj->unique_out()->isa_CallStaticJava(); - if (call != NULL && call->_name != NULL && strcmp(call->_name, "uncommon_trap") == 0) { + if (call != nullptr && call->_name != nullptr && strcmp(call->_name, "uncommon_trap") == 0) { ctrl_path.push(call); } } @@ -1009,7 +1009,7 @@ bool StringConcat::validate_control_flow() { // at the beginning. if (otherproj->outcnt() == 1) { CallStaticJavaNode* call = otherproj->unique_out()->isa_CallStaticJava(); - if (call != NULL && call->_name != NULL && strcmp(call->_name, "uncommon_trap") == 0) { + if (call != nullptr && call->_name != nullptr && strcmp(call->_name, "uncommon_trap") == 0) { // control flow leads to uct so should be ok _uncommon_traps.push(call); ctrl_path.push(call); @@ -1050,15 +1050,15 @@ bool StringConcat::validate_control_flow() { ptr = ptr->in(0)->in(0); } else if (ptr->is_Region()) { Node* copy = ptr->as_Region()->is_copy(); - if (copy != NULL) { + if (copy != nullptr) { ptr = copy; continue; } if (ptr->req() == 3 && - ptr->in(1) != NULL && ptr->in(1)->is_Proj() && - ptr->in(2) != NULL && ptr->in(2)->is_Proj() && + ptr->in(1) != nullptr && ptr->in(1)->is_Proj() && + ptr->in(2) != nullptr && ptr->in(2)->is_Proj() && ptr->in(1)->in(0) == ptr->in(2)->in(0) && - ptr->in(1)->in(0) != NULL && ptr->in(1)->in(0)->is_If()) { + ptr->in(1)->in(0) != nullptr && ptr->in(1)->in(0)->is_If()) { // Simple diamond. // XXX should check for possibly merging stores. simple data merges are ok. // The IGVN will make this simple diamond go away when it @@ -1111,16 +1111,16 @@ bool StringConcat::validate_control_flow() { Node* final_result = _end->proj_out_or_null(TypeFunc::Parms); for (uint i = 0; i < _control.size(); i++) { CallNode* cnode = _control.at(i)->isa_Call(); - if (cnode != NULL) { + if (cnode != nullptr) { _stringopts->_visited.test_set(cnode->_idx); } - Node* result = cnode != NULL ? cnode->proj_out_or_null(TypeFunc::Parms) : NULL; - if (result != NULL && result != final_result) { + Node* result = cnode != nullptr ? cnode->proj_out_or_null(TypeFunc::Parms) : nullptr; + if (result != nullptr && result != final_result) { worklist.push(result); } } - Node* last_result = NULL; + Node* last_result = nullptr; while (worklist.size() > 0) { Node* result = worklist.pop(); if (_stringopts->_visited.test_set(result->_idx)) @@ -1505,7 +1505,7 @@ void PhaseStringOpts::arraycopy(GraphKit& kit, IdealKit& ideal, Node* src_array, count = __ RShiftI(count, __ intcon(1)); } - Node* extra = NULL; + Node* extra = nullptr; #ifdef _LP64 count = __ ConvI2L(count); extra = C->top(); @@ -1682,7 +1682,7 @@ Node* PhaseStringOpts::copy_string(GraphKit& kit, Node* str, Node* dst_array, No // Compress copy the char into dst_array at index start. Node* PhaseStringOpts::copy_char(GraphKit& kit, Node* val, Node* dst_array, Node* dst_coder, Node* start) { - bool dcon = (dst_coder != NULL) && dst_coder->is_Con(); + bool dcon = (dst_coder != nullptr) && dst_coder->is_Con(); bool dbyte = dcon ? (dst_coder->get_int() == java_lang_String::CODER_LATIN1) : false; IdealKit ideal(&kit, true, true); @@ -1718,11 +1718,11 @@ Node* PhaseStringOpts::copy_char(GraphKit& kit, Node* val, Node* dst_array, Node // Allocate a byte array of specified length. Node* PhaseStringOpts::allocate_byte_array(GraphKit& kit, IdealKit* ideal, Node* length) { - if (ideal != NULL) { + if (ideal != nullptr) { // Sync IdealKit and graphKit. kit.sync_kit(*ideal); } - Node* byte_array = NULL; + Node* byte_array = nullptr; { PreserveReexecuteState preexecs(&kit); // The original jvms is for an allocation of either a String or @@ -1739,7 +1739,7 @@ Node* PhaseStringOpts::allocate_byte_array(GraphKit& kit, IdealKit* ideal, Node* AllocateArrayNode* byte_alloc = AllocateArrayNode::Ideal_array_allocation(byte_array, _gvn); byte_alloc->maybe_set_complete(_gvn); - if (ideal != NULL) { + if (ideal != nullptr) { // Sync IdealKit and graphKit. ideal->sync_kit(&kit); } @@ -1889,8 +1889,8 @@ void PhaseStringOpts::replace_string_concat(StringConcat* sc) { } case StringConcat::StringMode: { const Type* type = kit.gvn().type(arg); - Node* count = NULL; - Node* arg_coder = NULL; + Node* count = nullptr; + Node* arg_coder = nullptr; if (type == TypePtr::NULL_PTR) { // replace the argument with the null checked version arg = null_string; @@ -1948,7 +1948,7 @@ void PhaseStringOpts::replace_string_concat(StringConcat* sc) { coder = __ OrI(coder, arg_coder); } length = __ AddI(length, count); - string_sizes->init_req(argi, NULL); + string_sizes->init_req(argi, nullptr); break; } case StringConcat::CharMode: { @@ -2011,7 +2011,7 @@ void PhaseStringOpts::replace_string_concat(StringConcat* sc) { assert(CompactStrings || (coder->is_Con() && coder->get_int() == java_lang_String::CODER_UTF16), "Result string must be UTF16 encoded if CompactStrings is disabled"); - Node* dst_array = NULL; + Node* dst_array = nullptr; if (sc->num_arguments() == 1 && (sc->mode(0) == StringConcat::StringMode || sc->mode(0) == StringConcat::StringNullCheckMode)) { @@ -2020,7 +2020,7 @@ void PhaseStringOpts::replace_string_concat(StringConcat* sc) { dst_array = kit.load_String_value(sc->argument(0), true); } else { // Allocate destination byte array according to coder - dst_array = allocate_byte_array(kit, NULL, __ LShiftI(length, coder)); + dst_array = allocate_byte_array(kit, nullptr, __ LShiftI(length, coder)); // Now copy the string representations into the final byte[] Node* start = __ intcon(0); @@ -2066,7 +2066,7 @@ void PhaseStringOpts::replace_string_concat(StringConcat* sc) { // The value field is final. Emit a barrier here to ensure that the effect // of the initialization is committed to memory before any code publishes // a reference to the newly constructed object (see Parse::do_exits()). - assert(AllocateNode::Ideal_allocation(result, _gvn) != NULL, "should be newly allocated"); + assert(AllocateNode::Ideal_allocation(result, _gvn) != nullptr, "should be newly allocated"); kit.insert_mem_bar(Op_MemBarRelease, result); } else { result = C->top(); diff --git a/src/hotspot/share/opto/subnode.cpp b/src/hotspot/share/opto/subnode.cpp index e4e4717bd5566..98e0f5f2ed377 100644 --- a/src/hotspot/share/opto/subnode.cpp +++ b/src/hotspot/share/opto/subnode.cpp @@ -101,12 +101,12 @@ const Type* SubNode::Value_common(PhaseTransform *phase) const { if( t1 == Type::BOTTOM || t2 == Type::BOTTOM ) return bottom_type(); - return NULL; + return nullptr; } const Type* SubNode::Value(PhaseGVN* phase) const { const Type* t = Value_common(phase); - if (t != NULL) { + if (t != nullptr) { return t; } const Type* t1 = phase->type(in(1)); @@ -124,7 +124,7 @@ SubNode* SubNode::make(Node* in1, Node* in2, BasicType bt) { default: fatal("Not implemented for %s", type2name(bt)); } - return NULL; + return nullptr; } //============================================================================= @@ -177,7 +177,7 @@ Node *SubINode::Ideal(PhaseGVN *phase, bool can_reshape){ #endif const Type *t2 = phase->type( in2 ); - if( t2 == Type::TOP ) return NULL; + if( t2 == Type::TOP ) return nullptr; // Convert "x-c0" into "x+ -c0". if( t2->base() == Type::Int ){ // Might be bottom or top... const TypeInt *i = t2->is_int(); @@ -211,7 +211,7 @@ Node *SubINode::Ideal(PhaseGVN *phase, bool can_reshape){ } const Type *t1 = phase->type( in1 ); - if( t1 == Type::TOP ) return NULL; + if( t1 == Type::TOP ) return nullptr; #ifdef ASSERT // Check for dead loop @@ -281,7 +281,7 @@ Node *SubINode::Ideal(PhaseGVN *phase, bool can_reshape){ } } - return NULL; + return nullptr; } //------------------------------sub-------------------------------------------- @@ -321,7 +321,7 @@ Node *SubLNode::Ideal(PhaseGVN *phase, bool can_reshape) { } #endif - if( phase->type( in2 ) == Type::TOP ) return NULL; + if( phase->type( in2 ) == Type::TOP ) return nullptr; const TypeLong *i = phase->type( in2 )->isa_long(); // Convert "x-c0" into "x+ -c0". if( i && // Might be bottom or top... @@ -354,7 +354,7 @@ Node *SubLNode::Ideal(PhaseGVN *phase, bool can_reshape) { } const Type *t1 = phase->type( in1 ); - if( t1 == Type::TOP ) return NULL; + if( t1 == Type::TOP ) return nullptr; #ifdef ASSERT // Check for dead loop @@ -405,7 +405,7 @@ Node *SubLNode::Ideal(PhaseGVN *phase, bool can_reshape) { } } - return NULL; + return nullptr; } //------------------------------sub-------------------------------------------- @@ -469,7 +469,7 @@ Node *SubFNode::Ideal(PhaseGVN *phase, bool can_reshape) { //if( phase->type(in(1)) == TypeF::ZERO ) //return new (phase->C, 2) NegFNode(in(2)); - return NULL; + return nullptr; } //------------------------------sub-------------------------------------------- @@ -504,7 +504,7 @@ Node *SubDNode::Ideal(PhaseGVN *phase, bool can_reshape){ //if( phase->type(in(1)) == TypeD::ZERO ) //return new (phase->C, 2) NegDNode(in(2)); - return NULL; + return nullptr; } //------------------------------sub-------------------------------------------- @@ -581,7 +581,7 @@ CmpNode *CmpNode::make(Node *in1, Node *in2, BasicType bt, bool unsigned_comp) { default: fatal("Not implemented for %s", type2name(bt)); } - return NULL; + return nullptr; } //============================================================================= @@ -672,7 +672,7 @@ const Type *CmpUNode::sub( const Type *t1, const Type *t2 ) const { const Type* CmpUNode::Value(PhaseGVN* phase) const { const Type* t = SubNode::Value_common(phase); - if (t != NULL) { + if (t != nullptr) { return t; } const Node* in1 = in(1); @@ -683,6 +683,9 @@ const Type* CmpUNode::Value(PhaseGVN* phase) const { if (t2 == TypeInt::INT) { // Compare to bottom? return bottom_type(); } + + const Type* t_sub = sub(t1, t2); // compare based on immediate inputs + uint in1_op = in1->Opcode(); if (in1_op == Op_AddI || in1_op == Op_SubI) { // The problem rise when result of AddI(SubI) may overflow @@ -735,13 +738,15 @@ const Type* CmpUNode::Value(PhaseGVN* phase) const { const TypeInt* tr2 = TypeInt::make(lo_tr2, hi_tr2, w); const TypeInt* cmp1 = sub(tr1, t2)->is_int(); const TypeInt* cmp2 = sub(tr2, t2)->is_int(); - // compute union, so that cmp handles all possible results from the two cases - return cmp1->meet(cmp2); + // Compute union, so that cmp handles all possible results from the two cases + const Type* t_cmp = cmp1->meet(cmp2); + // Pick narrowest type, based on overflow computation and on immediate inputs + return t_sub->filter(t_cmp); } } } - return sub(t1, t2); // Local flavor of type subtraction + return t_sub; } bool CmpUNode::is_index_range_check() const { @@ -766,7 +771,7 @@ Node *CmpINode::Ideal( PhaseGVN *phase, bool can_reshape ) { // This is handled (with more general cases) by Ideal_sub_algebra. } } - return NULL; // No change + return nullptr; // No change } Node *CmpLNode::Ideal( PhaseGVN *phase, bool can_reshape ) { @@ -777,7 +782,7 @@ Node *CmpLNode::Ideal( PhaseGVN *phase, bool can_reshape ) { return new CmpINode(in(1)->in(1), phase->intcon((jint)con)); } } - return NULL; + return nullptr; } //============================================================================= @@ -882,9 +887,9 @@ const Type *CmpPNode::sub( const Type *t1, const Type *t2 ) const { if (both_oop_ptr) { Node* in1 = in(1)->uncast(); Node* in2 = in(2)->uncast(); - AllocateNode* alloc1 = AllocateNode::Ideal_allocation(in1, NULL); - AllocateNode* alloc2 = AllocateNode::Ideal_allocation(in2, NULL); - if (MemNode::detect_ptr_independence(in1, alloc1, in2, alloc2, NULL)) { + AllocateNode* alloc1 = AllocateNode::Ideal_allocation(in1, nullptr); + AllocateNode* alloc2 = AllocateNode::Ideal_allocation(in2, nullptr); + if (MemNode::detect_ptr_independence(in1, alloc1, in2, alloc2, nullptr)) { return TypeInt::CC_GT; // different pointers } } @@ -893,9 +898,9 @@ const Type *CmpPNode::sub( const Type *t1, const Type *t2 ) const { const TypeKlassPtr* klass_p1 = r1->isa_klassptr(); if (both_oop_ptr || (klass_p0 && klass_p1)) { // both or neither are klass pointers - ciKlass* klass0 = NULL; + ciKlass* klass0 = nullptr; bool xklass0 = false; - ciKlass* klass1 = NULL; + ciKlass* klass1 = nullptr; bool xklass1 = false; if (oop_p0) { @@ -968,25 +973,25 @@ const Type *CmpPNode::sub( const Type *t1, const Type *t2 ) const { static inline Node* isa_java_mirror_load(PhaseGVN* phase, Node* n) { // Return the klass node for (indirect load from OopHandle) // LoadBarrier?(LoadP(LoadP(AddP(foo:Klass, #java_mirror)))) - // or NULL if not matching. + // or null if not matching. BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); n = bs->step_over_gc_barrier(n); - if (n->Opcode() != Op_LoadP) return NULL; + if (n->Opcode() != Op_LoadP) return nullptr; const TypeInstPtr* tp = phase->type(n)->isa_instptr(); - if (!tp || tp->klass() != phase->C->env()->Class_klass()) return NULL; + if (!tp || tp->klass() != phase->C->env()->Class_klass()) return nullptr; Node* adr = n->in(MemNode::Address); // First load from OopHandle: ((OopHandle)mirror)->resolve(); may need barrier. - if (adr->Opcode() != Op_LoadP || !phase->type(adr)->isa_rawptr()) return NULL; + if (adr->Opcode() != Op_LoadP || !phase->type(adr)->isa_rawptr()) return nullptr; adr = adr->in(MemNode::Address); intptr_t off = 0; Node* k = AddPNode::Ideal_base_and_offset(adr, phase, off); - if (k == NULL) return NULL; + if (k == nullptr) return nullptr; const TypeKlassPtr* tkp = phase->type(k)->isa_klassptr(); - if (!tkp || off != in_bytes(Klass::java_mirror_offset())) return NULL; + if (!tkp || off != in_bytes(Klass::java_mirror_offset())) return nullptr; // We've found the klass node of a Java mirror load. return k; @@ -994,19 +999,19 @@ static inline Node* isa_java_mirror_load(PhaseGVN* phase, Node* n) { static inline Node* isa_const_java_mirror(PhaseGVN* phase, Node* n) { // for ConP(Foo.class) return ConP(Foo.klass) - // otherwise return NULL - if (!n->is_Con()) return NULL; + // otherwise return null + if (!n->is_Con()) return nullptr; const TypeInstPtr* tp = phase->type(n)->isa_instptr(); - if (!tp) return NULL; + if (!tp) return nullptr; ciType* mirror_type = tp->java_mirror_type(); - // TypeInstPtr::java_mirror_type() returns non-NULL for compile- + // TypeInstPtr::java_mirror_type() returns non-null for compile- // time Class constants only. - if (!mirror_type) return NULL; + if (!mirror_type) return nullptr; // x.getClass() == int.class can never be true (for all primitive types) - // Return a ConP(NULL) node for this case. + // Return a ConP(null) node for this case. if (mirror_type->is_classless()) { return phase->makecon(TypePtr::NULL_PTR); } @@ -1043,7 +1048,7 @@ Node *CmpPNode::Ideal( PhaseGVN *phase, bool can_reshape ) { if (k1 && (k2 || conk2)) { Node* lhs = k1; - Node* rhs = (k2 != NULL) ? k2 : conk2; + Node* rhs = (k2 != nullptr) ? k2 : conk2; set_req_X(1, lhs, phase); set_req_X(2, rhs, phase); return this; @@ -1052,8 +1057,8 @@ Node *CmpPNode::Ideal( PhaseGVN *phase, bool can_reshape ) { // Constant pointer on right? const TypeKlassPtr* t2 = phase->type(in(2))->isa_klassptr(); - if (t2 == NULL || !t2->klass_is_exact()) - return NULL; + if (t2 == nullptr || !t2->klass_is_exact()) + return nullptr; // Get the constant klass we are comparing to. ciKlass* superklass = t2->klass(); @@ -1062,15 +1067,15 @@ Node *CmpPNode::Ideal( PhaseGVN *phase, bool can_reshape ) { if (ldk1->is_DecodeNKlass()) { ldk1 = ldk1->in(1); if (ldk1->Opcode() != Op_LoadNKlass ) - return NULL; + return nullptr; } else if (ldk1->Opcode() != Op_LoadKlass ) - return NULL; + return nullptr; // Take apart the address of the LoadKlass: Node* adr1 = ldk1->in(MemNode::Address); intptr_t con2 = 0; Node* ldk2 = AddPNode::Ideal_base_and_offset(adr1, phase, con2); - if (ldk2 == NULL) - return NULL; + if (ldk2 == nullptr) + return nullptr; if (con2 == oopDesc::klass_offset_in_bytes()) { // We are inspecting an object's concrete class. // Short-circuit the check if the query is abstract. @@ -1087,13 +1092,13 @@ Node *CmpPNode::Ideal( PhaseGVN *phase, bool can_reshape ) { if (ldk2->is_DecodeNKlass()) { // Keep ldk2 as DecodeN since it could be used in CmpP below. if (ldk2->in(1)->Opcode() != Op_LoadNKlass ) - return NULL; + return nullptr; } else if (ldk2->Opcode() != Op_LoadKlass) - return NULL; + return nullptr; // Verify that we understand the situation if (con2 != (intptr_t) superklass->super_check_offset()) - return NULL; // Might be element-klass loading from array klass + return nullptr; // Might be element-klass loading from array klass // If 'superklass' has no subklasses and is not an interface, then we are // assured that the only input which will pass the type check is @@ -1111,7 +1116,7 @@ Node *CmpPNode::Ideal( PhaseGVN *phase, bool can_reshape ) { } if (superklass->is_instance_klass()) { ciInstanceKlass* ik = superklass->as_instance_klass(); - if (ik->has_subklass() || ik->is_interface()) return NULL; + if (ik->has_subklass() || ik->is_interface()) return nullptr; // Add a dependency if there is a chance that a subclass will be added later. if (!ik->is_final()) { phase->C->dependencies()->assert_leaf_type(ik); @@ -1135,7 +1140,7 @@ const Type *CmpNNode::sub( const Type *t1, const Type *t2 ) const { //------------------------------Ideal------------------------------------------ Node *CmpNNode::Ideal( PhaseGVN *phase, bool can_reshape ) { - return NULL; + return nullptr; } //============================================================================= @@ -1235,7 +1240,7 @@ Node *CmpDNode::Ideal(PhaseGVN *phase, bool can_reshape){ } // Testing value required the precision of a double } - return NULL; // No change + return nullptr; // No change } @@ -1326,7 +1331,7 @@ Node* BoolNode::make_predicate(Node* test_value, PhaseGVN* phase) { //--------------------------------as_int_value--------------------------------- Node* BoolNode::as_int_value(PhaseGVN* phase) { // Inverse to make_predicate. The CMove probably boils down to a Conv2B. - Node* cmov = CMoveNode::make(NULL, this, + Node* cmov = CMoveNode::make(nullptr, this, phase->intcon(0), phase->intcon(1), TypeInt::BOOL); return phase->transform(cmov); @@ -1348,8 +1353,8 @@ Node* BoolNode::fold_cmpI(PhaseGVN* phase, SubNode* cmp, Node* cmp1, int cmp_op, // Skip cases were inputs of add/sub are not integers or of bottom type const TypeInt* r0 = phase->type(cmp1->in(1))->isa_int(); const TypeInt* r1 = phase->type(cmp1->in(2))->isa_int(); - if ((r0 != NULL) && (r0 != TypeInt::INT) && - (r1 != NULL) && (r1 != TypeInt::INT) && + if ((r0 != nullptr) && (r0 != TypeInt::INT) && + (r1 != nullptr) && (r1 != TypeInt::INT) && (cmp2_type != TypeInt::INT)) { // Compute exact (long) type range of add/sub result jlong lo_long = r0->_lo; @@ -1383,14 +1388,14 @@ Node* BoolNode::fold_cmpI(PhaseGVN* phase, SubNode* cmp, Node* cmp1, int cmp_op, } } } - return NULL; + return nullptr; } static bool is_counted_loop_cmp(Node *cmp) { Node *n = cmp->in(1)->in(1); - return n != NULL && + return n != nullptr && n->is_Phi() && - n->in(0) != NULL && + n->in(0) != nullptr && n->in(0)->is_CountedLoop() && n->in(0)->as_CountedLoop()->phi() == n; } @@ -1400,15 +1405,15 @@ Node *BoolNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Change "bool tst (cmp con x)" into "bool ~tst (cmp x con)". // This moves the constant to the right. Helps value-numbering. Node *cmp = in(1); - if( !cmp->is_Sub() ) return NULL; + if( !cmp->is_Sub() ) return nullptr; int cop = cmp->Opcode(); - if( cop == Op_FastLock || cop == Op_FastUnlock || cmp->is_SubTypeCheck()) return NULL; + if( cop == Op_FastLock || cop == Op_FastUnlock || cmp->is_SubTypeCheck()) return nullptr; Node *cmp1 = cmp->in(1); Node *cmp2 = cmp->in(2); - if( !cmp1 ) return NULL; + if( !cmp1 ) return nullptr; if (_test._test == BoolTest::overflow || _test._test == BoolTest::no_overflow) { - return NULL; + return nullptr; } // Constant on left? @@ -1465,7 +1470,7 @@ Node *BoolNode::Ideal(PhaseGVN *phase, bool can_reshape) { // test instead. int cmp1_op = cmp1->Opcode(); const TypeInt* cmp2_type = phase->type(cmp2)->isa_int(); - if (cmp2_type == NULL) return NULL; + if (cmp2_type == nullptr) return nullptr; Node* j_xor = cmp1; if( cmp2_type == TypeInt::ZERO && cmp1_op == Op_XorI && @@ -1482,7 +1487,7 @@ Node *BoolNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Same with ((x & m) u< m+1) and ((m & x) u< m+1) if (cop == Op_CmpU && cmp1_op == Op_AndI) { - Node* bound = NULL; + Node* bound = nullptr; if (_test._test == BoolTest::le) { bound = cmp2; } else if (_test._test == BoolTest::lt && @@ -1578,7 +1583,7 @@ Node *BoolNode::Ideal(PhaseGVN *phase, bool can_reshape) { if ((_test._test == BoolTest::eq || _test._test == BoolTest::ne) && cop == Op_CmpI && cmp1_op == Op_AddI && - cmp1->in(2) != NULL && + cmp1->in(2) != nullptr && phase->type(cmp1->in(2))->isa_int() && phase->type(cmp1->in(2))->is_int()->is_con() && cmp2_type == TypeInt::ZERO && @@ -1651,18 +1656,18 @@ Node *BoolNode::Ideal(PhaseGVN *phase, bool can_reshape) { // // counter. If they use the PRE-incremented counter, then the counter has // // to be incremented in a private block on a loop backedge. // if( du && du->cnt(this) && du->out(this)[0]->Opcode() == Op_CountedLoopEnd ) - // return NULL; + // return nullptr; // #ifndef PRODUCT // // Do not do this in a wash GVN pass during verification. // // Gets triggered by too many simple optimizations to be bothered with // // re-trying it again and again. - // if( !phase->allow_progress() ) return NULL; + // if( !phase->allow_progress() ) return nullptr; // #endif // // Not valid for unsigned compare because of corner cases in involving zero. // // For example, replacing "X-1 Opcode() == Op_CmpU ) return NULL; + // if( cmp->Opcode() == Op_CmpU ) return nullptr; // int cmp2_op = cmp2->Opcode(); // if( _test._test == BoolTest::le ) { // if( cmp1_op == Op_AddI && diff --git a/src/hotspot/share/opto/subnode.hpp b/src/hotspot/share/opto/subnode.hpp index 4560c109a1ec7..74e4cebe61106 100644 --- a/src/hotspot/share/opto/subnode.hpp +++ b/src/hotspot/share/opto/subnode.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -252,7 +252,7 @@ class CmpFNode : public CmpNode { public: CmpFNode( Node *in1, Node *in2 ) : CmpNode(in1,in2) {} virtual int Opcode() const; - virtual const Type *sub( const Type *, const Type * ) const { ShouldNotReachHere(); return NULL; } + virtual const Type *sub( const Type *, const Type * ) const { ShouldNotReachHere(); return nullptr; } const Type* Value(PhaseGVN* phase) const; }; @@ -280,7 +280,7 @@ class CmpDNode : public CmpNode { public: CmpDNode( Node *in1, Node *in2 ) : CmpNode(in1,in2) {} virtual int Opcode() const; - virtual const Type *sub( const Type *, const Type * ) const { ShouldNotReachHere(); return NULL; } + virtual const Type *sub( const Type *, const Type * ) const { ShouldNotReachHere(); return nullptr; } const Type* Value(PhaseGVN* phase) const; virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); }; @@ -338,7 +338,7 @@ class BoolNode : public Node { int cmp1_op, const TypeInt* cmp2_type); public: const BoolTest _test; - BoolNode(Node *cc, BoolTest::mask t): Node(NULL,cc), _test(t) { + BoolNode(Node *cc, BoolTest::mask t): Node(nullptr,cc), _test(t) { init_class_id(Class_Bool); } // Convert an arbitrary int value to a Bool or other suitable predicate. @@ -512,7 +512,7 @@ class SqrtFNode : public Node { public: SqrtFNode(Compile* C, Node *c, Node *in1) : Node(c, in1) { init_flags(Flag_is_expensive); - if (c != NULL) { + if (c != nullptr) { // Treat node only as expensive if a control input is set because it might // be created from a SqrtDNode in ConvD2FNode::Ideal() that was found to // be unique and therefore has no control input. diff --git a/src/hotspot/share/opto/subtypenode.cpp b/src/hotspot/share/opto/subtypenode.cpp index 7aed5e65ef359..33de1e96b49d1 100644 --- a/src/hotspot/share/opto/subtypenode.cpp +++ b/src/hotspot/share/opto/subtypenode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -34,6 +34,7 @@ const Type* SubTypeCheckNode::sub(const Type* sub_t, const Type* super_t) const { ciKlass* superk = super_t->is_klassptr()->klass(); + assert(sub_t != Type::TOP && !TypePtr::NULL_PTR->higher_equal(sub_t), "should be not null"); ciKlass* subk = sub_t->isa_klassptr() ? sub_t->is_klassptr()->klass() : sub_t->is_oopptr()->klass(); bool xsubk = sub_t->isa_klassptr() ? sub_t->is_klassptr()->klass_is_exact() : sub_t->is_oopptr()->klass_is_exact(); @@ -85,7 +86,7 @@ const Type* SubTypeCheckNode::sub(const Type* sub_t, const Type* super_t) const } if (super_t->singleton()) { - if (subk != NULL) { + if (subk != nullptr) { switch (Compile::current()->static_subtype_check(superk, subk)) { case Compile::SSC_always_false: return TypeInt::CC_GT; @@ -107,9 +108,9 @@ Node *SubTypeCheckNode::Ideal(PhaseGVN* phase, bool can_reshape) { Node* obj_or_subklass = in(ObjOrSubKlass); Node* superklass = in(SuperKlass); - if (obj_or_subklass == NULL || - superklass == NULL) { - return NULL; + if (obj_or_subklass == nullptr || + superklass == nullptr) { + return nullptr; } const Type* sub_t = phase->type(obj_or_subklass); @@ -117,12 +118,12 @@ Node *SubTypeCheckNode::Ideal(PhaseGVN* phase, bool can_reshape) { if (!super_t->isa_klassptr() || (!sub_t->isa_klassptr() && !sub_t->isa_oopptr())) { - return NULL; + return nullptr; } - Node* addr = NULL; + Node* addr = nullptr; if (obj_or_subklass->is_DecodeNKlass()) { - if (obj_or_subklass->in(1) != NULL && + if (obj_or_subklass->in(1) != nullptr && obj_or_subklass->in(1)->Opcode() == Op_LoadNKlass) { addr = obj_or_subklass->in(1)->in(MemNode::Address); } @@ -130,10 +131,10 @@ Node *SubTypeCheckNode::Ideal(PhaseGVN* phase, bool can_reshape) { addr = obj_or_subklass->in(MemNode::Address); } - if (addr != NULL) { + if (addr != nullptr) { intptr_t con = 0; Node* obj = AddPNode::Ideal_base_and_offset(addr, phase, con); - if (con == oopDesc::klass_offset_in_bytes() && obj != NULL) { + if (con == oopDesc::klass_offset_in_bytes() && obj != nullptr) { assert(is_oop(phase, obj), "only for oop input"); set_req_X(ObjOrSubKlass, obj, phase); return this; @@ -142,7 +143,7 @@ Node *SubTypeCheckNode::Ideal(PhaseGVN* phase, bool can_reshape) { // AllocateNode might have more accurate klass input Node* allocated_klass = AllocateNode::Ideal_klass(obj_or_subklass, phase); - if (allocated_klass != NULL) { + if (allocated_klass != nullptr) { assert(is_oop(phase, obj_or_subklass), "only for oop input"); set_req_X(ObjOrSubKlass, allocated_klass, phase); return this; @@ -152,7 +153,7 @@ Node *SubTypeCheckNode::Ideal(PhaseGVN* phase, bool can_reshape) { // when possible would not constant fold better assert(verify(phase), "missing Value() optimization"); - return NULL; + return nullptr; } #ifdef ASSERT @@ -206,11 +207,11 @@ bool SubTypeCheckNode::verify(PhaseGVN* phase) { ciKlass* subk = sub_t->isa_klassptr() ? sub_t->is_klassptr()->klass() : sub_t->is_oopptr()->klass(); // can be NULL for bottom[] ciKlass* superk = super_t->is_klassptr()->klass(); - if (super_t->singleton() && subk != NULL) { - Node* subklass = NULL; + if (super_t->singleton() && subk != nullptr) { + Node* subklass = nullptr; if (sub_t->isa_oopptr()) { Node* adr = phase->transform(new AddPNode(obj_or_subklass, obj_or_subklass, phase->MakeConX(oopDesc::klass_offset_in_bytes()))); - subklass = phase->transform(LoadKlassNode::make(*phase, NULL, C->immutable_memory(), adr, TypeInstPtr::KLASS)); + subklass = phase->transform(LoadKlassNode::make(*phase, nullptr, C->immutable_memory(), adr, TypeInstPtr::KLASS)); record_for_cleanup(subklass, phase); } else { subklass = obj_or_subklass; @@ -223,7 +224,7 @@ bool SubTypeCheckNode::verify(PhaseGVN* phase) { } case Compile::SSC_full_test: { Node* p1 = phase->transform(new AddPNode(superklass, superklass, phase->MakeConX(in_bytes(Klass::super_check_offset_offset())))); - Node* chk_off = phase->transform(new LoadINode(NULL, C->immutable_memory(), p1, phase->type(p1)->is_ptr(), TypeInt::INT, MemNode::unordered)); + Node* chk_off = phase->transform(new LoadINode(nullptr, C->immutable_memory(), p1, phase->type(p1)->is_ptr(), TypeInt::INT, MemNode::unordered)); record_for_cleanup(chk_off, phase); int cacheoff_con = in_bytes(Klass::secondary_super_cache_offset()); @@ -234,7 +235,7 @@ bool SubTypeCheckNode::verify(PhaseGVN* phase) { chk_off_X = phase->transform(new ConvI2LNode(chk_off_X)); #endif Node* p2 = phase->transform(new AddPNode(subklass, subklass, chk_off_X)); - Node* nkls = phase->transform(LoadKlassNode::make(*phase, NULL, C->immutable_memory(), p2, phase->type(p2)->is_ptr(), TypeKlassPtr::OBJECT_OR_NULL)); + Node* nkls = phase->transform(LoadKlassNode::make(*phase, nullptr, C->immutable_memory(), p2, phase->type(p2)->is_ptr(), TypeKlassPtr::OBJECT_OR_NULL)); return verify_helper(phase, nkls, cached_t); } diff --git a/src/hotspot/share/opto/superword.cpp b/src/hotspot/share/opto/superword.cpp index 987ab27f309ad..0f82da86f7574 100644 --- a/src/hotspot/share/opto/superword.cpp +++ b/src/hotspot/share/opto/superword.cpp @@ -50,42 +50,42 @@ SuperWord::SuperWord(PhaseIdealLoop* phase) : _phase(phase), _arena(phase->C->comp_arena()), _igvn(phase->_igvn), - _packset(arena(), 8, 0, NULL), // packs for the current block + _packset(arena(), 8, 0, nullptr), // packs for the current block _bb_idx(arena(), (int)(1.10 * phase->C->unique()), 0, 0), // node idx to index in bb - _block(arena(), 8, 0, NULL), // nodes in current block - _post_block(arena(), 8, 0, NULL), // nodes common to current block which are marked as post loop vectorizable - _data_entry(arena(), 8, 0, NULL), // nodes with all inputs from outside - _mem_slice_head(arena(), 8, 0, NULL), // memory slice heads - _mem_slice_tail(arena(), 8, 0, NULL), // memory slice tails - _node_info(arena(), 8, 0, SWNodeInfo::initial), // info needed per node - _clone_map(phase->C->clone_map()), // map of nodes created in cloning - _cmovev_kit(_arena, this), // map to facilitate CMoveV creation - _align_to_ref(NULL), // memory reference to align vectors to - _disjoint_ptrs(arena(), 8, 0, OrderedPair::initial), // runtime disambiguated pointer pairs - _dg(_arena), // dependence graph - _visited(arena()), // visited node set - _post_visited(arena()), // post visited node set - _n_idx_list(arena(), 8), // scratch list of (node,index) pairs - _nlist(arena(), 8, 0, NULL), // scratch list of nodes - _stk(arena(), 8, 0, NULL), // scratch stack of nodes - _lpt(NULL), // loop tree node - _lp(NULL), // CountedLoopNode - _pre_loop_end(NULL), // Pre loop CountedLoopEndNode - _bb(NULL), // basic block - _iv(NULL), // induction var - _race_possible(false), // cases where SDMU is true - _early_return(true), // analysis evaluations routine - _do_vector_loop(phase->C->do_vector_loop()), // whether to do vectorization/simd style + _block(arena(), 8, 0, nullptr), // nodes in current block + _post_block(arena(), 8, 0, nullptr), // nodes common to current block which are marked as post loop vectorizable + _data_entry(arena(), 8, 0, nullptr), // nodes with all inputs from outside + _mem_slice_head(arena(), 8, 0, nullptr), // memory slice heads + _mem_slice_tail(arena(), 8, 0, nullptr), // memory slice tails + _node_info(arena(), 8, 0, SWNodeInfo::initial), // info needed per node + _clone_map(phase->C->clone_map()), // map of nodes created in cloning + _cmovev_kit(_arena, this), // map to facilitate CMoveV creation + _align_to_ref(nullptr), // memory reference to align vectors to + _disjoint_ptrs(arena(), 8, 0, OrderedPair::initial), // runtime disambiguated pointer pairs + _dg(_arena), // dependence graph + _visited(arena()), // visited node set + _post_visited(arena()), // post visited node set + _n_idx_list(arena(), 8), // scratch list of (node,index) pairs + _nlist(arena(), 8, 0, nullptr), // scratch list of nodes + _stk(arena(), 8, 0, nullptr), // scratch stack of nodes + _lpt(nullptr), // loop tree node + _lp(nullptr), // CountedLoopNode + _pre_loop_end(nullptr), // Pre loop CountedLoopEndNode + _bb(nullptr), // basic block + _iv(nullptr), // induction var + _race_possible(false), // cases where SDMU is true + _early_return(true), // analysis evaluations routine + _do_vector_loop(phase->C->do_vector_loop()), // whether to do vectorization/simd style _do_reserve_copy(DoReserveCopyInSuperWord), - _num_work_vecs(0), // amount of vector work we have - _num_reductions(0), // amount of reduction work we have - _ii_first(-1), // first loop generation index - only if do_vector_loop() - _ii_last(-1), // last loop generation index - only if do_vector_loop() + _num_work_vecs(0), // amount of vector work we have + _num_reductions(0), // amount of reduction work we have + _ii_first(-1), // first loop generation index - only if do_vector_loop() + _ii_last(-1), // last loop generation index - only if do_vector_loop() _ii_order(arena(), 8, 0, 0) { #ifndef PRODUCT _vector_loop_debug = 0; - if (_phase->C->method() != NULL) { + if (_phase->C->method() != nullptr) { _vector_loop_debug = phase->C->directive()->VectorizeDebugOption; } @@ -171,7 +171,7 @@ bool SuperWord::transform_loop(IdealLoopTree* lpt, bool do_optimization) { if (cl->is_main_loop()) { // Check for pre-loop ending with CountedLoopEnd(Bool(Cmp(x,Opaque1(limit)))) CountedLoopEndNode* pre_end = find_pre_loop_end(cl); - if (pre_end == NULL) { + if (pre_end == nullptr) { return false; } Node* pre_opaq1 = pre_end->limit(); @@ -296,7 +296,7 @@ void SuperWord::unrolling_analysis(int &local_loop_unroll_factor) { Node* n_ctrl = _phase->get_ctrl(adr); // save a queue of post process nodes - if (n_ctrl != NULL && lpt()->is_member(_phase->get_loop(n_ctrl))) { + if (n_ctrl != nullptr && lpt()->is_member(_phase->get_loop(n_ctrl))) { // Process the memory expression int stack_idx = 0; bool have_side_effects = true; @@ -526,7 +526,7 @@ bool SuperWord::SLP_extract() { find_adjacent_refs(); - if (align_to_ref() == NULL) { + if (align_to_ref() == nullptr) { return false; // Did not find memory reference to align vectors } @@ -615,16 +615,16 @@ void SuperWord::find_adjacent_refs() { Node_List align_to_refs; int max_idx; int best_iv_adjustment = 0; - MemNode* best_align_to_mem_ref = NULL; + MemNode* best_align_to_mem_ref = nullptr; while (memops.size() != 0) { // Find a memory reference to align to. MemNode* mem_ref = find_align_to_ref(memops, max_idx); - if (mem_ref == NULL) break; + if (mem_ref == nullptr) break; align_to_refs.push(mem_ref); int iv_adjustment = get_iv_adjustment(mem_ref); - if (best_align_to_mem_ref == NULL) { + if (best_align_to_mem_ref == nullptr) { // Set memory reference which is the best from all memory operations // to be used for alignment. The pre-loop trip count is modified to align // this reference to a vector-aligned address. @@ -633,13 +633,13 @@ void SuperWord::find_adjacent_refs() { NOT_PRODUCT(find_adjacent_refs_trace_1(best_align_to_mem_ref, best_iv_adjustment);) } - SWPointer align_to_ref_p(mem_ref, this, NULL, false); + SWPointer align_to_ref_p(mem_ref, this, nullptr, false); // Set alignment relative to "align_to_ref" for all related memory operations. for (int i = memops.size() - 1; i >= 0; i--) { MemNode* s = memops.at(i)->as_Mem(); if (isomorphic(s, mem_ref) && (!_do_vector_loop || same_origin_idx(s, mem_ref))) { - SWPointer p2(s, this, NULL, false); + SWPointer p2(s, this, nullptr, false); if (p2.comparable(align_to_ref_p)) { int align = memory_alignment(s, iv_adjustment); set_alignment(s, align); @@ -660,7 +660,7 @@ void SuperWord::find_adjacent_refs() { // iterations in pre-loop will be not enough to align it. create_pack = false; } else { - SWPointer p2(best_align_to_mem_ref, this, NULL, false); + SWPointer p2(best_align_to_mem_ref, this, nullptr, false); if (!align_to_ref_p.invar_equals(p2)) { // Do not vectorize memory accesses with different invariants // if unaligned memory accesses are not allowed. @@ -744,9 +744,9 @@ void SuperWord::find_adjacent_refs() { memops.push(s); } best_align_to_mem_ref = find_align_to_ref(memops, max_idx); - if (best_align_to_mem_ref == NULL) { + if (best_align_to_mem_ref == nullptr) { if (TraceSuperWord) { - tty->print_cr("SuperWord::find_adjacent_refs(): best_align_to_mem_ref == NULL"); + tty->print_cr("SuperWord::find_adjacent_refs(): best_align_to_mem_ref == nullptr"); } // best_align_to_mem_ref will be used for adjusting the pre-loop limit in // SuperWord::align_initial_loop_index. Find one with the biggest vector size, @@ -759,10 +759,10 @@ void SuperWord::find_adjacent_refs() { memops.remove(0); } best_align_to_mem_ref = find_align_to_ref(memops, max_idx); - assert(best_align_to_mem_ref == NULL, "sanity"); + assert(best_align_to_mem_ref == nullptr, "sanity"); best_align_to_mem_ref = memops.at(max_idx)->as_Mem(); } - assert(best_align_to_mem_ref != NULL, "sanity"); + assert(best_align_to_mem_ref != nullptr, "sanity"); } break; } @@ -811,7 +811,7 @@ MemNode* SuperWord::find_align_to_ref(Node_List &memops, int &idx) { // Count number of comparable memory ops for (uint i = 0; i < memops.size(); i++) { MemNode* s1 = memops.at(i)->as_Mem(); - SWPointer p1(s1, this, NULL, false); + SWPointer p1(s1, this, nullptr, false); // Only discard unalignable memory references if vector memory references // should be aligned on this platform. if (vectors_should_be_aligned() && !ref_is_alignable(p1)) { @@ -821,7 +821,7 @@ MemNode* SuperWord::find_align_to_ref(Node_List &memops, int &idx) { for (uint j = i+1; j < memops.size(); j++) { MemNode* s2 = memops.at(j)->as_Mem(); if (isomorphic(s1, s2)) { - SWPointer p2(s2, this, NULL, false); + SWPointer p2(s2, this, nullptr, false); if (p1.comparable(p2)) { (*cmp_ct.adr_at(i))++; (*cmp_ct.adr_at(j))++; @@ -842,7 +842,7 @@ MemNode* SuperWord::find_align_to_ref(Node_List &memops, int &idx) { if (s->is_Store()) { int vw = vector_width_in_bytes(s); assert(vw > 1, "sanity"); - SWPointer p(s, this, NULL, false); + SWPointer p(s, this, nullptr, false); if ( cmp_ct.at(j) > max_ct || (cmp_ct.at(j) == max_ct && ( vw > max_vw || @@ -865,7 +865,7 @@ MemNode* SuperWord::find_align_to_ref(Node_List &memops, int &idx) { if (s->is_Load()) { int vw = vector_width_in_bytes(s); assert(vw > 1, "sanity"); - SWPointer p(s, this, NULL, false); + SWPointer p(s, this, nullptr, false); if ( cmp_ct.at(j) > max_ct || (cmp_ct.at(j) == max_ct && ( vw > max_vw || @@ -903,7 +903,7 @@ MemNode* SuperWord::find_align_to_ref(Node_List &memops, int &idx) { #endif return memops.at(max_idx)->as_Mem(); } - return NULL; + return nullptr; } //------------------span_works_for_memory_size----------------------------- @@ -965,7 +965,7 @@ bool SuperWord::ref_is_alignable(SWPointer& p) { int vw = vector_width_in_bytes(p.mem()); assert(vw > 1, "sanity"); Node* init_nd = pre_end->init_trip(); - if (init_nd->is_Con() && p.invar() == NULL) { + if (init_nd->is_Con() && p.invar() == nullptr) { int init = init_nd->bottom_type()->is_int()->get_con(); int init_offset = init * p.scale_in_bytes() + offset; if (init_offset < 0) { // negative offset from object start? @@ -1021,7 +1021,7 @@ int SuperWord::get_vw_bytes_special(MemNode* s) { //---------------------------get_iv_adjustment--------------------------- // Calculate loop's iv adjustment for this memory ops. int SuperWord::get_iv_adjustment(MemNode* mem_ref) { - SWPointer align_to_ref_p(mem_ref, this, NULL, false); + SWPointer align_to_ref_p(mem_ref, this, nullptr, false); int offset = align_to_ref_p.offset_in_bytes(); int scale = align_to_ref_p.scale_in_bytes(); int elt_size = align_to_ref_p.memory_size(); @@ -1091,7 +1091,7 @@ void SuperWord::dependence_graph() { _dg.make_edge(_dg.root(), slice); // Create a sink for the slice - DepMem* slice_sink = _dg.make_node(NULL); + DepMem* slice_sink = _dg.make_node(nullptr); _dg.make_edge(slice_sink, _dg.tail()); // Now visit each pair of memory ops, creating the edges @@ -1102,13 +1102,13 @@ void SuperWord::dependence_graph() { if (_dg.dep(s1)->in_cnt() == 0) { _dg.make_edge(slice, s1); } - SWPointer p1(s1->as_Mem(), this, NULL, false); + SWPointer p1(s1->as_Mem(), this, nullptr, false); bool sink_dependent = true; for (int k = j - 1; k >= 0; k--) { Node* s2 = _nlist.at(k); if (s1->is_Load() && s2->is_Load()) continue; - SWPointer p2(s2->as_Mem(), this, NULL, false); + SWPointer p2(s2->as_Mem(), this, nullptr, false); int cmp = p1.cmp(p2); if (SuperWordRTDepCheck && @@ -1154,7 +1154,7 @@ void SuperWord::dependence_graph() { void SuperWord::mem_slice_preds(Node* start, Node* stop, GrowableArray &preds) { assert(preds.length() == 0, "start empty"); Node* n = start; - Node* prev = NULL; + Node* prev = nullptr; while (true) { NOT_PRODUCT( if(is_trace_mem_slice()) tty->print_cr("SuperWord::mem_slice_preds: n %d", n->_idx);) assert(in_bb(n), "must be in block"); @@ -1178,7 +1178,7 @@ void SuperWord::mem_slice_preds(Node* start, Node* stop, GrowableArray &p // StoreCM has an input edge used as a precedence edge. // Maybe an issue when oop stores are vectorized. } else { - assert(out == prev || prev == NULL, "no branches off of store slice"); + assert(out == prev || prev == nullptr, "no branches off of store slice"); } }//else }//for @@ -1254,8 +1254,8 @@ bool SuperWord::are_adjacent_refs(Node* s1, Node* s2) { if (_phase->C->get_alias_index(s1->as_Mem()->adr_type()) != _phase->C->get_alias_index(s2->as_Mem()->adr_type())) return false; - SWPointer p1(s1->as_Mem(), this, NULL, false); - SWPointer p2(s2->as_Mem(), this, NULL, false); + SWPointer p1(s1->as_Mem(), this, nullptr, false); + SWPointer p2(s2->as_Mem(), this, nullptr, false); if (p1.base() != p2.base() || !p1.comparable(p2)) return false; int diff = p2.offset_in_bytes() - p1.offset_in_bytes(); return diff == data_size(s1); @@ -1273,13 +1273,13 @@ bool SuperWord::isomorphic(Node* s1, Node* s2) { if (s1_ctrl == s2_ctrl) { return true; } else { - bool s1_ctrl_inv = ((s1_ctrl == NULL) ? true : lpt()->is_invariant(s1_ctrl)); - bool s2_ctrl_inv = ((s2_ctrl == NULL) ? true : lpt()->is_invariant(s2_ctrl)); + bool s1_ctrl_inv = ((s1_ctrl == nullptr) ? true : lpt()->is_invariant(s1_ctrl)); + bool s2_ctrl_inv = ((s2_ctrl == nullptr) ? true : lpt()->is_invariant(s2_ctrl)); // If the control nodes are not invariant for the loop, fail isomorphism test. if (!s1_ctrl_inv || !s2_ctrl_inv) { return false; } - if(s1_ctrl != NULL && s2_ctrl != NULL) { + if(s1_ctrl != nullptr && s2_ctrl != nullptr) { if (s1_ctrl->is_Proj()) { s1_ctrl = s1_ctrl->in(0); assert(lpt()->is_invariant(s1_ctrl), "must be invariant"); @@ -1393,14 +1393,14 @@ void SuperWord::set_alignment(Node* s1, Node* s2, int align) { //------------------------------data_size--------------------------- int SuperWord::data_size(Node* s) { - Node* use = NULL; //test if the node is a candidate for CMoveV optimization, then return the size of CMov + Node* use = nullptr; //test if the node is a candidate for CMoveV optimization, then return the size of CMov if (UseVectorCmov) { use = _cmovev_kit.is_Bool_candidate(s); - if (use != NULL) { + if (use != nullptr) { return data_size(use); } use = _cmovev_kit.is_CmpD_candidate(s); - if (use != NULL) { + if (use != nullptr) { return data_size(use); } } @@ -1489,8 +1489,8 @@ bool SuperWord::follow_def_uses(Node_List* p) { NOT_PRODUCT(if(is_trace_alignment()) tty->print_cr("SuperWord::follow_def_uses: s1 %d, align %d", s1->_idx, align);) int savings = -1; int num_s1_uses = 0; - Node* u1 = NULL; - Node* u2 = NULL; + Node* u1 = nullptr; + Node* u2 = nullptr; for (DUIterator_Fast imax, i = s1->fast_outs(imax); i < imax; i++) { Node* t1 = s1->fast_out(i); num_s1_uses++; @@ -1545,17 +1545,17 @@ void SuperWord::order_def_uses(Node_List* p) { } // Now find t1's packset - Node_List* p2 = NULL; + Node_List* p2 = nullptr; for (int j = 0; j < _packset.length(); j++) { p2 = _packset.at(j); Node* first = p2->at(0); if (t1 == first) { break; } - p2 = NULL; + p2 = nullptr; } // Arrange all sub components by the major component - if (p2 != NULL) { + if (p2 != nullptr) { for (uint j = 1; j < p->size(); j++) { Node* d1 = p->at(j); Node* u1 = p2->at(j); @@ -1678,17 +1678,17 @@ void SuperWord::combine_packs() { changed = false; for (int i = 0; i < _packset.length(); i++) { Node_List* p1 = _packset.at(i); - if (p1 == NULL) continue; + if (p1 == nullptr) continue; // Because of sorting we can start at i + 1 for (int j = i + 1; j < _packset.length(); j++) { Node_List* p2 = _packset.at(j); - if (p2 == NULL) continue; + if (p2 == nullptr) continue; if (i == j) continue; if (p1->at(p1->size()-1) == p2->at(0)) { for (uint k = 1; k < p2->size(); k++) { p1->push(p2->at(k)); } - _packset.at_put(j, NULL); + _packset.at_put(j, nullptr); changed = true; } } @@ -1698,7 +1698,7 @@ void SuperWord::combine_packs() { // Split packs which have size greater then max vector size. for (int i = 0; i < _packset.length(); i++) { Node_List* p1 = _packset.at(i); - if (p1 != NULL) { + if (p1 != nullptr) { BasicType bt = velt_basic_type(p1->at(0)); uint max_vlen = Matcher::max_vector_size(bt); // Max elements in vector assert(is_power_of_2(max_vlen), "sanity"); @@ -1707,7 +1707,7 @@ void SuperWord::combine_packs() { // Skip pack which can't be vector. // case1: for(...) { a[i] = i; } elements values are different (i+x) // case2: for(...) { a[i] = b[i+1]; } can't align both, load and store - _packset.at_put(i, NULL); + _packset.at_put(i, nullptr); continue; } if (psize > max_vlen) { @@ -1720,7 +1720,7 @@ void SuperWord::combine_packs() { pack = new Node_List(); } } - _packset.at_put(i, NULL); + _packset.at_put(i, nullptr); } } } @@ -1728,7 +1728,7 @@ void SuperWord::combine_packs() { // Compress list. for (int i = _packset.length() - 1; i >= 0; i--) { Node_List* p1 = _packset.at(i); - if (p1 == NULL) { + if (p1 == nullptr) { _packset.remove_at(i); } } @@ -1743,13 +1743,13 @@ void SuperWord::combine_packs() { // Construct the map from nodes to packs. Only valid after the // point where a node is only in one pack (after combine_packs). void SuperWord::construct_my_pack_map() { - Node_List* rslt = NULL; + Node_List* rslt = nullptr; for (int i = 0; i < _packset.length(); i++) { Node_List* p = _packset.at(i); for (uint j = 0; j < p->size(); j++) { Node* s = p->at(j); #ifdef ASSERT - if (my_pack(s) != NULL) { + if (my_pack(s) != nullptr) { s->dump(1); tty->print_cr("packs[%d]:", i); print_pack(p); @@ -1852,28 +1852,28 @@ void SuperWord::merge_packs_to_cmovd() { } Node* CMoveKit::is_Bool_candidate(Node* def) const { - Node* use = NULL; - if (!def->is_Bool() || def->in(0) != NULL || def->outcnt() != 1) { - return NULL; + Node* use = nullptr; + if (!def->is_Bool() || def->in(0) != nullptr || def->outcnt() != 1) { + return nullptr; } for (DUIterator_Fast jmax, j = def->fast_outs(jmax); j < jmax; j++) { use = def->fast_out(j); if (!_sw->same_generation(def, use) || !use->is_CMove()) { - return NULL; + return nullptr; } } return use; } Node* CMoveKit::is_CmpD_candidate(Node* def) const { - Node* use = NULL; - if (!def->is_Cmp() || def->in(0) != NULL || def->outcnt() != 1) { - return NULL; + Node* use = nullptr; + if (!def->is_Cmp() || def->in(0) != nullptr || def->outcnt() != 1) { + return nullptr; } for (DUIterator_Fast jmax, j = def->fast_outs(jmax); j < jmax; j++) { use = def->fast_out(j); - if (!_sw->same_generation(def, use) || (use = is_Bool_candidate(use)) == NULL || !_sw->same_generation(def, use)) { - return NULL; + if (!_sw->same_generation(def, use) || (use = is_Bool_candidate(use)) == nullptr || !_sw->same_generation(def, use)) { + return nullptr; } } return use; @@ -1882,50 +1882,50 @@ Node* CMoveKit::is_CmpD_candidate(Node* def) const { Node_List* CMoveKit::make_cmovevd_pack(Node_List* cmovd_pk) { Node *cmovd = cmovd_pk->at(0); if (!cmovd->is_CMove()) { - return NULL; + return nullptr; } if (cmovd->Opcode() != Op_CMoveF && cmovd->Opcode() != Op_CMoveD) { - return NULL; + return nullptr; } - if (pack(cmovd) != NULL) { // already in the cmov pack - return NULL; + if (pack(cmovd) != nullptr) { // already in the cmov pack + return nullptr; } - if (cmovd->in(0) != NULL) { + if (cmovd->in(0) != nullptr) { NOT_PRODUCT(if(_sw->is_trace_cmov()) {tty->print("CMoveKit::make_cmovevd_pack: CMoveD %d has control flow, escaping...", cmovd->_idx); cmovd->dump();}) - return NULL; + return nullptr; } Node* bol = cmovd->as_CMove()->in(CMoveNode::Condition); if (!bol->is_Bool() || bol->outcnt() != 1 || !_sw->same_generation(bol, cmovd) - || bol->in(0) != NULL // BoolNode has control flow!! - || _sw->my_pack(bol) == NULL) { + || bol->in(0) != nullptr // BoolNode has control flow!! + || _sw->my_pack(bol) == nullptr) { NOT_PRODUCT(if(_sw->is_trace_cmov()) {tty->print("CMoveKit::make_cmovevd_pack: Bool %d does not fit CMoveD %d for building vector, escaping...", bol->_idx, cmovd->_idx); bol->dump();}) - return NULL; + return nullptr; } Node_List* bool_pk = _sw->my_pack(bol); if (bool_pk->size() != cmovd_pk->size() ) { - return NULL; + return nullptr; } Node* cmpd = bol->in(1); if (!cmpd->is_Cmp() || cmpd->outcnt() != 1 || !_sw->same_generation(cmpd, cmovd) - || cmpd->in(0) != NULL // CmpDNode has control flow!! - || _sw->my_pack(cmpd) == NULL) { + || cmpd->in(0) != nullptr // CmpDNode has control flow!! + || _sw->my_pack(cmpd) == nullptr) { NOT_PRODUCT(if(_sw->is_trace_cmov()) {tty->print("CMoveKit::make_cmovevd_pack: CmpD %d does not fit CMoveD %d for building vector, escaping...", cmpd->_idx, cmovd->_idx); cmpd->dump();}) - return NULL; + return nullptr; } Node_List* cmpd_pk = _sw->my_pack(cmpd); if (cmpd_pk->size() != cmovd_pk->size() ) { - return NULL; + return nullptr; } if (!test_cmpd_pack(cmpd_pk, cmovd_pk)) { NOT_PRODUCT(if(_sw->is_trace_cmov()) {tty->print("CMoveKit::make_cmovevd_pack: cmpd pack for CmpD %d failed vectorization test", cmpd->_idx); cmpd->dump();}) - return NULL; + return nullptr; } Node_List* new_cmpd_pk = new Node_List(); @@ -1961,13 +1961,13 @@ bool CMoveKit::test_cmpd_pack(Node_List* cmpd_pk, Node_List* cmovd_pk) { Node_List* in1_pk = _sw->my_pack(in1); Node_List* in2_pk = _sw->my_pack(in2); - if ( (in1_pk != NULL && in1_pk->size() != cmpd_pk->size()) - || (in2_pk != NULL && in2_pk->size() != cmpd_pk->size()) ) { + if ( (in1_pk != nullptr && in1_pk->size() != cmpd_pk->size()) + || (in2_pk != nullptr && in2_pk->size() != cmpd_pk->size()) ) { return false; } // test if "all" in1 are in the same pack or the same node - if (in1_pk == NULL) { + if (in1_pk == nullptr) { for (uint j = 1; j < cmpd_pk->size(); j++) { if (cmpd_pk->at(j)->in(1) != in1) { return false; @@ -1975,7 +1975,7 @@ bool CMoveKit::test_cmpd_pack(Node_List* cmpd_pk, Node_List* cmovd_pk) { }//for: in1_pk is not pack but all CmpD nodes in the pack have the same in(1) } // test if "all" in2 are in the same pack or the same node - if (in2_pk == NULL) { + if (in2_pk == nullptr) { for (uint j = 1; j < cmpd_pk->size(); j++) { if (cmpd_pk->at(j)->in(2) != in2) { return false; @@ -2012,7 +2012,7 @@ bool CMoveKit::test_cmpd_pack(Node_List* cmpd_pk, Node_List* cmovd_pk) { bool SuperWord::implemented(Node_List* p) { bool retValue = false; Node* p0 = p->at(0); - if (p0 != NULL) { + if (p0 != nullptr) { int opc = p0->Opcode(); uint size = p->size(); if (p0->is_reduction()) { @@ -2037,7 +2037,7 @@ bool SuperWord::implemented(Node_List* p) { } bool SuperWord::is_cmov_pack(Node_List* p) { - return _cmovev_kit.pack(p->at(0)) != NULL; + return _cmovev_kit.pack(p->at(0)) != nullptr; } //------------------------------same_inputs-------------------------- // For pack p, are all idx operands the same? @@ -2076,7 +2076,7 @@ bool SuperWord::profitable(Node_List* p) { if (p0->is_reduction()) { Node* second_in = p0->in(2); Node_List* second_pk = my_pack(second_in); - if ((second_pk == NULL) || (_num_work_vecs == _num_reductions)) { + if ((second_pk == nullptr) || (_num_work_vecs == _num_reductions)) { // Remove reduction flag if no parent pack or if not enough work // to cover reduction expansion overhead p0->remove_flag(Node::Flag_is_reduction); @@ -2090,7 +2090,7 @@ bool SuperWord::profitable(Node_List* p) { // case (different shift counts) because it is not supported yet. Node* cnt = p0->in(2); Node_List* cnt_pk = my_pack(cnt); - if (cnt_pk != NULL) + if (cnt_pk != nullptr) return false; if (!same_inputs(p, 2)) return false; @@ -2239,7 +2239,7 @@ void SuperWord::co_locate_pack(Node_List* pk) { if (in_pack(s2, pk) || schedule_before_pack.member(s2)) { schedule_before_pack.push(s1); // s1 must be scheduled before Node_List* mem_pk = my_pack(s1); - if (mem_pk != NULL) { + if (mem_pk != nullptr) { for (uint ii = 0; ii < mem_pk->size(); ii++) { Node* s = mem_pk->at(ii); // follow partner if (memops.member(s) && !schedule_before_pack.member(s)) @@ -2448,7 +2448,7 @@ bool SuperWord::output() { if (p && n == executed_last(p)) { uint vlen = p->size(); uint vlen_in_bytes = 0; - Node* vn = NULL; + Node* vn = nullptr; Node* low_adr = p->at(0); Node* first = executed_first(p); if (can_process_post_loop) { @@ -2460,13 +2460,13 @@ bool SuperWord::output() { if (n->is_Load()) { Node* ctl = n->in(MemNode::Control); Node* mem = first->in(MemNode::Memory); - SWPointer p1(n->as_Mem(), this, NULL, false); + SWPointer p1(n->as_Mem(), this, nullptr, false); // Identify the memory dependency for the new loadVector node by // walking up through memory chain. // This is done to give flexibility to the new loadVector node so that // it can move above independent storeVector nodes. while (mem->is_StoreVector()) { - SWPointer p2(mem->as_Mem(), this, NULL, false); + SWPointer p2(mem->as_Mem(), this, nullptr, false); int cmp = p1.cmp(p2); if (SWPointer::not_equal(cmp) || !SWPointer::comparable(cmp)) { mem = mem->in(MemNode::Memory); @@ -2481,9 +2481,9 @@ bool SuperWord::output() { } else if (n->is_Store()) { // Promote value to be stored to vector Node* val = vector_opd(p, MemNode::ValueIn); - if (val == NULL) { + if (val == nullptr) { if (do_reserve_copy()) { - NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: val should not be NULL, exiting SuperWord");}) + NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: val should not be null, exiting SuperWord");}) return false; //and reverse to backup IG } ShouldNotReachHere(); @@ -2518,25 +2518,25 @@ bool SuperWord::output() { vlen_in_bytes = vn->as_Vector()->length_in_bytes(); } else if (n->req() == 3 && !is_cmov_pack(p)) { // Promote operands to vector - Node* in1 = NULL; + Node* in1 = nullptr; bool node_isa_reduction = n->is_reduction(); if (node_isa_reduction) { // the input to the first reduction operation is retained in1 = low_adr->in(1); } else { in1 = vector_opd(p, 1); - if (in1 == NULL) { + if (in1 == nullptr) { if (do_reserve_copy()) { - NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: in1 should not be NULL, exiting SuperWord");}) + NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: in1 should not be null, exiting SuperWord");}) return false; //and reverse to backup IG } ShouldNotReachHere(); } } Node* in2 = vector_opd(p, 2); - if (in2 == NULL) { + if (in2 == nullptr) { if (do_reserve_copy()) { - NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: in2 should not be NULL, exiting SuperWord");}) + NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: in2 should not be null, exiting SuperWord");}) return false; //and reverse to backup IG } ShouldNotReachHere(); @@ -2549,7 +2549,7 @@ bool SuperWord::output() { } if (node_isa_reduction) { const Type *arith_type = n->bottom_type(); - vn = ReductionNode::make(opc, NULL, in1, in2, arith_type->basic_type()); + vn = ReductionNode::make(opc, nullptr, in1, in2, arith_type->basic_type()); if (in2->is_Load()) { vlen_in_bytes = in2->as_LoadVector()->memory_size(); } else { @@ -2566,7 +2566,7 @@ bool SuperWord::output() { opc == Op_PopCountI) { assert(n->req() == 2, "only one input expected"); Node* in = vector_opd(p, 1); - vn = VectorNode::make(opc, in, NULL, vlen, velt_basic_type(n)); + vn = VectorNode::make(opc, in, nullptr, vlen, velt_basic_type(n)); vlen_in_bytes = vn->as_Vector()->length_in_bytes(); } else if (is_cmov_pack(p)) { if (can_process_post_loop) { @@ -2601,17 +2601,17 @@ bool SuperWord::output() { NOT_PRODUCT(if(is_trace_cmov()) {tty->print("SWPointer::output: created bool cc node %d", cc->_idx); cc->dump();}) Node* src1 = vector_opd(p, 2); //2=CMoveNode::IfFalse - if (src1 == NULL) { + if (src1 == nullptr) { if (do_reserve_copy()) { - NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: src1 should not be NULL, exiting SuperWord");}) + NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: src1 should not be null, exiting SuperWord");}) return false; //and reverse to backup IG } ShouldNotReachHere(); } Node* src2 = vector_opd(p, 3); //3=CMoveNode::IfTrue - if (src2 == NULL) { + if (src2 == nullptr) { if (do_reserve_copy()) { - NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: src2 should not be NULL, exiting SuperWord");}) + NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: src2 should not be null, exiting SuperWord");}) return false; //and reverse to backup IG } ShouldNotReachHere(); @@ -2641,10 +2641,10 @@ bool SuperWord::output() { ShouldNotReachHere(); } - assert(vn != NULL, "sanity"); - if (vn == NULL) { + assert(vn != nullptr, "sanity"); + if (vn == nullptr) { if (do_reserve_copy()){ - NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: got NULL node, cannot proceed, exiting SuperWord");}) + NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: got null node, cannot proceed, exiting SuperWord");}) return false; //and reverse to backup IG } ShouldNotReachHere(); @@ -2757,7 +2757,7 @@ Node* SuperWord::vector_opd(Node_List* p, int opd_idx) { assert(((opd_idx != 2) || !VectorNode::is_shift(p0)), "shift's count can't be vector"); if (opd_idx == 2 && VectorNode::is_shift(p0)) { NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("shift's count can't be vector");}) - return NULL; + return nullptr; } return opd; // input is matching vector } @@ -2767,13 +2767,13 @@ Node* SuperWord::vector_opd(Node_List* p, int opd_idx) { // Vector instructions do not mask shift count, do it here. juint mask = (p0->bottom_type() == TypeInt::INT) ? (BitsPerInt - 1) : (BitsPerLong - 1); const TypeInt* t = opd->find_int_type(); - if (t != NULL && t->is_con()) { + if (t != nullptr && t->is_con()) { juint shift = t->get_con(); if (shift > mask) { // Unsigned cmp cnt = ConNode::make(TypeInt::make(shift & mask)); } } else { - if (t == NULL || t->_lo < 0 || t->_hi > (int)mask) { + if (t == nullptr || t->_lo < 0 || t->_hi > (int)mask) { cnt = ConNode::make(TypeInt::make(mask)); _igvn.register_new_node_with_optimizer(cnt); cnt = new AndINode(opd, cnt); @@ -2783,7 +2783,7 @@ Node* SuperWord::vector_opd(Node_List* p, int opd_idx) { assert(opd->bottom_type()->isa_int(), "int type only"); if (!opd->bottom_type()->isa_int()) { NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("Should be int type only");}) - return NULL; + return nullptr; } } // Move shift count into vector register. @@ -2795,13 +2795,13 @@ Node* SuperWord::vector_opd(Node_List* p, int opd_idx) { assert(!opd->is_StoreVector(), "such vector is not expected here"); if (opd->is_StoreVector()) { NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("StoreVector is not expected here");}) - return NULL; + return nullptr; } // Convert scalar input to vector with the same number of elements as // p0's vector. Use p0's type because size of operand's container in // vector should match p0's size regardless operand's size. - const Type* p0_t = NULL; - VectorNode* vn = NULL; + const Type* p0_t = nullptr; + VectorNode* vn = nullptr; if (opd_idx == 2 && VectorNode::is_scalar_rotate(p0)) { Node* conv = opd; p0_t = TypeInt::INT; @@ -2836,19 +2836,19 @@ Node* SuperWord::vector_opd(Node_List* p, int opd_idx) { for (uint i = 1; i < vlen; i++) { Node* pi = p->at(i); Node* in = pi->in(opd_idx); - assert(my_pack(in) == NULL, "Should already have been unpacked"); - if (my_pack(in) != NULL) { + assert(my_pack(in) == nullptr, "Should already have been unpacked"); + if (my_pack(in) != nullptr) { NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("Should already have been unpacked");}) - return NULL; + return nullptr; } assert(opd_bt == in->bottom_type()->basic_type(), "all same type"); pk->add_opd(in); if (VectorNode::is_muladds2i(pi)) { Node* in2 = pi->in(opd_idx + 2); - assert(my_pack(in2) == NULL, "Should already have been unpacked"); - if (my_pack(in2) != NULL) { + assert(my_pack(in2) == nullptr, "Should already have been unpacked"); + if (my_pack(in2) != nullptr) { NOT_PRODUCT(if (is_trace_loop_reverse() || TraceLoopOpts) { tty->print_cr("Should already have been unpacked"); }) - return NULL; + return nullptr; } assert(opd_bt == in2->bottom_type()->basic_type(), "all same type"); pk->add_opd(in2); @@ -2883,7 +2883,7 @@ void SuperWord::insert_extracts(Node_List* p) { Node* n = use->in(k); if (def == n) { Node_List* u_pk = my_pack(use); - if ((u_pk == NULL || !is_cmov_pack(u_pk) || use->is_CMove()) && !is_vector_use(use, k)) { + if ((u_pk == nullptr || !is_cmov_pack(u_pk) || use->is_CMove()) && !is_vector_use(use, k)) { _n_idx_list.push(use, k); } } @@ -2918,11 +2918,11 @@ void SuperWord::insert_extracts(Node_List* p) { // Is use->in(u_idx) a vector use? bool SuperWord::is_vector_use(Node* use, int u_idx) { Node_List* u_pk = my_pack(use); - if (u_pk == NULL) return false; + if (u_pk == nullptr) return false; if (use->is_reduction()) return true; Node* def = use->in(u_idx); Node_List* d_pk = my_pack(def); - if (d_pk == NULL) { + if (d_pk == nullptr) { // check for scalar promotion Node* n = u_pk->at(0)->in(u_idx); for (uint i = 1; i < u_pk->size(); i++) { @@ -3268,7 +3268,7 @@ int SuperWord::memory_alignment(MemNode* s, int iv_adjust) { } #endif NOT_PRODUCT(SWPointer::Tracer::Depth ddd(0);) - SWPointer p(s, this, NULL, false); + SWPointer p(s, this, nullptr, false); if (!p.valid()) { NOT_PRODUCT(if(is_trace_alignment()) tty->print_cr("SWPointer::memory_alignment: SWPointer p invalid, return bottom_align");) return bottom_align; @@ -3349,7 +3349,7 @@ Node_List* SuperWord::in_pack(Node* s, Node_List* p) { return p; } } - return NULL; + return nullptr; } //------------------------------remove_pack_at--------------------------- @@ -3358,7 +3358,7 @@ void SuperWord::remove_pack_at(int pos) { Node_List* p = _packset.at(pos); for (uint i = 0; i < p->size(); i++) { Node* s = p->at(i); - set_my_pack(s, NULL); + set_my_pack(s, nullptr); } _packset.remove_at(pos); } @@ -3455,9 +3455,9 @@ void SuperWord::align_initial_loop_index(MemNode* align_to_ref) { // Ensure the original loop limit is available from the // pre-loop Opaque1 node. Node* orig_limit = pre_opaq->original_loop_limit(); - assert(orig_limit != NULL && _igvn.type(orig_limit) != Type::TOP, ""); + assert(orig_limit != nullptr && _igvn.type(orig_limit) != Type::TOP, ""); - SWPointer align_to_ref_p(align_to_ref, this, NULL, false); + SWPointer align_to_ref_p(align_to_ref, this, nullptr, false); assert(align_to_ref_p.valid(), "sanity"); // Given: @@ -3521,7 +3521,7 @@ void SuperWord::align_initial_loop_index(MemNode* align_to_ref) { Node *offsn = _igvn.intcon(offset); Node *e = offsn; - if (align_to_ref_p.invar() != NULL) { + if (align_to_ref_p.invar() != nullptr) { // incorporate any extra invariant piece producing (offset +/- invar) >>> log2(elt) Node* log2_elt = _igvn.intcon(exact_log2(elt_size)); Node* invar = align_to_ref_p.invar(); @@ -3533,7 +3533,7 @@ void SuperWord::align_initial_loop_index(MemNode* align_to_ref) { _igvn.register_new_node_with_optimizer(invar); } Node* invar_scale = align_to_ref_p.invar_scale(); - if (invar_scale != NULL) { + if (invar_scale != nullptr) { invar = new LShiftINode(invar, invar_scale); _igvn.register_new_node_with_optimizer(invar); } @@ -3550,7 +3550,7 @@ void SuperWord::align_initial_loop_index(MemNode* align_to_ref) { } if (vw > ObjectAlignmentInBytes || align_to_ref_p.base()->is_top()) { // incorporate base e +/- base && Mask >>> log2(elt) - Node* xbase = new CastP2XNode(NULL, align_to_ref_p.adr()); + Node* xbase = new CastP2XNode(nullptr, align_to_ref_p.adr()); _igvn.register_new_node_with_optimizer(xbase); #ifdef _LP64 xbase = new ConvL2INode(xbase); @@ -3613,16 +3613,16 @@ void SuperWord::align_initial_loop_index(MemNode* align_to_ref) { CountedLoopEndNode* SuperWord::find_pre_loop_end(CountedLoopNode* cl) const { // The loop cannot be optimized if the graph shape at // the loop entry is inappropriate. - if (cl->is_canonical_loop_entry() == NULL) { - return NULL; + if (cl->is_canonical_loop_entry() == nullptr) { + return nullptr; } Node* p_f = cl->skip_predicates()->in(0)->in(0); - if (!p_f->is_IfFalse()) return NULL; - if (!p_f->in(0)->is_CountedLoopEnd()) return NULL; + if (!p_f->is_IfFalse()) return nullptr; + if (!p_f->in(0)->is_CountedLoopEnd()) return nullptr; CountedLoopEndNode* pre_end = p_f->in(0)->as_CountedLoopEnd(); CountedLoopNode* loop_node = pre_end->loopnode(); - if (loop_node == NULL || !loop_node->is_pre_loop()) return NULL; + if (loop_node == nullptr || !loop_node->is_pre_loop()) return nullptr; return pre_end; } @@ -3639,11 +3639,11 @@ void SuperWord::init() { _iteration_first.clear(); _iteration_last.clear(); _node_info.clear(); - _align_to_ref = NULL; - _lpt = NULL; - _lp = NULL; - _bb = NULL; - _iv = NULL; + _align_to_ref = nullptr; + _lpt = nullptr; + _lp = nullptr; + _bb = nullptr; + _iv = nullptr; _race_possible = 0; _early_return = false; _num_work_vecs = 0; @@ -3720,9 +3720,9 @@ int SWPointer::Tracer::_depth = 0; #endif //----------------------------SWPointer------------------------ SWPointer::SWPointer(MemNode* mem, SuperWord* slp, Node_Stack *nstack, bool analyze_only) : - _mem(mem), _slp(slp), _base(NULL), _adr(NULL), - _scale(0), _offset(0), _invar(NULL), _negate_invar(false), - _invar_scale(NULL), + _mem(mem), _slp(slp), _base(nullptr), _adr(nullptr), + _scale(0), _offset(0), _invar(nullptr), _negate_invar(false), + _invar_scale(nullptr), _nstack(nstack), _analyze_only(analyze_only), _stack_idx(0) #ifndef PRODUCT @@ -3789,9 +3789,9 @@ SWPointer::SWPointer(MemNode* mem, SuperWord* slp, Node_Stack *nstack, bool anal // Following is used to create a temporary object during // the pattern match of an address expression. SWPointer::SWPointer(SWPointer* p) : - _mem(p->_mem), _slp(p->_slp), _base(NULL), _adr(NULL), - _scale(0), _offset(0), _invar(NULL), _negate_invar(false), - _invar_scale(NULL), + _mem(p->_mem), _slp(p->_slp), _base(nullptr), _adr(nullptr), + _scale(0), _offset(0), _invar(nullptr), _negate_invar(false), + _invar_scale(nullptr), _nstack(p->_nstack), _analyze_only(p->_analyze_only), _stack_idx(p->_stack_idx) #ifndef PRODUCT @@ -3905,7 +3905,7 @@ bool SWPointer::scaled_iv(Node* n) { return true; } } else if (opc == Op_LShiftL && n->in(2)->is_Con()) { - if (!has_iv() && _invar == NULL) { + if (!has_iv() && _invar == nullptr) { // Need to preserve the current _offset value, so // create a temporary object for this expression subtree. // Hacky, so should re-engineer the address pattern match. @@ -3918,7 +3918,7 @@ bool SWPointer::scaled_iv(Node* n) { _scale = tmp._scale << scale; _offset += tmp._offset << scale; _invar = tmp._invar; - if (_invar != NULL) { + if (_invar != nullptr) { _negate_invar = tmp._negate_invar; _invar_scale = n->in(2); } @@ -3956,7 +3956,7 @@ bool SWPointer::offset_plus_k(Node* n, bool negate) { NOT_PRODUCT(_tracer.offset_plus_k_4(n);) return false; } - if (_invar != NULL) { // already has an invariant + if (_invar != nullptr) { // already has an invariant NOT_PRODUCT(_tracer.offset_plus_k_5(n, _invar);) return false; } @@ -4022,10 +4022,10 @@ bool SWPointer::offset_plus_k(Node* n, bool negate) { void SWPointer::print() { #ifndef PRODUCT tty->print("base: [%d] adr: [%d] scale: %d offset: %d", - _base != NULL ? _base->_idx : 0, - _adr != NULL ? _adr->_idx : 0, + _base != nullptr ? _base->_idx : 0, + _adr != nullptr ? _adr->_idx : 0, _scale, _offset); - if (_invar != NULL) { + if (_invar != nullptr) { tty->print(" invar: %c[%d] << [%d]", _negate_invar?'-':'+', _invar->_idx, _invar_scale->_idx); } tty->cr(); @@ -4221,13 +4221,13 @@ void SWPointer::Tracer::scaled_iv_9(Node* n, int scale, int offset, Node* invar, print_depth(); tty->print_cr(" %d SWPointer::scaled_iv: Op_LShiftL PASSED, setting _scale = %d, _offset = %d", n->_idx, scale, offset); print_depth(); tty->print_cr(" \\ SWPointer::scaled_iv: in(1) [%d] is scaled_iv_plus_offset, in(2) [%d] used to scale: _scale = %d, _offset = %d", n->in(1)->_idx, n->in(2)->_idx, scale, offset); - if (invar != NULL) { + if (invar != nullptr) { print_depth(); tty->print_cr(" \\ SWPointer::scaled_iv: scaled invariant: %c[%d]", (negate_invar?'-':'+'), invar->_idx); } inc_depth(); inc_depth(); print_depth(); n->in(1)->dump(); print_depth(); n->in(2)->dump(); - if (invar != NULL) { + if (invar != nullptr) { print_depth(); invar->dump(); } dec_depth(); dec_depth(); @@ -4268,7 +4268,7 @@ void SWPointer::Tracer::offset_plus_k_4(Node* n) { void SWPointer::Tracer::offset_plus_k_5(Node* n, Node* _invar) { if(_slp->is_trace_alignment()) { print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: FAILED since another invariant has been detected before", n->_idx); - print_depth(); tty->print(" \\ %d SWPointer::offset_plus_k: _invar != NULL: ", _invar->_idx); _invar->dump(); + print_depth(); tty->print(" \\ %d SWPointer::offset_plus_k: _invar is not null: ", _invar->_idx); _invar->dump(); } } @@ -4336,8 +4336,8 @@ const SWNodeInfo SWNodeInfo::initial; // Make a new dependence graph node for an ideal node. DepMem* DepGraph::make_node(Node* node) { DepMem* m = new (_arena) DepMem(node); - if (node != NULL) { - assert(_map.at_grow(node->_idx) == NULL, "one init only"); + if (node != nullptr) { + assert(_map.at_grow(node->_idx) == nullptr, "one init only"); _map.at_put_grow(node->_idx, m); } return m; @@ -4357,14 +4357,14 @@ DepEdge* DepGraph::make_edge(DepMem* dpred, DepMem* dsucc) { //------------------------------in_cnt--------------------------- int DepMem::in_cnt() { int ct = 0; - for (DepEdge* e = _in_head; e != NULL; e = e->next_in()) ct++; + for (DepEdge* e = _in_head; e != nullptr; e = e->next_in()) ct++; return ct; } //------------------------------out_cnt--------------------------- int DepMem::out_cnt() { int ct = 0; - for (DepEdge* e = _out_head; e != NULL; e = e->next_out()) ct++; + for (DepEdge* e = _out_head; e != nullptr; e = e->next_out()) ct++; return ct; } @@ -4372,14 +4372,14 @@ int DepMem::out_cnt() { void DepMem::print() { #ifndef PRODUCT tty->print(" DepNode %d (", _node->_idx); - for (DepEdge* p = _in_head; p != NULL; p = p->next_in()) { + for (DepEdge* p = _in_head; p != nullptr; p = p->next_in()) { Node* pred = p->pred()->node(); - tty->print(" %d", pred != NULL ? pred->_idx : 0); + tty->print(" %d", pred != nullptr ? pred->_idx : 0); } tty->print(") ["); - for (DepEdge* s = _out_head; s != NULL; s = s->next_out()) { + for (DepEdge* s = _out_head; s != nullptr; s = s->next_out()) { Node* succ = s->succ()->node(); - tty->print(" %d", succ != NULL ? succ->_idx : 0); + tty->print(" %d", succ != nullptr ? succ->_idx : 0); } tty->print_cr(" ]"); #endif @@ -4412,14 +4412,14 @@ DepPreds::DepPreds(Node* n, DepGraph& dg) { } else { _next_idx = 1; _end_idx = _n->req(); - _dep_next = NULL; + _dep_next = nullptr; } next(); } //------------------------------next--------------------------- void DepPreds::next() { - if (_dep_next != NULL) { + if (_dep_next != nullptr) { _current = _dep_next->pred()->node(); _dep_next = _dep_next->next_in(); } else if (_next_idx < _end_idx) { @@ -4447,14 +4447,14 @@ DepSuccs::DepSuccs(Node* n, DepGraph& dg) { } else { _next_idx = 0; _end_idx = _n->outcnt(); - _dep_next = NULL; + _dep_next = nullptr; } next(); } //-------------------------------next--------------------------- void DepSuccs::next() { - if (_dep_next != NULL) { + if (_dep_next != nullptr) { _current = _dep_next->succ()->node(); _dep_next = _dep_next->next_out(); } else if (_next_idx < _end_idx) { @@ -4468,10 +4468,10 @@ void DepSuccs::next() { // --------------------------------- vectorization/simd ----------------------------------- // bool SuperWord::same_origin_idx(Node* a, Node* b) const { - return a != NULL && b != NULL && _clone_map.same_idx(a->_idx, b->_idx); + return a != nullptr && b != nullptr && _clone_map.same_idx(a->_idx, b->_idx); } bool SuperWord::same_generation(Node* a, Node* b) const { - return a != NULL && b != NULL && _clone_map.same_gen(a->_idx, b->_idx); + return a != nullptr && b != nullptr && _clone_map.same_gen(a->_idx, b->_idx); } Node* SuperWord::find_phi_for_mem_dep(LoadNode* ld) { @@ -4483,7 +4483,7 @@ Node* SuperWord::find_phi_for_mem_dep(LoadNode* ld) { _clone_map.gen(ld->_idx)); } #endif - return NULL; //we think that any ld in the first gen being vectorizable + return nullptr; //we think that any ld in the first gen being vectorizable } Node* mem = ld->in(MemNode::Memory); @@ -4497,7 +4497,7 @@ Node* SuperWord::find_phi_for_mem_dep(LoadNode* ld) { mem->dump(); } #endif - return NULL; + return nullptr; } if (!in_bb(mem) || same_generation(mem, ld)) { #ifndef PRODUCT @@ -4506,7 +4506,7 @@ Node* SuperWord::find_phi_for_mem_dep(LoadNode* ld) { _clone_map.gen(mem->_idx)); } #endif - return NULL; // does not depend on loop volatile node or depends on the same generation + return nullptr; // does not depend on loop volatile node or depends on the same generation } //otherwise first node should depend on mem-phi @@ -4521,7 +4521,7 @@ Node* SuperWord::find_phi_for_mem_dep(LoadNode* ld) { first->dump(); } #endif - return NULL; + return nullptr; } Node* tail = 0; @@ -4539,7 +4539,7 @@ Node* SuperWord::find_phi_for_mem_dep(LoadNode* ld) { phi->dump(); } #endif - return NULL; + return nullptr; } // now all conditions are met @@ -4586,7 +4586,7 @@ Node* SuperWord::last_node(Node* nd) { } int SuperWord::mark_generations() { - Node *ii_err = NULL, *tail_err = NULL; + Node *ii_err = nullptr, *tail_err = nullptr; for (int i = 0; i < _mem_slice_head.length(); i++) { Node* phi = _mem_slice_head.at(i); assert(phi->is_Phi(), "must be phi"); @@ -4849,7 +4849,7 @@ bool SuperWord::hoist_loads_in_graph() { for (int i = 0; i < loads.length(); i++) { LoadNode* ld = loads.at(i)->as_Load(); Node* phi = find_phi_for_mem_dep(ld); - if (phi != NULL) { + if (phi != nullptr) { #ifndef PRODUCT if (_vector_loop_debug) { tty->print_cr("SuperWord::hoist_loads_in_graph replacing MemNode::Memory(%d) edge in %d with one from %d", diff --git a/src/hotspot/share/opto/superword.hpp b/src/hotspot/share/opto/superword.hpp index 9c92365865a23..f53a25c5bfb12 100644 --- a/src/hotspot/share/opto/superword.hpp +++ b/src/hotspot/share/opto/superword.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2007, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -99,7 +99,7 @@ class DepMem : public ResourceObj { DepEdge* _out_head; // Head of list of out edges, null terminated public: - DepMem(Node* node) : _node(node), _in_head(NULL), _out_head(NULL) {} + DepMem(Node* node) : _node(node), _in_head(nullptr), _out_head(nullptr) {} Node* node() { return _node; } DepEdge* in_head() { return _in_head; } @@ -122,9 +122,9 @@ class DepGraph { DepMem* _tail; public: - DepGraph(Arena* a) : _arena(a), _map(a, 8, 0, NULL) { - _root = new (_arena) DepMem(NULL); - _tail = new (_arena) DepMem(NULL); + DepGraph(Arena* a) : _arena(a), _map(a, 8, 0, nullptr) { + _root = new (_arena) DepMem(nullptr); + _tail = new (_arena) DepMem(nullptr); } DepMem* root() { return _root; } @@ -197,7 +197,7 @@ class SWNodeInfo { const Type* _velt_type; // vector element type Node_List* _my_pack; // pack containing this node - SWNodeInfo() : _alignment(-1), _depth(0), _velt_type(NULL), _my_pack(NULL) {} + SWNodeInfo() : _alignment(-1), _depth(0), _velt_type(nullptr), _my_pack(nullptr) {} static const SWNodeInfo initial; }; @@ -210,11 +210,11 @@ class CMoveKit { CMoveKit(Arena* a, SuperWord* sw) : _sw(sw) {_dict = new Dict(cmpkey, hashkey, a);} void* _2p(Node* key) const { return (void*)(intptr_t)key; } // 2 conversion functions to make gcc happy Dict* dict() const { return _dict; } - void map(Node* key, Node_List* val) { assert(_dict->operator[](_2p(key)) == NULL, "key existed"); _dict->Insert(_2p(key), (void*)val); } + void map(Node* key, Node_List* val) { assert(_dict->operator[](_2p(key)) == nullptr, "key existed"); _dict->Insert(_2p(key), (void*)val); } void unmap(Node* key) { _dict->Delete(_2p(key)); } Node_List* pack(Node* key) const { return (Node_List*)_dict->operator[](_2p(key)); } Node* is_Bool_candidate(Node* nd) const; // if it is the right candidate return corresponding CMove* , - Node* is_CmpD_candidate(Node* nd) const; // otherwise return NULL + Node* is_CmpD_candidate(Node* nd) const; // otherwise return null Node_List* make_cmovevd_pack(Node_List* cmovd_pk); bool test_cmpd_pack(Node_List* cmpd_pk, Node_List* cmovd_pk); };//class CMoveKit @@ -227,7 +227,7 @@ class OrderedPair { Node* _p1; Node* _p2; public: - OrderedPair() : _p1(NULL), _p2(NULL) {} + OrderedPair() : _p1(nullptr), _p2(nullptr) {} OrderedPair(Node* p1, Node* p2) { if (p1->_idx < p2->_idx) { _p1 = p1; _p2 = p2; @@ -341,7 +341,7 @@ class SuperWord : public ResourceObj { int iv_stride() const { return lp()->stride_con(); } CountedLoopNode* pre_loop_head() const { - assert(_pre_loop_end != NULL && _pre_loop_end->loopnode() != NULL, "should find head from pre loop end"); + assert(_pre_loop_end != nullptr && _pre_loop_end->loopnode() != nullptr, "should find head from pre loop end"); return _pre_loop_end->loopnode(); } void set_pre_loop_end(CountedLoopEndNode* pre_loop_end) { @@ -350,8 +350,8 @@ class SuperWord : public ResourceObj { } CountedLoopEndNode* pre_loop_end() const { #ifdef ASSERT - assert(_lp != NULL, "sanity"); - assert(_pre_loop_end != NULL, "should be set when fetched"); + assert(_lp != nullptr, "sanity"); + assert(_pre_loop_end != nullptr, "should be set when fetched"); Node* found_pre_end = find_pre_loop_end(_lp); assert(_pre_loop_end == found_pre_end && _pre_loop_end == pre_loop_head()->loopexit(), "should find the pre loop end and must be the same result"); @@ -374,7 +374,7 @@ class SuperWord : public ResourceObj { Node* ctrl(Node* n) const { return _phase->has_ctrl(n) ? _phase->get_ctrl(n) : n; } // block accessors - bool in_bb(Node* n) { return n != NULL && n->outcnt() > 0 && ctrl(n) == _bb; } + bool in_bb(Node* n) { return n != nullptr && n->outcnt() > 0 && ctrl(n) == _bb; } int bb_idx(Node* n) { assert(in_bb(n), "must be"); return _bb_idx.at(n->_idx); } void set_bb_idx(Node* n, int i) { _bb_idx.at_put_grow(n->_idx, i); } @@ -408,7 +408,7 @@ class SuperWord : public ResourceObj { bool same_velt_type(Node* n1, Node* n2); // my_pack - Node_List* my_pack(Node* n) { return !in_bb(n) ? NULL : _node_info.adr_at(bb_idx(n))->_my_pack; } + Node_List* my_pack(Node* n) { return !in_bb(n) ? nullptr : _node_info.adr_at(bb_idx(n))->_my_pack; } void set_my_pack(Node* n, Node_List* p) { int i = bb_idx(n); grow_node_info(i); _node_info.adr_at(i)->_my_pack = p; } // is pack good for converting into one vector node replacing 12 nodes of Cmp, Bool, CMov bool is_cmov_pack(Node_List* p); @@ -577,12 +577,12 @@ class SWPointer { MemNode* _mem; // My memory reference node SuperWord* _slp; // SuperWord class - Node* _base; // NULL if unsafe nonheap reference + Node* _base; // null if unsafe nonheap reference Node* _adr; // address pointer int _scale; // multiplier for iv (in bytes), 0 if no loop iv int _offset; // constant offset (in bytes) - Node* _invar; // invariant offset (in bytes), NULL if none + Node* _invar; // invariant offset (in bytes), null if none bool _negate_invar; // if true then use: (0 - _invar) Node* _invar_scale; // multiplier for invariant @@ -618,7 +618,7 @@ class SWPointer { // the pattern match of an address expression. SWPointer(SWPointer* p); - bool valid() { return _adr != NULL; } + bool valid() { return _adr != nullptr; } bool has_iv() { return _scale != 0; } Node* base() { return _base; } diff --git a/src/hotspot/share/opto/type.cpp b/src/hotspot/share/opto/type.cpp index 4c6fe91aee9ec..a3762dc32ff4d 100644 --- a/src/hotspot/share/opto/type.cpp +++ b/src/hotspot/share/opto/type.cpp @@ -47,7 +47,7 @@ // Optimization - Graph Style // Dictionary of types shared among compilations. -Dict* Type::_shared_type_dict = NULL; +Dict* Type::_shared_type_dict = nullptr; // Array which maps compiler types to Basic Types const Type::TypeInfo Type::_type_info[Type::lastype] = { @@ -133,8 +133,8 @@ const Type *Type::TOP; // No values in set //------------------------------get_const_type--------------------------- const Type* Type::get_const_type(ciType* type) { - if (type == NULL) { - return NULL; + if (type == nullptr) { + return nullptr; } else if (type->is_primitive_type()) { return get_const_basic_type(type->basic_type()); } else { @@ -158,16 +158,16 @@ BasicType Type::array_element_basic_type() const { } // For two instance arrays of same dimension, return the base element types. -// Otherwise or if the arrays have different dimensions, return NULL. +// Otherwise or if the arrays have different dimensions, return null. void Type::get_arrays_base_elements(const Type *a1, const Type *a2, const TypeInstPtr **e1, const TypeInstPtr **e2) { - if (e1) *e1 = NULL; - if (e2) *e2 = NULL; - const TypeAryPtr* a1tap = (a1 == NULL) ? NULL : a1->isa_aryptr(); - const TypeAryPtr* a2tap = (a2 == NULL) ? NULL : a2->isa_aryptr(); + if (e1) *e1 = nullptr; + if (e2) *e2 = nullptr; + const TypeAryPtr* a1tap = (a1 == nullptr) ? nullptr : a1->isa_aryptr(); + const TypeAryPtr* a2tap = (a2 == nullptr) ? nullptr : a2->isa_aryptr(); - if (a1tap != NULL && a2tap != NULL) { + if (a1tap != nullptr && a2tap != nullptr) { // Handle multidimensional arrays const TypePtr* a1tp = a1tap->elem()->make_ptr(); const TypePtr* a2tp = a2tap->elem()->make_ptr(); @@ -246,7 +246,7 @@ const Type* Type::make_from_constant(ciConstant constant, bool require_constant, case T_DOUBLE: return TypeD::make(constant.as_double()); case T_ARRAY: case T_OBJECT: { - const Type* con_type = NULL; + const Type* con_type = nullptr; ciObject* oop_constant = constant.as_object(); if (oop_constant->is_null_object()) { con_type = Type::get_zero_type(T_OBJECT); @@ -270,10 +270,10 @@ const Type* Type::make_from_constant(ciConstant constant, bool require_constant, case T_ILLEGAL: // Invalid ciConstant returned due to OutOfMemoryError in the CI assert(Compile::current()->env()->failing(), "otherwise should not see this"); - return NULL; + return nullptr; default: // Fall through to failure - return NULL; + return nullptr; } } @@ -312,7 +312,7 @@ const Type* Type::make_constant_from_array_element(ciArray* array, int off, int // Decode the results of GraphKit::array_element_address. ciConstant element_value = array->element_value_by_offset(off); if (element_value.basic_type() == T_ILLEGAL) { - return NULL; // wrong offset + return nullptr; // wrong offset } ciConstant con = check_mismatched_access(element_value, loadbt, is_unsigned_load); @@ -324,21 +324,21 @@ const Type* Type::make_constant_from_array_element(ciArray* array, int off, int bool is_narrow_oop = (loadbt == T_NARROWOOP); return Type::make_from_constant(con, /*require_constant=*/true, stable_dimension, is_narrow_oop, /*is_autobox_cache=*/false); } - return NULL; + return nullptr; } const Type* Type::make_constant_from_field(ciInstance* holder, int off, bool is_unsigned_load, BasicType loadbt) { ciField* field; ciType* type = holder->java_mirror_type(); - if (type != NULL && type->is_instance_klass() && off >= InstanceMirrorKlass::offset_of_static_fields()) { + if (type != nullptr && type->is_instance_klass() && off >= InstanceMirrorKlass::offset_of_static_fields()) { // Static field field = type->as_instance_klass()->get_field_by_offset(off, /*is_static=*/true); } else { // Instance field field = holder->klass()->as_instance_klass()->get_field_by_offset(off, /*is_static=*/false); } - if (field == NULL) { - return NULL; // Wrong offset + if (field == nullptr) { + return nullptr; // Wrong offset } return Type::make_constant_from_field(field, holder, loadbt, is_unsigned_load); } @@ -346,13 +346,13 @@ const Type* Type::make_constant_from_field(ciInstance* holder, int off, bool is_ const Type* Type::make_constant_from_field(ciField* field, ciInstance* holder, BasicType loadbt, bool is_unsigned_load) { if (!field->is_constant()) { - return NULL; // Non-constant field + return nullptr; // Non-constant field } ciConstant field_value; if (field->is_static()) { // final static field field_value = field->constant_value(); - } else if (holder != NULL) { + } else if (holder != nullptr) { // final or stable non-static field // Treat final non-static fields of trusted classes (classes in // java.lang.invoke and sun.invoke packages and subpackages) as @@ -360,7 +360,7 @@ const Type* Type::make_constant_from_field(ciField* field, ciInstance* holder, field_value = field->constant_value_of(holder); } if (!field_value.is_valid()) { - return NULL; // Not a constant + return nullptr; // Not a constant } ciConstant con = check_mismatched_access(field_value, loadbt, is_unsigned_load); @@ -375,7 +375,7 @@ const Type* Type::make_constant_from_field(ciField* field, ciInstance* holder, const Type* con_type = make_from_constant(con, /*require_constant=*/ true, stable_dimension, is_narrow_oop, field->is_autobox_cache()); - if (con_type != NULL && field->is_call_site_target()) { + if (con_type != nullptr && field->is_call_site_target()) { ciCallSite* call_site = holder->as_call_site(); if (!call_site->is_fully_initialized_constant_call_site()) { ciMethodHandle* target = con.as_object()->as_method_handle(); @@ -547,7 +547,7 @@ void Type::Initialize_shared(Compile* current) { false, 0, oopDesc::klass_offset_in_bytes()); TypeOopPtr::BOTTOM = TypeOopPtr::make(TypePtr::BotPTR, OffsetBot, TypeOopPtr::InstanceBot); - TypeMetadataPtr::BOTTOM = TypeMetadataPtr::make(TypePtr::BotPTR, NULL, OffsetBot); + TypeMetadataPtr::BOTTOM = TypeMetadataPtr::make(TypePtr::BotPTR, nullptr, OffsetBot); TypeNarrowOop::NULL_PTR = TypeNarrowOop::make( TypePtr::NULL_PTR ); TypeNarrowOop::BOTTOM = TypeNarrowOop::make( TypeInstPtr::BOTTOM ); @@ -564,9 +564,9 @@ void Type::Initialize_shared(Compile* current) { mreg2type[Op_RegL] = TypeLong::LONG; mreg2type[Op_RegFlags] = TypeInt::CC; - TypeAryPtr::RANGE = TypeAryPtr::make( TypePtr::BotPTR, TypeAry::make(Type::BOTTOM,TypeInt::POS), NULL /* current->env()->Object_klass() */, false, arrayOopDesc::length_offset_in_bytes()); + TypeAryPtr::RANGE = TypeAryPtr::make( TypePtr::BotPTR, TypeAry::make(Type::BOTTOM,TypeInt::POS), nullptr /* current->env()->Object_klass() */, false, arrayOopDesc::length_offset_in_bytes()); - TypeAryPtr::NARROWOOPS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeNarrowOop::BOTTOM, TypeInt::POS), NULL /*ciArrayKlass::make(o)*/, false, Type::OffsetBot); + TypeAryPtr::NARROWOOPS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeNarrowOop::BOTTOM, TypeInt::POS), nullptr /*ciArrayKlass::make(o)*/, false, Type::OffsetBot); #ifdef _LP64 if (UseCompressedOops) { @@ -576,7 +576,7 @@ void Type::Initialize_shared(Compile* current) { #endif { // There is no shared klass for Object[]. See note in TypeAryPtr::klass(). - TypeAryPtr::OOPS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInstPtr::BOTTOM,TypeInt::POS), NULL /*ciArrayKlass::make(o)*/, false, Type::OffsetBot); + TypeAryPtr::OOPS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInstPtr::BOTTOM,TypeInt::POS), nullptr /*ciArrayKlass::make(o)*/, false, Type::OffsetBot); } TypeAryPtr::BYTES = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInt::BYTE ,TypeInt::POS), ciTypeArrayKlass::make(T_BYTE), true, Type::OffsetBot); TypeAryPtr::SHORTS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(TypeInt::SHORT ,TypeInt::POS), ciTypeArrayKlass::make(T_SHORT), true, Type::OffsetBot); @@ -586,8 +586,8 @@ void Type::Initialize_shared(Compile* current) { TypeAryPtr::FLOATS = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(Type::FLOAT ,TypeInt::POS), ciTypeArrayKlass::make(T_FLOAT), true, Type::OffsetBot); TypeAryPtr::DOUBLES = TypeAryPtr::make(TypePtr::BotPTR, TypeAry::make(Type::DOUBLE ,TypeInt::POS), ciTypeArrayKlass::make(T_DOUBLE), true, Type::OffsetBot); - // Nobody should ask _array_body_type[T_NARROWOOP]. Use NULL as assert. - TypeAryPtr::_array_body_type[T_NARROWOOP] = NULL; + // Nobody should ask _array_body_type[T_NARROWOOP]. Use null as assert. + TypeAryPtr::_array_body_type[T_NARROWOOP] = nullptr; TypeAryPtr::_array_body_type[T_OBJECT] = TypeAryPtr::OOPS; TypeAryPtr::_array_body_type[T_ARRAY] = TypeAryPtr::OOPS; // arrays are stored in oop arrays TypeAryPtr::_array_body_type[T_BYTE] = TypeAryPtr::BYTES; @@ -659,7 +659,7 @@ void Type::Initialize_shared(Compile* current) { _zero_type[T_VOID] = Type::TOP; // the only void value is no value at all // get_zero_type() should not happen for T_CONFLICT - _zero_type[T_CONFLICT]= NULL; + _zero_type[T_CONFLICT]= nullptr; TypeVect::VECTMASK = (TypeVect*)(new TypeVectMask(TypeInt::BOOL, MaxVectorSize))->hashcons(); mreg2type[Op_RegVectMask] = TypeVect::VECTMASK; @@ -694,14 +694,14 @@ void Type::Initialize_shared(Compile* current) { // Restore working type arena. current->set_type_arena(save); - current->set_type_dict(NULL); + current->set_type_dict(nullptr); } //------------------------------Initialize------------------------------------- void Type::Initialize(Compile* current) { - assert(current->type_arena() != NULL, "must have created type arena"); + assert(current->type_arena() != nullptr, "must have created type arena"); - if (_shared_type_dict == NULL) { + if (_shared_type_dict == nullptr) { Initialize_shared(current); } @@ -783,7 +783,7 @@ bool Type::interface_vs_oop_helper(const Type *t) const { const TypePtr* this_ptr = this->make_ptr(); // In case it is narrow_oop const TypePtr* t_ptr = t->make_ptr(); - if( this_ptr == NULL || t_ptr == NULL ) + if( this_ptr == nullptr || t_ptr == nullptr ) return result; const TypeInstPtr* this_inst = this_ptr->isa_instptr(); @@ -802,18 +802,18 @@ bool Type::interface_vs_oop(const Type *t) const { return true; } // Now check the speculative parts as well - const TypePtr* this_spec = isa_ptr() != NULL ? is_ptr()->speculative() : NULL; - const TypePtr* t_spec = t->isa_ptr() != NULL ? t->is_ptr()->speculative() : NULL; - if (this_spec != NULL && t_spec != NULL) { + const TypePtr* this_spec = isa_ptr() != nullptr ? is_ptr()->speculative() : nullptr; + const TypePtr* t_spec = t->isa_ptr() != nullptr ? t->is_ptr()->speculative() : nullptr; + if (this_spec != nullptr && t_spec != nullptr) { if (this_spec->interface_vs_oop_helper(t_spec)) { return true; } return false; } - if (this_spec != NULL && this_spec->interface_vs_oop_helper(t)) { + if (this_spec != nullptr && this_spec->interface_vs_oop_helper(t)) { return true; } - if (t_spec != NULL && interface_vs_oop_helper(t_spec)) { + if (t_spec != nullptr && interface_vs_oop_helper(t_spec)) { return true; } return false; @@ -1561,9 +1561,9 @@ const Type *TypeInt::widen( const Type *old, const Type* limit ) const { // Only happens for pessimistic optimizations. const Type *TypeInt::narrow( const Type *old ) const { if (_lo >= _hi) return this; // already narrow enough - if (old == NULL) return this; + if (old == nullptr) return this; const TypeInt* ot = old->isa_int(); - if (ot == NULL) return this; + if (ot == nullptr) return this; jint olo = ot->_lo; jint ohi = ot->_hi; @@ -1592,7 +1592,7 @@ const Type *TypeInt::narrow( const Type *old ) const { //-----------------------------filter------------------------------------------ const Type *TypeInt::filter_helper(const Type *kills, bool include_speculative) const { const TypeInt* ft = join_helper(kills, include_speculative)->isa_int(); - if (ft == NULL || ft->empty()) + if (ft == nullptr || ft->empty()) return Type::TOP; // Canonical empty value if (ft->_widen < this->_widen) { // Do not allow the value of kill->_widen to affect the outcome. @@ -1825,9 +1825,9 @@ const Type *TypeLong::widen( const Type *old, const Type* limit ) const { // Only happens for pessimistic optimizations. const Type *TypeLong::narrow( const Type *old ) const { if (_lo >= _hi) return this; // already narrow enough - if (old == NULL) return this; + if (old == nullptr) return this; const TypeLong* ot = old->isa_long(); - if (ot == NULL) return this; + if (ot == nullptr) return this; jlong olo = ot->_lo; jlong ohi = ot->_hi; @@ -1856,7 +1856,7 @@ const Type *TypeLong::narrow( const Type *old ) const { //-----------------------------filter------------------------------------------ const Type *TypeLong::filter_helper(const Type *kills, bool include_speculative) const { const TypeLong* ft = join_helper(kills, include_speculative)->isa_long(); - if (ft == NULL || ft->empty()) + if (ft == nullptr || ft->empty()) return Type::TOP; // Canonical empty value if (ft->_widen < this->_widen) { // Do not allow the value of kill->_widen to affect the outcome. @@ -1890,10 +1890,10 @@ bool TypeLong::is_finite() const { #ifndef PRODUCT static const char* longnamenear(jlong x, const char* xname, char* buf, jlong n) { if (n > x) { - if (n >= x + 10000) return NULL; + if (n >= x + 10000) return nullptr; sprintf(buf, "%s+" JLONG_FORMAT, xname, n - x); } else if (n < x) { - if (n <= x - 10000) return NULL; + if (n <= x - 10000) return nullptr; sprintf(buf, "%s-" JLONG_FORMAT, xname, x - n); } else { return xname; @@ -1911,11 +1911,11 @@ static const char* longname(char* buf, jlong n) { return "max"; else if (n > max_jlong - 10000) sprintf(buf, "max-" JLONG_FORMAT, max_jlong - n); - else if ((str = longnamenear(max_juint, "maxuint", buf, n)) != NULL) + else if ((str = longnamenear(max_juint, "maxuint", buf, n)) != nullptr) return str; - else if ((str = longnamenear(max_jint, "maxint", buf, n)) != NULL) + else if ((str = longnamenear(max_jint, "maxint", buf, n)) != nullptr) return str; - else if ((str = longnamenear(min_jint, "minint", buf, n)) != NULL) + else if ((str = longnamenear(min_jint, "minint", buf, n)) != nullptr) return str; else sprintf(buf, JLONG_FORMAT, n); @@ -2005,7 +2005,7 @@ const TypeTuple *TypeTuple::make_domain(ciInstanceKlass* recv, ciSignature* sig) uint pos = TypeFunc::Parms; const Type **field_array; - if (recv != NULL) { + if (recv != nullptr) { arg_cnt++; field_array = fields(arg_cnt); // Use get_const_type here because it respects UseUniqueSubclasses: @@ -2269,7 +2269,7 @@ bool TypeAry::interface_vs_oop(const Type *t) const { if (t_ary) { const TypePtr* this_ptr = _elem->make_ptr(); // In case we have narrow_oops const TypePtr* t_ptr = t_ary->_elem->make_ptr(); - if(this_ptr != NULL && t_ptr != NULL) { + if(this_ptr != nullptr && t_ptr != NULL) { return this_ptr->interface_vs_oop(t_ptr); } } @@ -2307,7 +2307,7 @@ bool TypeAry::ary_must_be_exact() const { // In such cases, an array built on this ary must have no subclasses. if (_elem == BOTTOM) return false; // general array not exact if (_elem == TOP ) return false; // inverted general array not exact - const TypeOopPtr* toop = NULL; + const TypeOopPtr* toop = nullptr; if (UseCompressedOops && _elem->isa_narrowoop()) { toop = _elem->make_ptr()->isa_oopptr(); } else { @@ -2336,13 +2336,13 @@ bool TypeAry::ary_must_be_exact() const { //==============================TypeVect======================================= // Convenience common pre-built types. -const TypeVect *TypeVect::VECTA = NULL; // vector length agnostic -const TypeVect *TypeVect::VECTS = NULL; // 32-bit vectors -const TypeVect *TypeVect::VECTD = NULL; // 64-bit vectors -const TypeVect *TypeVect::VECTX = NULL; // 128-bit vectors -const TypeVect *TypeVect::VECTY = NULL; // 256-bit vectors -const TypeVect *TypeVect::VECTZ = NULL; // 512-bit vectors -const TypeVect *TypeVect::VECTMASK = NULL; // predicate/mask vector +const TypeVect *TypeVect::VECTA = nullptr; // vector length agnostic +const TypeVect *TypeVect::VECTS = nullptr; // 32-bit vectors +const TypeVect *TypeVect::VECTD = nullptr; // 64-bit vectors +const TypeVect *TypeVect::VECTX = nullptr; // 128-bit vectors +const TypeVect *TypeVect::VECTY = nullptr; // 256-bit vectors +const TypeVect *TypeVect::VECTZ = nullptr; // 512-bit vectors +const TypeVect *TypeVect::VECTMASK = nullptr; // predicate/mask vector //------------------------------make------------------------------------------- const TypeVect* TypeVect::make(const Type *elem, uint length) { @@ -2367,7 +2367,7 @@ const TypeVect* TypeVect::make(const Type *elem, uint length) { return (TypeVect*)(new TypeVectZ(elem, length))->hashcons(); } ShouldNotReachHere(); - return NULL; + return nullptr; } const TypeVect *TypeVect::makemask(const Type* elem, uint length) { @@ -2527,18 +2527,18 @@ intptr_t TypePtr::get_con() const { // Compute the MEET of two types. It returns a new Type object. const Type *TypePtr::xmeet(const Type *t) const { const Type* res = xmeet_helper(t); - if (res->isa_ptr() == NULL) { + if (res->isa_ptr() == nullptr) { return res; } const TypePtr* res_ptr = res->is_ptr(); - if (res_ptr->speculative() != NULL) { - // type->speculative() == NULL means that speculation is no better + if (res_ptr->speculative() != nullptr) { + // type->speculative() is null means that speculation is no better // than type, i.e. type->speculative() == type. So there are 2 // ways to represent the fact that we have no useful speculative // data and we should use a single one to be able to test for // equality between types. Check whether type->speculative() == - // type and set speculative to NULL if it is the case. + // type and set speculative to null if it is the case. if (res_ptr->remove_speculative() == res_ptr->speculative()) { return res_ptr->remove_speculative(); } @@ -2653,11 +2653,11 @@ int TypePtr::hash(void) const { * Return same type without a speculative part */ const Type* TypePtr::remove_speculative() const { - if (_speculative == NULL) { + if (_speculative == nullptr) { return this; } assert(_inline_depth == InlineDepthTop || _inline_depth == InlineDepthBottom, "non speculative type shouldn't have inline depth"); - return make(AnyPtr, _ptr, _offset, NULL, _inline_depth); + return make(AnyPtr, _ptr, _offset, nullptr, _inline_depth); } /** @@ -2665,7 +2665,7 @@ const Type* TypePtr::remove_speculative() const { * it */ const Type* TypePtr::cleanup_speculative() const { - if (speculative() == NULL) { + if (speculative() == nullptr) { return this; } const Type* no_spec = remove_speculative(); @@ -2682,7 +2682,7 @@ const Type* TypePtr::cleanup_speculative() const { // If the speculative may be null and is an inexact klass then it // doesn't help if (speculative() != TypePtr::NULL_PTR && speculative()->maybe_null() && - (spec_oopptr == NULL || !spec_oopptr->klass_is_exact())) { + (spec_oopptr == nullptr || !spec_oopptr->klass_is_exact())) { return no_spec; } return this; @@ -2692,8 +2692,8 @@ const Type* TypePtr::cleanup_speculative() const { * dual of the speculative part of the type */ const TypePtr* TypePtr::dual_speculative() const { - if (_speculative == NULL) { - return NULL; + if (_speculative == nullptr) { + return nullptr; } return _speculative->dual()->is_ptr(); } @@ -2704,11 +2704,11 @@ const TypePtr* TypePtr::dual_speculative() const { * @param other type to meet with */ const TypePtr* TypePtr::xmeet_speculative(const TypePtr* other) const { - bool this_has_spec = (_speculative != NULL); - bool other_has_spec = (other->speculative() != NULL); + bool this_has_spec = (_speculative != nullptr); + bool other_has_spec = (other->speculative() != nullptr); if (!this_has_spec && !other_has_spec) { - return NULL; + return nullptr; } // If we are at a point where control flow meets and one branch has @@ -2753,7 +2753,7 @@ int TypePtr::meet_inline_depth(int depth) const { * @param other type to compare this one to */ bool TypePtr::eq_speculative(const TypePtr* other) const { - if (_speculative == NULL || other->speculative() == NULL) { + if (_speculative == nullptr || other->speculative() == nullptr) { return _speculative == other->speculative(); } @@ -2768,7 +2768,7 @@ bool TypePtr::eq_speculative(const TypePtr* other) const { * Hash of the speculative part of the type */ int TypePtr::hash_speculative() const { - if (_speculative == NULL) { + if (_speculative == nullptr) { return 0; } @@ -2781,8 +2781,8 @@ int TypePtr::hash_speculative() const { * @param offset offset to add */ const TypePtr* TypePtr::add_offset_speculative(intptr_t offset) const { - if (_speculative == NULL) { - return NULL; + if (_speculative == nullptr) { + return nullptr; } return _speculative->add_offset(offset)->is_ptr(); } @@ -2791,20 +2791,20 @@ const TypePtr* TypePtr::add_offset_speculative(intptr_t offset) const { * return exact klass from the speculative type if there's one */ ciKlass* TypePtr::speculative_type() const { - if (_speculative != NULL && _speculative->isa_oopptr()) { + if (_speculative != nullptr && _speculative->isa_oopptr()) { const TypeOopPtr* speculative = _speculative->join(this)->is_oopptr(); if (speculative->klass_is_exact()) { return speculative->klass(); } } - return NULL; + return nullptr; } /** * return true if speculative type may be null */ bool TypePtr::speculative_maybe_null() const { - if (_speculative != NULL) { + if (_speculative != nullptr) { const TypePtr* speculative = _speculative->join(this)->is_ptr(); return speculative->maybe_null(); } @@ -2812,7 +2812,7 @@ bool TypePtr::speculative_maybe_null() const { } bool TypePtr::speculative_always_null() const { - if (_speculative != NULL) { + if (_speculative != nullptr) { const TypePtr* speculative = _speculative->join(this)->is_ptr(); return speculative == TypePtr::NULL_PTR; } @@ -2825,7 +2825,7 @@ bool TypePtr::speculative_always_null() const { */ ciKlass* TypePtr::speculative_type_not_null() const { if (speculative_maybe_null()) { - return NULL; + return nullptr; } return speculative_type(); } @@ -2840,14 +2840,14 @@ ciKlass* TypePtr::speculative_type_not_null() const { */ bool TypePtr::would_improve_type(ciKlass* exact_kls, int inline_depth) const { // no profiling? - if (exact_kls == NULL) { + if (exact_kls == nullptr) { return false; } if (speculative() == TypePtr::NULL_PTR) { return false; } // no speculative type or non exact speculative type? - if (speculative_type() == NULL) { + if (speculative_type() == nullptr) { return true; } // If the node already has an exact speculative type keep it, @@ -2890,7 +2890,7 @@ bool TypePtr::would_improve_ptr(ProfilePtrKind ptr_kind) const { if (speculative_always_null()) { return false; } - if (ptr_kind == ProfileAlwaysNull && speculative() != NULL && speculative()->isa_oopptr()) { + if (ptr_kind == ProfileAlwaysNull && speculative() != nullptr && speculative()->isa_oopptr()) { return false; } return true; @@ -2898,12 +2898,12 @@ bool TypePtr::would_improve_ptr(ProfilePtrKind ptr_kind) const { //------------------------------dump2------------------------------------------ const char *const TypePtr::ptr_msg[TypePtr::lastPTR] = { - "TopPTR","AnyNull","Constant","NULL","NotNull","BotPTR" + "TopPTR","AnyNull","Constant","null","NotNull","BotPTR" }; #ifndef PRODUCT void TypePtr::dump2( Dict &d, uint depth, outputStream *st ) const { - if( _ptr == Null ) st->print("NULL"); + if( _ptr == Null ) st->print("null"); else st->print("%s *", ptr_msg[_ptr]); if( _offset == OffsetTop ) st->print("+top"); else if( _offset == OffsetBot ) st->print("+bot"); @@ -2916,7 +2916,7 @@ void TypePtr::dump2( Dict &d, uint depth, outputStream *st ) const { *dump the speculative part of the type */ void TypePtr::dump_speculative(outputStream *st) const { - if (_speculative != NULL) { + if (_speculative != nullptr) { st->print(" (speculative="); _speculative->dump_on(st); st->print(")"); @@ -2957,19 +2957,19 @@ const TypeRawPtr *TypeRawPtr::NOTNULL; //------------------------------make------------------------------------------- const TypeRawPtr *TypeRawPtr::make( enum PTR ptr ) { assert( ptr != Constant, "what is the constant?" ); - assert( ptr != Null, "Use TypePtr for NULL" ); + assert( ptr != Null, "Use TypePtr for null" ); return (TypeRawPtr*)(new TypeRawPtr(ptr,0))->hashcons(); } const TypeRawPtr *TypeRawPtr::make( address bits ) { - assert( bits, "Use TypePtr for NULL" ); + assert( bits, "Use TypePtr for null" ); return (TypeRawPtr*)(new TypeRawPtr(Constant,bits))->hashcons(); } //------------------------------cast_to_ptr_type------------------------------- const Type *TypeRawPtr::cast_to_ptr_type(PTR ptr) const { assert( ptr != Constant, "what is the constant?" ); - assert( ptr != Null, "Use TypePtr for NULL" ); + assert( ptr != Null, "Use TypePtr for null" ); assert( _bits==0, "Why cast a constant address?"); if( ptr == _ptr ) return this; return make(ptr); @@ -3057,7 +3057,7 @@ const TypePtr *TypeRawPtr::add_offset( intptr_t offset ) const { } default: ShouldNotReachHere(); } - return NULL; // Lint noise + return nullptr; // Lint noise } //------------------------------eq--------------------------------------------- @@ -3105,7 +3105,7 @@ TypeOopPtr::TypeOopPtr(TYPES t, PTR ptr, ciKlass* k, bool xk, ciObject* o, int o if (_offset > 0 || _offset == Type::OffsetTop || _offset == Type::OffsetBot) { if (_offset == oopDesc::klass_offset_in_bytes()) { _is_ptr_to_narrowklass = UseCompressedClassPointers; - } else if (klass() == NULL) { + } else if (klass() == nullptr) { // Array with unknown body type assert(this->isa_aryptr(), "only arrays without klass"); _is_ptr_to_narrowoop = UseCompressedOops; @@ -3132,12 +3132,12 @@ TypeOopPtr::TypeOopPtr(TYPES t, PTR ptr, ciKlass* k, bool xk, ciObject* o, int o } else if (klass() == ciEnv::current()->Class_klass() && _offset >= InstanceMirrorKlass::offset_of_static_fields()) { // Static fields - ciField* field = NULL; - if (const_oop() != NULL) { + ciField* field = nullptr; + if (const_oop() != nullptr) { ciInstanceKlass* k = const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass(); field = k->get_field_by_offset(_offset, true); } - if (field != NULL) { + if (field != nullptr) { BasicType basic_elem_type = field->layout_type(); _is_ptr_to_narrowoop = UseCompressedOops && is_reference_type(basic_elem_type); } else { @@ -3147,7 +3147,7 @@ TypeOopPtr::TypeOopPtr(TYPES t, PTR ptr, ciKlass* k, bool xk, ciObject* o, int o } else { // Instance fields which contains a compressed oop references. field = ik->get_field_by_offset(_offset, false); - if (field != NULL) { + if (field != nullptr) { BasicType basic_elem_type = field->layout_type(); _is_ptr_to_narrowoop = UseCompressedOops && is_reference_type(basic_elem_type); } else if (klass()->equals(ciEnv::current()->Object_klass())) { @@ -3171,7 +3171,7 @@ const TypeOopPtr *TypeOopPtr::make(PTR ptr, int offset, int instance_id, assert(ptr != Constant, "no constant generic pointers"); ciKlass* k = Compile::current()->env()->Object_klass(); bool xk = false; - ciObject* o = NULL; + ciObject* o = nullptr; return (TypeOopPtr*)(new TypeOopPtr(OopPtr, ptr, k, xk, o, offset, instance_id, speculative, inline_depth))->hashcons(); } @@ -3204,7 +3204,7 @@ const Type *TypeOopPtr::cast_to_exactness(bool klass_is_exact) const { const TypeKlassPtr* TypeOopPtr::as_klass_type() const { ciKlass* k = klass(); bool xk = klass_is_exact(); - if (k == NULL) + if (k == nullptr) return TypeKlassPtr::OBJECT; else return TypeKlassPtr::make(xk? Constant: NotNull, k, 0); @@ -3286,7 +3286,7 @@ const Type *TypeOopPtr::xmeet_helper(const Type *t) const { // Dual of a pure heap pointer. No relevant klass or oop information. const Type *TypeOopPtr::xdual() const { assert(klass() == Compile::current()->env()->Object_klass(), "no klasses here"); - assert(const_oop() == NULL, "no constants here"); + assert(const_oop() == nullptr, "no constants here"); return new TypeOopPtr(_base, dual_ptr(), klass(), klass_is_exact(), const_oop(), dual_offset(), dual_instance_id(), dual_speculative(), dual_inline_depth()); } @@ -3296,7 +3296,7 @@ const TypeOopPtr* TypeOopPtr::make_from_klass_common(ciKlass *klass, bool klass_ if (klass->is_instance_klass()) { Compile* C = Compile::current(); Dependencies* deps = C->dependencies(); - assert((deps != NULL) == (C->method() != NULL && C->method()->code_size() > 0), "sanity"); + assert((deps != nullptr) == (C->method() != nullptr && C->method()->code_size() > 0), "sanity"); // Element is an instance bool klass_is_exact = false; if (klass->is_loaded()) { @@ -3304,22 +3304,22 @@ const TypeOopPtr* TypeOopPtr::make_from_klass_common(ciKlass *klass, bool klass_ ciInstanceKlass* ik = klass->as_instance_klass(); klass_is_exact = ik->is_final(); if (!klass_is_exact && klass_change - && deps != NULL && UseUniqueSubclasses) { + && deps != nullptr && UseUniqueSubclasses) { ciInstanceKlass* sub = ik->unique_concrete_subklass(); - if (sub != NULL) { + if (sub != nullptr) { deps->assert_abstract_with_unique_concrete_subtype(ik, sub); klass = ik = sub; klass_is_exact = sub->is_final(); } } - if (!klass_is_exact && try_for_exact && deps != NULL && + if (!klass_is_exact && try_for_exact && deps != nullptr && !ik->is_interface() && !ik->has_subklass()) { // Add a dependence; if concrete subclass added we need to recompile deps->assert_leaf_type(ik); klass_is_exact = true; } } - return TypeInstPtr::make(TypePtr::BotPTR, klass, klass_is_exact, NULL, 0); + return TypeInstPtr::make(TypePtr::BotPTR, klass, klass_is_exact, nullptr, 0); } else if (klass->is_obj_array_klass()) { // Element is an object array. Recursively call ourself. const TypeOopPtr *etype = TypeOopPtr::make_from_klass_common(klass->as_obj_array_klass()->element_klass(), false, try_for_exact); @@ -3327,7 +3327,7 @@ const TypeOopPtr* TypeOopPtr::make_from_klass_common(ciKlass *klass, bool klass_ const TypeAry* arr0 = TypeAry::make(etype, TypeInt::POS); // We used to pass NotNull in here, asserting that the sub-arrays // are all not-null. This is not true in generally, as code can - // slam NULLs down in the subarrays. + // slam nulls down in the subarrays. const TypeAryPtr* arr = TypeAryPtr::make(TypePtr::BotPTR, arr0, klass, xk, 0); return arr; } else if (klass->is_type_array_klass()) { @@ -3340,7 +3340,7 @@ const TypeOopPtr* TypeOopPtr::make_from_klass_common(ciKlass *klass, bool klass_ return arr; } else { ShouldNotReachHere(); - return NULL; + return nullptr; } } @@ -3357,7 +3357,7 @@ const TypeOopPtr* TypeOopPtr::make_from_constant(ciObject* o, bool require_const if (make_constant) { return TypeInstPtr::make(o); } else { - return TypeInstPtr::make(TypePtr::NotNull, klass, true, NULL, 0); + return TypeInstPtr::make(TypePtr::NotNull, klass, true, nullptr, 0); } } else if (klass->is_obj_array_klass()) { // Element is an object array. Recursively call ourself. @@ -3366,7 +3366,7 @@ const TypeOopPtr* TypeOopPtr::make_from_constant(ciObject* o, bool require_const const TypeAry* arr0 = TypeAry::make(etype, TypeInt::make(o->as_array()->length())); // We used to pass NotNull in here, asserting that the sub-arrays // are all not-null. This is not true in generally, as code can - // slam NULLs down in the subarrays. + // slam nulls down in the subarrays. if (make_constant) { return TypeAryPtr::make(TypePtr::Constant, o, arr0, klass, true, 0); } else { @@ -3387,7 +3387,7 @@ const TypeOopPtr* TypeOopPtr::make_from_constant(ciObject* o, bool require_const } fatal("unhandled object type"); - return NULL; + return nullptr; } //------------------------------get_con---------------------------------------- @@ -3467,7 +3467,7 @@ bool TypeOopPtr::eq( const Type *t ) const { _instance_id != a->_instance_id) return false; ciObject* one = const_oop(); ciObject* two = a->const_oop(); - if (one == NULL || two == NULL) { + if (one == nullptr || two == nullptr) { return (one == two) && TypePtr::eq(t); } else { return one->equals(two) && TypePtr::eq(t); @@ -3522,11 +3522,11 @@ const TypePtr *TypeOopPtr::add_offset(intptr_t offset) const { * Return same type without a speculative part */ const Type* TypeOopPtr::remove_speculative() const { - if (_speculative == NULL) { + if (_speculative == nullptr) { return this; } assert(_inline_depth == InlineDepthTop || _inline_depth == InlineDepthBottom, "non speculative type shouldn't have inline depth"); - return make(_ptr, _offset, _instance_id, NULL, _inline_depth); + return make(_ptr, _offset, _instance_id, nullptr, _inline_depth); } /** @@ -3606,8 +3606,8 @@ TypeInstPtr::TypeInstPtr(PTR ptr, ciKlass* k, bool xk, ciObject* o, int off, int instance_id, const TypePtr* speculative, int inline_depth) : TypeOopPtr(InstPtr, ptr, k, xk, o, off, instance_id, speculative, inline_depth), _name(k->name()) { - assert(k != NULL && - (k->is_loaded() || o == NULL), + assert(k != nullptr && + (k->is_loaded() || o == nullptr), "cannot have constants with non-loaded klass"); }; @@ -3621,11 +3621,11 @@ const TypeInstPtr *TypeInstPtr::make(PTR ptr, const TypePtr* speculative, int inline_depth) { assert( !k->is_loaded() || k->is_instance_klass(), "Must be for instance"); - // Either const_oop() is NULL or else ptr is Constant + // Either const_oop() is null or else ptr is Constant assert( (!o && ptr != Constant) || (o && ptr == Constant), "constant pointers must have a value supplied" ); // Ptr is never Null - assert( ptr != Null, "NULL pointers are not typed" ); + assert( ptr != Null, "null pointers are not typed" ); assert(instance_id <= 0 || xk, "instances are always exactly typed"); if (ptr == Constant) { @@ -3649,7 +3649,7 @@ const TypeInstPtr *TypeInstPtr::make(PTR ptr, */ const Type* TypeInstPtr::get_const_boxed_value() const { assert(is_ptr_to_boxed_value(), "should be called only for boxed value"); - assert((const_oop() != NULL), "should be called only for constant object"); + assert((const_oop() != nullptr), "should be called only for constant object"); ciConstant constant = const_oop()->as_instance()->field_value_by_offset(offset()); BasicType bt = constant.basic_type(); switch (bt) { @@ -3664,7 +3664,7 @@ const Type* TypeInstPtr::get_const_boxed_value() const { default: break; } fatal("Invalid boxed value type '%s'", type2name(bt)); - return NULL; + return nullptr; } //------------------------------cast_to_ptr_type------------------------------- @@ -3721,7 +3721,7 @@ const TypeInstPtr *TypeInstPtr::xmeet_unloaded(const TypeInstPtr *tinst) const { assert(loaded->ptr() != TypePtr::Null, "insanity check"); // if( loaded->ptr() == TypePtr::TopPTR ) { return unloaded; } - else if (loaded->ptr() == TypePtr::AnyNull) { return TypeInstPtr::make(ptr, unloaded->klass(), false, NULL, off, instance_id, speculative, depth); } + else if (loaded->ptr() == TypePtr::AnyNull) { return TypeInstPtr::make(ptr, unloaded->klass(), false, nullptr, off, instance_id, speculative, depth); } else if (loaded->ptr() == TypePtr::BotPTR ) { return TypeInstPtr::BOTTOM; } else if (loaded->ptr() == TypePtr::Constant || loaded->ptr() == TypePtr::NotNull) { if (unloaded->ptr() == TypePtr::BotPTR ) { return TypeInstPtr::BOTTOM; } @@ -3789,7 +3789,7 @@ const Type *TypeInstPtr::xmeet_helper(const Type *t) const { const TypePtr* speculative = xmeet_speculative(tp); int depth = meet_inline_depth(tp->inline_depth()); return make(ptr, klass(), klass_is_exact(), - (ptr == Constant ? const_oop() : NULL), offset, instance_id, speculative, depth); + (ptr == Constant ? const_oop() : nullptr), offset, instance_id, speculative, depth); } case NotNull: case BotPTR: { @@ -3817,7 +3817,7 @@ const Type *TypeInstPtr::xmeet_helper(const Type *t) const { case TopPTR: case AnyNull: { return make(ptr, klass(), klass_is_exact(), - (ptr == Constant ? const_oop() : NULL), offset, instance_id, speculative, depth); + (ptr == Constant ? const_oop() : nullptr), offset, instance_id, speculative, depth); } case NotNull: case BotPTR: @@ -3856,7 +3856,7 @@ const Type *TypeInstPtr::xmeet_helper(const Type *t) const { // and we can handle the constants further down. This case handles // both-not-loaded or both-loaded classes if (ptr != Constant && klass()->equals(tinst->klass()) && klass_is_exact() == tinst->klass_is_exact()) { - return make(ptr, klass(), klass_is_exact(), NULL, off, instance_id, speculative, depth); + return make(ptr, klass(), klass_is_exact(), nullptr, off, instance_id, speculative, depth); } // Classes require inspection in the Java klass hierarchy. Must be loaded. @@ -3919,7 +3919,7 @@ const Type *TypeInstPtr::xmeet_helper(const Type *t) const { instance_id = InstanceBot; } } - ciObject* o = NULL; // the Constant value, if any + ciObject* o = nullptr; // the Constant value, if any if (ptr == Constant) { // Find out which constant. o = (this_klass == klass()) ? const_oop() : tinst->const_oop(); @@ -3954,7 +3954,7 @@ const Type *TypeInstPtr::xmeet_helper(const Type *t) const { // centerline and or-ed above it. (N.B. Constants are always exact.) // Check for subtyping: - ciKlass *subtype = NULL; + ciKlass *subtype = nullptr; bool subtype_exact = false; if( tinst_klass->equals(this_klass) ) { subtype = this_klass; @@ -3985,13 +3985,13 @@ const Type *TypeInstPtr::xmeet_helper(const Type *t) const { // Check for classes now being equal if (tinst_klass->equals(this_klass)) { // If the klasses are equal, the constants may still differ. Fall to - // NotNull if they do (neither constant is NULL; that is a special case + // NotNull if they do (neither constant is null; that is a special case // handled elsewhere). - ciObject* o = NULL; // Assume not constant when done + ciObject* o = nullptr; // Assume not constant when done ciObject* this_oop = const_oop(); ciObject* tinst_oop = tinst->const_oop(); if( ptr == Constant ) { - if (this_oop != NULL && tinst_oop != NULL && + if (this_oop != nullptr && tinst_oop != nullptr && this_oop->equals(tinst_oop) ) o = this_oop; else if (above_centerline(this ->_ptr)) @@ -4013,7 +4013,7 @@ const Type *TypeInstPtr::xmeet_helper(const Type *t) const { // Now we find the LCA of Java classes ciKlass* k = this_klass->least_common_ancestor(tinst_klass); - return make(ptr, k, false, NULL, off, instance_id, speculative, depth); + return make(ptr, k, false, nullptr, off, instance_id, speculative, depth); } // End of case InstPtr } // End of switch @@ -4024,10 +4024,10 @@ const Type *TypeInstPtr::xmeet_helper(const Type *t) const { //------------------------java_mirror_type-------------------------------------- ciType* TypeInstPtr::java_mirror_type() const { // must be a singleton type - if( const_oop() == NULL ) return NULL; + if( const_oop() == nullptr ) return nullptr; // must be of type java.lang.Class - if( klass() != ciEnv::current()->Class_klass() ) return NULL; + if( klass() != ciEnv::current()->Class_klass() ) return nullptr; return const_oop()->as_instance()->java_mirror_type(); } @@ -4116,12 +4116,12 @@ const TypePtr *TypeInstPtr::add_offset(intptr_t offset) const { } const Type *TypeInstPtr::remove_speculative() const { - if (_speculative == NULL) { + if (_speculative == nullptr) { return this; } assert(_inline_depth == InlineDepthTop || _inline_depth == InlineDepthBottom, "non speculative type shouldn't have inline depth"); return make(_ptr, klass(), klass_is_exact(), const_oop(), _offset, - _instance_id, NULL, _inline_depth); + _instance_id, nullptr, _inline_depth); } const TypePtr *TypeInstPtr::with_inline_depth(int depth) const { @@ -4152,21 +4152,21 @@ const TypeAryPtr *TypeAryPtr::DOUBLES; //------------------------------make------------------------------------------- const TypeAryPtr *TypeAryPtr::make(PTR ptr, const TypeAry *ary, ciKlass* k, bool xk, int offset, int instance_id, const TypePtr* speculative, int inline_depth) { - assert(!(k == NULL && ary->_elem->isa_int()), + assert(!(k == nullptr && ary->_elem->isa_int()), "integral arrays must be pre-equipped with a class"); if (!xk) xk = ary->ary_must_be_exact(); assert(instance_id <= 0 || xk, "instances are always exactly typed"); - return (TypeAryPtr*)(new TypeAryPtr(ptr, NULL, ary, k, xk, offset, instance_id, false, speculative, inline_depth))->hashcons(); + return (TypeAryPtr*)(new TypeAryPtr(ptr, nullptr, ary, k, xk, offset, instance_id, false, speculative, inline_depth))->hashcons(); } //------------------------------make------------------------------------------- const TypeAryPtr *TypeAryPtr::make(PTR ptr, ciObject* o, const TypeAry *ary, ciKlass* k, bool xk, int offset, int instance_id, const TypePtr* speculative, int inline_depth, bool is_autobox_cache) { - assert(!(k == NULL && ary->_elem->isa_int()), + assert(!(k == nullptr && ary->_elem->isa_int()), "integral arrays must be pre-equipped with a class"); assert( (ptr==Constant && o) || (ptr!=Constant && !o), "" ); - if (!xk) xk = (o != NULL) || ary->ary_must_be_exact(); + if (!xk) xk = (o != nullptr) || ary->ary_must_be_exact(); assert(instance_id <= 0 || xk, "instances are always exactly typed"); return (TypeAryPtr*)(new TypeAryPtr(ptr, o, ary, k, xk, offset, instance_id, is_autobox_cache, speculative, inline_depth))->hashcons(); } @@ -4209,7 +4209,7 @@ jint TypeAryPtr::max_array_length(BasicType etype) { //-----------------------------narrow_size_type------------------------------- // Narrow the given size type to the index range for the given array base type. -// Return NULL if the resulting int type becomes empty. +// Return null if the resulting int type becomes empty. const TypeInt* TypeAryPtr::narrow_size_type(const TypeInt* size) const { jint hi = size->_hi; jint lo = size->_lo; @@ -4241,7 +4241,7 @@ const TypeInt* TypeAryPtr::narrow_size_type(const TypeInt* size) const { //-------------------------------cast_to_size---------------------------------- const TypeAryPtr* TypeAryPtr::cast_to_size(const TypeInt* new_size) const { - assert(new_size != NULL, ""); + assert(new_size != nullptr, ""); new_size = narrow_size_type(new_size); if (new_size == size()) return this; const TypeAry* new_ary = TypeAry::make(elem(), new_size, is_stable()); @@ -4256,7 +4256,7 @@ const TypeAryPtr* TypeAryPtr::cast_to_stable(bool stable, int stable_dimension) const Type* elem = this->elem(); const TypePtr* elem_ptr = elem->make_ptr(); - if (stable_dimension > 1 && elem_ptr != NULL && elem_ptr->isa_aryptr()) { + if (stable_dimension > 1 && elem_ptr != nullptr && elem_ptr->isa_aryptr()) { // If this is widened from a narrow oop, TypeAry::make will re-narrow it. elem = elem_ptr = elem_ptr->is_aryptr()->cast_to_stable(stable, stable_dimension - 1); } @@ -4271,7 +4271,7 @@ int TypeAryPtr::stable_dimension() const { if (!is_stable()) return 0; int dim = 1; const TypePtr* elem_ptr = elem()->make_ptr(); - if (elem_ptr != NULL && elem_ptr->isa_aryptr()) + if (elem_ptr != nullptr && elem_ptr->isa_aryptr()) dim += elem_ptr->is_aryptr()->stable_dimension(); return dim; } @@ -4280,7 +4280,7 @@ int TypeAryPtr::stable_dimension() const { const TypeAryPtr* TypeAryPtr::cast_to_autobox_cache() const { if (is_autobox_cache()) return this; const TypeOopPtr* etype = elem()->make_oopptr(); - if (etype == NULL) return this; + if (etype == nullptr) return this; // The pointers in the autobox arrays are always non-null. etype = etype->cast_to_ptr_type(TypePtr::NotNull)->is_oopptr(); const TypeAry* new_ary = TypeAry::make(etype, size(), is_stable()); @@ -4340,7 +4340,7 @@ const Type *TypeAryPtr::xmeet_helper(const Type *t) const { case TopPTR: case AnyNull: { int instance_id = meet_instance_id(InstanceTop); - return make(ptr, (ptr == Constant ? const_oop() : NULL), + return make(ptr, (ptr == Constant ? const_oop() : nullptr), _ary, _klass, _klass_is_exact, offset, instance_id, speculative, depth); } case BotPTR: @@ -4370,7 +4370,7 @@ const Type *TypeAryPtr::xmeet_helper(const Type *t) const { // else fall through to AnyNull case AnyNull: { int instance_id = meet_instance_id(InstanceTop); - return make(ptr, (ptr == Constant ? const_oop() : NULL), + return make(ptr, (ptr == Constant ? const_oop() : nullptr), _ary, _klass, _klass_is_exact, offset, instance_id, speculative, depth); } default: ShouldNotReachHere(); @@ -4389,13 +4389,13 @@ const Type *TypeAryPtr::xmeet_helper(const Type *t) const { int instance_id = meet_instance_id(tap->instance_id()); const TypePtr* speculative = xmeet_speculative(tap); int depth = meet_inline_depth(tap->inline_depth()); - ciKlass* lazy_klass = NULL; + ciKlass* lazy_klass = nullptr; if (tary->_elem->isa_int()) { // Integral array element types have irrelevant lattice relations. // It is the klass that determines array layout, not the element type. - if (_klass == NULL) + if (_klass == nullptr) lazy_klass = tap->_klass; - else if (tap->_klass == NULL || tap->_klass == _klass) { + else if (tap->_klass == nullptr || tap->_klass == _klass) { lazy_klass = _klass; } else { // Something like byte[int+] meets char[int+]. @@ -4408,7 +4408,7 @@ const Type *TypeAryPtr::xmeet_helper(const Type *t) const { // are not equal or super klass is exact. if ((above_centerline(ptr) || ptr == Constant) && klass() != tap->klass() && // meet with top[] and bottom[] are processed further down: - tap->_klass != NULL && this->_klass != NULL && + tap->_klass != nullptr && this->_klass != nullptr && // both are exact and not equal: ((tap->_klass_is_exact && this->_klass_is_exact) || // 'tap' is exact and super or unrelated: @@ -4418,7 +4418,7 @@ const Type *TypeAryPtr::xmeet_helper(const Type *t) const { if (above_centerline(ptr) || (tary->_elem->make_ptr() && above_centerline(tary->_elem->make_ptr()->_ptr))) { tary = TypeAry::make(Type::BOTTOM, tary->_size, tary->_stable); } - return make(NotNull, NULL, tary, lazy_klass, false, off, InstanceBot, speculative, depth); + return make(NotNull, nullptr, tary, lazy_klass, false, off, InstanceBot, speculative, depth); } bool xk = false; @@ -4435,10 +4435,10 @@ const Type *TypeAryPtr::xmeet_helper(const Type *t) const { case Constant: { ciObject* o = const_oop(); if( _ptr == Constant ) { - if( tap->const_oop() != NULL && !o->equals(tap->const_oop()) ) { + if( tap->const_oop() != nullptr && !o->equals(tap->const_oop()) ) { xk = (klass() == tap->klass()); ptr = NotNull; - o = NULL; + o = nullptr; instance_id = InstanceBot; } else { xk = true; @@ -4461,7 +4461,7 @@ const Type *TypeAryPtr::xmeet_helper(const Type *t) const { xk = (tap->_klass_is_exact & this->_klass_is_exact) && (klass() == tap->klass()); // Only precise for identical arrays } - return make(ptr, NULL, tary, lazy_klass, xk, off, instance_id, speculative, depth); + return make(ptr, nullptr, tary, lazy_klass, xk, off, instance_id, speculative, depth); default: ShouldNotReachHere(); } } @@ -4486,7 +4486,7 @@ const Type *TypeAryPtr::xmeet_helper(const Type *t) const { // cannot subclass, so the meet has to fall badly below the centerline ptr = NotNull; instance_id = InstanceBot; - return TypeInstPtr::make(ptr, ciEnv::current()->Object_klass(), false, NULL,offset, instance_id, speculative, depth); + return TypeInstPtr::make(ptr, ciEnv::current()->Object_klass(), false, nullptr,offset, instance_id, speculative, depth); } case Constant: case NotNull: @@ -4500,7 +4500,7 @@ const Type *TypeAryPtr::xmeet_helper(const Type *t) const { // to do the same here. if (tp->klass()->equals(ciEnv::current()->Object_klass()) && !tp->klass_is_exact()) { // that is, my array type is a subtype of 'tp' klass - return make(ptr, (ptr == Constant ? const_oop() : NULL), + return make(ptr, (ptr == Constant ? const_oop() : nullptr), _ary, _klass, _klass_is_exact, offset, instance_id, speculative, depth); } } @@ -4512,7 +4512,7 @@ const Type *TypeAryPtr::xmeet_helper(const Type *t) const { if (instance_id > 0) { instance_id = InstanceBot; } - return TypeInstPtr::make(ptr, ciEnv::current()->Object_klass(), false, NULL, offset, instance_id, speculative, depth); + return TypeInstPtr::make(ptr, ciEnv::current()->Object_klass(), false, nullptr, offset, instance_id, speculative, depth); default: typerr(t); } } @@ -4594,11 +4594,11 @@ const TypePtr *TypeAryPtr::add_offset(intptr_t offset) const { } const Type *TypeAryPtr::remove_speculative() const { - if (_speculative == NULL) { + if (_speculative == nullptr) { return this; } assert(_inline_depth == InlineDepthTop || _inline_depth == InlineDepthBottom, "non speculative type shouldn't have inline depth"); - return make(_ptr, _const_oop, _ary->remove_speculative()->is_ary(), _klass, _klass_is_exact, _offset, _instance_id, NULL, _inline_depth); + return make(_ptr, _const_oop, _ary->remove_speculative()->is_ary(), _klass, _klass_is_exact, _offset, _instance_id, nullptr, _inline_depth); } const TypePtr *TypeAryPtr::with_inline_depth(int depth) const { @@ -4635,7 +4635,7 @@ intptr_t TypeNarrowPtr::get_con() const { bool TypeNarrowPtr::eq( const Type *t ) const { const TypeNarrowPtr* tc = isa_same_narrowptr(t); - if (tc != NULL) { + if (tc != nullptr) { if (_ptrtype->base() != tc->_ptrtype->base()) { return false; } @@ -4766,7 +4766,7 @@ bool TypeMetadataPtr::eq( const Type *t ) const { const TypeMetadataPtr *a = (const TypeMetadataPtr*)t; ciMetadata* one = metadata(); ciMetadata* two = a->metadata(); - if (one == NULL || two == NULL) { + if (one == nullptr || two == nullptr) { return (one == two) && TypePtr::eq(t); } else { return one->equals(two) && TypePtr::eq(t); @@ -4799,7 +4799,7 @@ const TypePtr *TypeMetadataPtr::add_offset( intptr_t offset ) const { // Do not allow interface-vs.-noninterface joins to collapse to top. const Type *TypeMetadataPtr::filter_helper(const Type *kills, bool include_speculative) const { const TypeMetadataPtr* ft = join_helper(kills, include_speculative)->isa_metadataptr(); - if (ft == NULL || ft->empty()) + if (ft == nullptr || ft->empty()) return Type::TOP; // Canonical empty value return ft; } @@ -4900,7 +4900,7 @@ const Type *TypeMetadataPtr::xmeet( const Type *t ) const { if( _ptr == Constant && tptr != Constant) return this; ptr = NotNull; // Fall down in lattice } - return make(ptr, NULL, offset); + return make(ptr, nullptr, offset); break; } } // End of switch @@ -4947,7 +4947,7 @@ const TypeMetadataPtr* TypeMetadataPtr::make(ciMethodData* m) { //------------------------------make------------------------------------------- // Create a meta data constant const TypeMetadataPtr *TypeMetadataPtr::make(PTR ptr, ciMetadata* m, int offset) { - assert(m == NULL || !m->is_klass(), "wrong type"); + assert(m == nullptr || !m->is_klass(), "wrong type"); return (TypeMetadataPtr*)(new TypeMetadataPtr(ptr, m, offset))->hashcons(); } @@ -4967,7 +4967,7 @@ TypeKlassPtr::TypeKlassPtr( PTR ptr, ciKlass* klass, int offset ) //------------------------------make------------------------------------------- // ptr to klass 'k', if Constant, or possibly to a sub-klass if not a Constant const TypeKlassPtr *TypeKlassPtr::make( PTR ptr, ciKlass* k, int offset ) { - assert( k != NULL, "Expect a non-NULL klass"); + assert( k != nullptr, "Expect a non-nullptr klass"); assert(k->is_instance_klass() || k->is_array_klass(), "Incorrect type of klass oop"); TypeKlassPtr *r = (TypeKlassPtr*)(new TypeKlassPtr(ptr, k, offset))->hashcons(); @@ -5008,7 +5008,7 @@ const Type *TypeKlassPtr::filter_helper(const Type *kills, bool include_speculat const TypeKlassPtr* ktkp = kills->isa_klassptr(); if (ft->empty()) { - if (!empty() && ktkp != NULL && ktkp->klass()->is_loaded() && ktkp->klass()->is_interface()) + if (!empty() && ktkp != nullptr && ktkp->klass()->is_loaded() && ktkp->klass()->is_interface()) return kills; // Uplift to interface return Type::TOP; // Canonical empty value @@ -5016,7 +5016,7 @@ const Type *TypeKlassPtr::filter_helper(const Type *kills, bool include_speculat // Interface klass type could be exact in opposite to interface type, // return it here instead of incorrect Constant ptr J/L/Object (6894807). - if (ftkp != NULL && ktkp != NULL && + if (ftkp != nullptr && ktkp != nullptr && ftkp->is_loaded() && ftkp->klass()->is_interface() && !ftkp->klass_is_exact() && // Keep exact interface klass ktkp->is_loaded() && !ktkp->klass()->is_interface()) { @@ -5030,7 +5030,7 @@ const Type *TypeKlassPtr::filter_helper(const Type *kills, bool include_speculat // Compute the defining klass for this class ciKlass* TypeAryPtr::compute_klass(DEBUG_ONLY(bool verify)) const { // Compute _klass based on element type. - ciKlass* k_ary = NULL; + ciKlass* k_ary = nullptr; const TypeInstPtr *tinst; const TypeAryPtr *tary; const Type* el = elem(); @@ -5039,20 +5039,20 @@ ciKlass* TypeAryPtr::compute_klass(DEBUG_ONLY(bool verify)) const { } // Get element klass - if ((tinst = el->isa_instptr()) != NULL) { + if ((tinst = el->isa_instptr()) != nullptr) { // Compute array klass from element klass k_ary = ciObjArrayKlass::make(tinst->klass()); - } else if ((tary = el->isa_aryptr()) != NULL) { + } else if ((tary = el->isa_aryptr()) != nullptr) { // Compute array klass from element klass ciKlass* k_elem = tary->klass(); // If element type is something like bottom[], k_elem will be null. - if (k_elem != NULL) + if (k_elem != nullptr) k_ary = ciObjArrayKlass::make(k_elem); } else if ((el->base() == Type::Top) || (el->base() == Type::Bottom)) { // element type of Bottom occurs from meet of basic type // and object; Top occurs when doing join on Bottom. - // Leave k_ary at NULL. + // Leave k_ary at null. } else { // Cannot compute array klass directly from basic type, // since subtypes of TypeInt all have basic type T_INT. @@ -5105,7 +5105,7 @@ ciKlass* TypeAryPtr::klass() const { // a bit less efficient than caching, but calls to // TypeAryPtr::OOPS->klass() are not common enough to matter. ((TypeAryPtr*)this)->_klass = k_ary; - if (UseCompressedOops && k_ary != NULL && k_ary->is_obj_array_klass() && + if (UseCompressedOops && k_ary != nullptr && k_ary->is_obj_array_klass() && _offset != 0 && _offset != arrayOopDesc::length_offset_in_bytes()) { ((TypeAryPtr*)this)->_is_ptr_to_narrowoop = true; } @@ -5141,9 +5141,9 @@ const Type *TypeKlassPtr::cast_to_exactness(bool klass_is_exact) const { const TypeOopPtr* TypeKlassPtr::as_instance_type() const { ciKlass* k = klass(); bool xk = klass_is_exact(); - //return TypeInstPtr::make(TypePtr::NotNull, k, xk, NULL, 0); + //return TypeInstPtr::make(TypePtr::NotNull, k, xk, nullptr, 0); const TypeOopPtr* toop = TypeOopPtr::make_from_klass_raw(k); - guarantee(toop != NULL, "need type for given klass"); + guarantee(toop != nullptr, "need type for given klass"); toop = toop->cast_to_ptr_type(TypePtr::NotNull)->is_oopptr(); return toop->cast_to_exactness(xk)->is_oopptr(); } @@ -5253,7 +5253,7 @@ const Type *TypeKlassPtr::xmeet( const Type *t ) const { // Check for classes now being equal if (tkls_klass->equals(this_klass)) { // If the klasses are equal, the constants may still differ. Fall to - // NotNull if they do (neither constant is NULL; that is a special case + // NotNull if they do (neither constant is null; that is a special case // handled elsewhere). if( ptr == Constant ) { if (this->_ptr == Constant && tkls->_ptr == Constant && @@ -5355,10 +5355,10 @@ const TypeFunc *TypeFunc::make( const TypeTuple *domain, const TypeTuple *range const TypeFunc *TypeFunc::make(ciMethod* method) { Compile* C = Compile::current(); const TypeFunc* tf = C->last_tf(method); // check cache - if (tf != NULL) return tf; // The hit rate here is almost 50%. + if (tf != nullptr) return tf; // The hit rate here is almost 50%. const TypeTuple *domain; if (method->is_static()) { - domain = TypeTuple::make_domain(NULL, method->signature()); + domain = TypeTuple::make_domain(nullptr, method->signature()); } else { domain = TypeTuple::make_domain(method->holder(), method->signature()); } diff --git a/src/hotspot/share/opto/type.hpp b/src/hotspot/share/opto/type.hpp index 928920c02c5a8..302fd71ac6a8d 100644 --- a/src/hotspot/share/opto/type.hpp +++ b/src/hotspot/share/opto/type.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -176,7 +176,7 @@ class Type { // Each class of type is also identified by its base. const TYPES _base; // Enum of Types type - Type( TYPES t ) : _dual(NULL), _base(t) {} // Simple types + Type( TYPES t ) : _dual(nullptr), _base(t) {} // Simple types // ~Type(); // Use fast deallocation const Type *hashcons(); // Hash-cons the type virtual const Type *filter_helper(const Type *kills, bool include_speculative) const; @@ -284,42 +284,42 @@ class Type { double getd() const; const TypeInt *is_int() const; - const TypeInt *isa_int() const; // Returns NULL if not an Int + const TypeInt *isa_int() const; // Returns null if not an Int const TypeInteger* is_integer(BasicType bt) const; const TypeInteger* isa_integer(BasicType bt) const; const TypeLong *is_long() const; - const TypeLong *isa_long() const; // Returns NULL if not a Long - const TypeD *isa_double() const; // Returns NULL if not a Double{Top,Con,Bot} + const TypeLong *isa_long() const; // Returns null if not a Long + const TypeD *isa_double() const; // Returns null if not a Double{Top,Con,Bot} const TypeD *is_double_constant() const; // Asserts it is a DoubleCon - const TypeD *isa_double_constant() const; // Returns NULL if not a DoubleCon - const TypeF *isa_float() const; // Returns NULL if not a Float{Top,Con,Bot} + const TypeD *isa_double_constant() const; // Returns null if not a DoubleCon + const TypeF *isa_float() const; // Returns null if not a Float{Top,Con,Bot} const TypeF *is_float_constant() const; // Asserts it is a FloatCon - const TypeF *isa_float_constant() const; // Returns NULL if not a FloatCon + const TypeF *isa_float_constant() const; // Returns null if not a FloatCon const TypeTuple *is_tuple() const; // Collection of fields, NOT a pointer const TypeAry *is_ary() const; // Array, NOT array pointer - const TypeAry *isa_ary() const; // Returns NULL of not ary + const TypeAry *isa_ary() const; // Returns null of not ary const TypeVect *is_vect() const; // Vector - const TypeVect *isa_vect() const; // Returns NULL if not a Vector + const TypeVect *isa_vect() const; // Returns null if not a Vector const TypeVectMask *is_vectmask() const; // Predicate/Mask Vector - const TypeVectMask *isa_vectmask() const; // Returns NULL if not a Vector Predicate/Mask + const TypeVectMask *isa_vectmask() const; // Returns null if not a Vector Predicate/Mask const TypePtr *is_ptr() const; // Asserts it is a ptr type - const TypePtr *isa_ptr() const; // Returns NULL if not ptr type + const TypePtr *isa_ptr() const; // Returns null if not ptr type const TypeRawPtr *isa_rawptr() const; // NOT Java oop const TypeRawPtr *is_rawptr() const; // Asserts is rawptr const TypeNarrowOop *is_narrowoop() const; // Java-style GC'd pointer - const TypeNarrowOop *isa_narrowoop() const; // Returns NULL if not oop ptr type + const TypeNarrowOop *isa_narrowoop() const; // Returns null if not oop ptr type const TypeNarrowKlass *is_narrowklass() const; // compressed klass pointer - const TypeNarrowKlass *isa_narrowklass() const;// Returns NULL if not oop ptr type - const TypeOopPtr *isa_oopptr() const; // Returns NULL if not oop ptr type + const TypeNarrowKlass *isa_narrowklass() const;// Returns null if not oop ptr type + const TypeOopPtr *isa_oopptr() const; // Returns null if not oop ptr type const TypeOopPtr *is_oopptr() const; // Java-style GC'd pointer - const TypeInstPtr *isa_instptr() const; // Returns NULL if not InstPtr + const TypeInstPtr *isa_instptr() const; // Returns null if not InstPtr const TypeInstPtr *is_instptr() const; // Instance - const TypeAryPtr *isa_aryptr() const; // Returns NULL if not AryPtr + const TypeAryPtr *isa_aryptr() const; // Returns null if not AryPtr const TypeAryPtr *is_aryptr() const; // Array oop - const TypeMetadataPtr *isa_metadataptr() const; // Returns NULL if not oop ptr type + const TypeMetadataPtr *isa_metadataptr() const; // Returns null if not oop ptr type const TypeMetadataPtr *is_metadataptr() const; // Java-style GC'd pointer - const TypeKlassPtr *isa_klassptr() const; // Returns NULL if not KlassPtr + const TypeKlassPtr *isa_klassptr() const; // Returns null if not KlassPtr const TypeKlassPtr *is_klassptr() const; // assert if not KlassPtr virtual bool is_finite() const; // Has a finite value @@ -385,12 +385,12 @@ class Type { // Create basic type static const Type* get_const_basic_type(BasicType type) { - assert((uint)type <= T_CONFLICT && _const_basic_type[type] != NULL, "bad type"); + assert((uint)type <= T_CONFLICT && _const_basic_type[type] != nullptr, "bad type"); return _const_basic_type[type]; } // For two instance arrays of same dimension, return the base element types. - // Otherwise or if the arrays have different dimensions, return NULL. + // Otherwise or if the arrays have different dimensions, return null. static void get_arrays_base_elements(const Type *a1, const Type *a2, const TypeInstPtr **e1, const TypeInstPtr **e2); @@ -402,7 +402,7 @@ class Type { // Create standard zero value: static const Type* get_zero_type(BasicType type) { - assert((uint)type <= T_CONFLICT && _zero_type[type] != NULL, "bad type"); + assert((uint)type <= T_CONFLICT && _zero_type[type] != nullptr, "bad type"); return _zero_type[type]; } @@ -460,14 +460,14 @@ class Type { bool is_unsigned_load); // Speculative type helper methods. See TypePtr. - virtual const TypePtr* speculative() const { return NULL; } - virtual ciKlass* speculative_type() const { return NULL; } - virtual ciKlass* speculative_type_not_null() const { return NULL; } + virtual const TypePtr* speculative() const { return nullptr; } + virtual ciKlass* speculative_type() const { return nullptr; } + virtual ciKlass* speculative_type_not_null() const { return nullptr; } virtual bool speculative_maybe_null() const { return true; } virtual bool speculative_always_null() const { return true; } virtual const Type* remove_speculative() const { return this; } virtual const Type* cleanup_speculative() const { return this; } - virtual bool would_improve_type(ciKlass* exact_kls, int inline_depth) const { return exact_kls != NULL; } + virtual bool would_improve_type(ciKlass* exact_kls, int inline_depth) const { return exact_kls != nullptr; } virtual bool would_improve_ptr(ProfilePtrKind ptr_kind) const { return ptr_kind == ProfileAlwaysNull || ptr_kind == ProfileNeverNull; } const Type* maybe_remove_speculative(bool include_speculative) const; @@ -875,7 +875,7 @@ class TypePtr : public Type { enum PTR { TopPTR, AnyNull, Constant, Null, NotNull, BotPTR, lastPTR }; protected: TypePtr(TYPES t, PTR ptr, int offset, - const TypePtr* speculative = NULL, + const TypePtr* speculative = nullptr, int inline_depth = InlineDepthBottom) : Type(t), _speculative(speculative), _inline_depth(inline_depth), _offset(offset), _ptr(ptr) {} @@ -924,7 +924,7 @@ class TypePtr : public Type { const PTR ptr() const { return _ptr; } static const TypePtr *make(TYPES t, PTR ptr, int offset, - const TypePtr* speculative = NULL, + const TypePtr* speculative = nullptr, int inline_depth = InlineDepthBottom); // Return a 'ptr' version of this type @@ -1030,9 +1030,9 @@ class TypeOopPtr : public TypePtr { }; protected: - // Oop is NULL, unless this is a constant oop. + // Oop is null, unless this is a constant oop. ciObject* _const_oop; // Constant oop - // If _klass is NULL, then so is _sig. This is an unloaded klass. + // If _klass is null, then so is _sig. This is an unloaded klass. ciKlass* _klass; // Klass object // Does the type exclude subclasses of the klass? (Inexact == polymorphic.) bool _klass_is_exact; @@ -1073,13 +1073,13 @@ class TypeOopPtr : public TypePtr { // Creates a singleton type given an object. // If the object cannot be rendered as a constant, // may return a non-singleton type. - // If require_constant, produce a NULL if a singleton is not possible. + // If require_constant, produce a null if a singleton is not possible. static const TypeOopPtr* make_from_constant(ciObject* o, bool require_constant = false); // Make a generic (unclassed) pointer to an oop. static const TypeOopPtr* make(PTR ptr, int offset, int instance_id, - const TypePtr* speculative = NULL, + const TypePtr* speculative = nullptr, int inline_depth = InlineDepthBottom); ciObject* const_oop() const { return _const_oop; } @@ -1154,29 +1154,29 @@ class TypeInstPtr : public TypeOopPtr { // Make a pointer to some value of type klass. static const TypeInstPtr *make(PTR ptr, ciKlass* klass) { - return make(ptr, klass, false, NULL, 0, InstanceBot); + return make(ptr, klass, false, nullptr, 0, InstanceBot); } // Make a pointer to some non-polymorphic value of exactly type klass. static const TypeInstPtr *make_exact(PTR ptr, ciKlass* klass) { - return make(ptr, klass, true, NULL, 0, InstanceBot); + return make(ptr, klass, true, nullptr, 0, InstanceBot); } // Make a pointer to some value of type klass with offset. static const TypeInstPtr *make(PTR ptr, ciKlass* klass, int offset) { - return make(ptr, klass, false, NULL, offset, InstanceBot); + return make(ptr, klass, false, nullptr, offset, InstanceBot); } // Make a pointer to an oop. static const TypeInstPtr *make(PTR ptr, ciKlass* k, bool xk, ciObject* o, int offset, int instance_id = InstanceBot, - const TypePtr* speculative = NULL, + const TypePtr* speculative = nullptr, int inline_depth = InlineDepthBottom); /** Create constant type for a constant boxed value */ const Type* get_const_boxed_value() const; - // If this is a java.lang.Class constant, return the type for it or NULL. + // If this is a java.lang.Class constant, return the type for it or null. // Pass to Type::get_const_type to turn it to a type, which will usually // be a TypeInstPtr, but may also be a TypeInt::INT for int.class, etc. ciType* java_mirror_type() const; @@ -1221,7 +1221,7 @@ class TypeAryPtr : public TypeOopPtr { _is_autobox_cache(is_autobox_cache) { #ifdef ASSERT - if (k != NULL) { + if (k != nullptr) { // Verify that specified klass and TypeAryPtr::klass() follow the same rules. ciKlass* ck = compute_klass(true); if (k != ck) { @@ -1229,8 +1229,8 @@ class TypeAryPtr : public TypeOopPtr { tty->print(" k: "); k->print(); tty->cr(); tty->print("ck: "); - if (ck != NULL) ck->print(); - else tty->print(""); + if (ck != nullptr) ck->print(); + else tty->print(""); tty->cr(); assert(false, "unexpected TypeAryPtr::_klass"); } @@ -1256,12 +1256,12 @@ class TypeAryPtr : public TypeOopPtr { static const TypeAryPtr *make(PTR ptr, const TypeAry *ary, ciKlass* k, bool xk, int offset, int instance_id = InstanceBot, - const TypePtr* speculative = NULL, + const TypePtr* speculative = nullptr, int inline_depth = InlineDepthBottom); // Constant pointer to array static const TypeAryPtr *make(PTR ptr, ciObject* o, const TypeAry *ary, ciKlass* k, bool xk, int offset, int instance_id = InstanceBot, - const TypePtr* speculative = NULL, + const TypePtr* speculative = nullptr, int inline_depth = InlineDepthBottom, bool is_autobox_cache = false); // Return a 'ptr' version of this type @@ -1306,7 +1306,7 @@ class TypeAryPtr : public TypeOopPtr { static const TypeAryPtr *DOUBLES; // selects one of the above: static const TypeAryPtr *get_array_body_type(BasicType elem) { - assert((uint)elem <= T_CONFLICT && _array_body_type[elem] != NULL, "bad elem type"); + assert((uint)elem <= T_CONFLICT && _array_body_type[elem] != nullptr, "bad elem type"); return _array_body_type[elem]; } static const TypeAryPtr *_array_body_type[T_CONFLICT+1]; @@ -1606,7 +1606,7 @@ class TypeFunc : public Type { //------------------------------accessors-------------------------------------- inline bool Type::is_ptr_to_narrowoop() const { #ifdef _LP64 - return (isa_oopptr() != NULL && is_oopptr()->is_ptr_to_narrowoop_nv()); + return (isa_oopptr() != nullptr && is_oopptr()->is_ptr_to_narrowoop_nv()); #else return false; #endif @@ -1614,7 +1614,7 @@ inline bool Type::is_ptr_to_narrowoop() const { inline bool Type::is_ptr_to_narrowklass() const { #ifdef _LP64 - return (isa_oopptr() != NULL && is_oopptr()->is_ptr_to_narrowklass_nv()); + return (isa_oopptr() != nullptr && is_oopptr()->is_ptr_to_narrowklass_nv()); #else return false; #endif @@ -1636,7 +1636,7 @@ inline const TypeInteger *Type::is_integer(BasicType bt) const { } inline const TypeInteger *Type::isa_integer(BasicType bt) const { - return (((bt == T_INT && _base == Int) || (bt == T_LONG && _base == Long)) ? (TypeInteger*)this : NULL); + return (((bt == T_INT && _base == Int) || (bt == T_LONG && _base == Long)) ? (TypeInteger*)this : nullptr); } inline const TypeInt *Type::is_int() const { @@ -1645,7 +1645,7 @@ inline const TypeInt *Type::is_int() const { } inline const TypeInt *Type::isa_int() const { - return ( _base == Int ? (TypeInt*)this : NULL); + return ( _base == Int ? (TypeInt*)this : nullptr); } inline const TypeLong *Type::is_long() const { @@ -1654,13 +1654,13 @@ inline const TypeLong *Type::is_long() const { } inline const TypeLong *Type::isa_long() const { - return ( _base == Long ? (TypeLong*)this : NULL); + return ( _base == Long ? (TypeLong*)this : nullptr); } inline const TypeF *Type::isa_float() const { return ((_base == FloatTop || _base == FloatCon || - _base == FloatBot) ? (TypeF*)this : NULL); + _base == FloatBot) ? (TypeF*)this : nullptr); } inline const TypeF *Type::is_float_constant() const { @@ -1669,13 +1669,13 @@ inline const TypeF *Type::is_float_constant() const { } inline const TypeF *Type::isa_float_constant() const { - return ( _base == FloatCon ? (TypeF*)this : NULL); + return ( _base == FloatCon ? (TypeF*)this : nullptr); } inline const TypeD *Type::isa_double() const { return ((_base == DoubleTop || _base == DoubleCon || - _base == DoubleBot) ? (TypeD*)this : NULL); + _base == DoubleBot) ? (TypeD*)this : nullptr); } inline const TypeD *Type::is_double_constant() const { @@ -1684,7 +1684,7 @@ inline const TypeD *Type::is_double_constant() const { } inline const TypeD *Type::isa_double_constant() const { - return ( _base == DoubleCon ? (TypeD*)this : NULL); + return ( _base == DoubleCon ? (TypeD*)this : nullptr); } inline const TypeTuple *Type::is_tuple() const { @@ -1698,7 +1698,7 @@ inline const TypeAry *Type::is_ary() const { } inline const TypeAry *Type::isa_ary() const { - return ((_base == Array) ? (TypeAry*)this : NULL); + return ((_base == Array) ? (TypeAry*)this : nullptr); } inline const TypeVectMask *Type::is_vectmask() const { @@ -1707,7 +1707,7 @@ inline const TypeVectMask *Type::is_vectmask() const { } inline const TypeVectMask *Type::isa_vectmask() const { - return (_base == VectorMask) ? (TypeVectMask*)this : NULL; + return (_base == VectorMask) ? (TypeVectMask*)this : nullptr; } inline const TypeVect *Type::is_vect() const { @@ -1716,7 +1716,7 @@ inline const TypeVect *Type::is_vect() const { } inline const TypeVect *Type::isa_vect() const { - return (_base >= VectorMask && _base <= VectorZ) ? (TypeVect*)this : NULL; + return (_base >= VectorMask && _base <= VectorZ) ? (TypeVect*)this : nullptr; } inline const TypePtr *Type::is_ptr() const { @@ -1727,7 +1727,7 @@ inline const TypePtr *Type::is_ptr() const { inline const TypePtr *Type::isa_ptr() const { // AnyPtr is the first Ptr and KlassPtr the last, with no non-ptrs between. - return (_base >= AnyPtr && _base <= KlassPtr) ? (TypePtr*)this : NULL; + return (_base >= AnyPtr && _base <= KlassPtr) ? (TypePtr*)this : nullptr; } inline const TypeOopPtr *Type::is_oopptr() const { @@ -1738,11 +1738,11 @@ inline const TypeOopPtr *Type::is_oopptr() const { inline const TypeOopPtr *Type::isa_oopptr() const { // OopPtr is the first and KlassPtr the last, with no non-oops between. - return (_base >= OopPtr && _base <= AryPtr) ? (TypeOopPtr*)this : NULL; + return (_base >= OopPtr && _base <= AryPtr) ? (TypeOopPtr*)this : nullptr; } inline const TypeRawPtr *Type::isa_rawptr() const { - return (_base == RawPtr) ? (TypeRawPtr*)this : NULL; + return (_base == RawPtr) ? (TypeRawPtr*)this : nullptr; } inline const TypeRawPtr *Type::is_rawptr() const { @@ -1751,7 +1751,7 @@ inline const TypeRawPtr *Type::is_rawptr() const { } inline const TypeInstPtr *Type::isa_instptr() const { - return (_base == InstPtr) ? (TypeInstPtr*)this : NULL; + return (_base == InstPtr) ? (TypeInstPtr*)this : nullptr; } inline const TypeInstPtr *Type::is_instptr() const { @@ -1760,7 +1760,7 @@ inline const TypeInstPtr *Type::is_instptr() const { } inline const TypeAryPtr *Type::isa_aryptr() const { - return (_base == AryPtr) ? (TypeAryPtr*)this : NULL; + return (_base == AryPtr) ? (TypeAryPtr*)this : nullptr; } inline const TypeAryPtr *Type::is_aryptr() const { @@ -1776,7 +1776,7 @@ inline const TypeNarrowOop *Type::is_narrowoop() const { inline const TypeNarrowOop *Type::isa_narrowoop() const { // OopPtr is the first and KlassPtr the last, with no non-oops between. - return (_base == NarrowOop) ? (TypeNarrowOop*)this : NULL; + return (_base == NarrowOop) ? (TypeNarrowOop*)this : nullptr; } inline const TypeNarrowKlass *Type::is_narrowklass() const { @@ -1785,7 +1785,7 @@ inline const TypeNarrowKlass *Type::is_narrowklass() const { } inline const TypeNarrowKlass *Type::isa_narrowklass() const { - return (_base == NarrowKlass) ? (TypeNarrowKlass*)this : NULL; + return (_base == NarrowKlass) ? (TypeNarrowKlass*)this : nullptr; } inline const TypeMetadataPtr *Type::is_metadataptr() const { @@ -1795,11 +1795,11 @@ inline const TypeMetadataPtr *Type::is_metadataptr() const { } inline const TypeMetadataPtr *Type::isa_metadataptr() const { - return (_base == MetadataPtr) ? (TypeMetadataPtr*)this : NULL; + return (_base == MetadataPtr) ? (TypeMetadataPtr*)this : nullptr; } inline const TypeKlassPtr *Type::isa_klassptr() const { - return (_base == KlassPtr) ? (TypeKlassPtr*)this : NULL; + return (_base == KlassPtr) ? (TypeKlassPtr*)this : nullptr; } inline const TypeKlassPtr *Type::is_klassptr() const { @@ -1819,12 +1819,12 @@ inline const TypeOopPtr* Type::make_oopptr() const { inline const TypeNarrowOop* Type::make_narrowoop() const { return (_base == NarrowOop) ? is_narrowoop() : - (isa_ptr() ? TypeNarrowOop::make(is_ptr()) : NULL); + (isa_ptr() ? TypeNarrowOop::make(is_ptr()) : nullptr); } inline const TypeNarrowKlass* Type::make_narrowklass() const { return (_base == NarrowKlass) ? is_narrowklass() : - (isa_ptr() ? TypeNarrowKlass::make(is_ptr()) : NULL); + (isa_ptr() ? TypeNarrowKlass::make(is_ptr()) : nullptr); } inline bool Type::is_floatingpoint() const { diff --git a/src/hotspot/share/opto/vector.cpp b/src/hotspot/share/opto/vector.cpp index 65f3217886562..6f0adce1ae82f 100644 --- a/src/hotspot/share/opto/vector.cpp +++ b/src/hotspot/share/opto/vector.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -201,7 +201,7 @@ void PhaseVector::scalarize_vbox_node(VectorBoxNode* vec_box) { } jvms = kit.sync_jvms(); - Node* new_vbox = NULL; + Node* new_vbox = nullptr; { Node* vect = vec_box->in(VectorBoxNode::Value); const TypeInstPtr* vbox_type = vec_box->box_type(); @@ -283,7 +283,7 @@ void PhaseVector::scalarize_vbox_node(VectorBoxNode* vec_box) { // to the allocated object with vector value. for (uint i = jvms->debug_start(); i < jvms->debug_end(); i++) { Node* debug = sfpt->in(i); - if (debug != NULL && debug->uncast(/*keep_deps*/false) == vec_box) { + if (debug != nullptr && debug->uncast(/*keep_deps*/false) == vec_box) { sfpt->set_req(i, sobj); } } @@ -390,7 +390,7 @@ Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc, ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(), ciSymbols::object_signature(), false); - assert(field != NULL, ""); + assert(field != nullptr, ""); Node* vec_field = kit.basic_plus_adr(vec_obj, field->offset_in_bytes()); const TypePtr* vec_adr_type = vec_field->bottom_type()->is_ptr(); @@ -431,7 +431,7 @@ void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) { ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(), ciSymbols::object_signature(), false); - assert(field != NULL, ""); + assert(field != nullptr, ""); int offset = field->offset_in_bytes(); Node* vec_adr = kit.basic_plus_adr(obj, offset); diff --git a/src/hotspot/share/opto/vectorIntrinsics.cpp b/src/hotspot/share/opto/vectorIntrinsics.cpp index 06f4914199d17..c4a99ed8f6d5f 100644 --- a/src/hotspot/share/opto/vectorIntrinsics.cpp +++ b/src/hotspot/share/opto/vectorIntrinsics.cpp @@ -43,14 +43,14 @@ static bool check_vbox(const TypeInstPtr* vbox_type) { assert(is_vector(ik), "not a vector"); ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true); - assert(fd1 != NULL, "element type info is missing"); + assert(fd1 != nullptr, "element type info is missing"); ciConstant val1 = fd1->constant_value(); BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type(); assert(is_java_primitive(elem_bt), "element type info is missing"); ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true); - assert(fd2 != NULL, "vector length info is missing"); + assert(fd2 != nullptr, "vector length info is missing"); ciConstant val2 = fd2->constant_value(); assert(val2.as_int() > 0, "vector length info is missing"); @@ -82,10 +82,10 @@ Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType el assert(EnableVectorSupport, ""); const TypeInstPtr* vbox_type_v = gvn().type(v)->is_instptr(); if (vbox_type->klass() != vbox_type_v->klass()) { - return NULL; // arguments don't agree on vector shapes + return nullptr; // arguments don't agree on vector shapes } if (vbox_type_v->maybe_null()) { - return NULL; // no nulls are allowed + return nullptr; // no nulls are allowed } assert(check_vbox(vbox_type), ""); const TypeVect* vt = TypeVect::make(elem_bt, num_elem); @@ -196,10 +196,10 @@ static bool is_vector_shuffle(ciKlass* klass) { } static bool is_klass_initialized(const TypeInstPtr* vec_klass) { - if (vec_klass->const_oop() == NULL) { + if (vec_klass->const_oop() == nullptr) { return false; // uninitialized or some kind of unsafe access } - assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != NULL, "klass instance expected"); + assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected"); ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass(); return klass->is_initialized(); } @@ -228,8 +228,8 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) { const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr(); const TypeInt* vlen = gvn().type(argument(3))->isa_int(); - if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL || - !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { + if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || + !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) { if (C->print_intrinsics()) { tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s", NodeClassNames[argument(0)->Opcode()], @@ -307,11 +307,11 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) { return false; // not supported } - Node* opd1 = NULL; Node* opd2 = NULL; Node* opd3 = NULL; + Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr; switch (n) { case 3: { opd3 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem); - if (opd3 == NULL) { + if (opd3 == nullptr) { if (C->print_intrinsics()) { tty->print_cr(" ** unbox failed v3=%s", NodeClassNames[argument(6)->Opcode()]); @@ -322,7 +322,7 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) { } case 2: { opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem); - if (opd2 == NULL) { + if (opd2 == nullptr) { if (C->print_intrinsics()) { tty->print_cr(" ** unbox failed v2=%s", NodeClassNames[argument(5)->Opcode()]); @@ -333,7 +333,7 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) { } case 1: { opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem); - if (opd1 == NULL) { + if (opd1 == nullptr) { if (C->print_intrinsics()) { tty->print_cr(" ** unbox failed v1=%s", NodeClassNames[argument(4)->Opcode()]); @@ -345,11 +345,11 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) { default: fatal("unsupported arity: %d", n); } - Node* operation = NULL; + Node* operation = nullptr; if (opc == Op_CallLeafVector) { assert(UseVectorStubs, "sanity"); operation = gen_call_to_svml(opr->get_con(), elem_bt, num_elem, opd1, opd2); - if (operation == NULL) { + if (operation == nullptr) { if (C->print_intrinsics()) { tty->print_cr(" ** svml call failed for %s_%s_%d", (elem_bt == T_FLOAT)?"float":"double", @@ -393,11 +393,11 @@ bool LibraryCallKit::inline_vector_shuffle_iota() { Node* start = argument(4); Node* step = argument(5); - if (shuffle_klass == NULL || vlen == NULL || start_val == NULL || step_val == NULL || wrap == NULL) { + if (shuffle_klass == nullptr || vlen == nullptr || start_val == nullptr || step_val == nullptr || wrap == nullptr) { return false; // dead code } if (!vlen->is_con() || !is_power_of_2(vlen->get_con()) || - shuffle_klass->const_oop() == NULL || !wrap->is_con()) { + shuffle_klass->const_oop() == nullptr || !wrap->is_con()) { return false; // not enough info for intrinsification } if (!is_klass_initialized(shuffle_klass)) { @@ -486,7 +486,7 @@ bool LibraryCallKit::inline_vector_mask_operation() { const TypeInt* vlen = gvn().type(argument(3))->isa_int(); Node* mask = argument(4); - if (mask_klass == NULL || elem_klass == NULL || mask->is_top() || vlen == NULL) { + if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) { return false; // dead code } @@ -540,10 +540,10 @@ bool LibraryCallKit::inline_vector_shuffle_to_vector() { Node* shuffle = argument(3); const TypeInt* vlen = gvn().type(argument(4))->isa_int(); - if (vector_klass == NULL || elem_klass == NULL || shuffle_klass == NULL || shuffle->is_top() || vlen == NULL) { + if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) { return false; // dead code } - if (!vlen->is_con() || vector_klass->const_oop() == NULL || shuffle_klass->const_oop() == NULL) { + if (!vlen->is_con() || vector_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr) { return false; // not enough info for intrinsification } if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) { @@ -600,8 +600,8 @@ bool LibraryCallKit::inline_vector_broadcast_coerced() { const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr(); const TypeInt* vlen = gvn().type(argument(2))->isa_int(); - if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || - vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { + if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || + vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) { if (C->print_intrinsics()) { tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s", NodeClassNames[argument(0)->Opcode()], @@ -642,7 +642,7 @@ bool LibraryCallKit::inline_vector_broadcast_coerced() { Node* bits = argument(3); // long - Node* elem = NULL; + Node* elem = nullptr; switch (elem_bt) { case T_BOOLEAN: // fall-through case T_BYTE: // fall-through @@ -678,7 +678,7 @@ bool LibraryCallKit::inline_vector_broadcast_coerced() { } static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type) { - assert(arr_type != NULL, "unexpected"); + assert(arr_type != nullptr, "unexpected"); BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type(); if (elem_bt == arr_elem_bt) { return true; @@ -712,8 +712,8 @@ bool LibraryCallKit::inline_vector_mem_operation(bool is_store) { const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr(); const TypeInt* vlen = gvn().type(argument(2))->isa_int(); - if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || - vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { + if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || + vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) { if (C->print_intrinsics()) { tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s", NodeClassNames[argument(0)->Opcode()], @@ -760,17 +760,17 @@ bool LibraryCallKit::inline_vector_mem_operation(bool is_store) { SafePointNode* old_map = clone_map(); Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true); - // Can base be NULL? Otherwise, always on-heap access. + // Can base be null? Otherwise, always on-heap access. bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base)); const TypePtr *addr_type = gvn().type(addr)->isa_ptr(); const TypeAryPtr* arr_type = addr_type->isa_aryptr(); // Now handle special case where load/store happens from/to byte array but element type is not byte. - bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE; + bool using_byte_array = arr_type != nullptr && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE; // Handle loading masks. // If there is no consistency between array and vector element types, it must be special byte array case or loading masks - if (arr_type != NULL && !using_byte_array && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type)) { + if (arr_type != nullptr && !using_byte_array && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type)) { if (C->print_intrinsics()) { tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no", is_store, is_store ? "store" : "load", @@ -829,7 +829,7 @@ bool LibraryCallKit::inline_vector_mem_operation(bool is_store) { if (is_store) { Node* val = unbox_vector(argument(6), vbox_type, elem_bt, num_elem); - if (val == NULL) { + if (val == nullptr) { set_map(old_map); set_sp(old_sp); return false; // operand unboxing failed @@ -848,7 +848,7 @@ bool LibraryCallKit::inline_vector_mem_operation(bool is_store) { set_memory(vstore, addr_type); } else { // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load. - Node* vload = NULL; + Node* vload = nullptr; if (using_byte_array) { int load_num_elem = num_elem * type2aelembytes(elem_bt); vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, load_num_elem, T_BYTE)); @@ -868,7 +868,7 @@ bool LibraryCallKit::inline_vector_mem_operation(bool is_store) { set_result(box); } - old_map->destruct(&_gvn); + destruct_map_clone(old_map); if (can_access_non_heap) { insert_mem_bar(Op_MemBarCPUOrder); @@ -898,8 +898,8 @@ bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) { const TypeInt* vlen = gvn().type(argument(2))->isa_int(); const TypeInstPtr* vector_idx_klass = gvn().type(argument(3))->isa_instptr(); - if (vector_klass == NULL || elem_klass == NULL || vector_idx_klass == NULL || vlen == NULL || - vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || vector_idx_klass->const_oop() == NULL || !vlen->is_con()) { + if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr || + vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) { if (C->print_intrinsics()) { tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s", NodeClassNames[argument(0)->Opcode()], @@ -958,7 +958,7 @@ bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) { const TypeAryPtr* arr_type = addr_type->isa_aryptr(); // The array must be consistent with vector type - if (arr_type == NULL || (arr_type != NULL && !elem_consistent_with_arr(elem_bt, arr_type))) { + if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type))) { if (C->print_intrinsics()) { tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no", is_scatter, is_scatter ? "scatter" : "gather", @@ -973,7 +973,7 @@ bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) { ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass(); - if (vbox_idx_klass == NULL) { + if (vbox_idx_klass == nullptr) { set_map(old_map); set_sp(old_sp); return false; @@ -982,7 +982,7 @@ bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) { const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass); Node* index_vect = unbox_vector(argument(7), vbox_idx_type, T_INT, num_elem); - if (index_vect == NULL) { + if (index_vect == nullptr) { set_map(old_map); set_sp(old_sp); return false; @@ -990,7 +990,7 @@ bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) { const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem); if (is_scatter) { Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem); - if (val == NULL) { + if (val == nullptr) { set_map(old_map); set_sp(old_sp); return false; // operand unboxing failed @@ -1006,7 +1006,7 @@ bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) { set_result(box); } - old_map->destruct(&_gvn); + destruct_map_clone(old_map); C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); return true; @@ -1023,8 +1023,8 @@ bool LibraryCallKit::inline_vector_reduction() { const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr(); const TypeInt* vlen = gvn().type(argument(3))->isa_int(); - if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL || - !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { + if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || + !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) { if (C->print_intrinsics()) { tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s", NodeClassNames[argument(0)->Opcode()], @@ -1066,14 +1066,14 @@ bool LibraryCallKit::inline_vector_reduction() { const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); Node* opd = unbox_vector(argument(4), vbox_type, elem_bt, num_elem); - if (opd == NULL) { + if (opd == nullptr) { return false; // operand unboxing failed } Node* init = ReductionNode::make_reduction_input(gvn(), opc, elem_bt); - Node* rn = gvn().transform(ReductionNode::make(opc, NULL, init, opd, elem_bt)); + Node* rn = gvn().transform(ReductionNode::make(opc, nullptr, init, opd, elem_bt)); - Node* bits = NULL; + Node* bits = nullptr; switch (elem_bt) { case T_BYTE: case T_SHORT: @@ -1111,8 +1111,8 @@ bool LibraryCallKit::inline_vector_test() { const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr(); const TypeInt* vlen = gvn().type(argument(3))->isa_int(); - if (cond == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL || - !cond->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { + if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || + !cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) { if (C->print_intrinsics()) { tty->print_cr(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s", NodeClassNames[argument(0)->Opcode()], @@ -1152,7 +1152,7 @@ bool LibraryCallKit::inline_vector_test() { Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem); Node* opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem); - if (opd1 == NULL || opd2 == NULL) { + if (opd1 == nullptr || opd2 == nullptr) { return false; // operand unboxing failed } Node* test = new VectorTestNode(opd1, opd2, booltest); @@ -1175,11 +1175,11 @@ bool LibraryCallKit::inline_vector_blend() { const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr(); const TypeInt* vlen = gvn().type(argument(3))->isa_int(); - if (mask_klass == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) { + if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) { return false; // dead code } - if (mask_klass->const_oop() == NULL || vector_klass->const_oop() == NULL || - elem_klass->const_oop() == NULL || !vlen->is_con()) { + if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr || + elem_klass->const_oop() == nullptr || !vlen->is_con()) { if (C->print_intrinsics()) { tty->print_cr(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s", NodeClassNames[argument(0)->Opcode()], @@ -1223,7 +1223,7 @@ bool LibraryCallKit::inline_vector_blend() { Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem); Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem); - if (v1 == NULL || v2 == NULL || mask == NULL) { + if (v1 == nullptr || v2 == nullptr || mask == nullptr) { return false; // operand unboxing failed } @@ -1249,11 +1249,11 @@ bool LibraryCallKit::inline_vector_compare() { const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr(); const TypeInt* vlen = gvn().type(argument(4))->isa_int(); - if (cond == NULL || vector_klass == NULL || mask_klass == NULL || elem_klass == NULL || vlen == NULL) { + if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) { return false; // dead code } - if (!cond->is_con() || vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL || - elem_klass->const_oop() == NULL || !vlen->is_con()) { + if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || + elem_klass->const_oop() == nullptr || !vlen->is_con()) { if (C->print_intrinsics()) { tty->print_cr(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s", NodeClassNames[argument(0)->Opcode()], @@ -1309,7 +1309,7 @@ bool LibraryCallKit::inline_vector_compare() { Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem); Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem); - if (v1 == NULL || v2 == NULL) { + if (v1 == nullptr || v2 == nullptr) { return false; // operand unboxing failed } BoolTest::mask pred = (BoolTest::mask)cond->get_con(); @@ -1336,11 +1336,11 @@ bool LibraryCallKit::inline_vector_rearrange() { const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr(); const TypeInt* vlen = gvn().type(argument(3))->isa_int(); - if (vector_klass == NULL || shuffle_klass == NULL || elem_klass == NULL || vlen == NULL) { + if (vector_klass == nullptr || shuffle_klass == nullptr || elem_klass == nullptr || vlen == nullptr) { return false; // dead code } - if (shuffle_klass->const_oop() == NULL || vector_klass->const_oop() == NULL || - elem_klass->const_oop() == NULL || !vlen->is_con()) { + if (shuffle_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr || + elem_klass->const_oop() == nullptr || !vlen->is_con()) { if (C->print_intrinsics()) { tty->print_cr(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s", NodeClassNames[argument(0)->Opcode()], @@ -1390,7 +1390,7 @@ bool LibraryCallKit::inline_vector_rearrange() { Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem); Node* shuffle = unbox_vector(argument(5), shbox_type, shuffle_bt, num_elem); - if (v1 == NULL || shuffle == NULL) { + if (v1 == nullptr || shuffle == nullptr) { return false; // operand unboxing failed } @@ -1403,9 +1403,9 @@ bool LibraryCallKit::inline_vector_rearrange() { } static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) { - address addr = NULL; + address addr = nullptr; assert(UseVectorStubs, "sanity"); - assert(name_ptr != NULL, "unexpected"); + assert(name_ptr != nullptr, "unexpected"); assert((vop >= VectorSupport::VECTOR_OP_SVML_START) && (vop <= VectorSupport::VECTOR_OP_SVML_END), "unexpected"); int op = vop - VectorSupport::VECTOR_OP_SVML_START; @@ -1425,7 +1425,7 @@ static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, break; default: snprintf(name_ptr, name_len, "invalid"); - addr = NULL; + addr = nullptr; Unimplemented(); break; } @@ -1436,19 +1436,19 @@ static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) { assert(UseVectorStubs, "sanity"); assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id"); - assert(opd1 != NULL, "must not be null"); + assert(opd1 != nullptr, "must not be null"); const TypeVect* vt = TypeVect::make(bt, num_elem); - const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != NULL ? 2 : 1, vt, vt); + const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != nullptr ? 2 : 1, vt, vt); char name[100] = ""; // Get address for svml method. address addr = get_svml_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100); - if (addr == NULL) { - return NULL; + if (addr == nullptr) { + return nullptr; } - assert(name != NULL, "name must not be null"); + assert(name[0] != '\0', "name must not be null"); Node* operation = make_runtime_call(RC_VECTOR, call_type, addr, @@ -1471,10 +1471,10 @@ bool LibraryCallKit::inline_vector_broadcast_int() { const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr(); const TypeInt* vlen = gvn().type(argument(3))->isa_int(); - if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) { + if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) { return false; // dead code } - if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) { + if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) { if (C->print_intrinsics()) { tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s", NodeClassNames[argument(0)->Opcode()], @@ -1525,7 +1525,7 @@ bool LibraryCallKit::inline_vector_broadcast_int() { } Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem); Node* opd2 = vector_shift_count(argument(5), opc, elem_bt, num_elem); - if (opd1 == NULL || opd2 == NULL) { + if (opd1 == nullptr || opd2 == nullptr) { return false; } Node* operation = gvn().transform(VectorNode::make(opc, opd1, opd2, num_elem, elem_bt)); @@ -1556,14 +1556,14 @@ bool LibraryCallKit::inline_vector_convert() { const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr(); const TypeInt* vlen_to = gvn().type(argument(6))->isa_int(); - if (opr == NULL || - vector_klass_from == NULL || elem_klass_from == NULL || vlen_from == NULL || - vector_klass_to == NULL || elem_klass_to == NULL || vlen_to == NULL) { + if (opr == nullptr || + vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr || + vector_klass_to == nullptr || elem_klass_to == nullptr || vlen_to == nullptr) { return false; // dead code } if (!opr->is_con() || - vector_klass_from->const_oop() == NULL || elem_klass_from->const_oop() == NULL || !vlen_from->is_con() || - vector_klass_to->const_oop() == NULL || elem_klass_to->const_oop() == NULL || !vlen_to->is_con()) { + vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() || + vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) { if (C->print_intrinsics()) { tty->print_cr(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s", NodeClassNames[argument(0)->Opcode()], @@ -1648,7 +1648,7 @@ bool LibraryCallKit::inline_vector_convert() { const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from); Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from); - if (opd1 == NULL) { + if (opd1 == nullptr) { return false; } @@ -1740,10 +1740,10 @@ bool LibraryCallKit::inline_vector_insert() { const TypeInt* vlen = gvn().type(argument(2))->isa_int(); const TypeInt* idx = gvn().type(argument(4))->isa_int(); - if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) { + if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) { return false; // dead code } - if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) { + if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) { if (C->print_intrinsics()) { tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s", NodeClassNames[argument(0)->Opcode()], @@ -1780,12 +1780,12 @@ bool LibraryCallKit::inline_vector_insert() { const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem); - if (opd == NULL) { + if (opd == nullptr) { return false; } Node* insert_val = argument(5); - assert(gvn().type(insert_val)->isa_long() != NULL, "expected to be long"); + assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long"); // Convert insert value back to its appropriate type. switch (elem_bt) { @@ -1833,10 +1833,10 @@ bool LibraryCallKit::inline_vector_extract() { const TypeInt* vlen = gvn().type(argument(2))->isa_int(); const TypeInt* idx = gvn().type(argument(4))->isa_int(); - if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) { + if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) { return false; // dead code } - if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) { + if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) { if (C->print_intrinsics()) { tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s", NodeClassNames[argument(0)->Opcode()], @@ -1874,13 +1874,13 @@ bool LibraryCallKit::inline_vector_extract() { const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem); - if (opd == NULL) { + if (opd == nullptr) { return false; } Node* operation = gvn().transform(ExtractNode::make(opd, idx->get_con(), elem_bt)); - Node* bits = NULL; + Node* bits = nullptr; switch (elem_bt) { case T_BYTE: case T_SHORT: diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 39ddc25a80647..b9ef5abd75a67 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -506,7 +506,7 @@ VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, const TypeVect* vt) { case Op_MulAddVS2VI: return new MulAddVS2VINode(n1, n2, vt); default: fatal("Missed vector creation for '%s'", NodeClassNames[vopc]); - return NULL; + return nullptr; } } @@ -528,7 +528,7 @@ VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, Node* n3, const TypeV case Op_FmaVF: return new FmaVFNode(n1, n2, n3, vt); default: fatal("Missed vector creation for '%s'", NodeClassNames[vopc]); - return NULL; + return nullptr; } } @@ -563,7 +563,7 @@ VectorNode* VectorNode::scalar2vector(Node* s, uint vlen, const Type* opd_t) { return new ReplicateDNode(s, vt); default: fatal("Type '%s' is not supported for vectors", type2name(bt)); - return NULL; + return nullptr; } } @@ -583,7 +583,7 @@ VectorNode* VectorNode::shift_count(int opc, Node* cnt, uint vlen, BasicType bt) return new RShiftCntVNode(cnt, vt); default: fatal("Missed vector creation for '%s'", NodeClassNames[opc]); - return NULL; + return nullptr; } } @@ -672,7 +672,7 @@ PackNode* PackNode::make(Node* s, uint vlen, BasicType bt) { return new PackDNode(s, vt); default: fatal("Type '%s' is not supported for vectors", type2name(bt)); - return NULL; + return nullptr; } } @@ -708,7 +708,7 @@ PackNode* PackNode::binary_tree_pack(int lo, int hi) { return new Pack2DNode(n1, n2, TypeVect::make(T_DOUBLE, 2)); default: fatal("Type '%s' is not supported for vectors", type2name(bt)); - return NULL; + return nullptr; } } } @@ -799,7 +799,7 @@ Node* ExtractNode::make(Node* v, uint position, BasicType bt) { case T_DOUBLE: return new ExtractDNode(v, pos); default: assert(false, "wrong type: %s", type2name(bt)); - return NULL; + return nullptr; } } @@ -980,7 +980,7 @@ ReductionNode* ReductionNode::make(int opc, Node *ctrl, Node* n1, Node* n2, Basi case Op_XorReductionV: return new XorReductionVNode(ctrl, n1, n2); default: assert(false, "unknown node: %s", NodeClassNames[vopc]); - return NULL; + return nullptr; } } @@ -1020,7 +1020,7 @@ VectorCastNode* VectorCastNode::make(int vopc, Node* n1, BasicType bt, uint vlen case Op_VectorCastD2X: return new VectorCastD2XNode(n1, vt); default: assert(false, "unknown node: %s", NodeClassNames[vopc]); - return NULL; + return nullptr; } } @@ -1064,7 +1064,7 @@ Node* ReductionNode::make_reduction_input(PhaseGVN& gvn, int opc, BasicType bt) return gvn.makecon(TypeLong::MINUS_1); default: fatal("Missed vector creation for '%s' as the basic type is not correct.", NodeClassNames[vopc]); - return NULL; + return nullptr; } break; case Op_AddReductionVI: // fallthrough @@ -1094,7 +1094,7 @@ Node* ReductionNode::make_reduction_input(PhaseGVN& gvn, int opc, BasicType bt) return gvn.makecon(TypeF::POS_INF); case T_DOUBLE: return gvn.makecon(TypeD::POS_INF); - default: Unimplemented(); return NULL; + default: Unimplemented(); return nullptr; } break; case Op_MaxReductionV: @@ -1109,12 +1109,12 @@ Node* ReductionNode::make_reduction_input(PhaseGVN& gvn, int opc, BasicType bt) return gvn.makecon(TypeF::NEG_INF); case T_DOUBLE: return gvn.makecon(TypeD::NEG_INF); - default: Unimplemented(); return NULL; + default: Unimplemented(); return nullptr; } break; default: fatal("Missed vector creation for '%s'", NodeClassNames[vopc]); - return NULL; + return nullptr; } } @@ -1149,8 +1149,8 @@ Node* VectorNode::degenerate_vector_rotate(Node* src, Node* cnt, bool is_rotate_ // Compute shift values for right rotation and // later swap them in case of left rotation. - Node* shiftRCnt = NULL; - Node* shiftLCnt = NULL; + Node* shiftRCnt = nullptr; + Node* shiftLCnt = nullptr; const TypeInt* cnt_type = cnt->bottom_type()->isa_int(); bool is_binary_vector_op = false; if (cnt_type && cnt_type->is_con()) { @@ -1219,7 +1219,7 @@ Node* RotateLeftVNode::Ideal(PhaseGVN* phase, bool can_reshape) { !Matcher::match_rule_supported_vector(Op_RotateLeftV, vlen, bt)) { return VectorNode::degenerate_vector_rotate(in(1), in(2), true, vlen, bt, phase); } - return NULL; + return nullptr; } Node* RotateRightVNode::Ideal(PhaseGVN* phase, bool can_reshape) { @@ -1229,7 +1229,7 @@ Node* RotateRightVNode::Ideal(PhaseGVN* phase, bool can_reshape) { !Matcher::match_rule_supported_vector(Op_RotateRightV, vlen, bt)) { return VectorNode::degenerate_vector_rotate(in(1), in(2), false, vlen, bt, phase); } - return NULL; + return nullptr; } #ifndef PRODUCT @@ -1299,7 +1299,7 @@ Node* VectorUnboxNode::Ideal(PhaseGVN* phase, bool can_reshape) { } } } - return NULL; + return nullptr; } Node* VectorUnboxNode::Identity(PhaseGVN* phase) { @@ -1345,7 +1345,7 @@ Node* VectorMaskOpNode::make(Node* mask, const Type* ty, int mopc) { default: assert(false, "Unhandled operation"); } - return NULL; + return nullptr; } diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index 7ba0149777fee..ccb5fd88bf6a1 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -1063,7 +1063,7 @@ class VectorLoadConstNode : public VectorNode { // Extract a scalar from a vector at position "pos" class ExtractNode : public Node { public: - ExtractNode(Node* src, ConINode* pos) : Node(NULL, src, (Node*)pos) { + ExtractNode(Node* src, ConINode* pos) : Node(nullptr, src, (Node*)pos) { assert(in(2)->get_int() >= 0, "positive constants"); } virtual int Opcode() const; @@ -1412,13 +1412,13 @@ class VectorBoxNode : public Node { }; VectorBoxNode(Compile* C, Node* box, Node* val, const TypeInstPtr* box_type, const TypeVect* vt) - : Node(NULL, box, val), _box_type(box_type), _vec_type(vt) { + : Node(nullptr, box, val), _box_type(box_type), _vec_type(vt) { init_flags(Flag_is_macro); C->add_macro_node(this); } - const TypeInstPtr* box_type() const { assert(_box_type != NULL, ""); return _box_type; }; - const TypeVect* vec_type() const { assert(_vec_type != NULL, ""); return _vec_type; }; + const TypeInstPtr* box_type() const { assert(_box_type != nullptr, ""); return _box_type; }; + const TypeVect* vec_type() const { assert(_vec_type != nullptr, ""); return _vec_type; }; virtual int Opcode() const; virtual const Type* bottom_type() const { return _box_type; } @@ -1431,7 +1431,7 @@ class VectorBoxNode : public Node { class VectorBoxAllocateNode : public CallStaticJavaNode { public: VectorBoxAllocateNode(Compile* C, const TypeInstPtr* vbox_type) - : CallStaticJavaNode(C, VectorBoxNode::vec_box_type(vbox_type), NULL, NULL) { + : CallStaticJavaNode(C, VectorBoxNode::vec_box_type(vbox_type), nullptr, nullptr) { init_flags(Flag_is_macro); C->add_macro_node(this); } diff --git a/src/hotspot/share/prims/forte.cpp b/src/hotspot/share/prims/forte.cpp index 9b22143caedb5..ac71146b5d404 100644 --- a/src/hotspot/share/prims/forte.cpp +++ b/src/hotspot/share/prims/forte.cpp @@ -598,6 +598,9 @@ void AsyncGetCallTrace(ASGCT_CallTrace *trace, jint depth, void* ucontext) { // !important! make sure all to call thread->set_in_asgct(false) before every return thread->set_in_asgct(true); + // signify to other code in the VM that we're in ASGCT + ThreadInAsgct tia(thread); + switch (thread->thread_state()) { case _thread_new: case _thread_uninitialized: diff --git a/src/hotspot/share/prims/jni.cpp b/src/hotspot/share/prims/jni.cpp index cd01152484132..68a29f8bc7890 100644 --- a/src/hotspot/share/prims/jni.cpp +++ b/src/hotspot/share/prims/jni.cpp @@ -625,7 +625,7 @@ JNI_ENTRY(void, jni_FatalError(JNIEnv *env, const char *msg)) HOTSPOT_JNI_FATALERROR_ENTRY(env, (char *) msg); tty->print_cr("FATAL ERROR in native method: %s", msg); - thread->print_stack(); + thread->print_jni_stack(); os::abort(); // Dump core and abort JNI_END @@ -2234,7 +2234,7 @@ JNI_ENTRY(const char*, jni_GetStringUTFChars(JNIEnv *env, jstring string, jboole if (s_value != NULL) { size_t length = java_lang_String::utf8_length(java_string, s_value); /* JNI Specification states return NULL on OOM */ - result = AllocateHeap(length + 1, mtInternal, 0, AllocFailStrategy::RETURN_NULL); + result = AllocateHeap(length + 1, mtInternal, AllocFailStrategy::RETURN_NULL); if (result != NULL) { java_lang_String::as_utf8_string(java_string, s_value, result, (int) length + 1); if (isCopy != NULL) { diff --git a/src/hotspot/share/prims/jniCheck.cpp b/src/hotspot/share/prims/jniCheck.cpp index 34b05d339b03b..52fa413fa1fba 100644 --- a/src/hotspot/share/prims/jniCheck.cpp +++ b/src/hotspot/share/prims/jniCheck.cpp @@ -145,7 +145,7 @@ static const char * fatal_non_utf8_class_name2 = "\""; // When in VM state: static void ReportJNIWarning(JavaThread* thr, const char *msg) { tty->print_cr("WARNING in native method: %s", msg); - thr->print_stack(); + thr->print_jni_stack(); } // When in NATIVE state: @@ -196,7 +196,7 @@ check_pending_exception(JavaThread* thr) { IN_VM( tty->print_cr("WARNING in native method: JNI call made without checking exceptions when required to from %s", thr->get_pending_jni_exception_check()); - thr->print_stack(); + thr->print_jni_stack(); ) thr->clear_pending_jni_exception_check(); // Just complain once } diff --git a/src/hotspot/share/prims/jniCheck.hpp b/src/hotspot/share/prims/jniCheck.hpp index d5de95ea8e444..3dc8c5c59398d 100644 --- a/src/hotspot/share/prims/jniCheck.hpp +++ b/src/hotspot/share/prims/jniCheck.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -35,7 +35,7 @@ extern "C" { // When in VM state: static inline void ReportJNIFatalError(JavaThread* thr, const char *msg) { tty->print_cr("FATAL ERROR in native method: %s", msg); - thr->print_stack(); + thr->print_jni_stack(); os::abort(true); } } diff --git a/src/hotspot/share/prims/jvm.cpp b/src/hotspot/share/prims/jvm.cpp index 8baba8c38b3c8..9e29c3300cfce 100644 --- a/src/hotspot/share/prims/jvm.cpp +++ b/src/hotspot/share/prims/jvm.cpp @@ -2976,12 +2976,6 @@ JVM_ENTRY(void, JVM_StopThread(JNIEnv* env, jobject jthread, jobject throwable)) JVM_END -JVM_ENTRY(jboolean, JVM_IsThreadAlive(JNIEnv* env, jobject jthread)) - oop thread_oop = JNIHandles::resolve_non_null(jthread); - return java_lang_Thread::is_alive(thread_oop); -JVM_END - - JVM_ENTRY(void, JVM_SuspendThread(JNIEnv* env, jobject jthread)) ThreadsListHandle tlh(thread); JavaThread* receiver = NULL; diff --git a/src/hotspot/share/prims/jvmtiEnv.cpp b/src/hotspot/share/prims/jvmtiEnv.cpp index 2107b186c5ecc..f04dc7dd27ca2 100644 --- a/src/hotspot/share/prims/jvmtiEnv.cpp +++ b/src/hotspot/share/prims/jvmtiEnv.cpp @@ -1326,28 +1326,19 @@ JvmtiEnv::RunAgentThread(jthread thread, jvmtiStartFunction proc, const void* ar } Handle thread_hndl(current_thread, thread_oop); - { - MutexLocker mu(current_thread, Threads_lock); // grab Threads_lock - JvmtiAgentThread *new_thread = new JvmtiAgentThread(this, proc, arg); + JvmtiAgentThread* new_thread = new JvmtiAgentThread(this, proc, arg); - // At this point it may be possible that no osthread was created for the - // JavaThread due to lack of memory. - if (new_thread == NULL || new_thread->osthread() == NULL) { - if (new_thread != NULL) { - new_thread->smr_delete(); - } - return JVMTI_ERROR_OUT_OF_MEMORY; - } - - java_lang_Thread::set_thread(thread_hndl(), new_thread); - java_lang_Thread::set_priority(thread_hndl(), (ThreadPriority)priority); - java_lang_Thread::set_daemon(thread_hndl()); + // At this point it may be possible that no osthread was created for the + // JavaThread due to lack of resources. + if (new_thread->osthread() == NULL) { + // The new thread is not known to Thread-SMR yet so we can just delete. + delete new_thread; + return JVMTI_ERROR_OUT_OF_MEMORY; + } - new_thread->set_threadObj(thread_hndl()); - Threads::add(new_thread); - Thread::start(new_thread); - } // unlock Threads_lock + JavaThread::start_internal_daemon(current_thread, new_thread, thread_hndl, + (ThreadPriority)priority); return JVMTI_ERROR_NONE; } /* end RunAgentThread */ diff --git a/src/hotspot/share/prims/jvmtiExport.cpp b/src/hotspot/share/prims/jvmtiExport.cpp index 0c8a43680fada..f5c5bacc9f3d7 100644 --- a/src/hotspot/share/prims/jvmtiExport.cpp +++ b/src/hotspot/share/prims/jvmtiExport.cpp @@ -808,8 +808,6 @@ JvmtiExport::cv_external_thread_to_JavaThread(ThreadsList * t_list, } // Looks like a live JavaThread at this point. - // We do not check the EnableThreadSMRExtraValidityChecks option - // for this includes() call because JVM/TI's spec is tighter. if (!t_list->includes(java_thread)) { // Not on the JavaThreads list so it is not alive. return JVMTI_ERROR_THREAD_NOT_ALIVE; @@ -851,8 +849,6 @@ JvmtiExport::cv_oop_to_JavaThread(ThreadsList * t_list, oop thread_oop, } // Looks like a live JavaThread at this point. - // We do not check the EnableThreadSMRExtraValidityChecks option - // for this includes() call because JVM/TI's spec is tighter. if (!t_list->includes(java_thread)) { // Not on the JavaThreads list so it is not alive. return JVMTI_ERROR_THREAD_NOT_ALIVE; @@ -2216,6 +2212,7 @@ void JvmtiExport::post_compiled_method_load(JvmtiEnv* env, nmethod *nm) { ResourceMark rm(thread); HandleMark hm(thread); + assert(!nm->is_zombie(), "nmethod zombie in post_compiled_method_load"); // Add inlining information jvmtiCompiledMethodLoadInlineRecord* inlinerecord = create_inline_record(nm); // Pass inlining information through the void pointer diff --git a/src/hotspot/share/prims/jvmtiImpl.cpp b/src/hotspot/share/prims/jvmtiImpl.cpp index ba79e19392c42..c542f110122c6 100644 --- a/src/hotspot/share/prims/jvmtiImpl.cpp +++ b/src/hotspot/share/prims/jvmtiImpl.cpp @@ -960,10 +960,10 @@ JvmtiDeferredEvent JvmtiDeferredEventQueue::dequeue() { } void JvmtiDeferredEventQueue::post(JvmtiEnv* env) { - // Post and destroy queue nodes + // Post events while nmethods are still in the queue and can't be unloaded or made zombie while (_queue_head != NULL) { - JvmtiDeferredEvent event = dequeue(); - event.post_compiled_method_load_event(env); + _queue_head->event().post_compiled_method_load_event(env); + dequeue(); } } diff --git a/src/hotspot/share/prims/jvmtiRedefineClasses.cpp b/src/hotspot/share/prims/jvmtiRedefineClasses.cpp index 8afec3c0f8e0b..5922dd979ae66 100644 --- a/src/hotspot/share/prims/jvmtiRedefineClasses.cpp +++ b/src/hotspot/share/prims/jvmtiRedefineClasses.cpp @@ -1318,7 +1318,7 @@ class RedefineVerifyMark : public StackObj { private: JvmtiThreadState* _state; Klass* _scratch_class; - Handle _scratch_mirror; + OopHandle _scratch_mirror; public: @@ -1326,14 +1326,14 @@ class RedefineVerifyMark : public StackObj { JvmtiThreadState* state) : _state(state), _scratch_class(scratch_class) { _state->set_class_versions_map(the_class, scratch_class); - _scratch_mirror = Handle(_state->get_thread(), _scratch_class->java_mirror()); - _scratch_class->replace_java_mirror(the_class->java_mirror()); + _scratch_mirror = the_class->java_mirror_handle(); // this is a copy that is swapped + _scratch_class->swap_java_mirror_handle(_scratch_mirror); } ~RedefineVerifyMark() { // Restore the scratch class's mirror, so when scratch_class is removed // the correct mirror pointing to it can be cleared. - _scratch_class->replace_java_mirror(_scratch_mirror()); + _scratch_class->swap_java_mirror_handle(_scratch_mirror); _state->clear_class_versions_map(); } }; diff --git a/src/hotspot/share/prims/unsafe.cpp b/src/hotspot/share/prims/unsafe.cpp index ee6c218f5331c..87658f1a4e7ef 100644 --- a/src/hotspot/share/prims/unsafe.cpp +++ b/src/hotspot/share/prims/unsafe.cpp @@ -829,21 +829,22 @@ UNSAFE_ENTRY(void, Unsafe_Park(JNIEnv *env, jobject unsafe, jboolean isAbsolute, UNSAFE_ENTRY(void, Unsafe_Unpark(JNIEnv *env, jobject unsafe, jobject jthread)) { if (jthread != NULL) { - ThreadsListHandle tlh; - JavaThread* thr = NULL; - oop java_thread = NULL; - (void) tlh.cv_internal_thread_to_JavaThread(jthread, &thr, &java_thread); - if (java_thread != NULL) { - // This is a valid oop. - if (thr != NULL) { - // The JavaThread is alive. - Parker* p = thr->parker(); - HOTSPOT_THREAD_UNPARK((uintptr_t) p); - p->unpark(); - } + oop thread_oop = JNIHandles::resolve_non_null(jthread); + // Get the JavaThread* stored in the java.lang.Thread object _before_ + // the embedded ThreadsListHandle is constructed so we know if the + // early life stage of the JavaThread* is protected. We use acquire + // here to ensure that if we see a non-nullptr value, then we also + // see the main ThreadsList updates from the JavaThread* being added. + FastThreadsListHandle ftlh(thread_oop, java_lang_Thread::thread_acquire(thread_oop)); + JavaThread* thr = ftlh.protected_java_thread(); + if (thr != nullptr) { + // The still live JavaThread* is protected by the FastThreadsListHandle + // so it is safe to access. + Parker* p = thr->parker(); + HOTSPOT_THREAD_UNPARK((uintptr_t) p); + p->unpark(); } - } // ThreadsListHandle is destroyed here. - + } // FastThreadsListHandle is destroyed here. } UNSAFE_END UNSAFE_ENTRY(jint, Unsafe_GetLoadAverage0(JNIEnv *env, jobject unsafe, jdoubleArray loadavg, jint nelem)) { diff --git a/src/hotspot/share/prims/whitebox.cpp b/src/hotspot/share/prims/whitebox.cpp index 296bfe9e49cf3..3fe1e9389de73 100644 --- a/src/hotspot/share/prims/whitebox.cpp +++ b/src/hotspot/share/prims/whitebox.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -115,6 +115,7 @@ #endif #ifdef LINUX +#include "os_linux.hpp" #include "osContainer_linux.hpp" #include "cgroupSubsystem_linux.hpp" #endif @@ -821,10 +822,9 @@ static bool is_excluded_for_compiler(AbstractCompiler* comp, methodHandle& mh) { return true; } DirectiveSet* directive = DirectivesStack::getMatchingDirective(mh, comp); - if (directive->ExcludeOption) { - return true; - } - return false; + bool exclude = directive->ExcludeOption; + DirectivesStack::release(directive); + return exclude; } static bool can_be_compiled_at_level(methodHandle& mh, jboolean is_osr, int level) { @@ -2201,6 +2201,18 @@ WB_ENTRY(jboolean, WB_IsContainerized(JNIEnv* env, jobject o)) return false; WB_END +// Physical memory of the host machine (including containers) +WB_ENTRY(jlong, WB_HostPhysicalMemory(JNIEnv* env, jobject o)) + LINUX_ONLY(return os::Linux::physical_memory();) + return os::physical_memory(); +WB_END + +// Physical swap of the host machine (including containers), Linux only. +WB_ENTRY(jlong, WB_HostPhysicalSwap(JNIEnv* env, jobject o)) + LINUX_ONLY(return (jlong)os::Linux::host_swap();) + return -1; // Not used/implemented on other platforms +WB_END + WB_ENTRY(jint, WB_ValidateCgroup(JNIEnv* env, jobject o, jstring proc_cgroups, @@ -2593,6 +2605,8 @@ static JNINativeMethod methods[] = { {CC"validateCgroup", CC"(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)I", (void*)&WB_ValidateCgroup }, + {CC"hostPhysicalMemory", CC"()J", (void*)&WB_HostPhysicalMemory }, + {CC"hostPhysicalSwap", CC"()J", (void*)&WB_HostPhysicalSwap }, {CC"printOsInfo", CC"()V", (void*)&WB_PrintOsInfo }, {CC"disableElfSectionCache", CC"()V", (void*)&WB_DisableElfSectionCache }, {CC"resolvedMethodItemsCount", CC"()J", (void*)&WB_ResolvedMethodItemsCount }, diff --git a/src/hotspot/share/runtime/abstract_vm_version.cpp b/src/hotspot/share/runtime/abstract_vm_version.cpp index bbe1043f0fb43..79c84f248fba7 100644 --- a/src/hotspot/share/runtime/abstract_vm_version.cpp +++ b/src/hotspot/share/runtime/abstract_vm_version.cpp @@ -183,7 +183,8 @@ const char* Abstract_VM_Version::jre_release_version() { AMD64_ONLY("amd64") \ IA32_ONLY("x86") \ IA64_ONLY("ia64") \ - S390_ONLY("s390") + S390_ONLY("s390") \ + RISCV64_ONLY("riscv64") #endif // !ZERO #endif // !CPU diff --git a/src/hotspot/share/runtime/abstract_vm_version.hpp b/src/hotspot/share/runtime/abstract_vm_version.hpp index fd57a35e28763..59c4b299d8e48 100644 --- a/src/hotspot/share/runtime/abstract_vm_version.hpp +++ b/src/hotspot/share/runtime/abstract_vm_version.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,6 +31,7 @@ typedef enum { NoDetectedVirtualization, XenHVM, + XenPVHVM, // mix-mode on Linux aarch64 KVM, VMWare, HyperV, @@ -71,9 +72,10 @@ class Abstract_VM_Version: AllStatic { static int _vm_build_number; static unsigned int _data_cache_line_flush_size; + public: + static VirtualizationType _detected_virtualization; - public: // Called as part of the runtime services initialization which is // called from the management module initialization (via init_globals()) // after argument parsing and attaching of the main thread has diff --git a/src/hotspot/share/runtime/flags/jvmFlagLimit.cpp b/src/hotspot/share/runtime/flags/jvmFlagLimit.cpp index 5b81444eb84ab..bcc1565e165bd 100644 --- a/src/hotspot/share/runtime/flags/jvmFlagLimit.cpp +++ b/src/hotspot/share/runtime/flags/jvmFlagLimit.cpp @@ -34,6 +34,7 @@ #include "gc/shared/referenceProcessor.hpp" #include "oops/markWord.hpp" #include "runtime/task.hpp" +#include "utilities/vmError.hpp" //---------------------------------------------------------------------- // Build flagLimitTable[] diff --git a/src/hotspot/share/runtime/globals.hpp b/src/hotspot/share/runtime/globals.hpp index 6bac359a7782e..d20e146187aec 100644 --- a/src/hotspot/share/runtime/globals.hpp +++ b/src/hotspot/share/runtime/globals.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -67,21 +67,22 @@ // option, you must first specify +UnlockDiagnosticVMOptions. // (This master switch also affects the behavior of -Xprintflags.) // -// EXPERIMENTAL flags are in support of features that are not -// part of the officially supported product, but are available +// EXPERIMENTAL flags are in support of features that may not be +// an officially supported part of a product, but may be available // for experimenting with. They could, for example, be performance // features that may not have undergone full or rigorous QA, but which may // help performance in some cases and released for experimentation // by the community of users and developers. This flag also allows one to // be able to build a fully supported product that nonetheless also // ships with some unsupported, lightly tested, experimental features. +// Refer to the documentation of any products using this code for details +// on support and fitness for production. // Like the UnlockDiagnosticVMOptions flag above, there is a corresponding // UnlockExperimentalVMOptions flag, which allows the control and // modification of the experimental flags. // // Nota bene: neither diagnostic nor experimental options should be used casually, -// and they are not supported on production loads, except under explicit -// direction from support engineers. +// Refer to the documentation of any products using this code for details. // // MANAGEABLE flags are writeable external product flags. // They are dynamically writeable through the JDK management interface @@ -700,6 +701,13 @@ const intx ObjectAlignmentInBytes = 8; "MonitorUsedDeflationThreshold is exceeded (0 is off).") \ range(0, max_jint) \ \ + /* notice: the max range value here is max_jint, not max_intx */ \ + /* because of overflow issue */ \ + product(intx, GuaranteedAsyncDeflationInterval, 60000, DIAGNOSTIC, \ + "Async deflate idle monitors every so many milliseconds even " \ + "when MonitorUsedDeflationThreshold is NOT exceeded (0 is off).") \ + range(0, max_jint) \ + \ product(size_t, AvgMonitorsPerThreadEstimate, 1024, DIAGNOSTIC, \ "Used to estimate a variable ceiling based on number of threads " \ "for use with MonitorUsedDeflationThreshold (0 is off).") \ @@ -714,8 +722,9 @@ const intx ObjectAlignmentInBytes = 8; \ product(intx, MonitorUsedDeflationThreshold, 90, DIAGNOSTIC, \ "Percentage of used monitors before triggering deflation (0 is " \ - "off). The check is performed on GuaranteedSafepointInterval " \ - "or AsyncDeflationInterval.") \ + "off). The check is performed on GuaranteedSafepointInterval, " \ + "AsyncDeflationInterval or GuaranteedAsyncDeflationInterval, " \ + "whichever is lower.") \ range(0, 100) \ \ product(uintx, NoAsyncDeflationProgressMax, 3, DIAGNOSTIC, \ @@ -908,10 +917,6 @@ const intx ObjectAlignmentInBytes = 8; develop(bool, FLSVerifyDictionary, false, \ "Do lots of (expensive) FLS dictionary verification") \ \ - \ - notproduct(bool, CheckMemoryInitialization, false, \ - "Check memory initialization") \ - \ product(uintx, ProcessDistributionStride, 4, \ "Stride through processors when distributing processes") \ range(0, max_juint) \ @@ -977,9 +982,6 @@ const intx ObjectAlignmentInBytes = 8; "null (+offset) will not raise a SIGSEGV, i.e.," \ "ImplicitNullChecks don't work (PPC64).") \ \ - product(bool, EnableThreadSMRExtraValidityChecks, true, DIAGNOSTIC, \ - "Enable Thread SMR extra validity checks") \ - \ product(bool, EnableThreadSMRStatistics, trueInDebug, DIAGNOSTIC, \ "Enable Thread SMR Statistics") \ \ @@ -1371,6 +1373,10 @@ const intx ObjectAlignmentInBytes = 8; develop(intx, StackPrintLimit, 100, \ "number of stack frames to print in VM-level stack dump") \ \ + product(int, ErrorLogPrintCodeLimit, 3, DIAGNOSTIC, \ + "max number of compiled code units to print in error log") \ + range(0, VMError::max_error_log_print_code) \ + \ notproduct(intx, MaxElementPrintSize, 256, \ "maximum number of elements to print") \ \ @@ -1998,10 +2004,10 @@ const intx ObjectAlignmentInBytes = 8; product(ccstr, ExtraSharedClassListFile, NULL, \ "Extra classlist for building the CDS archive file") \ \ - product(intx, ArchiveRelocationMode, 0, DIAGNOSTIC, \ + product(intx, ArchiveRelocationMode, 1, DIAGNOSTIC, \ "(0) first map at preferred address, and if " \ - "unsuccessful, map at alternative address (default); " \ - "(1) always map at alternative address; " \ + "unsuccessful, map at alternative address; " \ + "(1) always map at alternative address (default); " \ "(2) always map at preferred address, and if unsuccessful, " \ "do not map the archive") \ range(0, 2) \ diff --git a/src/hotspot/share/runtime/interfaceSupport.inline.hpp b/src/hotspot/share/runtime/interfaceSupport.inline.hpp index 7cb6eeff078be..ba246a46d8f5e 100644 --- a/src/hotspot/share/runtime/interfaceSupport.inline.hpp +++ b/src/hotspot/share/runtime/interfaceSupport.inline.hpp @@ -325,8 +325,6 @@ class VMNativeEntryWrapper { #define VM_LEAF_BASE(result_type, header) \ debug_only(NoHandleMark __hm;) \ - MACOS_AARCH64_ONLY(ThreadWXEnable __wx(WXWrite, \ - JavaThread::current())); \ os::verify_stack_alignment(); \ /* begin of body */ diff --git a/src/hotspot/share/runtime/monitorDeflationThread.cpp b/src/hotspot/share/runtime/monitorDeflationThread.cpp index 86fda7b846209..b2bce3ede064d 100644 --- a/src/hotspot/share/runtime/monitorDeflationThread.cpp +++ b/src/hotspot/share/runtime/monitorDeflationThread.cpp @@ -33,8 +33,6 @@ #include "runtime/monitorDeflationThread.hpp" #include "runtime/mutexLocker.hpp" -MonitorDeflationThread* MonitorDeflationThread::_instance = NULL; - void MonitorDeflationThread::initialize() { EXCEPTION_MARK; @@ -50,31 +48,45 @@ void MonitorDeflationThread::initialize() { string, CHECK); - { - MutexLocker mu(THREAD, Threads_lock); - MonitorDeflationThread* thread = new MonitorDeflationThread(&monitor_deflation_thread_entry); + MonitorDeflationThread* thread = new MonitorDeflationThread(&monitor_deflation_thread_entry); + JavaThread::vm_exit_on_osthread_failure(thread); - // At this point it may be possible that no osthread was created for the - // JavaThread due to lack of memory. We would have to throw an exception - // in that case. However, since this must work and we do not allow - // exceptions anyway, check and abort if this fails. - if (thread == NULL || thread->osthread() == NULL) { - vm_exit_during_initialization("java.lang.OutOfMemoryError", - os::native_thread_creation_failed_msg()); - } + JavaThread::start_internal_daemon(THREAD, thread, thread_oop, NearMaxPriority); +} - java_lang_Thread::set_thread(thread_oop(), thread); - java_lang_Thread::set_priority(thread_oop(), NearMaxPriority); - java_lang_Thread::set_daemon(thread_oop()); - thread->set_threadObj(thread_oop()); - _instance = thread; +void MonitorDeflationThread::monitor_deflation_thread_entry(JavaThread* jt, TRAPS) { - Threads::add(thread); - Thread::start(thread); + // We wait for the lowest of these three intervals: + // - GuaranteedSafepointInterval + // While deflation is not related to safepoint anymore, this keeps compatibility with + // the old behavior when deflation also happened at safepoints. Users who set this + // option to get more/less frequent deflations would be served with this option. + // - AsyncDeflationInterval + // Normal threshold-based deflation heuristic checks the conditions at this interval. + // See is_async_deflation_needed(). + // - GuaranteedAsyncDeflationInterval + // Backup deflation heuristic checks the conditions at this interval. + // See is_async_deflation_needed(). + // + intx wait_time = max_intx; + if (GuaranteedSafepointInterval > 0) { + wait_time = MIN2(wait_time, GuaranteedSafepointInterval); + } + if (AsyncDeflationInterval > 0) { + wait_time = MIN2(wait_time, AsyncDeflationInterval); + } + if (GuaranteedAsyncDeflationInterval > 0) { + wait_time = MIN2(wait_time, GuaranteedAsyncDeflationInterval); + } + + // If all options are disabled, then wait time is not defined, and the deflation + // is effectively disabled. In that case, exit the thread immediately after printing + // a warning message. + if (wait_time == max_intx) { + warning("Async deflation is disabled"); + return; } -} -void MonitorDeflationThread::monitor_deflation_thread_entry(JavaThread* jt, TRAPS) { while (true) { { // Need state transition ThreadBlockInVM so that this thread @@ -86,9 +98,7 @@ void MonitorDeflationThread::monitor_deflation_thread_entry(JavaThread* jt, TRAP MonitorLocker ml(MonitorDeflation_lock, Mutex::_no_safepoint_check_flag); while (!ObjectSynchronizer::is_async_deflation_needed()) { // Wait until notified that there is some work to do. - // We wait for GuaranteedSafepointInterval so that - // is_async_deflation_needed() is checked at the same interval. - ml.wait(GuaranteedSafepointInterval); + ml.wait(wait_time); } } diff --git a/src/hotspot/share/runtime/monitorDeflationThread.hpp b/src/hotspot/share/runtime/monitorDeflationThread.hpp index f76a32c26bd7a..ef57911c05b4e 100644 --- a/src/hotspot/share/runtime/monitorDeflationThread.hpp +++ b/src/hotspot/share/runtime/monitorDeflationThread.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,7 +32,6 @@ class MonitorDeflationThread : public JavaThread { friend class VMStructs; private: - static MonitorDeflationThread* _instance; static void monitor_deflation_thread_entry(JavaThread* thread, TRAPS); MonitorDeflationThread(ThreadFunction entry_point) : JavaThread(entry_point) {}; diff --git a/src/hotspot/share/runtime/nonJavaThread.cpp b/src/hotspot/share/runtime/nonJavaThread.cpp index b6291c6eabebc..2330fd4219a42 100644 --- a/src/hotspot/share/runtime/nonJavaThread.cpp +++ b/src/hotspot/share/runtime/nonJavaThread.cpp @@ -270,8 +270,8 @@ void WatcherThread::run() { os::die(); } - // Wait a second, then recheck for timeout. - os::naked_short_sleep(999); + // Wait a bit, then recheck for timeout. + os::naked_short_sleep(250); } } diff --git a/src/hotspot/share/runtime/notificationThread.cpp b/src/hotspot/share/runtime/notificationThread.cpp index a7c903a85261d..7a698cbb9b412 100644 --- a/src/hotspot/share/runtime/notificationThread.cpp +++ b/src/hotspot/share/runtime/notificationThread.cpp @@ -35,8 +35,6 @@ #include "services/gcNotifier.hpp" #include "services/lowMemoryDetector.hpp" -NotificationThread* NotificationThread::_instance = NULL; - void NotificationThread::initialize() { EXCEPTION_MARK; @@ -61,28 +59,11 @@ void NotificationThread::initialize() { vmSymbols::thread_void_signature(), thread_oop, THREAD); - { - MutexLocker mu(THREAD, Threads_lock); - NotificationThread* thread = new NotificationThread(¬ification_thread_entry); - - // At this point it may be possible that no osthread was created for the - // JavaThread due to lack of memory. We would have to throw an exception - // in that case. However, since this must work and we do not allow - // exceptions anyway, check and abort if this fails. - if (thread == NULL || thread->osthread() == NULL) { - vm_exit_during_initialization("java.lang.OutOfMemoryError", - os::native_thread_creation_failed_msg()); - } - java_lang_Thread::set_thread(thread_oop(), thread); - java_lang_Thread::set_priority(thread_oop(), NearMaxPriority); - java_lang_Thread::set_daemon(thread_oop()); - thread->set_threadObj(thread_oop()); - _instance = thread; + NotificationThread* thread = new NotificationThread(¬ification_thread_entry); + JavaThread::vm_exit_on_osthread_failure(thread); - Threads::add(thread); - Thread::start(thread); - } + JavaThread::start_internal_daemon(THREAD, thread, thread_oop, NearMaxPriority); } @@ -128,4 +109,3 @@ void NotificationThread::notification_thread_entry(JavaThread* jt, TRAPS) { } } - diff --git a/src/hotspot/share/runtime/notificationThread.hpp b/src/hotspot/share/runtime/notificationThread.hpp index a54d19709d834..25b88405d58ff 100644 --- a/src/hotspot/share/runtime/notificationThread.hpp +++ b/src/hotspot/share/runtime/notificationThread.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -36,8 +36,6 @@ class NotificationThread : public JavaThread { friend class VMStructs; private: - static NotificationThread* _instance; - static void notification_thread_entry(JavaThread* thread, TRAPS); NotificationThread(ThreadFunction entry_point) : JavaThread(entry_point) {}; diff --git a/src/hotspot/share/runtime/objectMonitor.cpp b/src/hotspot/share/runtime/objectMonitor.cpp index a9faccd9e209d..374816fd355e3 100644 --- a/src/hotspot/share/runtime/objectMonitor.cpp +++ b/src/hotspot/share/runtime/objectMonitor.cpp @@ -47,7 +47,7 @@ #include "runtime/orderAccess.hpp" #include "runtime/osThread.hpp" #include "runtime/perfData.hpp" -#include "runtime/safefetch.inline.hpp" +#include "runtime/safefetch.hpp" #include "runtime/safepointMechanism.inline.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/thread.inline.hpp" diff --git a/src/hotspot/share/runtime/os.cpp b/src/hotspot/share/runtime/os.cpp index ef43c3170401d..045ed007ab1a1 100644 --- a/src/hotspot/share/runtime/os.cpp +++ b/src/hotspot/share/runtime/os.cpp @@ -54,7 +54,7 @@ #include "runtime/mutexLocker.hpp" #include "runtime/os.inline.hpp" #include "runtime/osThread.hpp" -#include "runtime/safefetch.inline.hpp" +#include "runtime/safefetch.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/thread.inline.hpp" #include "runtime/threadSMR.hpp" @@ -485,26 +485,10 @@ void os::initialize_jdk_signal_support(TRAPS) { thread_oop, CHECK); - { MutexLocker mu(THREAD, Threads_lock); - JavaThread* signal_thread = new JavaThread(&signal_thread_entry); + JavaThread* thread = new JavaThread(&signal_thread_entry); + JavaThread::vm_exit_on_osthread_failure(thread); - // At this point it may be possible that no osthread was created for the - // JavaThread due to lack of memory. We would have to throw an exception - // in that case. However, since this must work and we do not allow - // exceptions anyway, check and abort if this fails. - if (signal_thread == NULL || signal_thread->osthread() == NULL) { - vm_exit_during_initialization("java.lang.OutOfMemoryError", - os::native_thread_creation_failed_msg()); - } - - java_lang_Thread::set_thread(thread_oop(), signal_thread); - java_lang_Thread::set_priority(thread_oop(), NearMaxPriority); - java_lang_Thread::set_daemon(thread_oop()); - - signal_thread->set_threadObj(thread_oop()); - Threads::add(signal_thread); - Thread::start(signal_thread); - } + JavaThread::start_internal_daemon(THREAD, thread, thread_oop, NearMaxPriority); } } @@ -1089,11 +1073,7 @@ void os::print_date_and_time(outputStream *st, char* buf, size_t buflen) { // Check if pointer can be read from (4-byte read access). // Helps to prove validity of a not-NULL pointer. // Returns true in very early stages of VM life when stub is not yet generated. -#define SAFEFETCH_DEFAULT true bool os::is_readable_pointer(const void* p) { - if (!CanUseSafeFetch32()) { - return SAFEFETCH_DEFAULT; - } int* const aligned = (int*) align_down((intptr_t)p, 4); int cafebabe = 0xcafebabe; // tester value 1 int deadbeef = 0xdeadbeef; // tester value 2 diff --git a/src/hotspot/share/runtime/os.hpp b/src/hotspot/share/runtime/os.hpp index 9e074a1d02379..53a411afcc257 100644 --- a/src/hotspot/share/runtime/os.hpp +++ b/src/hotspot/share/runtime/os.hpp @@ -366,6 +366,15 @@ class os: AllStatic { static bool uncommit_memory(char* addr, size_t bytes, bool executable = false); static bool release_memory(char* addr, size_t bytes); + // Does the platform support trimming the native heap? + static bool can_trim_native_heap(); + + // Trim the C-heap. Optionally returns working set size change (RSS+Swap) in *rss_change. + // Note: If trimming succeeded but no size change information could be obtained, + // rss_change.after will contain SIZE_MAX upon return. + struct size_change_t { size_t before; size_t after; }; + static bool trim_native_heap(size_change_t* rss_change = nullptr); + // A diagnostic function to print memory mappings in the given range. static void print_memory_mappings(char* addr, size_t bytes, outputStream* st); // Prints all mappings @@ -886,7 +895,7 @@ class os: AllStatic { #ifndef PLATFORM_PRINT_NATIVE_STACK // No platform-specific code for printing the native stack. static bool platform_print_native_stack(outputStream* st, const void* context, - char *buf, int buf_size) { + char *buf, int buf_size, address& lastpc) { return false; } #endif diff --git a/src/hotspot/share/runtime/relocator.cpp b/src/hotspot/share/runtime/relocator.cpp index a55dc644577ff..dd1c55d8efd9c 100644 --- a/src/hotspot/share/runtime/relocator.cpp +++ b/src/hotspot/share/runtime/relocator.cpp @@ -413,11 +413,24 @@ void Relocator::adjust_exception_table(int bci, int delta) { } } +static void print_linenumber_table(unsigned char* table) { + CompressedLineNumberReadStream stream(table); + tty->print_cr("-------------------------------------------------"); + while (stream.read_pair()) { + tty->print_cr(" - line %d: %d", stream.line(), stream.bci()); + } + tty->print_cr("-------------------------------------------------"); +} // The width of instruction at "bci" is changing by "delta". Adjust the line number table. void Relocator::adjust_line_no_table(int bci, int delta) { if (method()->has_linenumber_table()) { - CompressedLineNumberReadStream reader(method()->compressed_linenumber_table()); + // if we already made adjustments then use the updated table + unsigned char *table = compressed_line_number_table(); + if (table == nullptr) { + table = method()->compressed_linenumber_table(); + } + CompressedLineNumberReadStream reader(table); CompressedLineNumberWriteStream writer(64); // plenty big for most line number tables while (reader.read_pair()) { int adjustment = (reader.bci() > bci) ? delta : 0; @@ -426,6 +439,10 @@ void Relocator::adjust_line_no_table(int bci, int delta) { writer.write_terminator(); set_compressed_line_number_table(writer.buffer()); set_compressed_line_number_table_size(writer.position()); + if (TraceRelocator) { + tty->print_cr("Adjusted line number table"); + print_linenumber_table(compressed_line_number_table()); + } } } diff --git a/src/hotspot/share/runtime/safefetch.hpp b/src/hotspot/share/runtime/safefetch.hpp new file mode 100644 index 0000000000000..71a542e25e80f --- /dev/null +++ b/src/hotspot/share/runtime/safefetch.hpp @@ -0,0 +1,58 @@ +/* + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022 SAP SE. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_RUNTIME_SAFEFETCH_HPP +#define SHARE_RUNTIME_SAFEFETCH_HPP + +#include "utilities/macros.hpp" + +// Safefetch allows to load a value from a location that's not known +// to be valid. If the load causes a fault, the error value is returned. + +#ifdef _WIN32 + // Windows uses Structured Exception Handling + #include "safefetch_windows.hpp" +#elif defined(ZERO) || defined (_AIX) + // These platforms implement safefetch via Posix sigsetjmp/longjmp. + // This is slower than the other methods and uses more thread stack, + // but its safe and portable. + #include "safefetch_sigjmp.hpp" + #define SAFEFETCH_METHOD_SIGSETJMP +#else + // All other platforms use static assembly + #include "safefetch_static.hpp" + #define SAFEFETCH_METHOD_STATIC_ASSEMBLY +#endif + + +inline int SafeFetch32(int* adr, int errValue) { + return SafeFetch32_impl(adr, errValue); +} + +inline intptr_t SafeFetchN(intptr_t* adr, intptr_t errValue) { + return SafeFetchN_impl(adr, errValue); +} + +#endif // SHARE_RUNTIME_SAFEFETCH_HPP diff --git a/src/hotspot/share/runtime/safefetch.inline.hpp b/src/hotspot/share/runtime/safefetch.inline.hpp index cee0853573cf5..e69de29bb2d1d 100644 --- a/src/hotspot/share/runtime/safefetch.inline.hpp +++ b/src/hotspot/share/runtime/safefetch.inline.hpp @@ -1,74 +0,0 @@ -/* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_RUNTIME_SAFEFETCH_INLINE_HPP -#define SHARE_RUNTIME_SAFEFETCH_INLINE_HPP - -// No safefetch.hpp - -#include "runtime/stubRoutines.hpp" -#include "runtime/threadWXSetters.inline.hpp" - -// Safefetch allows to load a value from a location that's not known -// to be valid. If the load causes a fault, the error value is returned. -inline int SafeFetch32(int* adr, int errValue) { - assert(StubRoutines::SafeFetch32_stub(), "stub not yet generated"); -#if defined(__APPLE__) && defined(AARCH64) - Thread* thread = Thread::current_or_null_safe(); - assert(thread != NULL, "required for W^X management"); - ThreadWXEnable wx(WXExec, thread); -#endif // __APPLE__ && AARCH64 - return StubRoutines::SafeFetch32_stub()(adr, errValue); -} - -inline intptr_t SafeFetchN(intptr_t* adr, intptr_t errValue) { - assert(StubRoutines::SafeFetchN_stub(), "stub not yet generated"); -#if defined(__APPLE__) && defined(AARCH64) - Thread* thread = Thread::current_or_null_safe(); - assert(thread != NULL, "required for W^X management"); - ThreadWXEnable wx(WXExec, thread); -#endif // __APPLE__ && AARCH64 - return StubRoutines::SafeFetchN_stub()(adr, errValue); -} - -// returns true if SafeFetch32 and SafeFetchN can be used safely (stubroutines are already generated) -inline bool CanUseSafeFetch32() { -#if defined (__APPLE__) && defined(AARCH64) - if (Thread::current_or_null_safe() == NULL) { // workaround for JDK-8282475 - return false; - } -#endif // __APPLE__ && AARCH64 - return StubRoutines::SafeFetch32_stub() ? true : false; -} - -inline bool CanUseSafeFetchN() { -#if defined (__APPLE__) && defined(AARCH64) - if (Thread::current_or_null_safe() == NULL) { - return false; - } -#endif // __APPLE__ && AARCH64 - return StubRoutines::SafeFetchN_stub() ? true : false; -} - -#endif // SHARE_RUNTIME_SAFEFETCH_INLINE_HPP diff --git a/src/hotspot/share/runtime/safefetch_static.hpp b/src/hotspot/share/runtime/safefetch_static.hpp new file mode 100644 index 0000000000000..72b8db18e9385 --- /dev/null +++ b/src/hotspot/share/runtime/safefetch_static.hpp @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2022 SAP SE. All rights reserved. + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_RUNTIME_SAFEFETCH_STATIC_HPP +#define SHARE_RUNTIME_SAFEFETCH_STATIC_HPP + +#include "utilities/globalDefinitions.hpp" + +extern "C" int SafeFetch32_impl(int* adr, int errValue); + +#ifdef _LP64 +extern "C" intptr_t SafeFetchN_impl(intptr_t* adr, intptr_t errValue); +#else +inline intptr_t SafeFetchN_impl(intptr_t* adr, intptr_t errValue) { + return SafeFetch32_impl(adr, errValue); +} +#endif // _LP64 + +bool handle_safefetch(int sig, address pc, void* context); + +#endif // SHARE_RUNTIME_SAFEFETCH_STATIC_HPP diff --git a/src/hotspot/share/runtime/safepoint.cpp b/src/hotspot/share/runtime/safepoint.cpp index 3191275884c48..31918a79dfa11 100644 --- a/src/hotspot/share/runtime/safepoint.cpp +++ b/src/hotspot/share/runtime/safepoint.cpp @@ -506,19 +506,9 @@ bool SafepointSynchronize::is_cleanup_needed() { return false; } -class ParallelSPCleanupThreadClosure : public ThreadClosure { -public: - void do_thread(Thread* thread) { - if (thread->is_Java_thread()) { - StackWatermarkSet::start_processing(thread->as_Java_thread(), StackWatermarkKind::gc); - } - } -}; - -class ParallelSPCleanupTask : public AbstractGangTask { +class ParallelCleanupTask : public AbstractGangTask { private: SubTasksDone _subtasks; - uint _num_workers; bool _do_lazy_roots; class Tracer { @@ -538,32 +528,40 @@ class ParallelSPCleanupTask : public AbstractGangTask { }; public: - ParallelSPCleanupTask(uint num_workers) : + ParallelCleanupTask() : AbstractGangTask("Parallel Safepoint Cleanup"), _subtasks(SafepointSynchronize::SAFEPOINT_CLEANUP_NUM_TASKS), - _num_workers(num_workers), _do_lazy_roots(!VMThread::vm_operation()->skip_thread_oop_barriers() && Universe::heap()->uses_stack_watermark_barrier()) {} - void work(uint worker_id) { - if (_subtasks.try_claim_task(SafepointSynchronize::SAFEPOINT_CLEANUP_LAZY_ROOT_PROCESSING)) { - if (_do_lazy_roots) { - Tracer t("lazy partial thread root processing"); - ParallelSPCleanupThreadClosure cl; - Threads::threads_do(&cl); - } + uint expected_num_workers() const { + uint workers = 0; + + if (SymbolTable::rehash_table_expects_safepoint_rehashing()) { + workers++; } - if (_subtasks.try_claim_task(SafepointSynchronize::SAFEPOINT_CLEANUP_UPDATE_INLINE_CACHES)) { - Tracer t("updating inline caches"); - InlineCacheBuffer::update_inline_caches(); + if (StringTable::rehash_table_expects_safepoint_rehashing()) { + workers++; + } + + if (Dictionary::does_any_dictionary_needs_resizing()) { + workers++; } - if (_subtasks.try_claim_task(SafepointSynchronize::SAFEPOINT_CLEANUP_COMPILATION_POLICY)) { - Tracer t("compilation policy safepoint handler"); - CompilationPolicy::do_safepoint_work(); + if (InlineCacheBuffer::needs_update_inline_caches()) { + workers++; } + if (_do_lazy_roots) { + workers++; + } + + return MAX2(1, workers); + } + + void work(uint worker_id) { + // These tasks are ordered by relative length of time to execute so that potentially longer tasks start first. if (_subtasks.try_claim_task(SafepointSynchronize::SAFEPOINT_CLEANUP_SYMBOL_TABLE_REHASH)) { if (SymbolTable::needs_rehashing()) { Tracer t("rehashing symbol table"); @@ -585,6 +583,25 @@ class ParallelSPCleanupTask : public AbstractGangTask { } } + if (_subtasks.try_claim_task(SafepointSynchronize::SAFEPOINT_CLEANUP_LAZY_ROOT_PROCESSING)) { + if (_do_lazy_roots) { + Tracer t("lazy partial thread root processing"); + class LazyRootClosure : public ThreadClosure { + public: + void do_thread(Thread* thread) { + StackWatermarkSet::start_processing(thread->as_Java_thread(), StackWatermarkKind::gc); + } + }; + LazyRootClosure cl; + Threads::java_threads_do(&cl); + } + } + + if (_subtasks.try_claim_task(SafepointSynchronize::SAFEPOINT_CLEANUP_UPDATE_INLINE_CACHES)) { + Tracer t("updating inline caches"); + InlineCacheBuffer::update_inline_caches(); + } + if (_subtasks.try_claim_task(SafepointSynchronize::SAFEPOINT_CLEANUP_REQUEST_OOPSTORAGE_CLEANUP)) { // Don't bother reporting event or time for this very short operation. // To have any utility we'd also want to report whether needed. @@ -602,15 +619,15 @@ void SafepointSynchronize::do_cleanup_tasks() { CollectedHeap* heap = Universe::heap(); assert(heap != NULL, "heap not initialized yet?"); + ParallelCleanupTask cleanup; WorkGang* cleanup_workers = heap->safepoint_workers(); - if (cleanup_workers != NULL) { + const uint expected_num_workers = cleanup.expected_num_workers(); + if (cleanup_workers != nullptr && expected_num_workers > 1) { // Parallel cleanup using GC provided thread pool. - uint num_cleanup_workers = cleanup_workers->active_workers(); - ParallelSPCleanupTask cleanup(num_cleanup_workers); - cleanup_workers->run_task(&cleanup); + const uint num_workers = MIN2(expected_num_workers, cleanup_workers->active_workers()); + cleanup_workers->run_task(&cleanup, num_workers); } else { // Serial cleanup using VMThread. - ParallelSPCleanupTask cleanup(1); cleanup.work(0); } @@ -1011,6 +1028,7 @@ int SafepointTracing::_nof_running = 0; int SafepointTracing::_page_trap = 0; VM_Operation::VMOp_Type SafepointTracing::_current_type; jlong SafepointTracing::_max_sync_time = 0; +jlong SafepointTracing::_max_cleanup_time = 0; jlong SafepointTracing::_max_vmop_time = 0; uint64_t SafepointTracing::_op_count[VM_Operation::VMOp_Terminating] = {0}; @@ -1080,6 +1098,8 @@ void SafepointTracing::statistics_exit_log() { log_info(safepoint, stats)("Maximum sync time " INT64_FORMAT" ns", (int64_t)(_max_sync_time)); + log_info(safepoint, stats)("Maximum cleanup time " INT64_FORMAT" ns", + (int64_t)(_max_cleanup_time)); log_info(safepoint, stats)("Maximum vm operation time (except for Exit VM operation) " INT64_FORMAT " ns", (int64_t)(_max_vmop_time)); @@ -1118,6 +1138,9 @@ void SafepointTracing::end() { if (_max_sync_time < (_last_safepoint_sync_time_ns - _last_safepoint_begin_time_ns)) { _max_sync_time = _last_safepoint_sync_time_ns - _last_safepoint_begin_time_ns; } + if (_max_cleanup_time < (_last_safepoint_cleanup_time_ns - _last_safepoint_sync_time_ns)) { + _max_cleanup_time = _last_safepoint_cleanup_time_ns - _last_safepoint_sync_time_ns; + } if (_max_vmop_time < (_last_safepoint_end_time_ns - _last_safepoint_sync_time_ns)) { _max_vmop_time = _last_safepoint_end_time_ns - _last_safepoint_sync_time_ns; } @@ -1129,14 +1152,16 @@ void SafepointTracing::end() { "Safepoint \"%s\", " "Time since last: " JLONG_FORMAT " ns, " "Reaching safepoint: " JLONG_FORMAT " ns, " + "Cleanup: " JLONG_FORMAT " ns, " "At safepoint: " JLONG_FORMAT " ns, " "Total: " JLONG_FORMAT " ns", VM_Operation::name(_current_type), _last_app_time_ns, - _last_safepoint_cleanup_time_ns - _last_safepoint_begin_time_ns, + _last_safepoint_sync_time_ns - _last_safepoint_begin_time_ns, + _last_safepoint_cleanup_time_ns - _last_safepoint_sync_time_ns, _last_safepoint_end_time_ns - _last_safepoint_cleanup_time_ns, _last_safepoint_end_time_ns - _last_safepoint_begin_time_ns ); - RuntimeService::record_safepoint_end(_last_safepoint_end_time_ns - _last_safepoint_cleanup_time_ns); + RuntimeService::record_safepoint_end(_last_safepoint_end_time_ns - _last_safepoint_sync_time_ns); } diff --git a/src/hotspot/share/runtime/safepoint.hpp b/src/hotspot/share/runtime/safepoint.hpp index c35f55f0c276d..95d0e1fb18f40 100644 --- a/src/hotspot/share/runtime/safepoint.hpp +++ b/src/hotspot/share/runtime/safepoint.hpp @@ -72,7 +72,6 @@ class SafepointSynchronize : AllStatic { enum SafepointCleanupTasks { SAFEPOINT_CLEANUP_LAZY_ROOT_PROCESSING, SAFEPOINT_CLEANUP_UPDATE_INLINE_CACHES, - SAFEPOINT_CLEANUP_COMPILATION_POLICY, SAFEPOINT_CLEANUP_SYMBOL_TABLE_REHASH, SAFEPOINT_CLEANUP_STRING_TABLE_REHASH, SAFEPOINT_CLEANUP_SYSTEM_DICTIONARY_RESIZE, @@ -261,6 +260,7 @@ class SafepointTracing : public AllStatic { static VM_Operation::VMOp_Type _current_type; static jlong _max_sync_time; + static jlong _max_cleanup_time; static jlong _max_vmop_time; static uint64_t _op_count[VM_Operation::VMOp_Terminating]; diff --git a/src/hotspot/share/runtime/serviceThread.cpp b/src/hotspot/share/runtime/serviceThread.cpp index 1c6a7b7874e01..620267c68acec 100644 --- a/src/hotspot/share/runtime/serviceThread.cpp +++ b/src/hotspot/share/runtime/serviceThread.cpp @@ -51,7 +51,7 @@ #include "services/lowMemoryDetector.hpp" #include "services/threadIdTable.hpp" -ServiceThread* ServiceThread::_instance = NULL; +DEBUG_ONLY(JavaThread* ServiceThread::_instance = NULL;) JvmtiDeferredEvent* ServiceThread::_jvmti_event = NULL; // The service thread has it's own static deferred event queue. // Events can be posted before JVMTI vm_start, so it's too early to call JvmtiThreadState::state_for @@ -103,28 +103,11 @@ void ServiceThread::initialize() { string, CHECK); - { - MutexLocker mu(THREAD, Threads_lock); - ServiceThread* thread = new ServiceThread(&service_thread_entry); - - // At this point it may be possible that no osthread was created for the - // JavaThread due to lack of memory. We would have to throw an exception - // in that case. However, since this must work and we do not allow - // exceptions anyway, check and abort if this fails. - if (thread == NULL || thread->osthread() == NULL) { - vm_exit_during_initialization("java.lang.OutOfMemoryError", - os::native_thread_creation_failed_msg()); - } + ServiceThread* thread = new ServiceThread(&service_thread_entry); + JavaThread::vm_exit_on_osthread_failure(thread); - java_lang_Thread::set_thread(thread_oop(), thread); - java_lang_Thread::set_priority(thread_oop(), NearMaxPriority); - java_lang_Thread::set_daemon(thread_oop()); - thread->set_threadObj(thread_oop()); - _instance = thread; - - Threads::add(thread); - Thread::start(thread); - } + JavaThread::start_internal_daemon(THREAD, thread, thread_oop, NearMaxPriority); + DEBUG_ONLY(_instance = thread;) } static void cleanup_oopstorages() { diff --git a/src/hotspot/share/runtime/serviceThread.hpp b/src/hotspot/share/runtime/serviceThread.hpp index 99b65de9415ec..44dba40cc77fa 100644 --- a/src/hotspot/share/runtime/serviceThread.hpp +++ b/src/hotspot/share/runtime/serviceThread.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -36,7 +36,7 @@ class JvmtiDeferredEvent; class ServiceThread : public JavaThread { friend class VMStructs; private: - static ServiceThread* _instance; + DEBUG_ONLY(static JavaThread* _instance;) static JvmtiDeferredEvent* _jvmti_event; static JvmtiDeferredEventQueue _jvmti_service_queue; diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index 66613e5d8c6bb..9af4b513a9994 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -475,6 +475,9 @@ address SharedRuntime::raw_exception_handler_for_return_address(JavaThread* curr current->set_exception_pc(NULL); #endif // INCLUDE_JVMCI + // write lock needed because we might update the pc desc cache via PcDescCache::add_pc_desc + MACOS_AARCH64_ONLY(ThreadWXEnable wx(WXWrite, current)); + // The fastest case first CodeBlob* blob = CodeCache::find_blob(return_address); CompiledMethod* nm = (blob != NULL) ? blob->as_compiled_method_or_null() : NULL; @@ -1964,6 +1967,9 @@ JRT_LEAF(void, SharedRuntime::fixup_callers_callsite(Method* method, address cal return; } + // write lock needed because we might update the pc desc cache via PcDescCache::add_pc_desc + MACOS_AARCH64_ONLY(ThreadWXEnable __wx(WXWrite, JavaThread::current())); + CodeBlob* cb = CodeCache::find_blob(caller_pc); if (cb == NULL || !cb->is_compiled() || callee->is_unloading()) { return; diff --git a/src/hotspot/share/runtime/stubRoutines.cpp b/src/hotspot/share/runtime/stubRoutines.cpp index b231ce533fb74..e1b8a4e609949 100644 --- a/src/hotspot/share/runtime/stubRoutines.cpp +++ b/src/hotspot/share/runtime/stubRoutines.cpp @@ -31,8 +31,8 @@ #include "prims/vectorSupport.hpp" #include "runtime/interfaceSupport.inline.hpp" #include "runtime/timerTrace.hpp" -#include "runtime/safefetch.inline.hpp" #include "runtime/sharedRuntime.hpp" +#include "runtime/stubRoutines.hpp" #include "utilities/align.hpp" #include "utilities/copy.hpp" #ifdef COMPILER2 @@ -167,13 +167,6 @@ address StubRoutines::_dlibm_reduce_pi04l = NULL; address StubRoutines::_dlibm_tan_cot_huge = NULL; address StubRoutines::_dtan = NULL; -address StubRoutines::_safefetch32_entry = NULL; -address StubRoutines::_safefetch32_fault_pc = NULL; -address StubRoutines::_safefetch32_continuation_pc = NULL; -address StubRoutines::_safefetchN_entry = NULL; -address StubRoutines::_safefetchN_fault_pc = NULL; -address StubRoutines::_safefetchN_continuation_pc = NULL; - address StubRoutines::_vector_f_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_SVML_OP] = {{NULL}, {NULL}}; address StubRoutines::_vector_d_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_SVML_OP] = {{NULL}, {NULL}}; diff --git a/src/hotspot/share/runtime/stubRoutines.hpp b/src/hotspot/share/runtime/stubRoutines.hpp index 315c774a08edc..60cf386523cb8 100644 --- a/src/hotspot/share/runtime/stubRoutines.hpp +++ b/src/hotspot/share/runtime/stubRoutines.hpp @@ -249,14 +249,6 @@ class StubRoutines: AllStatic { static address _dlibm_tan_cot_huge; static address _dtan; - // Safefetch stubs. - static address _safefetch32_entry; - static address _safefetch32_fault_pc; - static address _safefetch32_continuation_pc; - static address _safefetchN_entry; - static address _safefetchN_fault_pc; - static address _safefetchN_continuation_pc; - // Vector Math Routines static address _vector_f_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_SVML_OP]; static address _vector_d_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_SVML_OP]; @@ -426,34 +418,6 @@ class StubRoutines: AllStatic { static address select_fill_function(BasicType t, bool aligned, const char* &name); - // - // Safefetch stub support - // - - typedef int (*SafeFetch32Stub)(int* adr, int errValue); - typedef intptr_t (*SafeFetchNStub) (intptr_t* adr, intptr_t errValue); - - static SafeFetch32Stub SafeFetch32_stub() { return CAST_TO_FN_PTR(SafeFetch32Stub, _safefetch32_entry); } - static SafeFetchNStub SafeFetchN_stub() { return CAST_TO_FN_PTR(SafeFetchNStub, _safefetchN_entry); } - - static bool is_safefetch_fault(address pc) { - return pc != NULL && - (pc == _safefetch32_fault_pc || - pc == _safefetchN_fault_pc); - } - - static address continuation_for_safefetch_fault(address pc) { - assert(_safefetch32_continuation_pc != NULL && - _safefetchN_continuation_pc != NULL, - "not initialized"); - - if (pc == _safefetch32_fault_pc) return _safefetch32_continuation_pc; - if (pc == _safefetchN_fault_pc) return _safefetchN_continuation_pc; - - ShouldNotReachHere(); - return NULL; - } - // // Default versions of the above arraycopy functions for platforms which do // not have specialized versions diff --git a/src/hotspot/share/runtime/synchronizer.cpp b/src/hotspot/share/runtime/synchronizer.cpp index b0e0059b16b29..0e208369849a1 100644 --- a/src/hotspot/share/runtime/synchronizer.cpp +++ b/src/hotspot/share/runtime/synchronizer.cpp @@ -213,6 +213,9 @@ void ObjectSynchronizer::initialize() { } // Start the ceiling with the estimate for one thread. set_in_use_list_ceiling(AvgMonitorsPerThreadEstimate); + + // Start the timer for deflations, so it does not trigger immediately. + _last_async_deflation_time_ns = os::javaTimeNanos(); } MonitorList ObjectSynchronizer::_in_use_list; @@ -241,6 +244,7 @@ bool volatile ObjectSynchronizer::_is_async_deflation_requested = false; bool volatile ObjectSynchronizer::_is_final_audit = false; jlong ObjectSynchronizer::_last_async_deflation_time_ns = 0; static uintx _no_progress_cnt = 0; +static bool _no_progress_skip_increment = false; // =====================> Quick functions @@ -729,14 +733,14 @@ struct SharedGlobals { static SharedGlobals GVars; static markWord read_stable_mark(oop obj) { - markWord mark = obj->mark(); + markWord mark = obj->mark_acquire(); if (!mark.is_being_inflated()) { return mark; // normal fast-path return } int its = 0; for (;;) { - markWord mark = obj->mark(); + markWord mark = obj->mark_acquire(); if (!mark.is_being_inflated()) { return mark; // normal fast-path return } @@ -770,7 +774,7 @@ static markWord read_stable_mark(oop obj) { int YieldThenBlock = 0; assert(ix >= 0 && ix < NINFLATIONLOCKS, "invariant"); gInflationLocks[ix]->lock(); - while (obj->mark() == markWord::INFLATING()) { + while (obj->mark_acquire() == markWord::INFLATING()) { // Beware: naked_yield() is advisory and has almost no effect on some platforms // so we periodically call current->_ParkEvent->park(1). // We use a mixed spin/yield/block mechanism. @@ -1106,7 +1110,14 @@ static bool monitors_used_above_threshold(MonitorList* list) { // Check if our monitor usage is above the threshold: size_t monitor_usage = (monitors_used * 100LL) / ceiling; - return int(monitor_usage) > MonitorUsedDeflationThreshold; + if (int(monitor_usage) > MonitorUsedDeflationThreshold) { + log_info(monitorinflation)("monitors_used=" SIZE_FORMAT ", ceiling=" SIZE_FORMAT + ", monitor_usage=" SIZE_FORMAT ", threshold=" INTX_FORMAT, + monitors_used, ceiling, monitor_usage, MonitorUsedDeflationThreshold); + return true; + } + + return false; } size_t ObjectSynchronizer::in_use_list_ceiling() { @@ -1128,17 +1139,49 @@ void ObjectSynchronizer::set_in_use_list_ceiling(size_t new_value) { bool ObjectSynchronizer::is_async_deflation_needed() { if (is_async_deflation_requested()) { // Async deflation request. + log_info(monitorinflation)("Async deflation needed: explicit request"); return true; } + + jlong time_since_last = time_since_last_async_deflation_ms(); + if (AsyncDeflationInterval > 0 && - time_since_last_async_deflation_ms() > AsyncDeflationInterval && + time_since_last > AsyncDeflationInterval && monitors_used_above_threshold(&_in_use_list)) { // It's been longer than our specified deflate interval and there // are too many monitors in use. We don't deflate more frequently // than AsyncDeflationInterval (unless is_async_deflation_requested) // in order to not swamp the MonitorDeflationThread. + log_info(monitorinflation)("Async deflation needed: monitors used are above the threshold"); return true; } + + if (GuaranteedAsyncDeflationInterval > 0 && + time_since_last > GuaranteedAsyncDeflationInterval) { + // It's been longer than our specified guaranteed deflate interval. + // We need to clean up the used monitors even if the threshold is + // not reached, to keep the memory utilization at bay when many threads + // touched many monitors. + log_info(monitorinflation)("Async deflation needed: guaranteed interval (" INTX_FORMAT " ms) " + "is greater than time since last deflation (" JLONG_FORMAT " ms)", + GuaranteedAsyncDeflationInterval, time_since_last); + + // If this deflation has no progress, then it should not affect the no-progress + // tracking, otherwise threshold heuristics would think it was triggered, experienced + // no progress, and needs to backoff more aggressively. In this "no progress" case, + // the generic code would bump the no-progress counter, and we compensate for that + // by telling it to skip the update. + // + // If this deflation has progress, then it should let non-progress tracking + // know about this, otherwise the threshold heuristics would kick in, potentially + // experience no-progress due to aggressive cleanup by this deflation, and think + // it is still in no-progress stride. In this "progress" case, the generic code would + // zero the counter, and we allow it to happen. + _no_progress_skip_increment = true; + + return true; + } + return false; } @@ -1189,7 +1232,7 @@ static void post_monitor_inflate_event(EventJavaMonitorInflate* event, // Fast path code shared by multiple functions void ObjectSynchronizer::inflate_helper(oop obj) { - markWord mark = obj->mark(); + markWord mark = obj->mark_acquire(); if (mark.has_monitor()) { ObjectMonitor* monitor = mark.monitor(); markWord dmw = monitor->header(); @@ -1204,7 +1247,7 @@ ObjectMonitor* ObjectSynchronizer::inflate(Thread* current, oop object, EventJavaMonitorInflate event; for (;;) { - const markWord mark = object->mark(); + const markWord mark = object->mark_acquire(); assert(!mark.has_bias_pattern(), "invariant"); // The mark can be in one of the following states: @@ -1533,6 +1576,8 @@ size_t ObjectSynchronizer::deflate_idle_monitors() { if (deflated_count != 0) { _no_progress_cnt = 0; + } else if (_no_progress_skip_increment) { + _no_progress_skip_increment = false; } else { _no_progress_cnt++; } @@ -1621,6 +1666,7 @@ void ObjectSynchronizer::do_final_audit_and_print_stats() { return; } set_is_final_audit(); + log_info(monitorinflation)("Starting the final audit."); if (log_is_enabled(Info, monitorinflation)) { // Do a deflation in order to reduce the in-use monitor population @@ -1630,7 +1676,7 @@ void ObjectSynchronizer::do_final_audit_and_print_stats() { ; // empty } // The other audit_and_print_stats() call is done at the Debug - // level at a safepoint in ObjectSynchronizer::do_safepoint_work(). + // level at a safepoint in SafepointSynchronize::do_cleanup_tasks. ObjectSynchronizer::audit_and_print_stats(true /* on_exit */); } } diff --git a/src/hotspot/share/runtime/thread.cpp b/src/hotspot/share/runtime/thread.cpp index 385f7a3fa99ba..6f64bc4908cf8 100644 --- a/src/hotspot/share/runtime/thread.cpp +++ b/src/hotspot/share/runtime/thread.cpp @@ -653,7 +653,9 @@ void Thread::print_on_error(outputStream* st, char* buf, int buflen) const { else if (is_GC_task_thread()) { st->print("GCTaskThread"); } else if (is_Watcher_thread()) { st->print("WatcherThread"); } else if (is_ConcurrentGC_thread()) { st->print("ConcurrentGCThread"); } - else { st->print("Thread"); } + else if (this == AsyncLogWriter::instance()) { + st->print("%s", this->name()); + } else { st->print("Thread"); } if (is_Named_thread()) { st->print(" \"%s\"", name()); @@ -1337,8 +1339,9 @@ static void ensure_join(JavaThread* thread) { // Thread is exiting. So set thread_status field in java.lang.Thread class to TERMINATED. java_lang_Thread::set_thread_status(threadObj(), JavaThreadStatus::TERMINATED); // Clear the native thread instance - this makes isAlive return false and allows the join() - // to complete once we've done the notify_all below - java_lang_Thread::set_thread(threadObj(), NULL); + // to complete once we've done the notify_all below. Needs a release() to obey Java Memory Model + // requirements. + java_lang_Thread::release_set_thread(threadObj(), NULL); lock.notify_all(thread); // Ignore pending exception (ThreadDeath), since we are exiting anyway thread->clear_pending_exception(); @@ -2257,7 +2260,6 @@ void JavaThread::prepare(jobject jni_thread, ThreadPriority prio) { assert(InstanceKlass::cast(thread_oop->klass())->is_linked(), "must be initialized"); set_threadObj(thread_oop()); - java_lang_Thread::set_thread(thread_oop(), this); if (prio == NoPriority) { prio = java_lang_Thread::priority(thread_oop()); @@ -2273,6 +2275,11 @@ void JavaThread::prepare(jobject jni_thread, ThreadPriority prio) { // added to the Threads list for if a GC happens, then the java_thread oop // will not be visited by GC. Threads::add(this); + // Publish the JavaThread* in java.lang.Thread after the JavaThread* is + // on a ThreadsList. We don't want to wait for the release when the + // Theads_lock is dropped somewhere in the caller since the JavaThread* + // is already visible to JVM/TI via the ThreadsList. + java_lang_Thread::release_set_thread(thread_oop(), this); } oop JavaThread::current_park_blocker() { @@ -2284,6 +2291,25 @@ oop JavaThread::current_park_blocker() { return NULL; } +// Print current stack trace for checked JNI warnings and JNI fatal errors. +// This is the external format, selecting the platform +// as applicable, and allowing for a native-only stack. +void JavaThread::print_jni_stack() { + assert(this == JavaThread::current(), "Can't print stack of other threads"); + if (!has_last_Java_frame()) { + ResourceMark rm(this); + char* buf = NEW_RESOURCE_ARRAY_RETURN_NULL(char, O_BUFLEN); + if (buf == nullptr) { + tty->print_cr("Unable to print native stack - out of memory"); + return; + } + frame f = os::current_frame(); + VMError::print_native_stack(tty, f, this, + buf, O_BUFLEN); + } else { + print_stack_on(tty); + } +} void JavaThread::print_stack_on(outputStream* st) { if (!has_last_Java_frame()) return; @@ -3972,3 +3998,46 @@ void JavaThread::verify_cross_modify_fence_failure(JavaThread *thread) { report_vm_error(__FILE__, __LINE__, "Cross modify fence failure", "%p", thread); } #endif + +// Starts the target JavaThread as a daemon of the given priority, and +// bound to the given java.lang.Thread instance. +// The Threads_lock is held for the duration. +void JavaThread::start_internal_daemon(JavaThread* current, JavaThread* target, + Handle thread_oop, ThreadPriority prio) { + + assert(target->osthread()!= NULL, "target thread is not properly initialized"); + + MutexLocker mu(current, Threads_lock); + + // Initialize the fields of the thread_oop first. + if (prio != NoPriority) { + java_lang_Thread::set_priority(thread_oop(), prio); + // Note: we don't call os::set_priority here. Possibly we should, + // else all threads should call it themselves when they first run. + } + + java_lang_Thread::set_daemon(thread_oop()); + + // Now bind the thread_oop to the target JavaThread. + target->set_threadObj(thread_oop()); + + Threads::add(target); // target is now visible for safepoint/handshake + // Publish the JavaThread* in java.lang.Thread after the JavaThread* is + // on a ThreadsList. We don't want to wait for the release when the + // Theads_lock is dropped when the 'mu' destructor is run since the + // JavaThread* is already visible to JVM/TI via the ThreadsList. + java_lang_Thread::release_set_thread(thread_oop(), target); // isAlive == true now + Thread::start(target); +} + +void JavaThread::vm_exit_on_osthread_failure(JavaThread* thread) { + // At this point it may be possible that no osthread was created for the + // JavaThread due to lack of resources. However, since this must work + // for critical system threads just check and abort if this fails. + if (thread->osthread() == nullptr) { + // This isn't really an OOM condition, but historically this is what + // we report. + vm_exit_during_initialization("java.lang.OutOfMemoryError", + os::native_thread_creation_failed_msg()); + } +} diff --git a/src/hotspot/share/runtime/thread.hpp b/src/hotspot/share/runtime/thread.hpp index 8cd0964019fd1..825b04cc272ca 100644 --- a/src/hotspot/share/runtime/thread.hpp +++ b/src/hotspot/share/runtime/thread.hpp @@ -54,7 +54,6 @@ #include "jfr/support/jfrThreadExtension.hpp" #endif - class SafeThreadsListPtr; class ThreadSafepointState; class ThreadsList; @@ -654,6 +653,31 @@ class Thread: public ThreadShadow { assert(_wx_state == expected, "wrong state"); } #endif // __APPLE__ && AARCH64 + + private: + bool _in_asgct = false; + public: + bool in_asgct() const { return _in_asgct; } + void set_in_asgct(bool value) { _in_asgct = value; } + static bool current_in_asgct() { + Thread *cur = Thread::current_or_null_safe(); + return cur != nullptr && cur->in_asgct(); + } +}; + +class ThreadInAsgct { + private: + Thread* _thread; + public: + ThreadInAsgct(Thread* thread) : _thread(thread) { + assert(thread != nullptr, "invariant"); + assert(!thread->in_asgct(), "invariant"); + thread->set_in_asgct(true); + } + ~ThreadInAsgct() { + assert(_thread->in_asgct(), "invariant"); + _thread->set_in_asgct(false); + } }; // Inline implementation of Thread::current() @@ -1428,6 +1452,10 @@ class JavaThread: public Thread { // Print stack trace in external format void print_stack_on(outputStream* st); void print_stack() { print_stack_on(tty); } + // Print current stack trace for checked JNI warnings and JNI fatal errors. + // This is the external format from above, but selecting the platform + // as applicable. + void print_jni_stack(); // Print stack traces in various internal formats void trace_stack() PRODUCT_RETURN; @@ -1615,6 +1643,15 @@ class JavaThread: public Thread { static void verify_cross_modify_fence_failure(JavaThread *thread) PRODUCT_RETURN; + // Helper function to start a VM-internal daemon thread. + // E.g. ServiceThread, NotificationThread, CompilerThread etc. + static void start_internal_daemon(JavaThread* current, JavaThread* target, + Handle thread_oop, ThreadPriority prio); + + // Helper function to do vm_exit_on_initialization for osthread + // resource allocation failure. + static void vm_exit_on_osthread_failure(JavaThread* thread); + // AsyncGetCallTrace support inline bool in_asgct(void) {return _in_asgct;} inline void set_in_asgct(bool value) {_in_asgct = value;} diff --git a/src/hotspot/share/runtime/thread.inline.hpp b/src/hotspot/share/runtime/thread.inline.hpp index 63101e7785592..d86fce3c8acc7 100644 --- a/src/hotspot/share/runtime/thread.inline.hpp +++ b/src/hotspot/share/runtime/thread.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2021, Azul Systems, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -132,7 +132,7 @@ inline void JavaThread::set_pending_async_exception(oop e) { } inline JavaThreadState JavaThread::thread_state() const { -#if defined(PPC64) || defined (AARCH64) +#if defined(PPC64) || defined (AARCH64) || defined(RISCV64) // Use membars when accessing volatile _thread_state. See // Threads::create_vm() for size checks. return (JavaThreadState) Atomic::load_acquire((volatile jint*)&_thread_state); @@ -144,7 +144,7 @@ inline JavaThreadState JavaThread::thread_state() const { inline void JavaThread::set_thread_state(JavaThreadState s) { assert(current_or_null() == NULL || current_or_null() == this, "state change should only be called by the current thread"); -#if defined(PPC64) || defined (AARCH64) +#if defined(PPC64) || defined (AARCH64) || defined(RISCV64) // Use membars when accessing volatile _thread_state. See // Threads::create_vm() for size checks. Atomic::release_store((volatile jint*)&_thread_state, (jint)s); diff --git a/src/hotspot/share/runtime/threadSMR.cpp b/src/hotspot/share/runtime/threadSMR.cpp index 7ff5e6fd1ae46..fc90da14ddcbd 100644 --- a/src/hotspot/share/runtime/threadSMR.cpp +++ b/src/hotspot/share/runtime/threadSMR.cpp @@ -792,19 +792,20 @@ bool ThreadsListHandle::cv_internal_thread_to_JavaThread(jobject jthread, *thread_oop_p = thread_oop; } - JavaThread *java_thread = java_lang_Thread::thread(thread_oop); + JavaThread *java_thread = java_lang_Thread::thread_acquire(thread_oop); if (java_thread == NULL) { - // The java.lang.Thread does not contain a JavaThread * so it has - // not yet run or it has died. + // The java.lang.Thread does not contain a JavaThread* so it has not + // run enough to be put on a ThreadsList or it has exited enough to + // make it past ensure_join() where the JavaThread* is cleared. return false; } // Looks like a live JavaThread at this point. if (java_thread != JavaThread::current()) { - // jthread is not for the current JavaThread so have to verify - // the JavaThread * against the ThreadsList. - if (EnableThreadSMRExtraValidityChecks && !includes(java_thread)) { - // Not on the JavaThreads list so it is not alive. + // java_thread is not the current JavaThread so we have to verify it + // against the ThreadsList. + if (!includes(java_thread)) { + // Not on this ThreadsList so it is not protected. return false; } } @@ -815,6 +816,20 @@ bool ThreadsListHandle::cv_internal_thread_to_JavaThread(jobject jthread, return true; } +FastThreadsListHandle::FastThreadsListHandle(oop thread_oop, JavaThread* java_thread) : _protected_java_thread(nullptr) { + assert(thread_oop != nullptr, "must be"); + if (java_thread != nullptr) { + // We captured a non-nullptr JavaThread* before the _tlh was created + // so that covers the early life stage of the target JavaThread. + _protected_java_thread = java_lang_Thread::thread(thread_oop); + assert(_protected_java_thread == nullptr || _tlh.includes(_protected_java_thread), "must be"); + // If we captured a non-nullptr JavaThread* after the _tlh was created + // then that covers the end life stage of the target JavaThread and we + // we know that _tlh protects the JavaThread*. The underlying atomic + // load is sufficient (no acquire necessary here). + } +} + void ThreadsSMRSupport::add_thread(JavaThread *thread){ ThreadsList *new_list = ThreadsList::add_thread(get_java_thread_list(), thread); if (EnableThreadSMRStatistics) { diff --git a/src/hotspot/share/runtime/threadSMR.hpp b/src/hotspot/share/runtime/threadSMR.hpp index a4df98cde0a6d..82e9296701286 100644 --- a/src/hotspot/share/runtime/threadSMR.hpp +++ b/src/hotspot/share/runtime/threadSMR.hpp @@ -320,6 +320,29 @@ class ThreadsListHandle : public StackObj { } }; +// This stack allocated FastThreadsListHandle implements the special case +// where we want to quickly determine if a JavaThread* is protected by the +// embedded ThreadsListHandle. +// +class FastThreadsListHandle : public StackObj { + JavaThread* _protected_java_thread; + ThreadsListHandle _tlh; + +public: + // The 'java_thread' parameter to the constructor must be provided + // by a java_lang_Thread::thread_acquire(thread_oop) call which gets + // us the JavaThread* stored in the java.lang.Thread object _before_ + // the embedded ThreadsListHandle is constructed. We use acquire there + // to ensure that if we see a non-nullptr value, then we also see the + // main ThreadsList updates from the JavaThread* being added. + // + FastThreadsListHandle(oop thread_oop, JavaThread* java_thread); + + JavaThread* protected_java_thread() { + return _protected_java_thread; + } +}; + // This stack allocated JavaThreadIterator is used to walk the // specified ThreadsList using the following style: // diff --git a/src/hotspot/share/runtime/vmOperation.hpp b/src/hotspot/share/runtime/vmOperation.hpp index 1fe9a60841e16..d331a0a0539bd 100644 --- a/src/hotspot/share/runtime/vmOperation.hpp +++ b/src/hotspot/share/runtime/vmOperation.hpp @@ -59,7 +59,8 @@ template(ParallelGCSystemGC) \ template(G1CollectForAllocation) \ template(G1CollectFull) \ - template(G1Concurrent) \ + template(G1PauseRemark) \ + template(G1PauseCleanup) \ template(G1TryInitiateConcMark) \ template(ZMarkStart) \ template(ZMarkEnd) \ diff --git a/src/hotspot/share/runtime/vmStructs.cpp b/src/hotspot/share/runtime/vmStructs.cpp index 2a38e09e67ffa..7bf40685cf63e 100644 --- a/src/hotspot/share/runtime/vmStructs.cpp +++ b/src/hotspot/share/runtime/vmStructs.cpp @@ -1598,7 +1598,7 @@ typedef HashtableEntry KlassHashtableEntry; declare_c2_type(MemBarVolatileNode, MemBarNode) \ declare_c2_type(MemBarCPUOrderNode, MemBarNode) \ declare_c2_type(OnSpinWaitNode, MemBarNode) \ - declare_c2_type(BlackholeNode, MemBarNode) \ + declare_c2_type(BlackholeNode, MultiNode) \ declare_c2_type(InitializeNode, MemBarNode) \ declare_c2_type(ThreadLocalNode, Node) \ declare_c2_type(Opaque1Node, Node) \ diff --git a/src/hotspot/share/services/attachListener.cpp b/src/hotspot/share/services/attachListener.cpp index 7f147c65d2742..527e5b606aa10 100644 --- a/src/hotspot/share/services/attachListener.cpp +++ b/src/hotspot/share/services/attachListener.cpp @@ -489,22 +489,10 @@ void AttachListener::init() { return; } - { MutexLocker mu(THREAD, Threads_lock); - JavaThread* listener_thread = new JavaThread(&attach_listener_thread_entry); + JavaThread* thread = new JavaThread(&attach_listener_thread_entry); + JavaThread::vm_exit_on_osthread_failure(thread); - // Check that thread and osthread were created - if (listener_thread == NULL || listener_thread->osthread() == NULL) { - vm_exit_during_initialization("java.lang.OutOfMemoryError", - os::native_thread_creation_failed_msg()); - } - - java_lang_Thread::set_thread(thread_oop(), listener_thread); - java_lang_Thread::set_daemon(thread_oop()); - - listener_thread->set_threadObj(thread_oop()); - Threads::add(listener_thread); - Thread::start(listener_thread); - } + JavaThread::start_internal_daemon(THREAD, thread, thread_oop, NoPriority); } // Performs clean-up tasks on platforms where we can detect that the last diff --git a/src/hotspot/share/services/diagnosticCommand.cpp b/src/hotspot/share/services/diagnosticCommand.cpp index 51f47421c5089..08e08a7f84466 100644 --- a/src/hotspot/share/services/diagnosticCommand.cpp +++ b/src/hotspot/share/services/diagnosticCommand.cpp @@ -800,7 +800,8 @@ void JMXStatusDCmd::execute(DCmdSource source, TRAPS) { if (str != NULL) { char* out = java_lang_String::as_utf8_string(str); if (out) { - output()->print_cr("%s", out); + // Avoid using print_cr() because length maybe longer than O_BUFLEN + output()->print_raw_cr(out); return; } } diff --git a/src/hotspot/share/services/heapDumperCompression.cpp b/src/hotspot/share/services/heapDumperCompression.cpp index 41c9726109494..222c6e383f2ff 100644 --- a/src/hotspot/share/services/heapDumperCompression.cpp +++ b/src/hotspot/share/services/heapDumperCompression.cpp @@ -54,10 +54,14 @@ char const* FileWriter::write_buf(char* buf, ssize_t size) { assert(_fd >= 0, "Must be open"); assert(size > 0, "Must write at least one byte"); - ssize_t n = (ssize_t) os::write(_fd, buf, (uint) size); + while (size > 0) { + ssize_t n = os::write(_fd, buf, (uint) size); + if (n <= 0) { + return os::strerror(errno); + } - if (n <= 0) { - return os::strerror(errno); + buf += n; + size -= n; } return NULL; diff --git a/src/hotspot/share/services/memReporter.cpp b/src/hotspot/share/services/memReporter.cpp index 92f3d4da4c959..f783560c4ce8c 100644 --- a/src/hotspot/share/services/memReporter.cpp +++ b/src/hotspot/share/services/memReporter.cpp @@ -731,6 +731,13 @@ void MemDetailDiffReporter::diff_virtual_memory_sites() const { } else if (compVal > 0) { old_virtual_memory_site(early_site); early_site = early_itr.next(); + } else if (early_site->flag() != current_site->flag()) { + // This site was originally allocated with one flag, then released, + // then re-allocated at the same site (as far as we can tell) with a different flag. + old_virtual_memory_site(early_site); + early_site = early_itr.next(); + new_virtual_memory_site(current_site); + current_site = current_itr.next(); } else { diff_virtual_memory_site(early_site, current_site); early_site = early_itr.next(); @@ -793,7 +800,6 @@ void MemDetailDiffReporter::old_virtual_memory_site(const VirtualMemoryAllocatio void MemDetailDiffReporter::diff_virtual_memory_site(const VirtualMemoryAllocationSite* early, const VirtualMemoryAllocationSite* current) const { - assert(early->flag() == current->flag(), "Should be the same"); diff_virtual_memory_site(current->call_stack(), current->reserved(), current->committed(), early->reserved(), early->committed(), current->flag()); } diff --git a/src/hotspot/share/services/memoryManager.cpp b/src/hotspot/share/services/memoryManager.cpp index a2c155b6fca6c..3f35604c60e7b 100644 --- a/src/hotspot/share/services/memoryManager.cpp +++ b/src/hotspot/share/services/memoryManager.cpp @@ -170,8 +170,8 @@ void GCStatInfo::clear() { } -GCMemoryManager::GCMemoryManager(const char* name, const char* gc_end_message) : - MemoryManager(name), _gc_end_message(gc_end_message) { +GCMemoryManager::GCMemoryManager(const char* name) : + MemoryManager(name) { _num_collections = 0; _last_gc_stat = NULL; _last_gc_lock = new Mutex(Mutex::leaf, "_last_gc_lock", true, @@ -236,9 +236,11 @@ void GCMemoryManager::gc_begin(bool recordGCBeginTime, bool recordPreGCUsage, // to ensure the current gc stat is placed in _last_gc_stat. void GCMemoryManager::gc_end(bool recordPostGCUsage, bool recordAccumulatedGCTime, - bool recordGCEndTime, bool countCollection, + bool recordGCEndTime, + bool countCollection, GCCause::Cause cause, - bool allMemoryPoolsAffected) { + bool allMemoryPoolsAffected, + const char* message) { if (recordAccumulatedGCTime) { _accumulated_timer.stop(); } @@ -288,7 +290,7 @@ void GCMemoryManager::gc_end(bool recordPostGCUsage, } if (is_notification_enabled()) { - GCNotifier::pushNotification(this, _gc_end_message, GCCause::to_string(cause)); + GCNotifier::pushNotification(this, message, GCCause::to_string(cause)); } } } diff --git a/src/hotspot/share/services/memoryManager.hpp b/src/hotspot/share/services/memoryManager.hpp index ca40425bf3f64..1518387ed0e4f 100644 --- a/src/hotspot/share/services/memoryManager.hpp +++ b/src/hotspot/share/services/memoryManager.hpp @@ -140,11 +140,10 @@ class GCMemoryManager : public MemoryManager { GCStatInfo* _current_gc_stat; int _num_gc_threads; volatile bool _notification_enabled; - const char* _gc_end_message; bool _pool_always_affected_by_gc[MemoryManager::max_num_pools]; public: - GCMemoryManager(const char* name, const char* gc_end_message); + GCMemoryManager(const char* name); ~GCMemoryManager(); void add_pool(MemoryPool* pool); @@ -167,7 +166,7 @@ class GCMemoryManager : public MemoryManager { bool recordAccumulatedGCTime); void gc_end(bool recordPostGCUsage, bool recordAccumulatedGCTime, bool recordGCEndTime, bool countCollection, GCCause::Cause cause, - bool allMemoryPoolsAffected); + bool allMemoryPoolsAffected, const char* message); void reset_gc_stat() { _num_collections = 0; _accumulated_timer.reset(); } diff --git a/src/hotspot/share/services/memoryService.cpp b/src/hotspot/share/services/memoryService.cpp index bb74092b15c18..b9edf887b8373 100644 --- a/src/hotspot/share/services/memoryService.cpp +++ b/src/hotspot/share/services/memoryService.cpp @@ -182,10 +182,10 @@ void MemoryService::gc_end(GCMemoryManager* manager, bool recordPostGCUsage, bool recordAccumulatedGCTime, bool recordGCEndTime, bool countCollection, GCCause::Cause cause, - bool allMemoryPoolsAffected) { + bool allMemoryPoolsAffected, const char* message) { // register the GC end statistics and memory usage manager->gc_end(recordPostGCUsage, recordAccumulatedGCTime, recordGCEndTime, - countCollection, cause, allMemoryPoolsAffected); + countCollection, cause, allMemoryPoolsAffected, message); } bool MemoryService::set_verbose(bool verbose) { @@ -219,6 +219,7 @@ Handle MemoryService::create_MemoryUsage_obj(MemoryUsage usage, TRAPS) { TraceMemoryManagerStats::TraceMemoryManagerStats(GCMemoryManager* gc_memory_manager, GCCause::Cause cause, + const char* end_message, bool allMemoryPoolsAffected, bool recordGCBeginTime, bool recordPreGCUsage, @@ -227,16 +228,17 @@ TraceMemoryManagerStats::TraceMemoryManagerStats(GCMemoryManager* gc_memory_mana bool recordAccumulatedGCTime, bool recordGCEndTime, bool countCollection) { - initialize(gc_memory_manager, cause, allMemoryPoolsAffected, - recordGCBeginTime, recordPreGCUsage, recordPeakUsage, - recordPostGCUsage, recordAccumulatedGCTime, recordGCEndTime, - countCollection); + initialize(gc_memory_manager, cause, end_message, + allMemoryPoolsAffected, recordGCBeginTime, recordPreGCUsage, + recordPeakUsage, recordPostGCUsage, recordAccumulatedGCTime, + recordGCEndTime, countCollection); } // for a subclass to create then initialize an instance before invoking // the MemoryService void TraceMemoryManagerStats::initialize(GCMemoryManager* gc_memory_manager, GCCause::Cause cause, + const char* end_message, bool allMemoryPoolsAffected, bool recordGCBeginTime, bool recordPreGCUsage, @@ -246,6 +248,8 @@ void TraceMemoryManagerStats::initialize(GCMemoryManager* gc_memory_manager, bool recordGCEndTime, bool countCollection) { _gc_memory_manager = gc_memory_manager; + _cause = cause; + _end_message = end_message; _allMemoryPoolsAffected = allMemoryPoolsAffected; _recordGCBeginTime = recordGCBeginTime; _recordPreGCUsage = recordPreGCUsage; @@ -254,7 +258,6 @@ void TraceMemoryManagerStats::initialize(GCMemoryManager* gc_memory_manager, _recordAccumulatedGCTime = recordAccumulatedGCTime; _recordGCEndTime = recordGCEndTime; _countCollection = countCollection; - _cause = cause; MemoryService::gc_begin(_gc_memory_manager, _recordGCBeginTime, _recordAccumulatedGCTime, _recordPreGCUsage, _recordPeakUsage); @@ -262,5 +265,6 @@ void TraceMemoryManagerStats::initialize(GCMemoryManager* gc_memory_manager, TraceMemoryManagerStats::~TraceMemoryManagerStats() { MemoryService::gc_end(_gc_memory_manager, _recordPostGCUsage, _recordAccumulatedGCTime, - _recordGCEndTime, _countCollection, _cause, _allMemoryPoolsAffected); + _recordGCEndTime, _countCollection, _cause, _allMemoryPoolsAffected, + _end_message); } diff --git a/src/hotspot/share/services/memoryService.hpp b/src/hotspot/share/services/memoryService.hpp index 3c7bc483dfbed..a00e49dd0a772 100644 --- a/src/hotspot/share/services/memoryService.hpp +++ b/src/hotspot/share/services/memoryService.hpp @@ -104,7 +104,7 @@ class MemoryService : public AllStatic { bool recordAccumulatedGCTime, bool recordGCEndTime, bool countCollection, GCCause::Cause cause, - bool allMemoryPoolsAffected); + bool allMemoryPoolsAffected, const char* notificationMessage = nullptr); static bool get_verbose() { return log_is_enabled(Info, gc); } static bool set_verbose(bool verbose); @@ -116,19 +116,21 @@ class MemoryService : public AllStatic { class TraceMemoryManagerStats : public StackObj { private: GCMemoryManager* _gc_memory_manager; - bool _allMemoryPoolsAffected; - bool _recordGCBeginTime; - bool _recordPreGCUsage; - bool _recordPeakUsage; - bool _recordPostGCUsage; - bool _recordAccumulatedGCTime; - bool _recordGCEndTime; - bool _countCollection; - GCCause::Cause _cause; + GCCause::Cause _cause; + const char* _end_message; + bool _allMemoryPoolsAffected; + bool _recordGCBeginTime; + bool _recordPreGCUsage; + bool _recordPeakUsage; + bool _recordPostGCUsage; + bool _recordAccumulatedGCTime; + bool _recordGCEndTime; + bool _countCollection; public: TraceMemoryManagerStats() {} TraceMemoryManagerStats(GCMemoryManager* gc_memory_manager, GCCause::Cause cause, + const char* end_message, bool allMemoryPoolsAffected = true, bool recordGCBeginTime = true, bool recordPreGCUsage = true, @@ -140,6 +142,7 @@ class TraceMemoryManagerStats : public StackObj { void initialize(GCMemoryManager* gc_memory_manager, GCCause::Cause cause, + const char* end_message, bool allMemoryPoolsAffected, bool recordGCBeginTime, bool recordPreGCUsage, diff --git a/src/hotspot/share/utilities/compilerWarnings.hpp b/src/hotspot/share/utilities/compilerWarnings.hpp index 98e66587e7479..fb796859b96ef 100644 --- a/src/hotspot/share/utilities/compilerWarnings.hpp +++ b/src/hotspot/share/utilities/compilerWarnings.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -55,6 +55,10 @@ #define ATTRIBUTE_SCANF(fmt, vargs) #endif +#ifndef PRAGMA_DANGLING_POINTER_IGNORED +#define PRAGMA_DANGLING_POINTER_IGNORED +#endif + #ifndef PRAGMA_FORMAT_NONLITERAL_IGNORED #define PRAGMA_FORMAT_NONLITERAL_IGNORED #endif @@ -66,4 +70,8 @@ #define PRAGMA_STRINGOP_TRUNCATION_IGNORED #endif +#ifndef PRAGMA_INFINITE_RECURSION_IGNORED +#define PRAGMA_INFINITE_RECURSION_IGNORED +#endif + #endif // SHARE_UTILITIES_COMPILERWARNINGS_HPP diff --git a/src/hotspot/share/utilities/compilerWarnings_gcc.hpp b/src/hotspot/share/utilities/compilerWarnings_gcc.hpp index 27ea42691a8ff..ecc13dc2b0739 100644 --- a/src/hotspot/share/utilities/compilerWarnings_gcc.hpp +++ b/src/hotspot/share/utilities/compilerWarnings_gcc.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -38,6 +38,14 @@ #define PRAGMA_DISABLE_GCC_WARNING(option_string) \ PRAGMA_DISABLE_GCC_WARNING_AUX(GCC diagnostic ignored option_string) +#if !defined(__clang_major__) && (__GNUC__ >= 12) +// Disable -Wdangling-pointer which is introduced in GCC 12. +#define PRAGMA_DANGLING_POINTER_IGNORED PRAGMA_DISABLE_GCC_WARNING("-Wdangling-pointer") + +// Disable -Winfinite-recursion which is introduced in GCC 12. +#define PRAGMA_INFINITE_RECURSION_IGNORED PRAGMA_DISABLE_GCC_WARNING("-Winfinite-recursion") +#endif + #define PRAGMA_FORMAT_NONLITERAL_IGNORED \ PRAGMA_DISABLE_GCC_WARNING("-Wformat-nonliteral") \ PRAGMA_DISABLE_GCC_WARNING("-Wformat-security") diff --git a/src/hotspot/share/utilities/debug.cpp b/src/hotspot/share/utilities/debug.cpp index f623d191a9bc6..809e8d97f8099 100644 --- a/src/hotspot/share/utilities/debug.cpp +++ b/src/hotspot/share/utilities/debug.cpp @@ -676,7 +676,8 @@ extern "C" JNIEXPORT void pns(void* sp, void* fp, void* pc) { // print native st extern "C" JNIEXPORT void pns2() { // print native stack Command c("pns2"); static char buf[O_BUFLEN]; - if (os::platform_print_native_stack(tty, NULL, buf, sizeof(buf))) { + address lastpc = nullptr; + if (os::platform_print_native_stack(tty, NULL, buf, sizeof(buf), lastpc)) { // We have printed the native stack in platform-specific code, // so nothing else to do in this case. } else { diff --git a/src/hotspot/share/utilities/globalDefinitions.hpp b/src/hotspot/share/utilities/globalDefinitions.hpp index 9b70dc2557018..dc756359967d9 100644 --- a/src/hotspot/share/utilities/globalDefinitions.hpp +++ b/src/hotspot/share/utilities/globalDefinitions.hpp @@ -324,6 +324,9 @@ inline T byte_size_in_proper_unit(T s) { } } +#define PROPERFMT SIZE_FORMAT "%s" +#define PROPERFMTARGS(s) byte_size_in_proper_unit(s), proper_unit_for_byte_size(s) + inline const char* exact_unit_for_byte_size(size_t s) { #ifdef _LP64 if (s >= G && (s % G) == 0) { diff --git a/src/hotspot/share/utilities/macros.hpp b/src/hotspot/share/utilities/macros.hpp index 501ba0dbabc25..33ecfe089f854 100644 --- a/src/hotspot/share/utilities/macros.hpp +++ b/src/hotspot/share/utilities/macros.hpp @@ -553,6 +553,32 @@ #define MACOS_AARCH64_ONLY(x) MACOS_ONLY(AARCH64_ONLY(x)) +#if defined(RISCV32) || defined(RISCV64) +#define RISCV +#define RISCV_ONLY(code) code +#define NOT_RISCV(code) +#else +#undef RISCV +#define RISCV_ONLY(code) +#define NOT_RISCV(code) code +#endif + +#ifdef RISCV32 +#define RISCV32_ONLY(code) code +#define NOT_RISCV32(code) +#else +#define RISCV32_ONLY(code) +#define NOT_RISCV32(code) code +#endif + +#ifdef RISCV64 +#define RISCV64_ONLY(code) code +#define NOT_RISCV64(code) +#else +#define RISCV64_ONLY(code) +#define NOT_RISCV64(code) code +#endif + #ifdef VM_LITTLE_ENDIAN #define LITTLE_ENDIAN_ONLY(code) code #define BIG_ENDIAN_ONLY(code) diff --git a/src/hotspot/share/utilities/nativeCallStack.hpp b/src/hotspot/share/utilities/nativeCallStack.hpp index 33ce0164d84d4..8387a1ce25977 100644 --- a/src/hotspot/share/utilities/nativeCallStack.hpp +++ b/src/hotspot/share/utilities/nativeCallStack.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -64,7 +64,7 @@ class NativeCallStack : public StackObj { memset(_stack, 0, sizeof(_stack)); } - NativeCallStack(int toSkip); + explicit NativeCallStack(int toSkip); NativeCallStack(address* pc, int frameCount); static inline const NativeCallStack& empty_stack() { return _empty_stack; } diff --git a/src/hotspot/share/utilities/vmError.cpp b/src/hotspot/share/utilities/vmError.cpp index 5d050693ffd6d..42e5fdf685d09 100644 --- a/src/hotspot/share/utilities/vmError.cpp +++ b/src/hotspot/share/utilities/vmError.cpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2017, 2020 SAP SE. All rights reserved. + * Copyright (c) 2023, Red Hat, Inc. and/or its affiliates. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -44,7 +45,7 @@ #include "runtime/init.hpp" #include "runtime/os.hpp" #include "runtime/osThread.hpp" -#include "runtime/safefetch.inline.hpp" +#include "runtime/safefetch.hpp" #include "runtime/safepointMechanism.hpp" #include "runtime/stackFrameStream.inline.hpp" #include "runtime/thread.inline.hpp" @@ -84,6 +85,7 @@ Thread* VMError::_thread; address VMError::_pc; void* VMError::_siginfo; void* VMError::_context; +bool VMError::_print_native_stack_used = false; const char* VMError::_filename; int VMError::_lineno; size_t VMError::_size; @@ -94,6 +96,8 @@ static const char* env_list[] = { "JAVA_HOME", "JAVA_TOOL_OPTIONS", "_JAVA_OPTIONS", "CLASSPATH", "PATH", "USERNAME", + "XDG_CACHE_HOME", "XDG_CONFIG_HOME", "FC_LANG", "FONTCONFIG_USE_MMAP", + // Env variables that are defined on Linux/BSD "LD_LIBRARY_PATH", "LD_PRELOAD", "SHELL", "DISPLAY", "HOSTTYPE", "OSTYPE", "ARCH", "MACHTYPE", @@ -239,6 +243,129 @@ void VMError::print_stack_trace(outputStream* st, JavaThread* jt, #endif // ZERO } +/** + * Adds `value` to `list` iff it's not already present and there is sufficient + * capacity (i.e. length(list) < `list_capacity`). The length of the list + * is the index of the first nullptr entry or `list_capacity` if there are + * no nullptr entries. + * + * @ return true if the value was added, false otherwise + */ +static bool add_if_absent(address value, address* list, int list_capacity) { + for (int i = 0; i < list_capacity; i++) { + if (list[i] == value) { + return false; + } + if (list[i] == nullptr) { + list[i] = value; + if (i + 1 < list_capacity) { + list[i + 1] = nullptr; + } + return true; + } + } + return false; +} + +/** + * Prints the VM generated code unit, if any, containing `pc` if it has not already + * been printed. If the code unit is an InterpreterCodelet or StubCodeDesc, it is + * only printed if `is_crash_pc` is true. + * + * @param printed array of code units that have already been printed (delimited by NULL entry) + * @param printed_capacity the capacity of `printed` + * @return true if the code unit was printed, false otherwise + */ +static bool print_code(outputStream* st, Thread* thread, address pc, bool is_crash_pc, + address* printed, int printed_capacity) { + if (Interpreter::contains(pc)) { + if (is_crash_pc) { + // The interpreter CodeBlob is very large so try to print the codelet instead. + InterpreterCodelet* codelet = Interpreter::codelet_containing(pc); + if (codelet != nullptr) { + if (add_if_absent((address) codelet, printed, printed_capacity)) { + codelet->print_on(st); + Disassembler::decode(codelet->code_begin(), codelet->code_end(), st); + return true; + } + } + } + } else { + StubCodeDesc* desc = StubCodeDesc::desc_for(pc); + if (desc != nullptr) { + if (is_crash_pc) { + if (add_if_absent((address) desc, printed, printed_capacity)) { + desc->print_on(st); + Disassembler::decode(desc->begin(), desc->end(), st); + return true; + } + } + } else if (thread != nullptr) { + CodeBlob* cb = CodeCache::find_blob(pc); + if (cb != nullptr && add_if_absent((address) cb, printed, printed_capacity)) { + // Disassembling nmethod will incur resource memory allocation, + // only do so when thread is valid. + ResourceMark rm(thread); + Disassembler::decode(cb, st); + st->cr(); + return true; + } + } + } + return false; +} + +// Like above, but only try to figure out a short name. Return nullptr if not found. +static const char* find_code_name(address pc) { + if (Interpreter::contains(pc)) { + InterpreterCodelet* codelet = Interpreter::codelet_containing(pc); + if (codelet != nullptr) { + return codelet->description(); + } + } else { + StubCodeDesc* desc = StubCodeDesc::desc_for(pc); + if (desc != nullptr) { + return desc->name(); + } else { + CodeBlob* cb = CodeCache::find_blob(pc); + if (cb != nullptr) { + return cb->name(); + } + } + } + return nullptr; +} + +/** + * Gets the caller frame of `fr`. + * + * @returns an invalid frame (i.e. fr.pc() === 0) if the caller cannot be obtained + */ +static frame next_frame(frame fr, Thread* t) { + // Compiled code may use EBP register on x86 so it looks like + // non-walkable C frame. Use frame.sender() for java frames. + frame invalid; + if (t != nullptr && t->is_Java_thread()) { + // Catch very first native frame by using stack address. + // For JavaThread stack_base and stack_size should be set. + if (!t->is_in_full_stack((address)(fr.real_fp() + 1))) { + return invalid; + } + if (fr.is_java_frame() || fr.is_native_frame() || fr.is_runtime_frame()) { + RegisterMap map(t->as_Java_thread(), false); // No update + return fr.sender(&map); + } else { + // is_first_C_frame() does only simple checks for frame pointer, + // it will pass if java compiled code has a pointer in EBP. + if (os::is_first_C_frame(&fr)) return invalid; + return os::get_sender_for_C_frame(&fr); + } + } else { + if (os::is_first_C_frame(&fr)) return invalid; + return os::get_sender_for_C_frame(&fr); + } +} + void VMError::print_native_stack(outputStream* st, frame fr, Thread* t, char* buf, int buf_size) { // see if it's a valid frame @@ -256,26 +383,9 @@ void VMError::print_native_stack(outputStream* st, frame fr, Thread* t, char* bu } } st->cr(); - // Compiled code may use EBP register on x86 so it looks like - // non-walkable C frame. Use frame.sender() for java frames. - if (t && t->is_Java_thread()) { - // Catch very first native frame by using stack address. - // For JavaThread stack_base and stack_size should be set. - if (!t->is_in_full_stack((address)(fr.real_fp() + 1))) { - break; - } - if (fr.is_java_frame() || fr.is_native_frame() || fr.is_runtime_frame()) { - RegisterMap map(t->as_Java_thread(), false); // No update - fr = fr.sender(&map); - } else { - // is_first_C_frame() does only simple checks for frame pointer, - // it will pass if java compiled code has a pointer in EBP. - if (os::is_first_C_frame(&fr)) break; - fr = os::get_sender_for_C_frame(&fr); - } - } else { - if (os::is_first_C_frame(&fr)) break; - fr = os::get_sender_for_C_frame(&fr); + fr = next_frame(fr, t); + if (fr.pc() == nullptr) { + break; } } @@ -441,6 +551,10 @@ void VMError::report(outputStream* st, bool _verbose) { // don't allocate large buffer on stack static char buf[O_BUFLEN]; + // Native stack trace may get stuck. We try to handle the last pc if it + // belongs to VM generated code. + address lastpc = nullptr; + BEGIN STEP("printing fatal error message") @@ -498,18 +612,14 @@ void VMError::report(outputStream* st, bool _verbose) { // to test that resetting the signal handler works correctly. if (_verbose && TestSafeFetchInErrorHandler) { st->print_cr("Will test SafeFetch..."); - if (CanUseSafeFetch32()) { - int* const invalid_pointer = (int*)segfault_address; - const int x = 0x76543210; - int i1 = SafeFetch32(invalid_pointer, x); - int i2 = SafeFetch32(invalid_pointer, x); - if (i1 == x && i2 == x) { - st->print_cr("SafeFetch OK."); // Correctly deflected and returned default pattern - } else { - st->print_cr("??"); - } + int* const invalid_pointer = (int*)segfault_address; + const int x = 0x76543210; + int i1 = SafeFetch32(invalid_pointer, x); + int i2 = SafeFetch32(invalid_pointer, x); + if (i1 == x && i2 == x) { + st->print_cr("SafeFetch OK."); // Correctly deflected and returned default pattern } else { - st->print_cr("not possible; skipped."); + st->print_cr("??"); } } #endif // ASSERT @@ -737,14 +847,22 @@ void VMError::report(outputStream* st, bool _verbose) { STEP("printing native stack") if (_verbose) { - if (os::platform_print_native_stack(st, _context, buf, sizeof(buf))) { + if (os::platform_print_native_stack(st, _context, buf, sizeof(buf), lastpc)) { // We have printed the native stack in platform-specific code // Windows/x64 needs special handling. + // Stack walking may get stuck. Try to find the calling code. + if (lastpc != nullptr) { + const char* name = find_code_name(lastpc); + if (name != nullptr) { + st->print_cr("The last pc belongs to %s (printed below).", name); + } + } } else { frame fr = _context ? os::fetch_frame_from_context(_context) : os::current_frame(); print_native_stack(st, fr, _thread, buf, sizeof(buf)); + _print_native_stack_used = true; } } @@ -819,29 +937,54 @@ void VMError::report(outputStream* st, bool _verbose) { st->cr(); } - STEP("printing code blob if possible") + STEP("printing code blobs if possible") - if (_verbose && _context) { - CodeBlob* cb = CodeCache::find_blob(_pc); - if (cb != NULL) { - if (Interpreter::contains(_pc)) { - // The interpreter CodeBlob is very large so try to print the codelet instead. - InterpreterCodelet* codelet = Interpreter::codelet_containing(_pc); - if (codelet != NULL) { - codelet->print_on(st); - Disassembler::decode(codelet->code_begin(), codelet->code_end(), st); + if (_verbose) { + const int printed_capacity = max_error_log_print_code; + address printed[printed_capacity]; + printed[0] = nullptr; + int printed_len = 0; + // Even though ErrorLogPrintCodeLimit is ranged checked + // during argument parsing, there's no way to prevent it + // subsequently (i.e., after parsing) being set to a + // value outside the range. + int limit = MIN2(ErrorLogPrintCodeLimit, printed_capacity); + if (limit > 0) { + // Check if a pc was found by native stack trace above. + if (lastpc != nullptr) { + if (print_code(st, _thread, lastpc, true, printed, printed_capacity)) { + printed_len++; + } + } + + // Scan the native stack + if (!_print_native_stack_used) { + // Only try to print code of the crashing frame since + // the native stack cannot be walked with next_frame. + if (print_code(st, _thread, _pc, true, printed, printed_capacity)) { + printed_len++; } } else { - StubCodeDesc* desc = StubCodeDesc::desc_for(_pc); - if (desc != NULL) { - desc->print_on(st); - Disassembler::decode(desc->begin(), desc->end(), st); - } else if (_thread != NULL) { - // Disassembling nmethod will incur resource memory allocation, - // only do so when thread is valid. - ResourceMark rm(_thread); - Disassembler::decode(cb, st); - st->cr(); + frame fr = _context ? os::fetch_frame_from_context(_context) + : os::current_frame(); + while (printed_len < limit && fr.pc() != nullptr) { + if (print_code(st, _thread, fr.pc(), fr.pc() == _pc, printed, printed_capacity)) { + printed_len++; + } + fr = next_frame(fr, _thread); + } + } + + // Scan the Java stack + if (_thread != nullptr && _thread->is_Java_thread()) { + JavaThread* jt = _thread->as_Java_thread(); + if (jt->has_last_Java_frame()) { + for (StackFrameStream sfs(jt, true /* update */, true /* process_frames */); printed_len < limit && !sfs.is_done(); sfs.next()) { + address pc = sfs.current()->pc(); + if (print_code(st, _thread, pc, pc == _pc, printed, printed_capacity)) { + printed_len++; + } + } } } } @@ -1713,43 +1856,68 @@ void VMError::show_message_box(char *buf, int buflen) { } while (yes); } -// Timeout handling: check if a timeout happened (either a single step did -// timeout or the whole of error reporting hit ErrorLogTimeout). Interrupt -// the reporting thread if that is the case. +// Fatal error handling is subject to several timeouts: +// - a global timeout (controlled via ErrorLogTimeout) +// - local error reporting step timeouts. +// +// The latter aims to "give the JVM a kick" if it gets stuck in one particular place during +// error reporting. This prevents one error reporting step from hogging all the time allotted +// to error reporting under ErrorLogTimeout. +// +// VMError::check_timeout() is called from the watcher thread and checks for either global +// or step timeout. If a timeout happened, we interrupt the reporting thread and set either +// _reporting_did_timeout or _step_did_timeout to signal which timeout fired. Function returns +// true if the *global* timeout fired, which will cause WatcherThread to shut down the JVM +// immediately. bool VMError::check_timeout() { + // This function is supposed to be called from watcher thread during fatal error handling only. + assert(VMError::is_error_reported(), "Only call during error handling"); + assert(Thread::current()->is_Watcher_thread(), "Only call from watcher thread"); + if (ErrorLogTimeout == 0) { return false; } - // Do not check for timeouts if we still have a message box to show to the - // user or if there are OnError handlers to be run. - if (ShowMessageBoxOnError - || (OnError != NULL && OnError[0] != '\0') - || Arguments::abort_hook() != NULL) { - return false; - } + // There are three situations where we suppress the *global* error timeout: + // - if the JVM is embedded and the launcher has its abort hook installed. + // That must be allowed to run. + // - if the user specified one or more OnError commands to run, and these + // did not yet run. These must have finished. + // - if the user (typically developer) specified ShowMessageBoxOnError, + // and the error box has not yet been shown + const bool ignore_global_timeout = + (ShowMessageBoxOnError + || (OnError != nullptr && OnError[0] != '\0') + || Arguments::abort_hook() != nullptr); - const jlong reporting_start_time_l = get_reporting_start_time(); const jlong now = get_current_timestamp(); - // Timestamp is stored in nanos. - if (reporting_start_time_l > 0) { - const jlong end = reporting_start_time_l + (jlong)ErrorLogTimeout * TIMESTAMP_TO_SECONDS_FACTOR; - if (end <= now && !_reporting_did_timeout) { - // We hit ErrorLogTimeout and we haven't interrupted the reporting - // thread yet. - _reporting_did_timeout = true; - interrupt_reporting_thread(); - return true; // global timeout + + // Global timeout hit? + if (!ignore_global_timeout) { + const jlong reporting_start_time = get_reporting_start_time(); + // Timestamp is stored in nanos. + if (reporting_start_time > 0) { + const jlong end = reporting_start_time + (jlong)ErrorLogTimeout * TIMESTAMP_TO_SECONDS_FACTOR; + if (end <= now && !_reporting_did_timeout) { + // We hit ErrorLogTimeout and we haven't interrupted the reporting + // thread yet. + _reporting_did_timeout = true; + interrupt_reporting_thread(); + return true; // global timeout + } } } - const jlong step_start_time_l = get_step_start_time(); - if (step_start_time_l > 0) { + // Reporting step timeout? + const jlong step_start_time = get_step_start_time(); + if (step_start_time > 0) { // A step times out after a quarter of the total timeout. Steps are mostly fast unless they // hang for some reason, so this simple rule allows for three hanging step and still // hopefully leaves time enough for the rest of the steps to finish. - const jlong end = step_start_time_l + (jlong)ErrorLogTimeout * TIMESTAMP_TO_SECONDS_FACTOR / 4; + const int max_step_timeout_secs = 5; + const jlong timeout_duration = MAX2((jlong)max_step_timeout_secs, (jlong)ErrorLogTimeout * TIMESTAMP_TO_SECONDS_FACTOR / 4); + const jlong end = step_start_time + timeout_duration; if (end <= now && !_step_did_timeout) { // The step timed out and we haven't interrupted the reporting // thread yet. diff --git a/src/hotspot/share/utilities/vmError.hpp b/src/hotspot/share/utilities/vmError.hpp index 8a6819d308b7c..b9b321a6563ac 100644 --- a/src/hotspot/share/utilities/vmError.hpp +++ b/src/hotspot/share/utilities/vmError.hpp @@ -51,6 +51,10 @@ class VMError : public AllStatic { static void* _context; // ContextRecord on Windows, // ucontext_t on Solaris/Linux + // records if VMError::print_native_stack was used to + // print the native stack instead of os::platform_print_native_stack + static bool _print_native_stack_used; + // additional info for VM internal errors static const char* _filename; static int _lineno; @@ -99,12 +103,6 @@ class VMError : public AllStatic { static void print_stack_trace(outputStream* st, JavaThread* jt, char* buf, int buflen, bool verbose = false); - // public for use by the internal non-product debugger. - NOT_PRODUCT(public:) - static void print_native_stack(outputStream* st, frame fr, Thread* t, - char* buf, int buf_size); - NOT_PRODUCT(private:) - static bool should_report_bug(unsigned int id) { return (id != OOM_MALLOC_ERROR) && (id != OOM_MMAP_ERROR); } @@ -134,6 +132,12 @@ class VMError : public AllStatic { public: + // print_source_info: if true, we try to resolve the source information on platforms that support it + // (useful but may slow down, timeout or misfunction in error situations) + // max_frames: if not -1, overrides StackPrintLimit + static void print_native_stack(outputStream* st, frame fr, Thread* t, + char* buf, int buf_size); + // return a string to describe the error static char* error_string(char* buf, int buflen); @@ -181,6 +185,9 @@ class VMError : public AllStatic { // which is not NULL and contains bits in every word. static const intptr_t segfault_address = LP64_ONLY(0xABC0000000000ABCULL) NOT_LP64(0x00000ABC); + // Max value for the ErrorLogPrintCodeLimit flag. + static const int max_error_log_print_code = 10; + // Needed when printing signal handlers. NOT_WINDOWS(static const void* crash_handler_address;) diff --git a/src/java.base/linux/classes/jdk/internal/platform/CgroupV1Metrics.java b/src/java.base/linux/classes/jdk/internal/platform/CgroupV1Metrics.java index cf8230b0bfc2e..c7e83faa59dda 100644 --- a/src/java.base/linux/classes/jdk/internal/platform/CgroupV1Metrics.java +++ b/src/java.base/linux/classes/jdk/internal/platform/CgroupV1Metrics.java @@ -53,16 +53,6 @@ public interface CgroupV1Metrics extends Metrics { */ public long getKernelMemoryFailCount(); - /** - * Returns the maximum amount of kernel physical memory, in bytes, that - * can be allocated in the Isolation Group. - * - * @return The maximum amount of memory in bytes or -1 if - * there is no limit set. - * - */ - public long getKernelMemoryLimit(); - /** * Returns the largest amount of kernel physical memory, in bytes, that * have been allocated in the Isolation Group. @@ -93,16 +83,6 @@ public interface CgroupV1Metrics extends Metrics { */ public long getTcpMemoryFailCount(); - /** - * Returns the maximum amount of networking physical memory, in bytes, - * that can be allocated in the Isolation Group. - * - * @return The maximum amount of memory in bytes or -1 if - * there is no limit. - * - */ - public long getTcpMemoryLimit(); - /** * Returns the largest amount of networking physical memory, in bytes, * that have been allocated in the Isolation Group. diff --git a/src/java.base/linux/classes/jdk/internal/platform/CgroupV1MetricsImpl.java b/src/java.base/linux/classes/jdk/internal/platform/CgroupV1MetricsImpl.java index 443b58f1f9dce..ce9b0c2cc20a0 100644 --- a/src/java.base/linux/classes/jdk/internal/platform/CgroupV1MetricsImpl.java +++ b/src/java.base/linux/classes/jdk/internal/platform/CgroupV1MetricsImpl.java @@ -48,11 +48,6 @@ public long getKernelMemoryFailCount() { return metrics.getKernelMemoryFailCount(); } - @Override - public long getKernelMemoryLimit() { - return metrics.getKernelMemoryLimit(); - } - @Override public long getKernelMemoryMaxUsage() { return metrics.getKernelMemoryMaxUsage(); @@ -68,11 +63,6 @@ public long getTcpMemoryFailCount() { return metrics.getTcpMemoryFailCount(); } - @Override - public long getTcpMemoryLimit() { - return metrics.getTcpMemoryLimit(); - } - @Override public long getTcpMemoryMaxUsage() { return metrics.getTcpMemoryMaxUsage(); diff --git a/src/java.base/linux/classes/jdk/internal/platform/cgroupv1/CgroupV1Subsystem.java b/src/java.base/linux/classes/jdk/internal/platform/cgroupv1/CgroupV1Subsystem.java index 07e9a168ef4d7..949a8622e560f 100644 --- a/src/java.base/linux/classes/jdk/internal/platform/cgroupv1/CgroupV1Subsystem.java +++ b/src/java.base/linux/classes/jdk/internal/platform/cgroupv1/CgroupV1Subsystem.java @@ -333,10 +333,6 @@ public long getKernelMemoryFailCount() { return getLongValue(memory, "memory.kmem.failcnt"); } - public long getKernelMemoryLimit() { - return CgroupV1SubsystemController.longValOrUnlimited(getLongValue(memory, "memory.kmem.limit_in_bytes")); - } - public long getKernelMemoryMaxUsage() { return getLongValue(memory, "memory.kmem.max_usage_in_bytes"); } @@ -349,10 +345,6 @@ public long getTcpMemoryFailCount() { return getLongValue(memory, "memory.kmem.tcp.failcnt"); } - public long getTcpMemoryLimit() { - return CgroupV1SubsystemController.longValOrUnlimited(getLongValue(memory, "memory.kmem.tcp.limit_in_bytes")); - } - public long getTcpMemoryMaxUsage() { return getLongValue(memory, "memory.kmem.tcp.max_usage_in_bytes"); } diff --git a/src/java.base/macosx/classes/apple/security/KeychainStore.java b/src/java.base/macosx/classes/apple/security/KeychainStore.java index e4a77e61cca5b..7cb280035e389 100644 --- a/src/java.base/macosx/classes/apple/security/KeychainStore.java +++ b/src/java.base/macosx/classes/apple/security/KeychainStore.java @@ -69,7 +69,7 @@ static class TrustedCertEntry { Certificate cert; long certRef; // SecCertificateRef for this key - // Each KeyStore.TrustedCertificateEntry have 2 attributes: + // Each KeyStore.TrustedCertificateEntry has 2 attributes: // 1. "trustSettings" -> trustSettings.toString() // 2. "2.16.840.1.113894.746875.1.1" -> trustedKeyUsageValue // The 1st one is mainly for debugging use. The 2nd one is similar @@ -660,7 +660,6 @@ public void engineStore(OutputStream stream, char[] password) _releaseKeychainItemRef(((TrustedCertEntry)entry).certRef); } } else { - Certificate certElem; KeyEntry keyEntry = (KeyEntry)entry; if (keyEntry.chain != null) { @@ -812,8 +811,26 @@ private void createTrustedCertEntry(String alias, List inputTrust, tce.cert = cert; tce.certRef = keychainItemRef; + // Check whether a certificate with same alias already exists and is the same + // If yes, we can return here - the existing entry must have the same + // properties and trust settings + if (entries.contains(alias.toLowerCase())) { + int uniqueVal = 1; + String originalAlias = alias; + var co = entries.get(alias.toLowerCase()); + while (co != null) { + if (co instanceof TrustedCertEntry tco) { + if (tco.cert.equals(tce.cert)) { + return; + } + } + alias = originalAlias + " " + uniqueVal++; + co = entries.get(alias.toLowerCase()); + } + } + tce.trustSettings = new ArrayList<>(); - Map tmpMap = new LinkedHashMap<>(); + Map tmpMap = new LinkedHashMap<>(); for (int i = 0; i < inputTrust.size(); i++) { if (inputTrust.get(i) == null) { tce.trustSettings.add(tmpMap); @@ -836,9 +853,10 @@ private void createTrustedCertEntry(String alias, List inputTrust, } catch (Exception e) { isSelfSigned = false; } + if (tce.trustSettings.isEmpty()) { if (isSelfSigned) { - // If a self-signed certificate has an empty trust settings, + // If a self-signed certificate has trust settings without specific entries, // trust it for all purposes tce.trustedKeyUsageValue = KnownOIDs.anyExtendedKeyUsage.value(); } else { @@ -851,11 +869,19 @@ private void createTrustedCertEntry(String alias, List inputTrust, for (var oneTrust : tce.trustSettings) { var result = oneTrust.get("kSecTrustSettingsResult"); // https://developer.apple.com/documentation/security/sectrustsettingsresult?language=objc - // 1 = kSecTrustSettingsResultTrustRoot, 2 = kSecTrustSettingsResultTrustAsRoot + // 1 = kSecTrustSettingsResultTrustRoot, 2 = kSecTrustSettingsResultTrustAsRoot, + // 3 = kSecTrustSettingsResultDeny // If missing, a default value of kSecTrustSettingsResultTrustRoot is assumed - // for self-signed certificates (see doc for SecTrustSettingsCopyTrustSettings). + // (see doc for SecTrustSettingsCopyTrustSettings). // Note that the same SecPolicyOid can appear in multiple trust settings // for different kSecTrustSettingsAllowedError and/or kSecTrustSettingsPolicyString. + + // If we find explicit distrust in some record, we ignore the certificate + if ("3".equals(result)) { + return; + } + + // Trust, if explicitly trusted or result is null and certificate is self signed if ((result == null && isSelfSigned) || "1".equals(result) || "2".equals(result)) { // When no kSecTrustSettingsPolicy, it means everything @@ -875,20 +901,13 @@ private void createTrustedCertEntry(String alias, List inputTrust, tce.trustedKeyUsageValue = values.toString(); } } + // Make a creation date. if (creationDate != 0) tce.date = new Date(creationDate); else tce.date = new Date(); - int uniqueVal = 1; - String originalAlias = alias; - - while (entries.containsKey(alias.toLowerCase())) { - alias = originalAlias + " " + uniqueVal; - uniqueVal++; - } - entries.put(alias.toLowerCase(), tce); } catch (Exception e) { // The certificate will be skipped. diff --git a/src/java.base/macosx/native/libosxsecurity/KeystoreImpl.m b/src/java.base/macosx/native/libosxsecurity/KeystoreImpl.m index 3f0c1de962d05..b4f13a80d96a9 100644 --- a/src/java.base/macosx/native/libosxsecurity/KeystoreImpl.m +++ b/src/java.base/macosx/native/libosxsecurity/KeystoreImpl.m @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -285,9 +285,14 @@ static void addIdentitiesToKeystore(JNIEnv *env, jobject keyStore) OSErr searchResult = noErr; jclass jc_KeychainStore = (*env)->FindClass(env, "apple/security/KeychainStore"); - CHECK_NULL(jc_KeychainStore); + if (jc_KeychainStore == NULL) { + goto errOut; + } jmethodID jm_createKeyEntry = (*env)->GetMethodID(env, jc_KeychainStore, "createKeyEntry", "(Ljava/lang/String;JJ[J[[B)V"); - CHECK_NULL(jm_createKeyEntry); + if (jm_createKeyEntry == NULL) { + goto errOut; + } + do { searchResult = SecIdentitySearchCopyNext(identitySearch, &theIdentity); @@ -358,7 +363,9 @@ static void addIdentitiesToKeystore(JNIEnv *env, jobject keyStore) // Call back to the Java object to create Java objects corresponding to this security object. jlong nativeKeyRef = ptr_to_jlong(privateKeyRef); (*env)->CallVoidMethod(env, keyStore, jm_createKeyEntry, alias, creationDate, nativeKeyRef, certRefArray, javaCertArray); - JNU_CHECK_EXCEPTION(env); + if ((*env)->ExceptionCheck(env)) { + goto errOut; + } } } while (searchResult == noErr); @@ -376,6 +383,35 @@ static void addIdentitiesToKeystore(JNIEnv *env, jobject keyStore) #define ADDNULL(list) (*env)->CallBooleanMethod(env, list, jm_listAdd, NULL) + +static void addTrustSettingsToInputTrust(JNIEnv *env, jmethodID jm_listAdd, CFArrayRef trustSettings, jobject inputTrust) +{ + CFIndex count = CFArrayGetCount(trustSettings); + for (int i = 0; i < count; i++) { + CFDictionaryRef oneTrust = (CFDictionaryRef) CFArrayGetValueAtIndex(trustSettings, i); + CFIndex size = CFDictionaryGetCount(oneTrust); + const void * keys [size]; + const void * values [size]; + CFDictionaryGetKeysAndValues(oneTrust, keys, values); + for (int j = 0; j < size; j++) { + NSString* s = [NSString stringWithFormat:@"%@", keys[j]]; + ADD(inputTrust, s); + s = [NSString stringWithFormat:@"%@", values[j]]; + ADD(inputTrust, s); + } + SecPolicyRef certPolicy; + certPolicy = (SecPolicyRef)CFDictionaryGetValue(oneTrust, kSecTrustSettingsPolicy); + if (certPolicy != NULL) { + CFDictionaryRef policyDict = SecPolicyCopyProperties(certPolicy); + ADD(inputTrust, @"SecPolicyOid"); + NSString* s = [NSString stringWithFormat:@"%@", CFDictionaryGetValue(policyDict, @"SecPolicyOid")]; + ADD(inputTrust, s); + CFRelease(policyDict); + } + ADDNULL(inputTrust); + } +} + static void addCertificatesToKeystore(JNIEnv *env, jobject keyStore) { // Search the user keychain list for all X509 certificates. @@ -385,16 +421,30 @@ static void addCertificatesToKeystore(JNIEnv *env, jobject keyStore) OSErr searchResult = noErr; jclass jc_KeychainStore = (*env)->FindClass(env, "apple/security/KeychainStore"); - CHECK_NULL(jc_KeychainStore); + if (jc_KeychainStore == NULL) { + goto errOut; + } + jmethodID jm_createTrustedCertEntry = (*env)->GetMethodID( env, jc_KeychainStore, "createTrustedCertEntry", "(Ljava/lang/String;Ljava/util/List;JJ[B)V"); - CHECK_NULL(jm_createTrustedCertEntry); + if (jm_createTrustedCertEntry == NULL) { + goto errOut; + } + jclass jc_arrayListClass = (*env)->FindClass(env, "java/util/ArrayList"); - CHECK_NULL(jc_arrayListClass); + if (jc_arrayListClass == NULL) { + goto errOut; + } + jmethodID jm_arrayListCons = (*env)->GetMethodID(env, jc_arrayListClass, "", "()V"); - CHECK_NULL(jm_arrayListCons); + if (jm_arrayListCons == NULL) { + goto errOut; + } + jmethodID jm_listAdd = (*env)->GetMethodID(env, jc_arrayListClass, "add", "(Ljava/lang/Object;)Z"); - CHECK_NULL(jm_listAdd); + if (jm_listAdd == NULL) { + goto errOut; + } do { searchResult = SecKeychainSearchCopyNext(keychainItemSearch, &theItem); @@ -416,43 +466,40 @@ static void addCertificatesToKeystore(JNIEnv *env, jobject keyStore) goto errOut; } - // Only add certificates with trusted settings - CFArrayRef trustSettings; - if (SecTrustSettingsCopyTrustSettings(certRef, kSecTrustSettingsDomainUser, &trustSettings) - == errSecItemNotFound) { - continue; - } - // See KeychainStore::createTrustedCertEntry for content of inputTrust - jobject inputTrust = (*env)->NewObject(env, jc_arrayListClass, jm_arrayListCons); - CHECK_NULL(inputTrust); - - // Dump everything inside trustSettings into inputTrust - CFIndex count = CFArrayGetCount(trustSettings); - for (int i = 0; i < count; i++) { - CFDictionaryRef oneTrust = (CFDictionaryRef) CFArrayGetValueAtIndex(trustSettings, i); - CFIndex size = CFDictionaryGetCount(oneTrust); - const void * keys [size]; - const void * values [size]; - CFDictionaryGetKeysAndValues(oneTrust, keys, values); - for (int j = 0; j < size; j++) { - NSString* s = [NSString stringWithFormat:@"%@", keys[j]]; - ADD(inputTrust, s); - s = [NSString stringWithFormat:@"%@", values[j]]; - ADD(inputTrust, s); + // We load trust settings from domains kSecTrustSettingsDomainUser and kSecTrustSettingsDomainAdmin + // kSecTrustSettingsDomainSystem is ignored because it seems to only contain data for root certificates + jobject inputTrust = NULL; + CFArrayRef trustSettings = NULL; + + // Load user trustSettings into inputTrust + if (SecTrustSettingsCopyTrustSettings(certRef, kSecTrustSettingsDomainUser, &trustSettings) == errSecSuccess && trustSettings != NULL) { + inputTrust = (*env)->NewObject(env, jc_arrayListClass, jm_arrayListCons); + if (inputTrust == NULL) { + CFRelease(trustSettings); + goto errOut; } - SecPolicyRef certPolicy; - certPolicy = (SecPolicyRef)CFDictionaryGetValue(oneTrust, kSecTrustSettingsPolicy); - if (certPolicy != NULL) { - CFDictionaryRef policyDict = SecPolicyCopyProperties(certPolicy); - ADD(inputTrust, @"SecPolicyOid"); - NSString* s = [NSString stringWithFormat:@"%@", CFDictionaryGetValue(policyDict, @"SecPolicyOid")]; - ADD(inputTrust, s); - CFRelease(policyDict); + addTrustSettingsToInputTrust(env, jm_listAdd, trustSettings, inputTrust); + CFRelease(trustSettings); + } + // Load admin trustSettings into inputTrust + trustSettings = NULL; + if (SecTrustSettingsCopyTrustSettings(certRef, kSecTrustSettingsDomainAdmin, &trustSettings) == errSecSuccess && trustSettings != NULL) { + if (inputTrust == NULL) { + inputTrust = (*env)->NewObject(env, jc_arrayListClass, jm_arrayListCons); + } + if (inputTrust == NULL) { + CFRelease(trustSettings); + goto errOut; } - ADDNULL(inputTrust); + addTrustSettingsToInputTrust(env, jm_listAdd, trustSettings, inputTrust); + CFRelease(trustSettings); + } + + // Only add certificates with trust settings + if (inputTrust == NULL) { + continue; } - CFRelease(trustSettings); // Find the creation date. jlong creationDate = getModDateFromItem(env, theItem); @@ -460,7 +507,9 @@ static void addCertificatesToKeystore(JNIEnv *env, jobject keyStore) // Call back to the Java object to create Java objects corresponding to this security object. jlong nativeRef = ptr_to_jlong(certRef); (*env)->CallVoidMethod(env, keyStore, jm_createTrustedCertEntry, alias, inputTrust, nativeRef, creationDate, certData); - JNU_CHECK_EXCEPTION(env); + if ((*env)->ExceptionCheck(env)) { + goto errOut; + } } } while (searchResult == noErr); diff --git a/src/java.base/share/classes/java/io/DataInputStream.java b/src/java.base/share/classes/java/io/DataInputStream.java index c8f49caa7fcfe..8d406cf360730 100644 --- a/src/java.base/share/classes/java/io/DataInputStream.java +++ b/src/java.base/share/classes/java/io/DataInputStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1994, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1994, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -57,8 +57,8 @@ public DataInputStream(InputStream in) { /** * working arrays initialized on demand by readUTF */ - private byte bytearr[] = new byte[80]; - private char chararr[] = new char[80]; + private byte[] bytearr = new byte[80]; + private char[] chararr = new char[80]; /** * Reads some number of bytes from the contained input stream and @@ -245,10 +245,7 @@ public final int skipBytes(int n) throws IOException { * @see java.io.FilterInputStream#in */ public final boolean readBoolean() throws IOException { - int ch = in.read(); - if (ch < 0) - throw new EOFException(); - return (ch != 0); + return readUnsignedByte() != 0; } /** @@ -268,10 +265,7 @@ public final boolean readBoolean() throws IOException { * @see java.io.FilterInputStream#in */ public final byte readByte() throws IOException { - int ch = in.read(); - if (ch < 0) - throw new EOFException(); - return (byte)(ch); + return (byte) readUnsignedByte(); } /** @@ -315,11 +309,7 @@ public final int readUnsignedByte() throws IOException { * @see java.io.FilterInputStream#in */ public final short readShort() throws IOException { - int ch1 = in.read(); - int ch2 = in.read(); - if ((ch1 | ch2) < 0) - throw new EOFException(); - return (short)((ch1 << 8) + (ch2 << 0)); + return (short) readUnsignedShort(); } /** @@ -340,6 +330,7 @@ public final short readShort() throws IOException { * @see java.io.FilterInputStream#in */ public final int readUnsignedShort() throws IOException { + InputStream in = this.in; int ch1 = in.read(); int ch2 = in.read(); if ((ch1 | ch2) < 0) @@ -365,11 +356,7 @@ public final int readUnsignedShort() throws IOException { * @see java.io.FilterInputStream#in */ public final char readChar() throws IOException { - int ch1 = in.read(); - int ch2 = in.read(); - if ((ch1 | ch2) < 0) - throw new EOFException(); - return (char)((ch1 << 8) + (ch2 << 0)); + return (char) readUnsignedShort(); } /** @@ -390,6 +377,7 @@ public final char readChar() throws IOException { * @see java.io.FilterInputStream#in */ public final int readInt() throws IOException { + InputStream in = this.in; int ch1 = in.read(); int ch2 = in.read(); int ch3 = in.read(); @@ -399,7 +387,7 @@ public final int readInt() throws IOException { return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4 << 0)); } - private byte readBuffer[] = new byte[8]; + private final byte[] readBuffer = new byte[8]; /** * See the general contract of the {@code readLong} @@ -474,7 +462,7 @@ public final double readDouble() throws IOException { return Double.longBitsToDouble(readLong()); } - private char lineBuffer[]; + private char[] lineBuffer; /** * See the general contract of the {@code readLine} @@ -505,7 +493,7 @@ public final double readDouble() throws IOException { */ @Deprecated public final String readLine() throws IOException { - char buf[] = lineBuffer; + char[] buf = lineBuffer; if (buf == null) { buf = lineBuffer = new char[128]; @@ -634,7 +622,7 @@ public static final String readUTF(DataInput in) throws IOException { if (count > utflen) throw new UTFDataFormatException( "malformed input: partial character at end"); - char2 = (int) bytearr[count-1]; + char2 = bytearr[count-1]; if ((char2 & 0xC0) != 0x80) throw new UTFDataFormatException( "malformed input around byte " + count); @@ -647,8 +635,8 @@ public static final String readUTF(DataInput in) throws IOException { if (count > utflen) throw new UTFDataFormatException( "malformed input: partial character at end"); - char2 = (int) bytearr[count-2]; - char3 = (int) bytearr[count-1]; + char2 = bytearr[count-2]; + char3 = bytearr[count-1]; if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) throw new UTFDataFormatException( "malformed input around byte " + (count-1)); diff --git a/src/java.base/share/classes/java/lang/ClassLoader.java b/src/java.base/share/classes/java/lang/ClassLoader.java index 7ad0f61910131..c1caeb18cd1b5 100644 --- a/src/java.base/share/classes/java/lang/ClassLoader.java +++ b/src/java.base/share/classes/java/lang/ClassLoader.java @@ -406,6 +406,11 @@ private static String nameAndId(ClassLoader ld) { return nid; } + // Returns nameAndId string for exception message printing + String nameAndId() { + return nameAndId; + } + /** * Creates a new class loader of the specified name and using the * specified parent class loader for delegation. diff --git a/src/java.base/share/classes/java/lang/ProcessBuilder.java b/src/java.base/share/classes/java/lang/ProcessBuilder.java index d096bdad7b08d..1bbbec2a8476b 100644 --- a/src/java.base/share/classes/java/lang/ProcessBuilder.java +++ b/src/java.base/share/classes/java/lang/ProcessBuilder.java @@ -1100,8 +1100,8 @@ private Process start(Redirect[] redirects) throws IOException { String dir = directory == null ? null : directory.toString(); - for (int i = 1; i < cmdarray.length; i++) { - if (cmdarray[i].indexOf('\u0000') >= 0) { + for (String s : cmdarray) { + if (s.indexOf('\u0000') >= 0) { throw new IOException("invalid null character in command"); } } diff --git a/src/java.base/share/classes/java/lang/String.java b/src/java.base/share/classes/java/lang/String.java index 905f61c4b123f..de62d82a331c2 100644 --- a/src/java.base/share/classes/java/lang/String.java +++ b/src/java.base/share/classes/java/lang/String.java @@ -660,7 +660,13 @@ public String(byte[] bytes, int offset, int length, Charset charset) { offset = 0; } - int caLen = decodeWithDecoder(cd, ca, bytes, offset, length); + int caLen; + try { + caLen = decodeWithDecoder(cd, ca, bytes, offset, length); + } catch (CharacterCodingException x) { + // Substitution is enabled, so this shouldn't happen + throw new Error(x); + } if (COMPACT_STRINGS) { byte[] bs = StringUTF16.compress(ca, 0, caLen); if (bs != null) { @@ -781,7 +787,13 @@ private static String newStringNoRepl1(byte[] src, Charset cs) { System.getSecurityManager() != null) { src = Arrays.copyOf(src, len); } - int caLen = decodeWithDecoder(cd, ca, src, 0, src.length); + int caLen; + try { + caLen = decodeWithDecoder(cd, ca, src, 0, src.length); + } catch (CharacterCodingException x) { + // throw via IAE + throw new IllegalArgumentException(x); + } if (COMPACT_STRINGS) { byte[] bs = StringUTF16.compress(ca, 0, caLen); if (bs != null) { @@ -835,7 +847,8 @@ private static byte[] encodeWithEncoder(Charset cs, byte coder, byte[] val, bool CharsetEncoder ce = cs.newEncoder(); int len = val.length >> coder; // assume LATIN1=0/UTF16=1; int en = scale(len, ce.maxBytesPerChar()); - if (ce instanceof ArrayEncoder ae) { + // fastpath with ArrayEncoder implies `doReplace`. + if (doReplace && ce instanceof ArrayEncoder ae) { // fastpath for ascii compatible if (coder == LATIN1 && ae.isASCIICompatible() && @@ -846,10 +859,6 @@ private static byte[] encodeWithEncoder(Charset cs, byte coder, byte[] val, bool if (len == 0) { return ba; } - if (doReplace) { - ce.onMalformedInput(CodingErrorAction.REPLACE) - .onUnmappableCharacter(CodingErrorAction.REPLACE); - } int blen = (coder == LATIN1) ? ae.encodeFromLatin1(val, 0, len, ba) : ae.encodeFromUTF16(val, 0, len, ba); @@ -1194,21 +1203,16 @@ private static int decodeUTF8_UTF16(byte[] src, int sp, int sl, byte[] dst, int return dp; } - private static int decodeWithDecoder(CharsetDecoder cd, char[] dst, byte[] src, int offset, int length) { + private static int decodeWithDecoder(CharsetDecoder cd, char[] dst, byte[] src, int offset, int length) + throws CharacterCodingException { ByteBuffer bb = ByteBuffer.wrap(src, offset, length); CharBuffer cb = CharBuffer.wrap(dst, 0, dst.length); - try { - CoderResult cr = cd.decode(bb, cb, true); - if (!cr.isUnderflow()) - cr.throwException(); - cr = cd.flush(cb); - if (!cr.isUnderflow()) - cr.throwException(); - } catch (CharacterCodingException x) { - // Substitution is always enabled, - // so this shouldn't happen - throw new Error(x); - } + CoderResult cr = cd.decode(bb, cb, true); + if (!cr.isUnderflow()) + cr.throwException(); + cr = cd.flush(cb); + if (!cr.isUnderflow()) + cr.throwException(); return cb.position(); } diff --git a/src/java.base/share/classes/java/lang/System.java b/src/java.base/share/classes/java/lang/System.java index 74cc41870ee2a..c78014a9c951a 100644 --- a/src/java.base/share/classes/java/lang/System.java +++ b/src/java.base/share/classes/java/lang/System.java @@ -2451,6 +2451,10 @@ public long findNative(ClassLoader loader, String entry) { public void exit(int statusCode) { Shutdown.exit(statusCode); } + + public String getLoaderNameID(ClassLoader loader) { + return loader.nameAndId(); + } }); } } diff --git a/src/java.base/share/classes/java/lang/Thread.java b/src/java.base/share/classes/java/lang/Thread.java index c13b42c2f6f2e..e1ea45cafc2f2 100644 --- a/src/java.base/share/classes/java/lang/Thread.java +++ b/src/java.base/share/classes/java/lang/Thread.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1994, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1994, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -157,7 +157,14 @@ public class Thread implements Runnable { /* Fields reserved for exclusive use by the JVM */ private boolean stillborn = false; - private long eetop; + + /* + * Reserved for exclusive use by the JVM. The historically named + * `eetop` holds the address of the underlying VM JavaThread, and is set to + * non-zero when the thread is started, and reset to zero when the thread terminates. + * A non-zero value indicates this thread isAlive(). + */ + private volatile long eetop; /* What will be run. */ private Runnable target; @@ -1050,7 +1057,9 @@ public boolean isInterrupted() { * @return {@code true} if this thread is alive; * {@code false} otherwise. */ - public final native boolean isAlive(); + public final boolean isAlive() { + return eetop != 0; + } /** * Suspends this thread. diff --git a/src/java.base/share/classes/java/lang/reflect/Proxy.java b/src/java.base/share/classes/java/lang/reflect/Proxy.java index 32aed2448b88e..a56f2ddef9d76 100644 --- a/src/java.base/share/classes/java/lang/reflect/Proxy.java +++ b/src/java.base/share/classes/java/lang/reflect/Proxy.java @@ -881,7 +881,7 @@ private static void ensureVisible(ClassLoader ld, Class c) { } if (type != c) { throw new IllegalArgumentException(c.getName() + - " referenced from a method is not visible from class loader"); + " referenced from a method is not visible from class loader: " + JLA.getLoaderNameID(ld)); } } diff --git a/src/java.base/share/classes/java/net/InetAddress.java b/src/java.base/share/classes/java/net/InetAddress.java index a7b57a15f2f9b..ffc1e3cdbb2f4 100644 --- a/src/java.base/share/classes/java/net/InetAddress.java +++ b/src/java.base/share/classes/java/net/InetAddress.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1995, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1995, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -930,6 +930,7 @@ private static final class PlatformNameService implements NameService { public InetAddress[] lookupAllHostAddr(String host) throws UnknownHostException { + validate(host); return impl.lookupAllHostAddr(host); } @@ -1314,6 +1315,7 @@ private static InetAddress[] getAllByName(String host, InetAddress reqAddr) return ret; } + validate(host); boolean ipv6Expected = false; if (host.charAt(0) == '[') { // This is supposed to be an IPv6 literal @@ -1321,44 +1323,45 @@ private static InetAddress[] getAllByName(String host, InetAddress reqAddr) host = host.substring(1, host.length() -1); ipv6Expected = true; } else { - // This was supposed to be a IPv6 address, but it's not! - throw new UnknownHostException(host + ": invalid IPv6 address"); + // This was supposed to be a IPv6 literal, but it's not + throw invalidIPv6LiteralException(host, false); } } - // if host is an IP address, we won't do further lookup + // Check and try to parse host string as an IP address literal if (IPAddressUtil.digit(host.charAt(0), 16) != -1 || (host.charAt(0) == ':')) { - byte[] addr; + byte[] addr = null; int numericZone = -1; String ifname = null; - // see if it is IPv4 address - try { - addr = IPAddressUtil.validateNumericFormatV4(host); - } catch (IllegalArgumentException iae) { - var uhe = new UnknownHostException(host); - uhe.initCause(iae); - throw uhe; + + if (!ipv6Expected) { + // check if it is IPv4 address only if host is not wrapped in '[]' + try { + addr = IPAddressUtil.validateNumericFormatV4(host); + } catch (IllegalArgumentException iae) { + var uhe = new UnknownHostException(host); + uhe.initCause(iae); + throw uhe; + } } if (addr == null) { - // This is supposed to be an IPv6 literal - // Check if a numeric or string zone id is present + // Try to parse host string as an IPv6 literal + // Check if a numeric or string zone id is present first int pos; - if ((pos=host.indexOf ('%')) != -1) { - numericZone = checkNumericZone (host); + if ((pos = host.indexOf('%')) != -1) { + numericZone = checkNumericZone(host); if (numericZone == -1) { /* remainder of string must be an ifname */ - ifname = host.substring (pos+1); + ifname = host.substring(pos + 1); } } - if ((addr = IPAddressUtil.textToNumericFormatV6(host)) == null && host.contains(":")) { - throw new UnknownHostException(host + ": invalid IPv6 address"); + if ((addr = IPAddressUtil.textToNumericFormatV6(host)) == null && + (host.contains(":") || ipv6Expected)) { + throw invalidIPv6LiteralException(host, ipv6Expected); } - } else if (ipv6Expected) { - // Means an IPv4 literal between brackets! - throw new UnknownHostException("["+host+"]"); } - InetAddress[] ret = new InetAddress[1]; if(addr != null) { + InetAddress[] ret = new InetAddress[1]; if (addr.length == Inet4Address.INADDRSZ) { if (numericZone != -1 || ifname != null) { // IPv4-mapped address must not contain zone-id @@ -1375,12 +1378,18 @@ private static InetAddress[] getAllByName(String host, InetAddress reqAddr) return ret; } } else if (ipv6Expected) { - // We were expecting an IPv6 Literal, but got something else - throw new UnknownHostException("["+host+"]"); + // We were expecting an IPv6 Literal since host string starts + // and ends with square brackets, but we got something else. + throw invalidIPv6LiteralException(host, true); } return getAllByName0(host, reqAddr, true, true); } + private static UnknownHostException invalidIPv6LiteralException(String host, boolean wrapInBrackets) { + String hostString = wrapInBrackets ? "[" + host + "]" : host; + return new UnknownHostException(hostString + ": invalid IPv6 address literal"); + } + /** * Returns the loopback address. *

@@ -1802,6 +1811,12 @@ private void writeObject (ObjectOutputStream s) throws pf.put("family", holder().getFamily()); s.writeFields(); } + + private static void validate(String host) throws UnknownHostException { + if (host.indexOf(0) != -1) { + throw new UnknownHostException("NUL character not allowed in hostname"); + } + } } /* diff --git a/src/java.base/share/classes/java/net/URI.java b/src/java.base/share/classes/java/net/URI.java index 90e7b2fe9421e..644fc05bdc12e 100644 --- a/src/java.base/share/classes/java/net/URI.java +++ b/src/java.base/share/classes/java/net/URI.java @@ -3256,6 +3256,7 @@ private int parseAuthority(int start, int n) boolean serverChars; boolean regChars; + boolean skipParseException; if (scan(p, n, "]") > p) { // contains a literal IPv6 address, therefore % is allowed @@ -3271,15 +3272,28 @@ private int parseAuthority(int start, int n) return n; } + // When parsing a URI, skip creating exception objects if the server-based + // authority is not required and the registry parse is successful. + // + skipParseException = (!requireServerAuthority && regChars); if (serverChars) { // Might be (probably is) a server-based authority, so attempt // to parse it as such. If the attempt fails, try to treat it // as a registry-based authority. try { - q = parseServer(p, n); - if (q < n) - failExpecting("end of authority", q); - authority = input.substring(p, n); + q = parseServer(p, n, skipParseException); + if (q < n) { + if (skipParseException) { + userInfo = null; + host = null; + port = -1; + q = p; + } else { + failExpecting("end of authority", q); + } + } else { + authority = input.substring(p, n); + } } catch (URISyntaxException x) { // Undo results of failed parse userInfo = null; @@ -3317,7 +3331,7 @@ private int parseAuthority(int start, int n) // [@][:] // - private int parseServer(int start, int n) + private int parseServer(int start, int n, boolean skipParseException) throws URISyntaxException { int p = start; @@ -3357,7 +3371,7 @@ private int parseServer(int start, int n) } else { q = parseIPv4Address(p, n); if (q <= p) - q = parseHostname(p, n); + q = parseHostname(p, n, skipParseException); p = q; } @@ -3374,7 +3388,10 @@ private int parseServer(int start, int n) } p = q; } + } else if (p < n && skipParseException) { + return p; } + if (p < n) failExpecting("port number", p); @@ -3479,7 +3496,7 @@ private int parseIPv4Address(int start, int n) { // domainlabel = alphanum | alphanum *( alphanum | "-" ) alphanum // toplabel = alpha | alpha *( alphanum | "-" ) alphanum // - private int parseHostname(int start, int n) + private int parseHostname(int start, int n, boolean skipParseException) throws URISyntaxException { int p = start; @@ -3507,9 +3524,12 @@ private int parseHostname(int start, int n) p = q; } while (p < n); - if ((p < n) && !at(p, n, ':')) + if ((p < n) && !at(p, n, ':')) { + if (skipParseException) { + return p; + } fail("Illegal character in hostname", p); - + } if (l < 0) failExpecting("hostname", start); diff --git a/src/java.base/share/classes/java/nio/Direct-X-Buffer.java.template b/src/java.base/share/classes/java/nio/Direct-X-Buffer.java.template index 4c540d324a0e9..91ad40b7b7eaa 100644 --- a/src/java.base/share/classes/java/nio/Direct-X-Buffer.java.template +++ b/src/java.base/share/classes/java/nio/Direct-X-Buffer.java.template @@ -134,7 +134,14 @@ class Direct$Type$Buffer$RW$$BO$ } else { address = base; } - cleaner = Cleaner.create(this, new Deallocator(base, size, cap)); + try { + cleaner = Cleaner.create(this, new Deallocator(base, size, cap)); + } catch (Throwable t) { + // Prevent leak if the Deallocator or Cleaner fail for any reason + UNSAFE.freeMemory(base); + Bits.unreserveMemory(size, cap); + throw t; + } att = null; #else[rw] super(cap); diff --git a/src/java.base/share/classes/java/security/Provider.java b/src/java.base/share/classes/java/security/Provider.java index ca8c96c3e04f7..af8ebeeda5784 100644 --- a/src/java.base/share/classes/java/security/Provider.java +++ b/src/java.base/share/classes/java/security/Provider.java @@ -25,6 +25,8 @@ package java.security; +import jdk.internal.event.SecurityProviderServiceEvent; + import java.io.*; import java.util.*; import static java.util.Locale.ENGLISH; @@ -1234,19 +1236,28 @@ public Service getService(String type, String algorithm) { key = new ServiceKey(type, algorithm, false); previousKey = key; } + Service s = null; if (!serviceMap.isEmpty()) { - Service s = serviceMap.get(key); - if (s != null) { - return s; - } + s = serviceMap.get(key); } - synchronized (this) { - ensureLegacyParsed(); - if (legacyMap != null && !legacyMap.isEmpty()) { - return legacyMap.get(key); + if (s == null) { + synchronized (this) { + ensureLegacyParsed(); + if (legacyMap != null && !legacyMap.isEmpty()) { + s = legacyMap.get(key); + } } } - return null; + + if (s != null && SecurityProviderServiceEvent.isTurnedOn()) { + var e = new SecurityProviderServiceEvent(); + e.provider = getName(); + e.type = type; + e.algorithm = algorithm; + e.commit(); + } + + return s; } // ServiceKey from previous getService() call diff --git a/src/java.base/share/classes/java/time/format/DateTimeFormatterBuilder.java b/src/java.base/share/classes/java/time/format/DateTimeFormatterBuilder.java index 3a0d75956163f..7ba85cf56c9c1 100644 --- a/src/java.base/share/classes/java/time/format/DateTimeFormatterBuilder.java +++ b/src/java.base/share/classes/java/time/format/DateTimeFormatterBuilder.java @@ -4169,11 +4169,12 @@ public boolean format(DateTimePrintContext context, StringBuilder buf) { return true; } - // cache per instance for now + // Cache per instance for now. These HashMaps normally contain a single + // element, initialize them with initial capacity = 2 to avoid resizes due to load factor. private final Map>> - cachedTree = new HashMap<>(); + cachedTree = new HashMap<>(2); private final Map>> - cachedTreeCI = new HashMap<>(); + cachedTreeCI = new HashMap<>(2); @Override protected PrefixTree getTree(DateTimeParseContext context) { @@ -4182,9 +4183,8 @@ protected PrefixTree getTree(DateTimeParseContext context) { } Locale locale = context.getLocale(); boolean isCaseSensitive = context.isCaseSensitive(); - Set regionIds = new HashSet<>(ZoneRulesProvider.getAvailableZoneIds()); - Set nonRegionIds = new HashSet<>(64); - int regionIdsSize = regionIds.size(); + Set availableZoneIds = ZoneRulesProvider.getAvailableZoneIds(); + int regionIdsSize = availableZoneIds.size(); Map>> cached = isCaseSensitive ? cachedTree : cachedTreeCI; @@ -4197,6 +4197,8 @@ protected PrefixTree getTree(DateTimeParseContext context) { (tree = entry.getValue().get()) == null)) { tree = PrefixTree.newTree(context); zoneStrings = TimeZoneNameUtility.getZoneStrings(locale); + Set nonRegionIds = new HashSet<>(64); + Set regionIds = new HashSet<>(availableZoneIds); for (String[] names : zoneStrings) { String zid = names[0]; if (!regionIds.remove(zid)) { @@ -4321,8 +4323,10 @@ public int parse(DateTimeParseContext context, CharSequence text, int position) if (length >= position + 3 && context.charEquals(text.charAt(position + 2), 'C')) { // There are localized zone texts that start with "UTC", e.g. // "UTC\u221210:00" (MINUS SIGN instead of HYPHEN-MINUS) in French. - // Exclude those ZoneText cases. - if (!(this instanceof ZoneTextPrinterParser)) { + // Exclude those cases. + if (length == position + 3 || + context.charEquals(text.charAt(position + 3), '+') || + context.charEquals(text.charAt(position + 3), '-')) { return parseOffsetBased(context, text, position, position + 3, OffsetIdPrinterParser.INSTANCE_ID_ZERO); } } else { diff --git a/src/java.base/share/classes/java/util/concurrent/ConcurrentHashMap.java b/src/java.base/share/classes/java/util/concurrent/ConcurrentHashMap.java index 117bb72a1a669..71c59d3ae17a1 100644 --- a/src/java.base/share/classes/java/util/concurrent/ConcurrentHashMap.java +++ b/src/java.base/share/classes/java/util/concurrent/ConcurrentHashMap.java @@ -2862,22 +2862,20 @@ private final void unlockRoot() { * Possibly blocks awaiting root lock. */ private final void contendedLock() { - boolean waiting = false; + Thread current = Thread.currentThread(), w; for (int s;;) { if (((s = lockState) & ~WAITER) == 0) { if (U.compareAndSetInt(this, LOCKSTATE, s, WRITER)) { - if (waiting) - waiter = null; + if (waiter == current) + U.compareAndSetReference(this, WAITERTHREAD, current, null); return; } } - else if ((s & WAITER) == 0) { - if (U.compareAndSetInt(this, LOCKSTATE, s, s | WAITER)) { - waiting = true; - waiter = Thread.currentThread(); - } - } - else if (waiting) + else if ((s & WAITER) == 0) + U.compareAndSetInt(this, LOCKSTATE, s, s | WAITER); + else if ((w = waiter) == null) + U.compareAndSetReference(this, WAITERTHREAD, null, current); + else if (w == current) LockSupport.park(this); } } @@ -3296,6 +3294,8 @@ static boolean checkInvariants(TreeNode t) { private static final long LOCKSTATE = U.objectFieldOffset(TreeBin.class, "lockState"); + private static final long WAITERTHREAD + = U.objectFieldOffset(TreeBin.class, "waiter"); } /* ----------------Table Traversal -------------- */ diff --git a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java index 323a151b4ce6c..1f029bc022dd2 100644 --- a/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java +++ b/src/java.base/share/classes/java/util/concurrent/ForkJoinPool.java @@ -2759,8 +2759,10 @@ public List> invokeAll(Collection> tasks, ForkJoinTask.cancelIgnoringExceptions(f); else { ((ForkJoinTask)f).awaitPoolInvoke(this, ns); - if ((ns = nanos - (System.nanoTime() - startTime)) < 0L) + if ((ns = nanos - (System.nanoTime() - startTime)) < 0L) { timedOut = true; + ForkJoinTask.cancelIgnoringExceptions(f); + } } } } diff --git a/src/java.base/share/classes/java/util/concurrent/ThreadLocalRandom.java b/src/java.base/share/classes/java/util/concurrent/ThreadLocalRandom.java index de8dd5f85809e..7a254ebe39bae 100644 --- a/src/java.base/share/classes/java/util/concurrent/ThreadLocalRandom.java +++ b/src/java.base/share/classes/java/util/concurrent/ThreadLocalRandom.java @@ -422,6 +422,10 @@ public int nextInt() { public long nextLong() { return ThreadLocalRandom.current().nextLong(); } + + public double nextDouble() { + return ThreadLocalRandom.current().nextDouble(); + } } /** diff --git a/src/java.base/share/classes/java/util/jar/JarFile.java b/src/java.base/share/classes/java/util/jar/JarFile.java index f0cd0e5c3820f..bd538649a4fff 100644 --- a/src/java.base/share/classes/java/util/jar/JarFile.java +++ b/src/java.base/share/classes/java/util/jar/JarFile.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,6 +29,7 @@ import jdk.internal.access.JavaUtilZipFileAccess; import sun.security.action.GetPropertyAction; import sun.security.util.ManifestEntryVerifier; +import sun.security.util.SignatureFileVerifier; import java.io.ByteArrayInputStream; import java.io.EOFException; @@ -151,8 +152,6 @@ public class JarFile extends ZipFile { private static final boolean MULTI_RELEASE_ENABLED; private static final boolean MULTI_RELEASE_FORCED; private static final ThreadLocal isInitializing = new ThreadLocal<>(); - // The maximum size of array to allocate. Some VMs reserve some header words in an array. - private static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8; private SoftReference manRef; private JarEntry manEntry; @@ -800,8 +799,11 @@ private void initializeVerifier() { private byte[] getBytes(ZipEntry ze) throws IOException { try (InputStream is = super.getInputStream(ze)) { long uncompressedSize = ze.getSize(); - if (uncompressedSize > MAX_ARRAY_SIZE) { - throw new IOException("Unsupported size: " + uncompressedSize); + if (uncompressedSize > SignatureFileVerifier.MAX_SIG_FILE_SIZE) { + throw new IOException("Unsupported size: " + uncompressedSize + + " for JarEntry " + ze.getName() + + ". Allowed max size: " + + SignatureFileVerifier.MAX_SIG_FILE_SIZE + " bytes"); } int len = (int)uncompressedSize; int bytesRead; diff --git a/src/java.base/share/classes/java/util/zip/ZipFile.java b/src/java.base/share/classes/java/util/zip/ZipFile.java index f073fe8fb1b75..a2573f83f668b 100644 --- a/src/java.base/share/classes/java/util/zip/ZipFile.java +++ b/src/java.base/share/classes/java/util/zip/ZipFile.java @@ -69,6 +69,7 @@ import jdk.internal.ref.CleanerFactory; import jdk.internal.vm.annotation.Stable; import sun.nio.cs.UTF_8; +import sun.security.action.GetBooleanAction; import sun.security.util.SignatureFileVerifier; import static java.util.zip.ZipConstants64.*; @@ -121,6 +122,12 @@ public class ZipFile implements ZipConstants, Closeable { */ public static final int OPEN_DELETE = 0x4; + /** + * Flag which specifies whether the validation of the Zip64 extra + * fields should be disabled + */ + private static final boolean disableZip64ExtraFieldValidation = + GetBooleanAction.privilegedGetProperty("jdk.util.zip.disableZip64ExtraFieldValidation"); /** * Opens a zip file for reading. * @@ -1195,6 +1202,16 @@ private int checkAndAddEntry(int pos, int index) if (entryPos + nlen > cen.length - ENDHDR) { zerror("invalid CEN header (bad header size)"); } + + int elen = CENEXT(cen, pos); + if (elen > 0 && !disableZip64ExtraFieldValidation) { + long extraStartingOffset = pos + CENHDR + nlen; + if ((int)extraStartingOffset != extraStartingOffset) { + zerror("invalid CEN header (bad extra offset)"); + } + checkExtraFields(pos, (int)extraStartingOffset, elen); + } + try { ZipCoder zcp = zipCoderForPos(pos); int hash = zcp.checkedHash(cen, entryPos, nlen); @@ -1211,6 +1228,119 @@ private int checkAndAddEntry(int pos, int index) return nlen; } + /** + * Validate the Zip64 Extra block fields + * @param startingOffset Extra Field starting offset within the CEN + * @param extraFieldLen Length of this Extra field + * @throws ZipException If an error occurs validating the Zip64 Extra + * block + */ + private void checkExtraFields(int cenPos, int startingOffset, + int extraFieldLen) throws ZipException { + // Extra field Length cannot exceed 65,535 bytes per the PKWare + // APP.note 4.4.11 + if (extraFieldLen > 0xFFFF) { + zerror("invalid extra field length"); + } + // CEN Offset where this Extra field ends + int extraEndOffset = startingOffset + extraFieldLen; + if (extraEndOffset > cen.length) { + zerror("Invalid CEN header (extra data field size too long)"); + } + int currentOffset = startingOffset; + while (currentOffset < extraEndOffset) { + int tag = get16(cen, currentOffset); + currentOffset += Short.BYTES; + + int tagBlockSize = get16(cen, currentOffset); + int tagBlockEndingOffset = currentOffset + tagBlockSize; + + // The ending offset for this tag block should not go past the + // offset for the end of the extra field + if (tagBlockEndingOffset > extraEndOffset) { + zerror("Invalid CEN header (invalid zip64 extra data field size)"); + } + currentOffset += Short.BYTES; + + if (tag == ZIP64_EXTID) { + // Get the compressed size; + long csize = CENSIZ(cen, cenPos); + // Get the uncompressed size; + long size = CENLEN(cen, cenPos); + checkZip64ExtraFieldValues(currentOffset, tagBlockSize, + csize, size); + } + currentOffset += tagBlockSize; + } + } + + /** + * Validate the Zip64 Extended Information Extra Field (0x0001) block + * size and that the uncompressed size and compressed size field + * values are not negative. + * Note: As we do not use the LOC offset or Starting disk number + * field value we will not validate them + * @param off the starting offset for the Zip64 field value + * @param blockSize the size of the Zip64 Extended Extra Field + * @param csize CEN header compressed size value + * @param size CEN header uncompressed size value + * @throws ZipException if an error occurs + */ + private void checkZip64ExtraFieldValues(int off, int blockSize, long csize, + long size) + throws ZipException { + byte[] cen = this.cen; + // Validate the Zip64 Extended Information Extra Field (0x0001) + // length. + if (!isZip64ExtBlockSizeValid(blockSize)) { + zerror("Invalid CEN header (invalid zip64 extra data field size)"); + } + // Check the uncompressed size is not negative + // Note we do not need to check blockSize is >= 8 as + // we know its length is at least 8 from the call to + // isZip64ExtBlockSizeValid() + if ((size == ZIP64_MAGICVAL)) { + if(get64(cen, off) < 0) { + zerror("Invalid zip64 extra block size value"); + } + } + // Check the compressed size is not negative + if ((csize == ZIP64_MAGICVAL) && (blockSize >= 16)) { + if (get64(cen, off + 8) < 0) { + zerror("Invalid zip64 extra block compressed size value"); + } + } + } + + /** + * Validate the size and contents of a Zip64 extended information field + * The order of the Zip64 fields is fixed, but the fields MUST + * only appear if the corresponding LOC or CEN field is set to 0xFFFF: + * or 0xFFFFFFFF: + * Uncompressed Size - 8 bytes + * Compressed Size - 8 bytes + * LOC Header offset - 8 bytes + * Disk Start Number - 4 bytes + * See PKWare APP.Note Section 4.5.3 for more details + * + * @param blockSize the Zip64 Extended Information Extra Field size + * @return true if the extra block size is valid; false otherwise + */ + private static boolean isZip64ExtBlockSizeValid(int blockSize) { + /* + * As the fields must appear in order, the block size indicates which + * fields to expect: + * 8 - uncompressed size + * 16 - uncompressed size, compressed size + * 24 - uncompressed size, compressed sise, LOC Header offset + * 28 - uncompressed size, compressed sise, LOC Header offset, + * and Disk start number + */ + return switch(blockSize) { + case 8, 16, 24, 28 -> true; + default -> false; + }; + } private int getEntryHash(int index) { return entries[index]; } private int getEntryNext(int index) { return entries[index + 1]; } private int getEntryPos(int index) { return entries[index + 2]; } diff --git a/src/java.base/share/classes/javax/crypto/JceSecurity.java.template b/src/java.base/share/classes/javax/crypto/JceSecurity.java.template index 69ca07dbc6d92..7a344e8e30665 100644 --- a/src/java.base/share/classes/javax/crypto/JceSecurity.java.template +++ b/src/java.base/share/classes/javax/crypto/JceSecurity.java.template @@ -235,7 +235,11 @@ final class JceSecurity { // return whether this provider is properly signed and can be used by JCE static boolean canUseProvider(Provider p) { - return getVerificationResult(p) == null; + Exception e = getVerificationResult(p); + if (debug != null && e != null) { + debug.println("Provider verification result: " + e); + } + return e == null; } // dummy object to represent null diff --git a/src/java.base/share/classes/jdk/internal/access/JavaLangAccess.java b/src/java.base/share/classes/jdk/internal/access/JavaLangAccess.java index b68490ad7a397..1a76efc206dd1 100644 --- a/src/java.base/share/classes/jdk/internal/access/JavaLangAccess.java +++ b/src/java.base/share/classes/jdk/internal/access/JavaLangAccess.java @@ -410,4 +410,10 @@ public interface JavaLangAccess { * @param statusCode the status code */ void exit(int statusCode); + + /** + * Returns '' @ if classloader has a name + * explicitly set otherwise @ + */ + String getLoaderNameID(ClassLoader loader); } diff --git a/src/java.base/share/classes/jdk/internal/event/SecurityProviderServiceEvent.java b/src/java.base/share/classes/jdk/internal/event/SecurityProviderServiceEvent.java new file mode 100644 index 0000000000000..2c2c487849b22 --- /dev/null +++ b/src/java.base/share/classes/jdk/internal/event/SecurityProviderServiceEvent.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package jdk.internal.event; + +/** + * Event recording details of Provider.getService(String type, String algorithm) calls + */ + +public final class SecurityProviderServiceEvent extends Event { + private final static SecurityProviderServiceEvent EVENT = new SecurityProviderServiceEvent(); + + /** + * Returns {@code true} if event is enabled, {@code false} otherwise. + */ + public static boolean isTurnedOn() { + return EVENT.isEnabled(); + } + + public String type; + public String algorithm; + public String provider; +} diff --git a/src/java.base/share/classes/jdk/internal/misc/Unsafe.java b/src/java.base/share/classes/jdk/internal/misc/Unsafe.java index 22aa09c9d6269..9c88f40e55613 100644 --- a/src/java.base/share/classes/jdk/internal/misc/Unsafe.java +++ b/src/java.base/share/classes/jdk/internal/misc/Unsafe.java @@ -3436,16 +3436,14 @@ public final void loadLoadFence() { * Ensures that stores before the fence will not be reordered with * stores after the fence. * - * @implNote - * This method is operationally equivalent to {@link #storeFence()}. - * * @since 9 */ + @IntrinsicCandidate public final void storeStoreFence() { + // If storeStoreFence intrinsic is not available, fall back to storeFence. storeFence(); } - /** * Throws IllegalAccessError; for use by the VM for access control * error support. diff --git a/src/java.base/share/classes/sun/launcher/LauncherHelper.java b/src/java.base/share/classes/sun/launcher/LauncherHelper.java index 82b73d01c6b63..4154f7a5d6640 100644 --- a/src/java.base/share/classes/sun/launcher/LauncherHelper.java +++ b/src/java.base/share/classes/sun/launcher/LauncherHelper.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2007, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -87,6 +87,7 @@ import jdk.internal.module.Modules; import jdk.internal.platform.Container; import jdk.internal.platform.Metrics; +import sun.util.calendar.ZoneInfoFile; public final class LauncherHelper { @@ -280,6 +281,8 @@ private static void printLocale() { Locale.getDefault(Category.DISPLAY).getDisplayName()); ostream.println(INDENT + "default format locale = " + Locale.getDefault(Category.FORMAT).getDisplayName()); + ostream.println(INDENT + "tzdata version = " + + ZoneInfoFile.getVersion()); printLocales(); ostream.println(); } diff --git a/src/java.base/share/classes/sun/net/www/HeaderParser.java b/src/java.base/share/classes/sun/net/www/HeaderParser.java index a4b2ab50497f3..e6d7435a45864 100644 --- a/src/java.base/share/classes/sun/net/www/HeaderParser.java +++ b/src/java.base/share/classes/sun/net/www/HeaderParser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1996, 2011, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1996, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,6 +26,7 @@ package sun.net.www; import java.util.Iterator; +import java.util.OptionalInt; /* This is useful for the nightmare of parsing multi-part HTTP/RFC822 headers * sensibly: @@ -246,6 +247,19 @@ public int findInt(String k, int Default) { return Default; } } + + public OptionalInt findInt(String k) { + try { + String s = findValue(k); + if (s == null) { + return OptionalInt.empty(); + } + return OptionalInt.of(Integer.parseInt(s)); + } catch (Throwable t) { + return OptionalInt.empty(); + } + } + /* public static void main(String[] a) throws Exception { System.out.print("enter line to parse> "); diff --git a/src/java.base/share/classes/sun/net/www/http/HttpClient.java b/src/java.base/share/classes/sun/net/www/http/HttpClient.java index a603badb0c6cb..ba5e38827c73a 100644 --- a/src/java.base/share/classes/sun/net/www/http/HttpClient.java +++ b/src/java.base/share/classes/sun/net/www/http/HttpClient.java @@ -29,6 +29,7 @@ import java.net.*; import java.util.Locale; import java.util.Objects; +import java.util.OptionalInt; import java.util.Properties; import java.util.concurrent.locks.ReentrantLock; @@ -127,6 +128,7 @@ private static int getDefaultPort(String proto) { * 0: the server specified no keep alive headers * -1: the server provided "Connection: keep-alive" but did not specify a * a particular time in a "Keep-Alive:" headers + * -2: the server provided "Connection: keep-alive" and timeout=0 * Positive values are the number of seconds specified by the server * in a "Keep-Alive" header */ @@ -897,7 +899,23 @@ private boolean parseHTTPHeader(MessageHeader responses, ProgressSource pi, Http responses.findValue("Keep-Alive")); /* default should be larger in case of proxy */ keepAliveConnections = p.findInt("max", usingProxy?50:5); - keepAliveTimeout = p.findInt("timeout", -1); + if (keepAliveConnections < 0) { + keepAliveConnections = usingProxy?50:5; + } + OptionalInt timeout = p.findInt("timeout"); + if (timeout.isEmpty()) { + keepAliveTimeout = -1; + } else { + keepAliveTimeout = timeout.getAsInt(); + if (keepAliveTimeout < 0) { + // if the server specified a negative (invalid) value + // then we set to -1, which is equivalent to no value + keepAliveTimeout = -1; + } else if (keepAliveTimeout == 0) { + // handled specially to mean close connection immediately + keepAliveTimeout = -2; + } + } } } else if (b[7] != '0') { /* diff --git a/src/java.base/share/classes/sun/net/www/http/KeepAliveCache.java b/src/java.base/share/classes/sun/net/www/http/KeepAliveCache.java index 6a61932d9832d..5bdd6f2c009f5 100644 --- a/src/java.base/share/classes/sun/net/www/http/KeepAliveCache.java +++ b/src/java.base/share/classes/sun/net/www/http/KeepAliveCache.java @@ -172,6 +172,8 @@ public Void run() { // different default for server and proxy keepAliveTimeout = http.getUsingProxy() ? 60 : 5; } + } else if (keepAliveTimeout == -2) { + keepAliveTimeout = 0; } // at this point keepAliveTimeout is the number of seconds to keep // alive, which could be 0, if the user specified 0 for the property diff --git a/src/java.base/share/classes/sun/net/www/protocol/http/AuthenticationInfo.java b/src/java.base/share/classes/sun/net/www/protocol/http/AuthenticationInfo.java index c88c178c13e90..e563f561d651e 100644 --- a/src/java.base/share/classes/sun/net/www/protocol/http/AuthenticationInfo.java +++ b/src/java.base/share/classes/sun/net/www/protocol/http/AuthenticationInfo.java @@ -528,4 +528,13 @@ private synchronized void writeObject(java.io.ObjectOutputStream s) s2 = new String (pw.getPassword()); s.defaultWriteObject (); } + + /** + * Releases any system or cryptographic resources. + * It is up to implementors to override disposeContext() + * to take necessary action. + */ + public void disposeContext() { + // do nothing + } } diff --git a/src/java.base/share/classes/sun/net/www/protocol/http/HttpURLConnection.java b/src/java.base/share/classes/sun/net/www/protocol/http/HttpURLConnection.java index d67caf77c16cd..96543280747dd 100644 --- a/src/java.base/share/classes/sun/net/www/protocol/http/HttpURLConnection.java +++ b/src/java.base/share/classes/sun/net/www/protocol/http/HttpURLConnection.java @@ -2029,6 +2029,12 @@ private InputStream getInputStream0() throws IOException { if (serverAuthKey != null) { AuthenticationInfo.endAuthRequest(serverAuthKey); } + if (proxyAuthentication != null) { + proxyAuthentication.disposeContext(); + } + if (serverAuthentication != null) { + serverAuthentication.disposeContext(); + } } } @@ -2274,6 +2280,9 @@ private void doTunneling0() throws IOException { if (proxyAuthKey != null) { AuthenticationInfo.endAuthRequest(proxyAuthKey); } + if (proxyAuthentication != null) { + proxyAuthentication.disposeContext(); + } } // restore original request headers @@ -2372,7 +2381,8 @@ private void setPreemptiveProxyAuthentication(MessageHeader requests) throws IOE * the connection. */ @SuppressWarnings({"removal","fallthrough"}) - private AuthenticationInfo getHttpProxyAuthentication(AuthenticationHeader authhdr) { + private AuthenticationInfo getHttpProxyAuthentication(AuthenticationHeader authhdr) + throws IOException { assert isLockHeldByCurrentThread(); @@ -2473,6 +2483,7 @@ public InetAddress run() authenticator, host, null, port, url.getProtocol(), "", scheme, url, RequestorType.PROXY); + validateNTLMCredentials(a); } /* If we are not trying transparent authentication then * we need to have a PasswordAuthentication instance. For @@ -2523,6 +2534,7 @@ public InetAddress run() } if (ret != null) { if (!ret.setHeaders(this, p, raw)) { + ret.disposeContext(); ret = null; } } @@ -2540,7 +2552,8 @@ public InetAddress run() * preferred. */ @SuppressWarnings("fallthrough") - private AuthenticationInfo getServerAuthentication(AuthenticationHeader authhdr) { + private AuthenticationInfo getServerAuthentication(AuthenticationHeader authhdr) + throws IOException { // Only called from getInputStream0 assert isLockHeldByCurrentThread(); @@ -2652,6 +2665,7 @@ private AuthenticationInfo getServerAuthentication(AuthenticationHeader authhdr) authenticator, url.getHost(), addr, port, url.getProtocol(), "", scheme, url, RequestorType.SERVER); + validateNTLMCredentials(a); } /* If we are not trying transparent authentication then @@ -2695,6 +2709,7 @@ private AuthenticationInfo getServerAuthentication(AuthenticationHeader authhdr) if (ret != null ) { if (!ret.setHeaders(this, p, raw)) { + ret.disposeContext(); ret = null; } } @@ -2721,6 +2736,7 @@ private void checkResponseCredentials (boolean inClose) throws IOException { DigestAuthentication da = (DigestAuthentication) currentProxyCredentials; da.checkResponse (raw, method, getRequestURI()); + currentProxyCredentials.disposeContext(); currentProxyCredentials = null; } } @@ -2731,6 +2747,7 @@ private void checkResponseCredentials (boolean inClose) throws IOException { DigestAuthentication da = (DigestAuthentication) currentServerCredentials; da.checkResponse (raw, method, url); + currentServerCredentials.disposeContext(); currentServerCredentials = null; } } @@ -4003,6 +4020,27 @@ public void close() throws IOException { } } } + + // ensure there are no null characters in username or password + private static void validateNTLMCredentials(PasswordAuthentication pw) + throws IOException { + + if (pw == null) { + return; + } + char[] password = pw.getPassword(); + if (password != null) { + for (int i=0; i MILLISECONDS.toNanos(millis)) { + // Round up any excess nanos to the nearest millisecond to + // avoid parking for less than requested. + millis++; + } } Net.poll(fd, event, millis); } diff --git a/src/java.base/share/classes/sun/nio/ch/SelChImpl.java b/src/java.base/share/classes/sun/nio/ch/SelChImpl.java index d5102aec3147b..6917158022e61 100644 --- a/src/java.base/share/classes/sun/nio/ch/SelChImpl.java +++ b/src/java.base/share/classes/sun/nio/ch/SelChImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,6 +29,7 @@ import java.io.FileDescriptor; import java.io.IOException; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; /** @@ -88,6 +89,11 @@ default void park(int event, long nanos) throws IOException { millis = -1; } else { millis = NANOSECONDS.toMillis(nanos); + if (nanos > MILLISECONDS.toNanos(millis)) { + // Round up any excess nanos to the nearest millisecond to + // avoid parking for less than requested. + millis++; + } } Net.poll(getFD(), event, millis); } diff --git a/src/jdk.charsets/share/classes/sun/nio/cs/ext/GB18030.java.template b/src/java.base/share/classes/sun/nio/cs/GB18030.java similarity index 99% rename from src/jdk.charsets/share/classes/sun/nio/cs/ext/GB18030.java.template rename to src/java.base/share/classes/sun/nio/cs/GB18030.java index 378e4857ba53e..362553a5dbba0 100644 --- a/src/jdk.charsets/share/classes/sun/nio/cs/ext/GB18030.java.template +++ b/src/java.base/share/classes/sun/nio/cs/GB18030.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,7 +26,7 @@ /* */ -package $PACKAGE$; +package sun.nio.cs; import java.nio.ByteBuffer; import java.nio.CharBuffer; @@ -34,7 +34,9 @@ import java.nio.charset.CharsetDecoder; import java.nio.charset.CharsetEncoder; import java.nio.charset.CoderResult; +import jdk.internal.misc.VM; import sun.nio.cs.Surrogate; +import sun.security.action.GetPropertyAction; public class GB18030 extends Charset @@ -43,8 +45,14 @@ public class GB18030 private static final int GB18030_DOUBLE_BYTE = 2; private static final int GB18030_FOUR_BYTE = 3; + // Assumes non-2000 standard if initialized during System.initPhase1(), + // as the system property is not ready to be read in that case. + static final boolean IS_2000 = + VM.initLevel() >= 1 && + "2000".equals(GetPropertyAction.privilegedGetProperty("jdk.charset.GB18030", "")); + public GB18030() { - super("GB18030", $ALIASES$); + super("GB18030", StandardCharsets.aliases_GB18030()); } public boolean contains(Charset cs) { @@ -1046,7 +1054,8 @@ public CharsetEncoder newEncoder() { "\u1E26\u1E27\u1E28\u1E29\u1E2A\u1E2B\u1E2C\u1E2D"+ "\u1E2E\u1E2F\u1E30\u1E31\u1E32\u1E33\u1E34\u1E35"+ "\u1E36\u1E37\u1E38\u1E39\u1E3A\u1E3B\u1E3C\u1E3D"+ - "\u1E3E\u1E3F\u1E40\u1E41\u1E42\u1E43\u1E44\u1E45"+ + (IS_2000 ? "\u1E3E\u1E3F\u1E40\u1E41\u1E42\u1E43\u1E44\u1E45" : + "\u1E3E\uE7C7\u1E40\u1E41\u1E42\u1E43\u1E44\u1E45")+ "\u1E46\u1E47\u1E48\u1E49\u1E4A\u1E4B\u1E4C\u1E4D"+ "\u1E4E\u1E4F\u1E50\u1E51\u1E52\u1E53\u1E54\u1E55"+ "\u1E56\u1E57\u1E58\u1E59\u1E5A\u1E5B\u1E5C\u1E5D"+ @@ -2502,8 +2511,10 @@ public CharsetEncoder newEncoder() { "\u4DF5\u4DF6\u4DF7\u4DF8\u4DF9\u4DFA\u4DFB\u4DFC"+ "\u4DFD\u4DFE\u4DFF\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"+ "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"+ - "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"+ - "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"+ + (IS_2000 ? "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"+ + "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD" : + "\uFFFD\uE81E\uE826\uE82B\uE82C\uE832\uE843\uE854"+ + "\uE864\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD")+ "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"+ "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"+ "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD"+ @@ -2766,8 +2777,10 @@ public CharsetEncoder newEncoder() { "\uFDF4\uFDF5\uFDF6\uFDF7\uFDF8\uFDF9\uFDFA\uFDFB"+ "\uFDFC\uFDFD\uFDFE\uFDFF\uFE00\uFE01\uFE02\uFE03"+ "\uFE04\uFE05\uFE06\uFE07\uFE08\uFE09\uFE0A\uFE0B"+ - "\uFE0C\uFE0D\uFE0E\uFE0F\uFE10\uFE11\uFE12\uFE13"+ - "\uFE14\uFE15\uFE16\uFE17\uFE18\uFE19\uFE1A\uFE1B"+ + (IS_2000 ? "\uFE0C\uFE0D\uFE0E\uFE0F\uFE10\uFE11\uFE12\uFE13"+ + "\uFE14\uFE15\uFE16\uFE17\uFE18\uFE19\uFE1A\uFE1B" : + "\uFE0C\uFE0D\uFE0E\uFE0F\uE78D\uE78F\uE78E\uE790"+ + "\uE791\uE792\uE793\uE794\uE795\uE796\uFE1A\uFE1B")+ "\uFE1C\uFE1D\uFE1E\uFE1F\uFE20\uFE21\uFE22\uFE23"+ "\uFE24\uFE25\uFE26\uFE27\uFE28\uFE29\uFE2A\uFE2B"+ "\uFE2C\uFE2D\uFE2E\uFE2F\uFE32\uFE45\uFE46\uFE47"+ @@ -3773,10 +3786,14 @@ public CharsetEncoder newEncoder() { "\uE78A\uE78B\uE78C\u03B1\u03B2\u03B3\u03B4\u03B5"+ "\u03B6\u03B7\u03B8\u03B9\u03BA\u03BB\u03BC\u03BD"+ "\u03BE\u03BF\u03C0\u03C1\u03C3\u03C4\u03C5\u03C6"+ - "\u03C7\u03C8\u03C9\uE78D\uE78E\uE78F\uE790\uE791"+ - "\uE792\uE793\uFE35\uFE36\uFE39\uFE3A\uFE3F\uFE40"+ - "\uFE3D\uFE3E\uFE41\uFE42\uFE43\uFE44\uE794\uE795"+ - "\uFE3B\uFE3C\uFE37\uFE38\uFE31\uE796\uFE33\uFE34"+ + (IS_2000 ? "\u03C7\u03C8\u03C9\uE78D\uE78E\uE78F\uE790\uE791"+ + "\uE792\uE793\uFE35\uFE36\uFE39\uFE3A\uFE3F\uFE40"+ + "\uFE3D\uFE3E\uFE41\uFE42\uFE43\uFE44\uE794\uE795"+ + "\uFE3B\uFE3C\uFE37\uFE38\uFE31\uE796\uFE33\uFE34" : + "\u03C7\u03C8\u03C9\uFE10\uFE12\uFE11\uFE13\uFE14"+ + "\uFE15\uFE16\uFE35\uFE36\uFE39\uFE3A\uFE3F\uFE40"+ + "\uFE3D\uFE3E\uFE41\uFE42\uFE43\uFE44\uFE17\uFE18"+ + "\uFE3B\uFE3C\uFE37\uFE38\uFE31\uFE19\uFE33\uFE34")+ "\uE797\uE798\uE799\uE79A\uE79B\uE79C\uE79D\uE79E"+ "\uE79F\uE706\uE707\uE708\uE709\uE70A\uE70B\uE70C"+ "\uE70D\uE70E\uE70F\uE710\uE711\uE712\uE713\uE714"+ @@ -3817,7 +3834,8 @@ public CharsetEncoder newEncoder() { "\uE7C6\u0101\u00E1\u01CE\u00E0\u0113\u00E9\u011B"+ "\u00E8\u012B\u00ED\u01D0\u00EC\u014D\u00F3\u01D2"+ "\u00F2\u016B\u00FA\u01D4\u00F9\u01D6\u01D8\u01DA"+ - "\u01DC\u00FC\u00EA\u0251\uE7C7\u0144\u0148\u01F9"+ + (IS_2000 ? "\u01DC\u00FC\u00EA\u0251\uE7C7\u0144\u0148\u01F9" : + "\u01DC\u00FC\u00EA\u0251\u1E3F\u0144\u0148\u01F9")+ "\u0261\uE7C9\uE7CA\uE7CB\uE7CC\u3105\u3106\u3107"+ "\u3108\u3109\u310A\u310B\u310C\u310D\u310E\u310F"+ "\u3110\u3111\u3112\u3113\u3114\u3115\u3116\u3117"+ @@ -5868,16 +5886,22 @@ public CharsetEncoder newEncoder() { "\uE466\uE467\uFA0C\uFA0D\uFA0E\uFA0F\uFA11\uFA13"+ "\uFA14\uFA18\uFA1F\uFA20\uFA21\uFA23\uFA24\uFA27"+ "\uFA28\uFA29\u2E81\uE816\uE817\uE818\u2E84\u3473"+ - "\u3447\u2E88\u2E8B\uE81E\u359E\u361A\u360E\u2E8C"+ - "\u2E97\u396E\u3918\uE826\u39CF\u39DF\u3A73\u39D0"+ - "\uE82B\uE82C\u3B4E\u3C6E\u3CE0\u2EA7\uE831\uE832"+ + (IS_2000 ? "\u3447\u2E88\u2E8B\uE81E\u359E\u361A\u360E\u2E8C"+ + "\u2E97\u396E\u3918\uE826\u39CF\u39DF\u3A73\u39D0"+ + "\uE82B\uE82C\u3B4E\u3C6E\u3CE0\u2EA7\uE831\uE832" : + "\u3447\u2E88\u2E8B\u9FB4\u359E\u361A\u360E\u2E8C"+ + "\u2E97\u396E\u3918\u9FB5\u39CF\u39DF\u3A73\u39D0"+ + "\u9FB6\u9FB7\u3B4E\u3C6E\u3CE0\u2EA7\uE831\u9FB8")+ "\u2EAA\u4056\u415F\u2EAE\u4337\u2EB3\u2EB6\u2EB7"+ "\uE83B\u43B1\u43AC\u2EBB\u43DD\u44D6\u4661\u464C"+ - "\uE843\uFFFD\u4723\u4729\u477C\u478D\u2ECA\u4947"+ + (IS_2000 ? "\uE843\uFFFD\u4723\u4729\u477C\u478D\u2ECA\u4947" : + "\u9FB9\uFFFD\u4723\u4729\u477C\u478D\u2ECA\u4947")+ "\u497A\u497D\u4982\u4983\u4985\u4986\u499F\u499B"+ - "\u49B7\u49B6\uE854\uE855\u4CA3\u4C9F\u4CA0\u4CA1"+ + (IS_2000 ? "\u49B7\u49B6\uE854\uE855\u4CA3\u4C9F\u4CA0\u4CA1" : + "\u49B7\u49B6\u9FBA\uE855\u4CA3\u4C9F\u4CA0\u4CA1")+ "\u4C77\u4CA2\u4D13\u4D14\u4D15\u4D16\u4D17\u4D18"+ - "\u4D19\u4DAE\uE864\uE468\uE469\uE46A\uE46B\uE46C"+ + (IS_2000 ? "\u4D19\u4DAE\uE864\uE468\uE469\uE46A\uE46B\uE46C" : + "\u4D19\u4DAE\u9FBB\uE468\uE469\uE46A\uE46B\uE46C")+ "\uE46D\uE46E\uE46F\uE470\uE471\uE472\uE473\uE474"+ "\uE475\uE476\uE477\uE478\uE479\uE47A\uE47B\uE47C"+ "\uE47D\uE47E\uE47F\uE480\uE481\uE482\uE483\uE484"+ @@ -6895,7 +6919,8 @@ public CharsetEncoder newEncoder() { "\u3D02\u3D03\u3D04\u3D05\u3D06\u3D07\u3D08\u3D09"+ "\u3D0A\u3D0B\u3D0C\u3D0D\u3D0E\u3D0F\u3D10\u3D11"+ "\u3D12\u3D13\u3D14\u3D15\u3D16\u3D17\u3D18\u3D19"+ - "\u3D1A\u3D1B\u3D1C\u3D1D\u3D1E\u3D1F\u3D20\u3D21"+ + (IS_2000 ? "\u3D1A\u3D1B\u3D1C\u3D1D\u3D1E\u3D1F\u3D20\u3D21" : + "\u3D1A\u3D1B\u3D1C\u3D1D\u3D1E\u3D1F\u3D20\uA8BC")+ "\u3D22\u3D23\u3D24\u3D25\u3D26\u3D27\u3D28\u3D29"+ "\u3D2A\u3D2B\u3D2C\u3D2D\u3D2E\u3D2F\u3D30\u3D31"+ "\u3D32\u3D33\u3D34\u3D35\u3D36\u3D37\u3D38\u3D39"+ @@ -11054,8 +11079,10 @@ public CharsetEncoder newEncoder() { "\uFD93\uC1FA\uB9A8\uEDE8\uFD94\uFD95\uFD96\uB9EA"+ "\uD9DF\uFD97\uFD98\uFD99\uFD9A\uFD9B\u6A63\u6A64"+ "\u6A65\u6A66\u6A67\u6A68\u6A69\u6A6A\u6A6B\u6A6C"+ - "\u6A6D\u6A6E\u6A6F\u6A70\u6A71\u6A72\u6A73\u6A74"+ - "\u6A75\u6A76\u6A77\u6A78\u6A79\u6A7A\u6A7B\u6A7C"+ + (IS_2000 ? "\u6A6D\u6A6E\u6A6F\u6A70\u6A71\u6A72\u6A73\u6A74"+ + "\u6A75\u6A76\u6A77\u6A78\u6A79\u6A7A\u6A7B\u6A7C" : + "\u6A6D\u6A6E\u6A6F\u6A70\uFE59\uFE61\uFE66\uFE67"+ + "\uFE6D\uFE7E\uFE90\uFEA0\u6A79\u6A7A\u6A7B\u6A7C")+ "\u6A7D\u6A7E\u6A7F\u6A80\u6A81\u6A82\u6A83\u6A84"+ "\u6A85\u6A86\u6A87\u6A88\u6A89\u6A8A\u6A8B\u6A8C"+ "\u6A8D\u6A8E\u6A8F\u6A90\u6A91\u6A92\u6A93\u6A94"+ @@ -11467,14 +11494,17 @@ public CharsetEncoder newEncoder() { "\uA2FD\uA2FE\uA4F4\uA4F5\uA4F6\uA4F7\uA4F8\uA4F9"+ "\uA4FA\uA4FB\uA4FC\uA4FD\uA4FE\uA5F7\uA5F8\uA5F9"+ "\uA5FA\uA5FB\uA5FC\uA5FD\uA5FE\uA6B9\uA6BA\uA6BB"+ - "\uA6BC\uA6BD\uA6BE\uA6BF\uA6C0\uA6D9\uA6DA\uA6DB"+ - "\uA6DC\uA6DD\uA6DE\uA6DF\uA6EC\uA6ED\uA6F3\uA6F6"+ + (IS_2000 ? "\uA6BC\uA6BD\uA6BE\uA6BF\uA6C0\uA6D9\uA6DA\uA6DB"+ + "\uA6DC\uA6DD\uA6DE\uA6DF\uA6EC\uA6ED\uA6F3\uA6F6" : + "\uA6BC\uA6BD\uA6BE\uA6BF\uA6C0\u35E7\u35E9\u35E8"+ + "\u35EA\u35EB\u35EC\u35ED\u35EE\u35EF\u35F0\uA6F6")+ "\uA6F7\uA6F8\uA6F9\uA6FA\uA6FB\uA6FC\uA6FD\uA6FE"+ "\uA7C2\uA7C3\uA7C4\uA7C5\uA7C6\uA7C7\uA7C8\uA7C9"+ "\uA7CA\uA7CB\uA7CC\uA7CD\uA7CE\uA7CF\uA7D0\uA7F2"+ "\uA7F3\uA7F4\uA7F5\uA7F6\uA7F7\uA7F8\uA7F9\uA7FA"+ "\uA7FB\uA7FC\uA7FD\uA7FE\uA896\uA897\uA898\uA899"+ - "\uA89A\uA89B\uA89C\uA89D\uA89E\uA89F\uA8A0\uA8BC"+ + (IS_2000 ? "\uA89A\uA89B\uA89C\uA89D\uA89E\uA89F\uA8A0\uA8BC" : + "\uA89A\uA89B\uA89C\uA89D\uA89E\uA89F\uA8A0\u3D21")+ "\u2001\uA8C1\uA8C2\uA8C3\uA8C4\uA8EA\uA8EB\uA8EC"+ "\uA8ED\uA8EE\uA8EF\uA8F0\uA8F1\uA8F2\uA8F3\uA8F4"+ "\uA8F5\uA8F6\uA8F7\uA8F8\uA8F9\uA8FA\uA8FB\uA8FC"+ @@ -11485,16 +11515,23 @@ public CharsetEncoder newEncoder() { "\uA9A3\uA9F0\uA9F1\uA9F2\uA9F3\uA9F4\uA9F5\uA9F6"+ "\uA9F7\uA9F8\uA9F9\uA9FA\uA9FB\uA9FC\uA9FD\uA9FE"+ "\uD7FA\uD7FB\uD7FC\uD7FD\uD7FE\u200F\uFE51\uFE52"+ - "\uFE53\u2010\u2011\u2012\u2013\u2014\uFE59\u2015"+ - "\u2016\u2017\u2018\u2019\u201A\u201B\uFE61\u201C"+ - "\u201D\u201E\u201F\uFE66\uFE67\u2020\u2021\u2022"+ - "\u2023\uFE6C\uFE6D\u2024\u2025\u2026\u2027\u2028"+ + (IS_2000 ? "\uFE53\u2010\u2011\u2012\u2013\u2014\uFE59\u2015"+ + "\u2016\u2017\u2018\u2019\u201A\u201B\uFE61\u201C"+ + "\u201D\u201E\u201F\uFE66\uFE67\u2020\u2021\u2022"+ + "\u2023\uFE6C\uFE6D\u2024\u2025\u2026\u2027\u2028" : + "\uFE53\u2010\u2011\u2012\u2013\u2014\u6A71\u2015"+ + "\u2016\u2017\u2018\u2019\u201A\u201B\u6A72\u201C"+ + "\u201D\u201E\u201F\u6A73\u6A74\u2020\u2021\u2022"+ + "\u2023\uFE6C\u6A75\u2024\u2025\u2026\u2027\u2028")+ "\u2029\u202A\u202B\uFE76\u202C\u202D\u202E\u202F"+ - "\u2030\u2031\u2032\uFE7E\u2033\u2034\u2035\u2036"+ + (IS_2000 ? "\u2030\u2031\u2032\uFE7E\u2033\u2034\u2035\u2036" : + "\u2030\u2031\u2032\u6A76\u2033\u2034\u2035\u2036")+ "\u2037\u2038\u2039\u203A\u203B\u203C\u203D\u203E"+ - "\u203F\u2040\u2041\u2042\uFE90\uFE91\u2043\u2044"+ + (IS_2000 ? "\u203F\u2040\u2041\u2042\uFE90\uFE91\u2043\u2044" : + "\u203F\u2040\u2041\u2042\u6A77\uFE91\u2043\u2044")+ "\u2045\u2046\u2047\u2048\u2049\u204A\u204B\u204C"+ - "\u204D\u204E\u204F\u2050\uFEA0\u2051\u2052\u2053"+ + (IS_2000 ? "\u204D\u204E\u204F\u2050\uFEA0\u2051\u2052\u2053" : + "\u204D\u204E\u204F\u2050\u6A78\u2051\u2052\u2053")+ "\u2054\u2055\u2056\u2057\u2058\u2059\u205A\u205B"+ "\u205C\u205D\u205E\u205F\u2060\u2061\u2062\u2063"+ "\u2064\u2065\u2066\u2067\u2068\u2069\u206A\u206B"+ @@ -12192,8 +12229,10 @@ public CharsetEncoder newEncoder() { "\u24E3\u24E4\u24E5\u24E6\u24E7\u24E8\u24E9\u24EA"+ "\u24EB\u24EC\u24ED\u24EE\u24EF\u24F0\u24F1\u24F2"+ "\u24F3\u24F4\u24F5\u24F6\u24F7\u24F8\u24F9\u24FA"+ - "\u24FB\u24FC\u24FD\u24FE\u24FF\u2500\u2501\u2502"+ - "\u2503\u2504\u2505\u2506\u2507\u2508\u2509\u250A"+ + (IS_2000 ? "\u24FB\u24FC\u24FD\u24FE\u24FF\u2500\u2501\u2502"+ + "\u2503\u2504\u2505\u2506\u2507\u2508\u2509\u250A" : + "\uA6D9\uA6DB\uA6DA\uA6DC\uA6DD\uA6DE\uA6DF\uA6EC"+ + "\uA6ED\uA6F3\u2505\u2506\u2507\u2508\u2509\u250A")+ "\u250B\u250C\u250D\u250E\u250F\u2510\u2511\u2512"+ "\u2513\u2514\u2515\u2516\u2517\u2518\u2519\u251A"+ "\uA955\uA6F2\u251B\uA6F4\uA6F5\uA6E0\uA6E1\uA6F0"+ @@ -12406,8 +12445,13 @@ else if (byte2 == 0x7f || byte2 == 0xff || // BMP Ranges if (offset <= 0x4A62) da[dp++] = getChar(offset); - else if (offset > 0x4A62 && offset <= 0x82BC) - da[dp++] = (char)(offset + 0x5543); + else if (offset > 0x4A62 && offset <= 0x82BC) { + if (offset >= 0x4A71 && offset <= 0x4A78 && !IS_2000) { + da[dp++] = getChar(offset); + } else { + da[dp++] = (char)(offset + 0x5543); + } + } else if (offset >= 0x82BD && offset <= 0x830D) da[dp++] = getChar(offset); else if (offset >= 0x830D && offset <= 0x93A8) @@ -12509,8 +12553,13 @@ else if (byte2 == 0x7f || byte2 == 0xff || // BMP Ranges if (offset <= 0x4A62) dst.put(getChar(offset)); - else if (offset > 0x4A62 && offset <= 0x82BC) - dst.put((char)(offset + 0x5543)); + else if (offset > 0x4A62 && offset <= 0x82BC) { + if (offset >= 0x4A71 && offset <= 0x4A78 && !IS_2000) { + dst.put(getChar(offset)); + } else { + dst.put((char)(offset + 0x5543)); + } + } else if (offset >= 0x82BD && offset <= 0x830D) dst.put(getChar(offset)); else if (offset >= 0x830D && offset <= 0x93A8) @@ -12620,15 +12669,24 @@ else if (c <= 0xA4C6 || c >= 0xE000) { condensedKey = (hiByte - 0x20) * 256 + loByte; - if (c >= 0xE000 && c < 0xF900) + if (c >= 0xE000 && c < 0xF900) { + if (IS_2000) { condensedKey += 0x82BD; + } else { + condensedKey = switch (c) { + case 0xE7C7, 0xE81E, 0xE826, 0xE82B, 0xE82C, 0xE832, + 0xE843, 0xE854, 0xE864 -> condensedKey; + default -> condensedKey + 0x82BD; + }; + } + } else if (c >= 0xF900) - condensedKey += 0x93A9; + condensedKey += 0x93A9; if (hiByte > 0x80) - currentState = GB18030_DOUBLE_BYTE; + currentState = GB18030_DOUBLE_BYTE; else - currentState = GB18030_FOUR_BYTE; + currentState = GB18030_FOUR_BYTE; } else if (c >= 0xA4C7 && c <= 0xD7FF) { condensedKey = c - 0x5543; @@ -12671,7 +12729,7 @@ else if (c >= 0xA4C7 && c <= 0xD7FF) { } sp += inputSize; } - return CoderResult.UNDERFLOW; + return CoderResult.UNDERFLOW; } finally { src.position(sp - src.arrayOffset()); dst.position(dp - dst.arrayOffset()); @@ -12711,15 +12769,24 @@ else if (c <= 0xA4C6 || c >= 0xE000) { condensedKey = (hiByte - 0x20) * 256 + loByte; - if (c >= 0xE000 && c < 0xF900) + if (c >= 0xE000 && c < 0xF900) { + if (IS_2000) { condensedKey += 0x82BD; + } else { + condensedKey = switch (c) { + case 0xE7C7, 0xE81E, 0xE826, 0xE82B, 0xE82C, 0xE832, + 0xE843, 0xE854, 0xE864 -> condensedKey; + default -> condensedKey + 0x82BD; + }; + } + } else if (c >= 0xF900) - condensedKey += 0x93A9; + condensedKey += 0x93A9; if (hiByte > 0x80) - currentState = GB18030_DOUBLE_BYTE; + currentState = GB18030_DOUBLE_BYTE; else - currentState = GB18030_FOUR_BYTE; + currentState = GB18030_FOUR_BYTE; } else if (c >= 0xA4C7 && c <= 0xD7FF) { condensedKey = c - 0x5543; diff --git a/src/java.base/share/classes/sun/nio/cs/StandardCharsets.java.template b/src/java.base/share/classes/sun/nio/cs/StandardCharsets.java.template index b5590880e2d1c..339e01de39e6b 100644 --- a/src/java.base/share/classes/sun/nio/cs/StandardCharsets.java.template +++ b/src/java.base/share/classes/sun/nio/cs/StandardCharsets.java.template @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. * * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -58,8 +58,13 @@ public class StandardCharsets extends CharsetProvider { } private String canonicalize(String csn) { - String acn = aliasMap().get(csn); - return (acn != null) ? acn : csn; + if (csn.startsWith("gb18030-")) { + return csn.equals("gb18030-2022") && !GB18030.IS_2000 || + csn.equals("gb18030-2000") && GB18030.IS_2000 ? "gb18030" : csn; + } else { + String acn = aliasMap().get(csn); + return (acn != null) ? acn : csn; + } } private Map aliasMap() { diff --git a/src/java.base/share/classes/sun/security/pkcs/PKCS7.java b/src/java.base/share/classes/sun/security/pkcs/PKCS7.java index 0dca814eb76d0..f7df9934f5538 100644 --- a/src/java.base/share/classes/sun/security/pkcs/PKCS7.java +++ b/src/java.base/share/classes/sun/security/pkcs/PKCS7.java @@ -37,6 +37,7 @@ import java.security.*; import java.util.function.Function; +import sun.security.jca.JCAUtil; import sun.security.provider.SHAKE256; import sun.security.timestamp.*; import sun.security.util.*; @@ -67,23 +68,6 @@ public class PKCS7 { private Principal[] certIssuerNames; - /* - * Random number generator for creating nonce values - * (Lazy initialization) - */ - private static class SecureRandomHolder { - static final SecureRandom RANDOM; - static { - SecureRandom tmp = null; - try { - tmp = SecureRandom.getInstance("SHA1PRNG"); - } catch (NoSuchAlgorithmException e) { - // should not happen - } - RANDOM = tmp; - } - } - /** * Unmarshals a PKCS7 block from its encoded form, parsing the * encoded bytes from the InputStream. @@ -1027,11 +1011,9 @@ public static byte[] generateTimestampToken(Timestamper tsa, } // Generate a nonce - BigInteger nonce = null; - if (SecureRandomHolder.RANDOM != null) { - nonce = new BigInteger(64, SecureRandomHolder.RANDOM); - tsQuery.setNonce(nonce); - } + BigInteger nonce = new BigInteger(64, JCAUtil.getDefSecureRandom()); + tsQuery.setNonce(nonce); + tsQuery.requestCertificate(true); TSResponse tsReply = tsa.generateTimestamp(tsQuery); diff --git a/src/java.base/share/classes/sun/security/provider/certpath/AdjacencyList.java b/src/java.base/share/classes/sun/security/provider/certpath/AdjacencyList.java index 9039d4fe75baa..905a1d3e13c46 100644 --- a/src/java.base/share/classes/sun/security/provider/certpath/AdjacencyList.java +++ b/src/java.base/share/classes/sun/security/provider/certpath/AdjacencyList.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -87,7 +87,7 @@ public class AdjacencyList { // the actual set of steps the AdjacencyList represents private ArrayList mStepList; - // the original list, just for the toString method + // the original list private List> mOrigList; /** @@ -114,6 +114,13 @@ public Iterator iterator() { return Collections.unmodifiableList(mStepList).iterator(); } + /** + * Returns the number of attempted paths (useful for debugging). + */ + public int numAttemptedPaths() { + return mOrigList.size(); + } + /** * Recursive, private method which actually builds the step list from * the given adjacency list. Follow is the parent BuildStep diff --git a/src/java.base/share/classes/sun/security/provider/certpath/Builder.java b/src/java.base/share/classes/sun/security/provider/certpath/Builder.java index 186d1269de315..c920ad0282689 100644 --- a/src/java.base/share/classes/sun/security/provider/certpath/Builder.java +++ b/src/java.base/share/classes/sun/security/provider/certpath/Builder.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -412,8 +412,7 @@ Set getMatchingPolicies() { /** * Search the specified CertStores and add all certificates matching - * selector to resultCerts. Self-signed certs are not useful here - * and therefore ignored. + * selector to resultCerts. * * If the targetCert criterion of the selector is set, only that cert * is examined and the CertStores are not searched. @@ -432,8 +431,7 @@ boolean addMatchingCerts(X509CertSelector selector, X509Certificate targetCert = selector.getCertificate(); if (targetCert != null) { // no need to search CertStores - if (selector.match(targetCert) && !X509CertImpl.isSelfSigned - (targetCert, buildParams.sigProvider())) { + if (selector.match(targetCert)) { if (debug != null) { debug.println("Builder.addMatchingCerts: " + "adding target cert" + @@ -452,11 +450,8 @@ boolean addMatchingCerts(X509CertSelector selector, Collection certs = store.getCertificates(selector); for (Certificate cert : certs) { - if (!X509CertImpl.isSelfSigned - ((X509Certificate)cert, buildParams.sigProvider())) { - if (resultCerts.add((X509Certificate)cert)) { - add = true; - } + if (resultCerts.add((X509Certificate)cert)) { + add = true; } } if (!checkAll && add) { diff --git a/src/java.base/share/classes/sun/security/provider/certpath/ForwardBuilder.java b/src/java.base/share/classes/sun/security/provider/certpath/ForwardBuilder.java index 6122fdaa97077..f05a924440ba1 100644 --- a/src/java.base/share/classes/sun/security/provider/certpath/ForwardBuilder.java +++ b/src/java.base/share/classes/sun/security/provider/certpath/ForwardBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -48,6 +48,7 @@ import sun.security.x509.AuthorityInfoAccessExtension; import sun.security.x509.AuthorityKeyIdentifierExtension; import static sun.security.x509.PKIXExtensions.*; +import sun.security.x509.SubjectAlternativeNameExtension; import sun.security.x509.X500Name; import sun.security.x509.X509CertImpl; @@ -294,9 +295,7 @@ private void getMatchingCACerts(ForwardState currentState, "\n Issuer: " + trustedCert.getIssuerX500Principal()); } - if (caCerts.add(trustedCert) && !searchAllCertStores) { - return; - } + caCerts.add(trustedCert); } } @@ -675,8 +674,7 @@ public int compare(X509Certificate oCert1, X509Certificate oCert2) { * only be executed in a reverse direction are deferred until the * complete path has been built. * - * Trust anchor certs are not validated, but are used to verify the - * signature and revocation status of the previous cert. + * Trust anchor certs are not validated. * * If the last certificate is being verified (the one whose subject * matches the target subject, then steps in 6.1.4 of the PKIX @@ -707,17 +705,15 @@ void verifyCert(X509Certificate cert, State currentState, currState.untrustedChecker.check(cert, Collections.emptySet()); /* - * check for looping - abort a loop if we encounter the same - * certificate twice + * Abort if we encounter the same certificate or a certificate with + * the same public key, subject DN, and subjectAltNames as a cert + * that is already in path. */ - if (certPathList != null) { - for (X509Certificate cpListCert : certPathList) { - if (cert.equals(cpListCert)) { - if (debug != null) { - debug.println("loop detected!!"); - } - throw new CertPathValidatorException("loop detected"); - } + for (X509Certificate cpListCert : certPathList) { + if (repeated(cpListCert, cert)) { + throw new CertPathValidatorException( + "cert with repeated subject, public key, and " + + "subjectAltNames detected"); } } @@ -796,21 +792,48 @@ void verifyCert(X509Certificate cert, State currentState, */ KeyChecker.verifyCAKeyUsage(cert); } + } - /* - * the following checks are performed even when the cert - * is a trusted cert, since we are only extracting the - * subjectDN, and publicKey from the cert - * in order to verify a previous cert - */ + /** + * Return true if two certificates are equal or have the same subject, + * public key, and subject alternative names. + */ + private static boolean repeated( + X509Certificate currCert, X509Certificate nextCert) { + if (currCert.equals(nextCert)) { + return true; + } + return (currCert.getSubjectX500Principal().equals( + nextCert.getSubjectX500Principal()) && + currCert.getPublicKey().equals(nextCert.getPublicKey()) && + altNamesEqual(currCert, nextCert)); + } - /* - * Check signature only if no key requiring key parameters has been - * encountered. - */ - if (!currState.keyParamsNeeded()) { - (currState.cert).verify(cert.getPublicKey(), - buildParams.sigProvider()); + /** + * Return true if two certificates have the same subject alternative names. + */ + private static boolean altNamesEqual( + X509Certificate currCert, X509Certificate nextCert) { + X509CertImpl curr, next; + try { + curr = X509CertImpl.toImpl(currCert); + next = X509CertImpl.toImpl(nextCert); + } catch (CertificateException ce) { + return false; + } + + SubjectAlternativeNameExtension currAltNameExt = + curr.getSubjectAlternativeNameExtension(); + SubjectAlternativeNameExtension nextAltNameExt = + next.getSubjectAlternativeNameExtension(); + if (currAltNameExt != null) { + if (nextAltNameExt == null) { + return false; + } + return Arrays.equals(currAltNameExt.getExtensionValue(), + nextAltNameExt.getExtensionValue()); + } else { + return (nextAltNameExt == null); } } diff --git a/src/java.base/share/classes/sun/security/provider/certpath/ForwardState.java b/src/java.base/share/classes/sun/security/provider/certpath/ForwardState.java index 2dc9e208e924f..9d7af9b169baa 100644 --- a/src/java.base/share/classes/sun/security/provider/certpath/ForwardState.java +++ b/src/java.base/share/classes/sun/security/provider/certpath/ForwardState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2012, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -80,10 +80,8 @@ class ForwardState implements State { /* The list of user-defined checkers that support forward checking */ ArrayList forwardCheckers; - /* Flag indicating if key needing to inherit key parameters has been - * encountered. - */ - boolean keyParamsNeededFlag = false; + /* Flag indicating if last cert in path is self-issued */ + boolean selfIssued; /** * Returns a boolean flag indicating if the state is initial @@ -96,18 +94,6 @@ public boolean isInitial() { return init; } - /** - * Return boolean flag indicating whether a public key that needs to inherit - * key parameters has been encountered. - * - * @return boolean true if key needing to inherit parameters has been - * encountered; false otherwise. - */ - @Override - public boolean keyParamsNeeded() { - return keyParamsNeededFlag; - } - /** * Display state for debugging purposes */ @@ -118,10 +104,10 @@ public String toString() { sb.append("\n issuerDN of last cert: ").append(issuerDN); sb.append("\n traversedCACerts: ").append(traversedCACerts); sb.append("\n init: ").append(String.valueOf(init)); - sb.append("\n keyParamsNeeded: ").append - (String.valueOf(keyParamsNeededFlag)); sb.append("\n subjectNamesTraversed: \n").append (subjectNamesTraversed); + sb.append("\n selfIssued: ").append + (String.valueOf(selfIssued)); sb.append("]\n"); return sb.toString(); } @@ -166,18 +152,14 @@ public void updateState(X509Certificate cert) X509CertImpl icert = X509CertImpl.toImpl(cert); - /* see if certificate key has null parameters */ - if (PKIX.isDSAPublicKeyWithoutParams(icert.getPublicKey())) { - keyParamsNeededFlag = true; - } - /* update certificate */ this.cert = icert; /* update issuer DN */ issuerDN = cert.getIssuerX500Principal(); - if (!X509CertImpl.isSelfIssued(cert)) { + selfIssued = X509CertImpl.isSelfIssued(cert); + if (!selfIssued) { /* * update traversedCACerts only if this is a non-self-issued @@ -190,7 +172,7 @@ public void updateState(X509Certificate cert) /* update subjectNamesTraversed only if this is the EE cert or if this cert is not self-issued */ - if (init || !X509CertImpl.isSelfIssued(cert)){ + if (init || !selfIssued) { X500Principal subjName = cert.getSubjectX500Principal(); subjectNamesTraversed.add(X500Name.asX500Name(subjName)); diff --git a/src/java.base/share/classes/sun/security/provider/certpath/OCSP.java b/src/java.base/share/classes/sun/security/provider/certpath/OCSP.java index 859823b937e1b..98575120ef953 100644 --- a/src/java.base/share/classes/sun/security/provider/certpath/OCSP.java +++ b/src/java.base/share/classes/sun/security/provider/certpath/OCSP.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2009, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -267,20 +267,23 @@ public static byte[] getOCSPBytes(List certIds, URI responderURI, out.flush(); } - // Check the response - if (debug != null && - con.getResponseCode() != HttpURLConnection.HTTP_OK) { - debug.println("Received HTTP error: " + con.getResponseCode() - + " - " + con.getResponseMessage()); + // Check the response. Non-200 codes will generate an exception + // but path validation may complete successfully if revocation info + // can be obtained elsewhere (e.g. CRL). + int respCode = con.getResponseCode(); + if (respCode != HttpURLConnection.HTTP_OK) { + String msg = "Received HTTP error: " + respCode + " - " + + con.getResponseMessage(); + if (debug != null) { + debug.println(msg); + } + throw new IOException(msg); } int contentLength = con.getContentLength(); - if (contentLength == -1) { - contentLength = Integer.MAX_VALUE; - } - - return IOUtils.readExactlyNBytes(con.getInputStream(), - contentLength); + return (contentLength == -1) ? con.getInputStream().readAllBytes() : + IOUtils.readExactlyNBytes(con.getInputStream(), + contentLength); } finally { if (con != null) { con.disconnect(); diff --git a/src/java.base/share/classes/sun/security/provider/certpath/State.java b/src/java.base/share/classes/sun/security/provider/certpath/State.java index 93a153fec3890..3292d65488071 100644 --- a/src/java.base/share/classes/sun/security/provider/certpath/State.java +++ b/src/java.base/share/classes/sun/security/provider/certpath/State.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2001, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -62,12 +62,4 @@ public void updateState(X509Certificate cert) * @return boolean flag indicating if the state is initial (just starting) */ public boolean isInitial(); - - /** - * Returns a boolean flag indicating if a key lacking necessary key - * algorithm parameters has been encountered. - * - * @return boolean flag indicating if key lacking parameters encountered. - */ - public boolean keyParamsNeeded(); } diff --git a/src/java.base/share/classes/sun/security/provider/certpath/SunCertPathBuilder.java b/src/java.base/share/classes/sun/security/provider/certpath/SunCertPathBuilder.java index c1c03d86b7eb0..fd4eb9543e92e 100644 --- a/src/java.base/share/classes/sun/security/provider/certpath/SunCertPathBuilder.java +++ b/src/java.base/share/classes/sun/security/provider/certpath/SunCertPathBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -42,6 +42,7 @@ import sun.security.provider.certpath.PKIX.BuilderParams; import static sun.security.x509.PKIXExtensions.*; +import sun.security.x509.X509CertImpl; import sun.security.util.Debug; /** @@ -130,18 +131,21 @@ private PKIXCertPathBuilderResult build() throws CertPathBuilderException { List> adjList = new ArrayList<>(); PKIXCertPathBuilderResult result = buildCertPath(false, adjList); if (result == null) { - if (debug != null) { - debug.println("SunCertPathBuilder.engineBuild: 2nd pass; " + + if (buildParams.certStores().size() > 1 || Builder.USE_AIA) { + if (debug != null) { + debug.println("SunCertPathBuilder.engineBuild: 2nd pass; " + "try building again searching all certstores"); + } + // try again + adjList.clear(); + result = buildCertPath(true, adjList); + if (result != null) { + return result; + } } - // try again - adjList.clear(); - result = buildCertPath(true, adjList); - if (result == null) { - throw new SunCertPathBuilderException("unable to find valid " - + "certification path to requested target", - new AdjacencyList(adjList)); - } + throw new SunCertPathBuilderException("unable to find valid " + + "certification path to requested target", + new AdjacencyList(adjList)); } return result; } @@ -270,8 +274,8 @@ private void depthFirstSearchForward(X500Principal dN, /* * For each cert in the collection, verify anything * that hasn't been checked yet (signature, revocation, etc) - * and check for loops. Call depthFirstSearchForward() - * recursively for each good cert. + * and check for certs with repeated public key and subject. + * Call depthFirstSearchForward() recursively for each good cert. */ vertices: @@ -346,26 +350,24 @@ private void depthFirstSearchForward(X500Principal dN, checkers.add(new AlgorithmChecker(builder.trustAnchor, buildParams.timestamp(), buildParams.variant())); - BasicChecker basicChecker = null; - if (nextState.keyParamsNeeded()) { - PublicKey rootKey = cert.getPublicKey(); - if (builder.trustAnchor.getTrustedCert() == null) { - rootKey = builder.trustAnchor.getCAPublicKey(); - if (debug != null) - debug.println( - "SunCertPathBuilder.depthFirstSearchForward " + - "using buildParams public key: " + - rootKey.toString()); - } - TrustAnchor anchor = new TrustAnchor - (cert.getSubjectX500Principal(), rootKey, null); + PublicKey rootKey = cert.getPublicKey(); + if (builder.trustAnchor.getTrustedCert() == null) { + rootKey = builder.trustAnchor.getCAPublicKey(); + if (debug != null) + debug.println( + "SunCertPathBuilder.depthFirstSearchForward " + + "using buildParams public key: " + + rootKey.toString()); + } + TrustAnchor anchor = new TrustAnchor + (cert.getSubjectX500Principal(), rootKey, null); - // add the basic checker - basicChecker = new BasicChecker(anchor, buildParams.date(), + // add the basic checker + BasicChecker basicChecker = new BasicChecker(anchor, + buildParams.date(), buildParams.sigProvider(), true); - checkers.add(basicChecker); - } + checkers.add(basicChecker); buildParams.setCertPath(cf.generateCertPath(appendedCerts)); @@ -511,6 +513,14 @@ private void depthFirstSearchForward(X500Principal dN, policyTreeResult = policyChecker.getPolicyTree(); return; } else { + // If successive certs are self-issued, don't continue search + // on this branch. + if (currentState.selfIssued && X509CertImpl.isSelfIssued(cert)) { + if (debug != null) { + debug.println("Successive certs are self-issued"); + } + return; + } builder.addCertToPath(cert, cpList); } diff --git a/src/java.base/share/classes/sun/security/ssl/DHKeyExchange.java b/src/java.base/share/classes/sun/security/ssl/DHKeyExchange.java index bf8ef82b4b322..a59df9b751bcb 100644 --- a/src/java.base/share/classes/sun/security/ssl/DHKeyExchange.java +++ b/src/java.base/share/classes/sun/security/ssl/DHKeyExchange.java @@ -327,45 +327,36 @@ public SSLPossession createPossession(HandshakeContext context) { } /* - * 768 bits ephemeral DH private keys were used to be used in + * 768 bit ephemeral DH private keys used to be used in * ServerKeyExchange except that exportable ciphers max out at 512 - * bits modulus values. We still adhere to this behavior in legacy + * bit modulus values. We still adhere to this behavior in legacy * mode (system property "jdk.tls.ephemeralDHKeySize" is defined * as "legacy"). * - * Old JDK (JDK 7 and previous) releases don't support DH keys - * bigger than 1024 bits. We have to consider the compatibility - * requirement. 1024 bits DH key is always used for non-exportable - * cipher suites in default mode (system property + * Only very old JDK releases don't support DH keys bigger than + * 1024 bits (JDK 1.5 and 6u/7u releases prior to adding support + * for DH keys > 1024 bits - see JDK-8062834). A 2048 bit + * DH key is always used for non-exportable cipher suites in + * default mode (when the system property * "jdk.tls.ephemeralDHKeySize" is not defined). * - * However, if applications want more stronger strength, setting - * system property "jdk.tls.ephemeralDHKeySize" to "matched" - * is a workaround to use ephemeral DH key which size matches the - * corresponding authentication key. For example, if the public key - * size of an authentication certificate is 2048 bits, then the - * ephemeral DH key size should be 2048 bits accordingly unless - * the cipher suite is exportable. This key sizing scheme keeps - * the cryptographic strength consistent between authentication - * keys and key-exchange keys. - * * Applications may also want to customize the ephemeral DH key * size to a fixed length for non-exportable cipher suites. This - * can be approached by setting system property + * can be done by setting the system property * "jdk.tls.ephemeralDHKeySize" to a valid positive integer between * 1024 and 8192 bits, inclusive. * - * Note that the minimum acceptable key size is 1024 bits except - * exportable cipher suites or legacy mode. + * Note that the minimum acceptable key size is 2048 bits except + * for exportable cipher suites or legacy mode. * * Note that per RFC 2246, the key size limit of DH is 512 bits for * exportable cipher suites. Because of the weakness, exportable * cipher suites are deprecated since TLS v1.1 and they are not * enabled by default in Oracle provider. The legacy behavior is - * reserved and 512 bits DH key is always used for exportable + * preserved and a 512 bit DH key is always used for exportable * cipher suites. */ - int keySize = exportable ? 512 : 1024; // default mode + int keySize = exportable ? 512 : 2048; // default mode if (!exportable) { if (useLegacyEphemeralDHKeys) { // legacy mode keySize = 768; @@ -391,7 +382,7 @@ public SSLPossession createPossession(HandshakeContext context) { // limit in the future when the compatibility and // interoperability impact is limited. keySize = ks <= 1024 ? 1024 : 2048; - } // Otherwise, anonymous cipher suites, 1024-bit is used. + } // Otherwise, anonymous cipher suites, 2048-bit is used. } else if (customizedDHKeySize > 0) { // customized mode keySize = customizedDHKeySize; } diff --git a/src/java.base/share/classes/sun/security/ssl/KeyUpdate.java b/src/java.base/share/classes/sun/security/ssl/KeyUpdate.java index 13063444ee6e0..9e921e6363310 100644 --- a/src/java.base/share/classes/sun/security/ssl/KeyUpdate.java +++ b/src/java.base/share/classes/sun/security/ssl/KeyUpdate.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -169,7 +169,9 @@ private KeyUpdateKickstartProducer() { public byte[] produce(ConnectionContext context) throws IOException { PostHandshakeContext hc = (PostHandshakeContext)context; return handshakeProducer.produce(context, - new KeyUpdateMessage(hc, KeyUpdateRequest.REQUESTED)); + new KeyUpdateMessage(hc, hc.conContext.isInboundClosed() ? + KeyUpdateRequest.NOTREQUESTED : + KeyUpdateRequest.REQUESTED)); } } diff --git a/src/java.base/share/classes/sun/security/ssl/SSLEngineImpl.java b/src/java.base/share/classes/sun/security/ssl/SSLEngineImpl.java index 9ebb55fd86a09..8bb1602be56f8 100644 --- a/src/java.base/share/classes/sun/security/ssl/SSLEngineImpl.java +++ b/src/java.base/share/classes/sun/security/ssl/SSLEngineImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -394,11 +394,11 @@ private HandshakeStatus tryToFinishHandshake(byte contentType) { */ private HandshakeStatus tryKeyUpdate( HandshakeStatus currentHandshakeStatus) throws IOException { - // Don't bother to kickstart if handshaking is in progress, or if the - // connection is not duplex-open. + // Don't bother to kickstart if handshaking is in progress, or if + // the write side of the connection is not open. We allow a half- + // duplex write-only connection for key updates. if ((conContext.handshakeContext == null) && !conContext.isOutboundClosed() && - !conContext.isInboundClosed() && !conContext.isBroken) { if (SSLLogger.isOn && SSLLogger.isOn("ssl")) { SSLLogger.finest("trigger key update"); diff --git a/src/java.base/share/classes/sun/security/ssl/SSLSocketImpl.java b/src/java.base/share/classes/sun/security/ssl/SSLSocketImpl.java index 0a888f0557e7c..f288e210aa30c 100644 --- a/src/java.base/share/classes/sun/security/ssl/SSLSocketImpl.java +++ b/src/java.base/share/classes/sun/security/ssl/SSLSocketImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1996, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1996, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1541,11 +1541,11 @@ private Plaintext decode(ByteBuffer destination) throws IOException { * wrapped. */ private void tryKeyUpdate() throws IOException { - // Don't bother to kickstart if handshaking is in progress, or if the - // connection is not duplex-open. + // Don't bother to kickstart if handshaking is in progress, or if + // the write side of the connection is not open. We allow a half- + // duplex write-only connection for key updates. if ((conContext.handshakeContext == null) && !conContext.isOutboundClosed() && - !conContext.isInboundClosed() && !conContext.isBroken) { if (SSLLogger.isOn && SSLLogger.isOn("ssl")) { SSLLogger.finest("trigger key update"); diff --git a/src/java.base/share/classes/sun/security/ssl/TransportContext.java b/src/java.base/share/classes/sun/security/ssl/TransportContext.java index c087514c5e258..b0572d8a56767 100644 --- a/src/java.base/share/classes/sun/security/ssl/TransportContext.java +++ b/src/java.base/share/classes/sun/security/ssl/TransportContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -219,7 +219,14 @@ void kickstart() throws IOException { throw new IllegalStateException("Client/Server mode not yet set."); } - if (outputRecord.isClosed() || inputRecord.isClosed() || isBroken) { + // The threshold for allowing the method to continue processing + // depends on whether we are doing a key update or kickstarting + // a handshake. In the former case, we only require the write-side + // to be open where a handshake would require a full duplex connection. + boolean isNotUsable = outputRecord.writeCipher.atKeyLimit() ? + (outputRecord.isClosed() || isBroken) : + (outputRecord.isClosed() || inputRecord.isClosed() || isBroken); + if (isNotUsable) { if (closeReason != null) { throw new SSLException( "Cannot kickstart, the connection is broken or closed", @@ -247,7 +254,7 @@ void kickstart() throws IOException { // // Need no kickstart message on server side unless the connection // has been established. - if(isNegotiated || sslConfig.isClientMode) { + if (isNegotiated || sslConfig.isClientMode) { handshakeContext.kickstart(); } } diff --git a/src/java.base/share/classes/sun/security/util/SignatureFileVerifier.java b/src/java.base/share/classes/sun/security/util/SignatureFileVerifier.java index d07152eeb18be..4ea9255ba0a83 100644 --- a/src/java.base/share/classes/sun/security/util/SignatureFileVerifier.java +++ b/src/java.base/share/classes/sun/security/util/SignatureFileVerifier.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -52,6 +52,7 @@ import java.util.jar.JarFile; import java.util.jar.Manifest; +import sun.security.action.GetIntegerAction; import sun.security.jca.Providers; import sun.security.pkcs.PKCS7; import sun.security.pkcs.SignerInfo; @@ -97,6 +98,12 @@ public class SignatureFileVerifier { /** ConstraintsParameters for checking disabled algorithms */ private JarConstraintsParameters params; + // the maximum allowed size in bytes for the signature-related files + public static final int MAX_SIG_FILE_SIZE = initializeMaxSigFileSize(); + + // The maximum size of array to allocate. Some VMs reserve some header words in an array. + private static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8; + /** * Create the named SignatureFileVerifier. * @@ -842,4 +849,24 @@ void updateSigners(CodeSigner[] newSigners, signerCache.add(cachedSigners); signers.put(name, cachedSigners); } + + private static int initializeMaxSigFileSize() { + /* + * System property "jdk.jar.maxSignatureFileSize" used to configure + * the maximum allowed number of bytes for the signature-related files + * in a JAR file. + */ + Integer tmp = GetIntegerAction.privilegedGetProperty( + "jdk.jar.maxSignatureFileSize", 8000000); + if (tmp < 0 || tmp > MAX_ARRAY_SIZE) { + if (debug != null) { + debug.println("Default signature file size 8000000 bytes " + + "is used as the specified size for the " + + "jdk.jar.maxSignatureFileSize system property " + + "is out of range: " + tmp); + } + tmp = 8000000; + } + return tmp; + } } diff --git a/src/java.base/share/classes/sun/util/calendar/ZoneInfoFile.java b/src/java.base/share/classes/sun/util/calendar/ZoneInfoFile.java index 6f6e190efcd0e..3762eb820bb98 100644 --- a/src/java.base/share/classes/sun/util/calendar/ZoneInfoFile.java +++ b/src/java.base/share/classes/sun/util/calendar/ZoneInfoFile.java @@ -609,33 +609,16 @@ private static ZoneInfo getZoneInfo(String zoneId, params[9] = toSTZTime[endRule.timeDefinition]; dstSavings = (startRule.offsetAfter - startRule.offsetBefore) * 1000; - // Note: known mismatching -> Asia/Amman + // Note: known mismatching -> Africa/Cairo // ZoneInfo : startDayOfWeek=5 <= Thursday - // startTime=86400000 <= 24 hours - // This: startDayOfWeek=6 - // startTime=0 - // Similar workaround needs to be applied to Africa/Cairo and - // its endDayOfWeek and endTime - // Below is the workarounds, it probably slows down everyone a little - if (params[2] == 6 && params[3] == 0 && - (zoneId.equals("Asia/Amman"))) { - params[2] = 5; - params[3] = 86400000; + // startTime=86400000 <= 24:00 + // This: startDayOfWeek=6 <= Friday + // startTime=0 <= 0:00 + if (zoneId.equals("Africa/Cairo") && + params[7] == Calendar.FRIDAY && params[8] == 0) { + params[7] = Calendar.THURSDAY; + params[8] = SECONDS_PER_DAY * 1000; } - // Additional check for startDayOfWeek=6 and starTime=86400000 - // is needed for Asia/Amman; - if (params[2] == 7 && params[3] == 0 && - (zoneId.equals("Asia/Amman"))) { - params[2] = 6; // Friday - params[3] = 86400000; // 24h - } - //endDayOfWeek and endTime workaround - if (params[7] == 6 && params[8] == 0 && - (zoneId.equals("Africa/Cairo"))) { - params[7] = 5; - params[8] = 86400000; - } - } else if (nTrans > 0) { // only do this if there is something in table already if (lastyear < LASTYEAR) { // ZoneInfo has an ending entry for 2037 @@ -908,7 +891,6 @@ private static class ZoneOffsetTransitionRule { this.dow = dowByte == 0 ? -1 : dowByte; this.secondOfDay = timeByte == 31 ? in.readInt() : timeByte * 3600; this.timeDefinition = (data & (3 << 12)) >>> 12; - this.standardOffset = stdByte == 255 ? in.readInt() : (stdByte - 128) * 900; this.offsetBefore = beforeByte == 3 ? in.readInt() : standardOffset + beforeByte * 1800; this.offsetAfter = afterByte == 3 ? in.readInt() : standardOffset + afterByte * 1800; diff --git a/src/java.base/share/classes/sun/util/cldr/CLDRLocaleProviderAdapter.java b/src/java.base/share/classes/sun/util/cldr/CLDRLocaleProviderAdapter.java index d2ade7bb58e2c..ba71f5bf8ff1f 100644 --- a/src/java.base/share/classes/sun/util/cldr/CLDRLocaleProviderAdapter.java +++ b/src/java.base/share/classes/sun/util/cldr/CLDRLocaleProviderAdapter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -238,6 +238,8 @@ public List getCandidateLocales(String baseName, Locale locale) { return applyParentLocales(baseName, candidates); } + private static final Locale NB = Locale.forLanguageTag("nb"); + private static final Locale NO = Locale.forLanguageTag("no"); private List applyParentLocales(String baseName, List candidates) { // check irregular parents for (int i = 0; i < candidates.size(); i++) { @@ -247,11 +249,15 @@ private List applyParentLocales(String baseName, List candidates if (p != null && !candidates.get(i+1).equals(p)) { List applied = candidates.subList(0, i+1); - if (applied.contains(p)) { - // avoid circular recursion (could happen with nb/no case) - continue; + // Tweak for Norwegian locales, CLDR switched the canonical form of + // Norwegian Bokmal language code from "nb" to "no" in CLDR 39 + // (https://unicode-org.atlassian.net/browse/CLDR-2698) + if (p.equals(NB) || p.equals(NO)) { + applied.add(NO); + applied.add(Locale.ROOT); + } else { + applied.addAll(applyParentLocales(baseName, super.getCandidateLocales(baseName, p))); } - applied.addAll(applyParentLocales(baseName, super.getCandidateLocales(baseName, p))); return applied; } } diff --git a/src/java.base/share/classes/sun/util/cldr/CLDRTimeZoneNameProviderImpl.java b/src/java.base/share/classes/sun/util/cldr/CLDRTimeZoneNameProviderImpl.java index 6d78d77f64d1d..f97888a697783 100644 --- a/src/java.base/share/classes/sun/util/cldr/CLDRTimeZoneNameProviderImpl.java +++ b/src/java.base/share/classes/sun/util/cldr/CLDRTimeZoneNameProviderImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -82,7 +82,7 @@ protected String[] getDisplayNameArray(String id, Locale locale) { } if (namesSuper != null) { - // CLDR's resource bundle has an translated entry for this id. + // CLDR's resource bundle has a translated entry for this id. // Fix up names if needed, either missing or no-inheritance namesSuper[INDEX_TZID] = id; @@ -91,7 +91,7 @@ protected String[] getDisplayNameArray(String id, Locale locale) { case "": // Fill in empty elements deriveFallbackName(namesSuper, i, locale, - !TimeZone.getTimeZone(id).useDaylightTime()); + TimeZone.getTimeZone(id).toZoneId().getRules().isFixedOffset()); break; case NO_INHERITANCE_MARKER: // CLDR's "no inheritance marker" @@ -129,7 +129,7 @@ protected String[][] getZoneStrings(Locale locale) { // Derive fallback time zone name according to LDML's logic private void deriveFallbackNames(String[] names, Locale locale) { - boolean noDST = !TimeZone.getTimeZone(names[0]).useDaylightTime(); + boolean noDST = TimeZone.getTimeZone(names[0]).toZoneId().getRules().isFixedOffset(); for (int i = INDEX_STD_LONG; i <= INDEX_GEN_SHORT; i++) { deriveFallbackName(names, i, locale, noDST); @@ -149,13 +149,12 @@ private void deriveFallbackName(String[] names, int index, Locale locale, boolea return; } - // Check parent locale first + // Check parent locales first if (!exists(names, index)) { - CLDRLocaleProviderAdapter clpa = (CLDRLocaleProviderAdapter)LocaleProviderAdapter.forType(Type.CLDR); - var cands = clpa.getCandidateLocales("", locale); - if (cands.size() > 1) { - var parentLoc = cands.get(1); // immediate parent locale - String[] parentNames = super.getDisplayNameArray(id, parentLoc); + var cands = ((CLDRLocaleProviderAdapter)LocaleProviderAdapter.forType(Type.CLDR)) + .getCandidateLocales("", locale); + for (int i = 1; i < cands.size() ; i++) { + String[] parentNames = super.getDisplayNameArray(id, cands.get(i)); if (parentNames != null && !parentNames[index].isEmpty()) { names[index] = parentNames[index]; return; @@ -163,18 +162,20 @@ private void deriveFallbackName(String[] names, int index, Locale locale, boolea } } + // Type Fallback + if (noDST && typeFallback(names, index)) { + return; + } + // Check if COMPAT can substitute the name - if (LocaleProviderAdapter.getAdapterPreference().contains(Type.JRE)) { + if (!exists(names, index) && + LocaleProviderAdapter.getAdapterPreference().contains(Type.JRE)) { String[] compatNames = (String[])LocaleProviderAdapter.forJRE() - .getLocaleResources(mapChineseLocale(locale)) - .getTimeZoneNames(id); + .getLocaleResources(mapChineseLocale(locale)) + .getTimeZoneNames(id); if (compatNames != null) { - for (int i = INDEX_STD_LONG; i <= INDEX_GEN_SHORT; i++) { - // Assumes COMPAT has no empty slots - if (i == index || !exists(names, i)) { - names[i] = compatNames[i]; - } - } + // Assumes COMPAT has no empty slots + names[index] = compatNames[index]; return; } } @@ -184,11 +185,6 @@ private void deriveFallbackName(String[] names, int index, Locale locale, boolea return; } - // Type Fallback - if (noDST && typeFallback(names, index)) { - return; - } - // last resort names[index] = toGMTFormat(id, index == INDEX_DST_LONG || index == INDEX_DST_SHORT, @@ -234,6 +230,11 @@ private boolean typeFallback(String[] names, int index) { } private boolean regionFormatFallback(String[] names, int index, Locale l) { + if (index % 2 == 0) { + // ignore short names + return false; + } + String id = names[INDEX_TZID]; LocaleResources lr = LocaleProviderAdapter.forType(Type.CLDR).getLocaleResources(l); ResourceBundle fd = lr.getJavaTimeFormatData(); diff --git a/src/java.base/share/classes/sun/util/resources/TimeZoneNames.java b/src/java.base/share/classes/sun/util/resources/TimeZoneNames.java index bf7918659aef3..2763ac30ca73d 100644 --- a/src/java.base/share/classes/sun/util/resources/TimeZoneNames.java +++ b/src/java.base/share/classes/sun/util/resources/TimeZoneNames.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1996, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1996, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -845,9 +845,7 @@ protected final Object[][] getContents() { {"Europe/Jersey", GMTBST}, {"Europe/Kaliningrad", EET}, {"Europe/Kiev", EET}, - {"Europe/Kirov", new String[] {"Kirov Standard Time", "GMT+03:00", - "Kirov Daylight Time", "GMT+03:00", - "Kirov Time", "GMT+03:00"}}, + {"Europe/Kirov", MSK}, {"Europe/Lisbon", WET}, {"Europe/Ljubljana", CET}, {"Europe/London", GMTBST}, diff --git a/src/java.base/share/conf/security/java.security b/src/java.base/share/conf/security/java.security index 29337576f37e7..4dd055a9ccfb0 100644 --- a/src/java.base/share/conf/security/java.security +++ b/src/java.base/share/conf/security/java.security @@ -947,7 +947,8 @@ jdk.tls.legacyAlgorithms=NULL, anon, RC4, DES, 3DES_EDE_CBC # Note: This property is currently used by OpenJDK's JSSE implementation. It # is not guaranteed to be examined and used by other implementations. # -jdk.tls.keyLimits=AES/GCM/NoPadding KeyUpdate 2^37 +jdk.tls.keyLimits=AES/GCM/NoPadding KeyUpdate 2^37, \ + ChaCha20-Poly1305 KeyUpdate 2^37 # # Cryptographic Jurisdiction Policy defaults diff --git a/src/java.base/share/native/libjava/Thread.c b/src/java.base/share/native/libjava/Thread.c index 3f176f814e406..2a5db4efdeae4 100644 --- a/src/java.base/share/native/libjava/Thread.c +++ b/src/java.base/share/native/libjava/Thread.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1994, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1994, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -43,7 +43,6 @@ static JNINativeMethod methods[] = { {"start0", "()V", (void *)&JVM_StartThread}, {"stop0", "(" OBJ ")V", (void *)&JVM_StopThread}, - {"isAlive", "()Z", (void *)&JVM_IsThreadAlive}, {"suspend0", "()V", (void *)&JVM_SuspendThread}, {"resume0", "()V", (void *)&JVM_ResumeThread}, {"setPriority0", "(I)V", (void *)&JVM_SetThreadPriority}, diff --git a/src/java.base/share/native/libjava/jni_util.c b/src/java.base/share/native/libjava/jni_util.c index 3652e384e1647..7699bdf4637f2 100644 --- a/src/java.base/share/native/libjava/jni_util.c +++ b/src/java.base/share/native/libjava/jni_util.c @@ -23,6 +23,7 @@ * questions. */ +#include #include #include @@ -35,8 +36,13 @@ * such as "z:" need to be appended with a "." so we * must allocate at least 4 bytes to allow room for * this expansion. See 4235353 for details. + * This macro returns NULL if the requested size is + * negative, or the size is INT_MAX as the macro adds 1 + * that overflows into negative value. */ -#define MALLOC_MIN4(len) ((char *)malloc((len) + 1 < 4 ? 4 : (len) + 1)) +#define MALLOC_MIN4(len) ((unsigned)(len) >= INT_MAX ? \ + NULL : \ + ((char *)malloc((len) + 1 < 4 ? 4 : (len) + 1))) /** * Throw a Java exception by name. Similar to SignalError. @@ -179,7 +185,7 @@ JNU_ThrowByNameWithMessageAndLastError } /* - * Convenience method. + * Convenience function. * Call JNU_ThrowByNameWithLastError for java.io.IOException. */ JNIEXPORT void JNICALL @@ -188,6 +194,16 @@ JNU_ThrowIOExceptionWithLastError(JNIEnv *env, const char *defaultDetail) JNU_ThrowByNameWithLastError(env, "java/io/IOException", defaultDetail); } +/* + * Throw java.io.IOException using a given message and the string + * returned by getLastErrorString to construct the detail string. + */ +JNIEXPORT void JNICALL +JNU_ThrowIOExceptionWithMessageAndLastError(JNIEnv *env, const char *message) +{ + JNU_ThrowByNameWithMessageAndLastError(env, "java/io/IOException", message); +} + JNIEXPORT jvalue JNICALL JNU_CallStaticMethodByName(JNIEnv *env, @@ -885,17 +901,10 @@ getStringUTF8(JNIEnv *env, jstring jstr) } } - // Check `jint` overflow - if (rlen < 0) { - (*env)->ReleasePrimitiveArrayCritical(env, value, str, 0); - JNU_ThrowOutOfMemoryError(env, "requested array size exceeds VM limit"); - return NULL; - } - result = MALLOC_MIN4(rlen); if (result == NULL) { (*env)->ReleasePrimitiveArrayCritical(env, value, str, 0); - JNU_ThrowOutOfMemoryError(env, 0); + JNU_ThrowOutOfMemoryError(env, "requested array size exceeds VM limit"); return NULL; } diff --git a/src/java.base/share/native/libjava/jni_util.h b/src/java.base/share/native/libjava/jni_util.h index bd815e6f1b904..4e4d687b71ee4 100644 --- a/src/java.base/share/native/libjava/jni_util.h +++ b/src/java.base/share/native/libjava/jni_util.h @@ -92,6 +92,9 @@ JNU_ThrowByNameWithMessageAndLastError JNIEXPORT void JNICALL JNU_ThrowIOExceptionWithLastError(JNIEnv *env, const char *defaultDetail); +JNIEXPORT void JNICALL +JNU_ThrowIOExceptionWithMessageAndLastError(JNIEnv *env, const char *message); + /* Convert between Java strings and i18n C strings */ JNIEXPORT const char * GetStringPlatformChars(JNIEnv *env, jstring jstr, jboolean *isCopy); diff --git a/src/java.base/unix/classes/sun/nio/fs/UnixUriUtils.java b/src/java.base/unix/classes/sun/nio/fs/UnixUriUtils.java index 133e123b636e5..08efc2b35a445 100644 --- a/src/java.base/unix/classes/sun/nio/fs/UnixUriUtils.java +++ b/src/java.base/unix/classes/sun/nio/fs/UnixUriUtils.java @@ -88,6 +88,10 @@ static Path fromUri(UnixFileSystem fs, URI uri) { throw new IllegalArgumentException("Bad escape"); b = (byte)c; } + if (b == '/' && rlen > 0 && result[rlen-1] == '/') { + // skip redundant slashes + continue; + } result[rlen++] = b; } if (rlen != result.length) diff --git a/src/java.base/unix/native/libnet/Inet4AddressImpl.c b/src/java.base/unix/native/libnet/Inet4AddressImpl.c index b165be7cea1f6..8cde515b7d41d 100644 --- a/src/java.base/unix/native/libnet/Inet4AddressImpl.c +++ b/src/java.base/unix/native/libnet/Inet4AddressImpl.c @@ -342,8 +342,8 @@ ping4(JNIEnv *env, jint fd, SOCKETADDRESS *sa, SOCKETADDRESS *netif, struct ip *ip; struct sockaddr_in sa_recv; jchar pid; - struct timeval tv; - size_t plen = ICMP_ADVLENMIN + sizeof(tv); + struct timeval tv = { 0, 0 }; + const size_t plen = ICMP_MINLEN + sizeof(tv); setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &size, sizeof(size)); @@ -415,7 +415,7 @@ ping4(JNIEnv *env, jint fd, SOCKETADDRESS *sa, SOCKETADDRESS *netif, ip = (struct ip *)recvbuf; hlen = ((jint)(unsigned int)(ip->ip_hl)) << 2; // check if we received enough data - if (n < (jint)(hlen + sizeof(struct icmp))) { + if (n < (jint)(hlen + plen)) { continue; } icmp = (struct icmp *)(recvbuf + hlen); diff --git a/src/java.base/unix/native/libnet/Inet6AddressImpl.c b/src/java.base/unix/native/libnet/Inet6AddressImpl.c index 058f3d3a7e47d..b5c4288339eb7 100644 --- a/src/java.base/unix/native/libnet/Inet6AddressImpl.c +++ b/src/java.base/unix/native/libnet/Inet6AddressImpl.c @@ -545,7 +545,7 @@ ping6(JNIEnv *env, jint fd, SOCKETADDRESS *sa, SOCKETADDRESS *netif, struct icmp6_hdr *icmp6; struct sockaddr_in6 sa_recv; jchar pid; - struct timeval tv; + struct timeval tv = { 0, 0 }; size_t plen = sizeof(struct icmp6_hdr) + sizeof(tv); #if defined(__linux__) diff --git a/src/java.base/unix/native/libnio/MappedMemoryUtils.c b/src/java.base/unix/native/libnio/MappedMemoryUtils.c index e90acd2863fa1..4c9b72e51ad3d 100644 --- a/src/java.base/unix/native/libnio/MappedMemoryUtils.c +++ b/src/java.base/unix/native/libnio/MappedMemoryUtils.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -110,7 +110,7 @@ Java_java_nio_MappedMemoryUtils_load0(JNIEnv *env, jobject obj, jlong address, char *a = (char *)jlong_to_ptr(address); int result = madvise((caddr_t)a, (size_t)len, MADV_WILLNEED); if (result == -1) { - JNU_ThrowIOExceptionWithLastError(env, "madvise failed"); + JNU_ThrowIOExceptionWithMessageAndLastError(env, "madvise with advise MADV_WILLNEED failed"); } } @@ -121,7 +121,7 @@ Java_java_nio_MappedMemoryUtils_unload0(JNIEnv *env, jobject obj, jlong address, char *a = (char *)jlong_to_ptr(address); int result = madvise((caddr_t)a, (size_t)len, MADV_DONTNEED); if (result == -1) { - JNU_ThrowIOExceptionWithLastError(env, "madvise failed"); + JNU_ThrowIOExceptionWithMessageAndLastError(env, "madvise with advise MADV_DONTNEED failed"); } } @@ -132,6 +132,6 @@ Java_java_nio_MappedMemoryUtils_force0(JNIEnv *env, jobject obj, jobject fdo, void* a = (void *)jlong_to_ptr(address); int result = msync(a, (size_t)len, MS_SYNC); if (result == -1) { - JNU_ThrowIOExceptionWithLastError(env, "msync failed"); + JNU_ThrowIOExceptionWithMessageAndLastError(env, "msync with parameter MS_SYNC failed"); } } diff --git a/src/java.base/unix/native/libnio/ch/Net.c b/src/java.base/unix/native/libnio/ch/Net.c index 42a07359dde3e..d4a15b34ea6f3 100644 --- a/src/java.base/unix/native/libnio/ch/Net.c +++ b/src/java.base/unix/native/libnio/ch/Net.c @@ -122,6 +122,51 @@ static jboolean isSourceFilterSupported(){ static jclass isa_class; /* java.net.InetSocketAddress */ static jmethodID isa_ctorID; /* InetSocketAddress(InetAddress, int) */ +static jint handleSocketErrorWithMessage(JNIEnv *env, jint errorValue, + const char* message) +{ + char *xn; + switch (errorValue) { + case EINPROGRESS: /* Non-blocking connect */ + return 0; +#ifdef EPROTO + case EPROTO: + xn = JNU_JAVANETPKG "ProtocolException"; + break; +#endif + case ECONNREFUSED: + case ETIMEDOUT: + case ENOTCONN: + xn = JNU_JAVANETPKG "ConnectException"; + break; + + case EHOSTUNREACH: + xn = JNU_JAVANETPKG "NoRouteToHostException"; + break; + case EADDRINUSE: /* Fall through */ + case EADDRNOTAVAIL: + case EACCES: + xn = JNU_JAVANETPKG "BindException"; + break; + default: + xn = JNU_JAVANETPKG "SocketException"; + break; + } + errno = errorValue; + if (message == NULL) { + JNU_ThrowByNameWithLastError(env, xn, "NioSocketError"); + } else { + JNU_ThrowByNameWithMessageAndLastError(env, xn, message); + } + return IOS_THROWN; +} + +/* Declared in nio_util.h */ +jint handleSocketError(JNIEnv *env, jint errorValue) +{ + return handleSocketErrorWithMessage(env, errorValue, NULL); +} + JNIEXPORT void JNICALL Java_sun_nio_ch_Net_initIDs(JNIEnv *env, jclass clazz) { @@ -615,7 +660,7 @@ Java_sun_nio_ch_Net_joinOrDrop4(JNIEnv *env, jobject this, jboolean join, jobjec if (n < 0) { if (join && (errno == ENOPROTOOPT || errno == EOPNOTSUPP)) return IOS_UNAVAILABLE; - handleSocketError(env, errno); + handleSocketErrorWithMessage(env, errno, "setsockopt failed"); } return 0; } @@ -692,7 +737,7 @@ Java_sun_nio_ch_Net_joinOrDrop6(JNIEnv *env, jobject this, jboolean join, jobjec if (n < 0) { if (join && (errno == ENOPROTOOPT || errno == EOPNOTSUPP)) return IOS_UNAVAILABLE; - handleSocketError(env, errno); + handleSocketErrorWithMessage(env, errno, "setsockopt failed"); } return 0; } @@ -913,38 +958,3 @@ Java_sun_nio_ch_Net_sendOOB(JNIEnv* env, jclass this, jobject fdo, jbyte b) return convertReturnVal(env, n, JNI_FALSE); } -/* Declared in nio_util.h */ - -jint handleSocketError(JNIEnv *env, jint errorValue) -{ - char *xn; - switch (errorValue) { - case EINPROGRESS: /* Non-blocking connect */ - return 0; -#ifdef EPROTO - case EPROTO: - xn = JNU_JAVANETPKG "ProtocolException"; - break; -#endif - case ECONNREFUSED: - case ETIMEDOUT: - case ENOTCONN: - xn = JNU_JAVANETPKG "ConnectException"; - break; - - case EHOSTUNREACH: - xn = JNU_JAVANETPKG "NoRouteToHostException"; - break; - case EADDRINUSE: /* Fall through */ - case EADDRNOTAVAIL: - case EACCES: - xn = JNU_JAVANETPKG "BindException"; - break; - default: - xn = JNU_JAVANETPKG "SocketException"; - break; - } - errno = errorValue; - JNU_ThrowByNameWithLastError(env, xn, "NioSocketError"); - return IOS_THROWN; -} diff --git a/src/java.desktop/macosx/classes/com/apple/eio/FileManager.java b/src/java.desktop/macosx/classes/com/apple/eio/FileManager.java index 2ccdf0eca15d3..8967a99f15e39 100644 --- a/src/java.desktop/macosx/classes/com/apple/eio/FileManager.java +++ b/src/java.desktop/macosx/classes/com/apple/eio/FileManager.java @@ -53,9 +53,13 @@ * * @since 1.4 */ -@SuppressWarnings("removal") public class FileManager { static { + loadOSXLibrary(); + } + + @SuppressWarnings("removal") + private static void loadOSXLibrary() { java.security.AccessController.doPrivileged( new java.security.PrivilegedAction() { public Void run() { @@ -132,6 +136,7 @@ public static int OSTypeToInt(String type) { * @since 1.4 */ public static void setFileTypeAndCreator(String filename, int type, int creator) throws IOException { + @SuppressWarnings("removal") SecurityManager security = System.getSecurityManager(); if (security != null) { security.checkWrite(filename); @@ -146,6 +151,7 @@ public static void setFileTypeAndCreator(String filename, int type, int creator) * @since 1.4 */ public static void setFileType(String filename, int type) throws IOException { + @SuppressWarnings("removal") SecurityManager security = System.getSecurityManager(); if (security != null) { security.checkWrite(filename); @@ -160,6 +166,7 @@ public static void setFileType(String filename, int type) throws IOException { * @since 1.4 */ public static void setFileCreator(String filename, int creator) throws IOException { + @SuppressWarnings("removal") SecurityManager security = System.getSecurityManager(); if (security != null) { security.checkWrite(filename); @@ -174,6 +181,7 @@ public static void setFileCreator(String filename, int creator) throws IOExcepti * @since 1.4 */ public static int getFileType(String filename) throws IOException { + @SuppressWarnings("removal") SecurityManager security = System.getSecurityManager(); if (security != null) { security.checkRead(filename); @@ -188,6 +196,7 @@ public static int getFileType(String filename) throws IOException { * @since 1.4 */ public static int getFileCreator(String filename) throws IOException { + @SuppressWarnings("removal") SecurityManager security = System.getSecurityManager(); if (security != null) { security.checkRead(filename); @@ -251,6 +260,7 @@ public static String findFolder(short domain, int folderType) throws FileNotFoun * @since 1.4 */ public static String findFolder(short domain, int folderType, boolean createIfNeeded) throws FileNotFoundException { + @SuppressWarnings("removal") final SecurityManager security = System.getSecurityManager(); if (security != null) { security.checkPermission(new RuntimePermission("canExamineFileSystem")); @@ -278,6 +288,7 @@ public static String findFolder(short domain, int folderType, boolean createIfNe */ @Deprecated public static void openURL(String url) throws IOException { + @SuppressWarnings("removal") SecurityManager security = System.getSecurityManager(); if (security != null) { security.checkPermission(new RuntimePermission("canOpenURLs")); @@ -329,6 +340,7 @@ public static String getResource(String resourceName, String subDirName, String private static native String getNativeResourceFromBundle(String resourceName, String subDirName, String type) throws FileNotFoundException; private static String getResourceFromBundle(String resourceName, String subDirName, String type) throws FileNotFoundException { + @SuppressWarnings("removal") final SecurityManager security = System.getSecurityManager(); if (security != null) security.checkPermission(new RuntimePermission("canReadBundle")); @@ -347,6 +359,7 @@ private static String getResourceFromBundle(String resourceName, String subDirNa * @since Java for Mac OS X 10.5 Update 2 - 1.5 */ public static String getPathToApplicationBundle() { + @SuppressWarnings("removal") SecurityManager security = System.getSecurityManager(); if (security != null) security.checkPermission(new RuntimePermission("canReadBundle")); return getNativePathToApplicationBundle(); @@ -368,6 +381,7 @@ public static boolean moveToTrash(final File file) throws FileNotFoundException if (file == null) throw new FileNotFoundException(); final String fileName = file.getAbsolutePath(); + @SuppressWarnings("removal") final SecurityManager security = System.getSecurityManager(); if (security != null) security.checkDelete(fileName); @@ -391,6 +405,7 @@ public static boolean revealInFinder(final File file) throws FileNotFoundExcepti if (file == null || !file.exists()) throw new FileNotFoundException(); final String fileName = file.getAbsolutePath(); + @SuppressWarnings("removal") final SecurityManager security = System.getSecurityManager(); if (security != null) security.checkRead(fileName); diff --git a/src/java.desktop/macosx/classes/com/apple/laf/AquaFileView.java b/src/java.desktop/macosx/classes/com/apple/laf/AquaFileView.java index e4e9a383153f1..d9f9a66e44c28 100644 --- a/src/java.desktop/macosx/classes/com/apple/laf/AquaFileView.java +++ b/src/java.desktop/macosx/classes/com/apple/laf/AquaFileView.java @@ -34,7 +34,7 @@ import com.apple.laf.AquaUtils.RecyclableSingleton; -@SuppressWarnings({"removal","serial"}) // JDK implementation class +@SuppressWarnings("serial") // JDK implementation class class AquaFileView extends FileView { private static final boolean DEBUG = false; @@ -57,6 +57,11 @@ class AquaFileView extends FileView { static final int kLSItemInfoExtensionIsHidden = 0x00100000; /* Item has a hidden extension*/ static { + loadOSXUILibrary(); + } + + @SuppressWarnings("removal") + private static void loadOSXUILibrary() { java.security.AccessController.doPrivileged( new java.security.PrivilegedAction() { public Void run() { diff --git a/src/java.desktop/macosx/classes/com/apple/laf/ScreenMenu.java b/src/java.desktop/macosx/classes/com/apple/laf/ScreenMenu.java index 66748944d3113..4f6e006faf828 100644 --- a/src/java.desktop/macosx/classes/com/apple/laf/ScreenMenu.java +++ b/src/java.desktop/macosx/classes/com/apple/laf/ScreenMenu.java @@ -36,12 +36,17 @@ import sun.lwawt.LWToolkit; import sun.lwawt.macosx.*; -@SuppressWarnings({"removal","serial"}) // JDK implementation class +@SuppressWarnings("serial") // JDK implementation class final class ScreenMenu extends Menu implements ContainerListener, ComponentListener, ScreenMenuPropertyHandler { static { + loadAWTLibrary(); + } + + @SuppressWarnings("removal") + private static void loadAWTLibrary() { java.security.AccessController.doPrivileged( new java.security.PrivilegedAction() { public Void run() { diff --git a/src/java.desktop/macosx/classes/sun/lwawt/macosx/CAccessibility.java b/src/java.desktop/macosx/classes/sun/lwawt/macosx/CAccessibility.java index 4f4578caecd2a..2f23d8d0fdf25 100644 --- a/src/java.desktop/macosx/classes/sun/lwawt/macosx/CAccessibility.java +++ b/src/java.desktop/macosx/classes/sun/lwawt/macosx/CAccessibility.java @@ -67,12 +67,16 @@ import sun.awt.AWTAccessor; import sun.lwawt.LWWindowPeer; -@SuppressWarnings("removal") class CAccessibility implements PropertyChangeListener { private static Set ignoredRoles; static { - // Need to load the native library for this code. + loadAWTLibrary(); + } + + @SuppressWarnings("removal") + private static void loadAWTLibrary() { + // Need to load the native library for this code. java.security.AccessController.doPrivileged( new java.security.PrivilegedAction() { public Void run() { diff --git a/src/java.desktop/macosx/classes/sun/lwawt/macosx/LWCToolkit.java b/src/java.desktop/macosx/classes/sun/lwawt/macosx/LWCToolkit.java index b742ac9e38d1b..1b037510299e4 100644 --- a/src/java.desktop/macosx/classes/sun/lwawt/macosx/LWCToolkit.java +++ b/src/java.desktop/macosx/classes/sun/lwawt/macosx/LWCToolkit.java @@ -131,7 +131,6 @@ final class NamedCursor extends Cursor { /** * Mac OS X Cocoa-based AWT Toolkit. */ -@SuppressWarnings("removal") public final class LWCToolkit extends LWToolkit { // While it is possible to enumerate all mouse devices // and query them for the number of buttons, the code @@ -147,6 +146,7 @@ public final class LWCToolkit extends LWToolkit { static { System.err.flush(); + @SuppressWarnings("removal") ResourceBundle platformResources = java.security.AccessController.doPrivileged( new java.security.PrivilegedAction() { @Override @@ -176,20 +176,23 @@ public ResourceBundle run() { if (!GraphicsEnvironment.isHeadless()) { initIDs(); } - inAWT = AccessController.doPrivileged(new PrivilegedAction() { - @Override - public Boolean run() { - return !Boolean.parseBoolean(System.getProperty("javafx.embed.singleThread", "false")); - } - }); } /* * If true we operate in normal mode and nested runloop is executed in JavaRunLoopMode * If false we operate in singleThreaded FX/AWT interop mode and nested loop uses NSDefaultRunLoopMode */ - private static final boolean inAWT; + @SuppressWarnings("removal") + private static final boolean inAWT + = AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Boolean run() { + return !Boolean.parseBoolean( + System.getProperty("javafx.embed.singleThread", "false")); + } + }); + @SuppressWarnings("removal") public LWCToolkit() { final String extraButtons = "sun.awt.enableExtraMouseButtons"; AccessController.doPrivileged((PrivilegedAction) () -> { @@ -248,6 +251,7 @@ public static Color getAppleColor(int color) { } // This is only called from native code. + @SuppressWarnings("removal") static void systemColorsChanged() { EventQueue.invokeLater(() -> { AccessController.doPrivileged( (PrivilegedAction) () -> { @@ -586,6 +590,7 @@ public boolean isAlwaysOnTopSupported() { private static final String APPKIT_THREAD_NAME = "AppKit Thread"; // Intended to be called from the LWCToolkit.m only. + @SuppressWarnings("removal") private static void installToolkitThreadInJava() { Thread.currentThread().setName(APPKIT_THREAD_NAME); AccessController.doPrivileged((PrivilegedAction) () -> { diff --git a/src/java.desktop/macosx/native/libawt_lwawt/awt/CRobot.m b/src/java.desktop/macosx/native/libawt_lwawt/awt/CRobot.m index 4f495e31e362d..147af2b7a7f0b 100644 --- a/src/java.desktop/macosx/native/libawt_lwawt/awt/CRobot.m +++ b/src/java.desktop/macosx/native/libawt_lwawt/awt/CRobot.m @@ -47,10 +47,6 @@ #define k_JAVA_ROBOT_WHEEL_COUNT 1 -#if !defined(kCGBitmapByteOrder32Host) -#define kCGBitmapByteOrder32Host 0 -#endif - // In OS X, left and right mouse button share the same click count. // That is, if one starts clicking the left button rapidly and then // switches to the right button, then the click count will continue @@ -355,7 +351,7 @@ static inline void autoDelay(BOOL isMove) { 8, picWidth * sizeof(jint), picColorSpace, kCGBitmapByteOrder32Host | - kCGImageAlphaPremultipliedFirst); + kCGImageAlphaNoneSkipFirst); CGColorSpaceRelease(picColorSpace); diff --git a/src/java.desktop/macosx/native/libawt_lwawt/awt/QuartzSurfaceData.h b/src/java.desktop/macosx/native/libawt_lwawt/awt/QuartzSurfaceData.h index d0312504e104f..6eac8458f6c60 100644 --- a/src/java.desktop/macosx/native/libawt_lwawt/awt/QuartzSurfaceData.h +++ b/src/java.desktop/macosx/native/libawt_lwawt/awt/QuartzSurfaceData.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2012, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,14 +29,6 @@ #import #import "JNIUtilities.h" -// these flags are not defined on Tiger on PPC, so we need to make them a no-op -#if !defined(kCGBitmapByteOrder32Host) -#define kCGBitmapByteOrder32Host 0 -#endif -#if !defined(kCGBitmapByteOrder16Host) -#define kCGBitmapByteOrder16Host 0 -#endif - // NOTE : Modify the printSurfaceDataDiagnostics API if you change this enum enum SDRenderType { diff --git a/src/java.desktop/share/classes/com/sun/imageio/plugins/bmp/BMPImageReader.java b/src/java.desktop/share/classes/com/sun/imageio/plugins/bmp/BMPImageReader.java index 4eeb8b06d9b1f..8e9b08af72566 100644 --- a/src/java.desktop/share/classes/com/sun/imageio/plugins/bmp/BMPImageReader.java +++ b/src/java.desktop/share/classes/com/sun/imageio/plugins/bmp/BMPImageReader.java @@ -615,8 +615,10 @@ else if (size == 124) height = Math.abs(height); } - if (metadata.compression == BI_RGB) { - long imageDataSize = ((long)width * height * (bitsPerPixel / 8)); + if (metadata.compression == BI_RGB && + metadata.paletteSize == 0 && + metadata.bitsPerPixel >= 16) { + long imageDataSize = (((long)width * height * bitsPerPixel) / 8); if (imageDataSize > (bitmapFileSize - bitmapOffset)) { throw new IIOException(I18N.getString("BMPImageReader9")); } diff --git a/src/java.desktop/share/classes/com/sun/media/sound/DirectAudioDevice.java b/src/java.desktop/share/classes/com/sun/media/sound/DirectAudioDevice.java index e7aca8d3682fe..2f4a25f141ac7 100644 --- a/src/java.desktop/share/classes/com/sun/media/sound/DirectAudioDevice.java +++ b/src/java.desktop/share/classes/com/sun/media/sound/DirectAudioDevice.java @@ -1297,8 +1297,9 @@ public void run() { } } while (doIO && thread == curThread) { - if (newFramePosition >= 0) { - clipBytePosition = newFramePosition * frameSize; + int npf = newFramePosition; // copy into local variable + if (npf >= 0) { + clipBytePosition = npf * frameSize; newFramePosition = -1; } int endFrame = getFrameLength() - 1; diff --git a/src/java.desktop/share/classes/java/awt/EventQueue.java b/src/java.desktop/share/classes/java/awt/EventQueue.java index eec41c454e191..e7d1dcc82863c 100644 --- a/src/java.desktop/share/classes/java/awt/EventQueue.java +++ b/src/java.desktop/share/classes/java/awt/EventQueue.java @@ -94,7 +94,6 @@ * * @since 1.1 */ -@SuppressWarnings("removal") public class EventQueue { private static final AtomicInteger threadInitNumber = new AtomicInteger(); @@ -192,8 +191,6 @@ private static final PlatformLogger getEventLog() { return eventLog; } - private static boolean fxAppThreadIsDispatchThread; - static { AWTAccessor.setEventQueueAccessor( new AWTAccessor.EventQueueAccessor() { @@ -230,15 +227,16 @@ public long getMostRecentEventTime(EventQueue eventQueue) { return eventQueue.getMostRecentEventTimeImpl(); } }); - AccessController.doPrivileged(new PrivilegedAction() { - public Object run() { - fxAppThreadIsDispatchThread = - "true".equals(System.getProperty("javafx.embed.singleThread")); - return null; - } - }); } + @SuppressWarnings("removal") + private static boolean fxAppThreadIsDispatchThread = + AccessController.doPrivileged(new PrivilegedAction() { + public Boolean run() { + return "true".equals(System.getProperty("javafx.embed.singleThread")); + } + }); + /** * Initializes a new instance of {@code EventQueue}. */ @@ -734,8 +732,11 @@ public void run() { } }; + @SuppressWarnings("removal") final AccessControlContext stack = AccessController.getContext(); + @SuppressWarnings("removal") final AccessControlContext srcAcc = getAccessControlContextFrom(src); + @SuppressWarnings("removal") final AccessControlContext eventAcc = event.getAccessControlContext(); if (srcAcc == null) { javaSecurityAccess.doIntersectionPrivilege(action, stack, eventAcc); @@ -750,6 +751,7 @@ public Void run() { } } + @SuppressWarnings("removal") private static AccessControlContext getAccessControlContextFrom(Object src) { return src instanceof Component ? ((Component)src).getAccessControlContext() : diff --git a/src/java.desktop/share/classes/java/awt/Font.java b/src/java.desktop/share/classes/java/awt/Font.java index b13e6e2007437..78373aa4f377d 100644 --- a/src/java.desktop/share/classes/java/awt/Font.java +++ b/src/java.desktop/share/classes/java/awt/Font.java @@ -2633,8 +2633,10 @@ public Rectangle2D getStringBounds(char [] chars, // quick check for simple text, assume GV ok to use if simple boolean simple = values == null || - (values.getKerning() == 0 && values.getLigatures() == 0 && - values.getBaselineTransform() == null); + (values.getKerning() == 0 + && values.getLigatures() == 0 + && values.getTracking() == 0 + && values.getBaselineTransform() == null); if (simple) { simple = ! FontUtilities.isComplexText(chars, beginIndex, limit); } diff --git a/src/java.desktop/share/classes/java/beans/Introspector.java b/src/java.desktop/share/classes/java/beans/Introspector.java index f73444a54ef9d..c3a0dd92e95d8 100644 --- a/src/java.desktop/share/classes/java/beans/Introspector.java +++ b/src/java.desktop/share/classes/java/beans/Introspector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1996, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1996, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1158,6 +1158,8 @@ private static Method internalFindMethod(Class start, String methodName, // For overriden methods we need to find the most derived version. // So we start with the given class and walk up the superclass chain. for (Class cl = start; cl != null; cl = cl.getSuperclass()) { + Class type = null; + Method foundMethod = null; for (Method method : ClassInfo.get(cl).getMethods()) { // make sure method signature matches. if (method.getName().equals(methodName)) { @@ -1177,10 +1179,17 @@ private static Method internalFindMethod(Class start, String methodName, } } } - return method; + Class rt = method.getReturnType(); + if (foundMethod == null || type.isAssignableFrom(rt)) { + foundMethod = method; + type = rt; + } } } } + if (foundMethod != null) { + return foundMethod; + } } // Now check any inherited interfaces. This is necessary both when // the argument class is itself an interface, and when the argument diff --git a/src/java.desktop/share/classes/javax/print/DocFlavor.java b/src/java.desktop/share/classes/javax/print/DocFlavor.java index 746c5ed0fa6eb..557a1799a241f 100644 --- a/src/java.desktop/share/classes/javax/print/DocFlavor.java +++ b/src/java.desktop/share/classes/javax/print/DocFlavor.java @@ -385,7 +385,6 @@ * * @author Alan Kaminsky */ -@SuppressWarnings("removal") public class DocFlavor implements Serializable, Cloneable { /** @@ -405,13 +404,10 @@ public class DocFlavor implements Serializable, Cloneable { * This is the charset for all the "HOST" pre-defined {@code DocFlavors} in * the executing VM. */ - public static final String hostEncoding; - - static { - hostEncoding = + @SuppressWarnings("removal") + public static final String hostEncoding = java.security.AccessController.doPrivileged( new sun.security.action.GetPropertyAction("file.encoding")); - } /** * MIME type. diff --git a/src/java.desktop/share/classes/javax/swing/ImageIcon.java b/src/java.desktop/share/classes/javax/swing/ImageIcon.java index 1ddca79477961..566e1c8f870c8 100644 --- a/src/java.desktop/share/classes/javax/swing/ImageIcon.java +++ b/src/java.desktop/share/classes/javax/swing/ImageIcon.java @@ -85,7 +85,7 @@ * @author Lynn Monsanto * @since 1.2 */ -@SuppressWarnings({"removal","serial"}) // Same-version serialization only +@SuppressWarnings("serial") // Same-version serialization only public class ImageIcon implements Icon, Serializable, Accessible { /* Keep references to the filename and location so that * alternate persistence schemes have the option to archive @@ -105,8 +105,27 @@ public class ImageIcon implements Icon, Serializable, Accessible { * It is left for backward compatibility only. * @deprecated since 1.8 */ + @SuppressWarnings("removal") @Deprecated - protected static final Component component; + protected static final Component component + = AccessController.doPrivileged(new PrivilegedAction() { + public Component run() { + try { + final Component component = createNoPermsComponent(); + + // 6482575 - clear the appContext field so as not to leak it + AWTAccessor.getComponentAccessor(). + setAppContext(component, null); + + return component; + } catch (Throwable e) { + // We don't care about component. + // So don't prevent class initialisation. + e.printStackTrace(); + return null; + } + } + }); /** * Do not use this shared media tracker, which is used to load images. @@ -114,30 +133,9 @@ public class ImageIcon implements Icon, Serializable, Accessible { * @deprecated since 1.8 */ @Deprecated - protected static final MediaTracker tracker; - - static { - component = AccessController.doPrivileged(new PrivilegedAction() { - public Component run() { - try { - final Component component = createNoPermsComponent(); - - // 6482575 - clear the appContext field so as not to leak it - AWTAccessor.getComponentAccessor(). - setAppContext(component, null); - - return component; - } catch (Throwable e) { - // We don't care about component. - // So don't prevent class initialisation. - e.printStackTrace(); - return null; - } - } - }); - tracker = new MediaTracker(component); - } + protected static final MediaTracker tracker = new MediaTracker(component); + @SuppressWarnings("removal") private static Component createNoPermsComponent() { // 7020198 - set acc field to no permissions and no subject // Note, will have appContext set. diff --git a/src/java.desktop/share/classes/javax/swing/JPopupMenu.java b/src/java.desktop/share/classes/javax/swing/JPopupMenu.java index 639c640862d09..e4eb45c28a9dc 100644 --- a/src/java.desktop/share/classes/javax/swing/JPopupMenu.java +++ b/src/java.desktop/share/classes/javax/swing/JPopupMenu.java @@ -101,7 +101,7 @@ */ @JavaBean(defaultProperty = "UI", description = "A small window that pops up and displays a series of choices.") @SwingContainer(false) -@SuppressWarnings({"removal","serial"}) +@SuppressWarnings("serial") public class JPopupMenu extends JComponent implements Accessible,MenuElement { /** @@ -117,14 +117,11 @@ public class JPopupMenu extends JComponent implements Accessible,MenuElement { new StringBuffer("JPopupMenu.defaultLWPopupEnabledKey"); /** Bug#4425878-Property javax.swing.adjustPopupLocationToFit introduced */ - static boolean popupPostionFixDisabled = false; - - static { - popupPostionFixDisabled = java.security.AccessController.doPrivileged( + @SuppressWarnings("removal") + static boolean popupPostionFixDisabled = + java.security.AccessController.doPrivileged( new sun.security.action.GetPropertyAction( - "javax.swing.adjustPopupLocationToFit","")).equals("false"); - - } + "javax.swing.adjustPopupLocationToFit","")).equals("false"); transient Component invoker; transient Popup popup; diff --git a/src/java.desktop/share/classes/javax/swing/JRootPane.java b/src/java.desktop/share/classes/javax/swing/JRootPane.java index 0a2e0c3a89aae..e3eb7ac75a4d3 100644 --- a/src/java.desktop/share/classes/javax/swing/JRootPane.java +++ b/src/java.desktop/share/classes/javax/swing/JRootPane.java @@ -199,7 +199,7 @@ * @since 1.2 */ /// PENDING(klobad) Who should be opaque in this component? -@SuppressWarnings({"removal","serial"}) +@SuppressWarnings("serial") public class JRootPane extends JComponent implements Accessible { private static final String uiClassID = "RootPaneUI"; @@ -208,13 +208,19 @@ public class JRootPane extends JComponent implements Accessible { * Whether or not we should dump the stack when true double buffering * is disabled. Default is false. */ - private static final boolean LOG_DISABLE_TRUE_DOUBLE_BUFFERING; + @SuppressWarnings("removal") + private static final boolean LOG_DISABLE_TRUE_DOUBLE_BUFFERING + = AccessController.doPrivileged(new GetBooleanAction( + "swing.logDoubleBufferingDisable")); /** * Whether or not we should ignore requests to disable true double * buffering. Default is false. */ - private static final boolean IGNORE_DISABLE_TRUE_DOUBLE_BUFFERING; + @SuppressWarnings("removal") + private static final boolean IGNORE_DISABLE_TRUE_DOUBLE_BUFFERING + = AccessController.doPrivileged(new GetBooleanAction( + "swing.ignoreDoubleBufferingDisable")); /** * Constant used for the windowDecorationStyle property. Indicates that @@ -326,15 +332,6 @@ public class JRootPane extends JComponent implements Accessible { */ boolean useTrueDoubleBuffering = true; - static { - LOG_DISABLE_TRUE_DOUBLE_BUFFERING = - AccessController.doPrivileged(new GetBooleanAction( - "swing.logDoubleBufferingDisable")); - IGNORE_DISABLE_TRUE_DOUBLE_BUFFERING = - AccessController.doPrivileged(new GetBooleanAction( - "swing.ignoreDoubleBufferingDisable")); - } - /** * Creates a JRootPane, setting up its * glassPane, layeredPane, diff --git a/src/java.desktop/share/classes/javax/swing/SortingFocusTraversalPolicy.java b/src/java.desktop/share/classes/javax/swing/SortingFocusTraversalPolicy.java index 4eb8fef420054..5305be37e86b3 100644 --- a/src/java.desktop/share/classes/javax/swing/SortingFocusTraversalPolicy.java +++ b/src/java.desktop/share/classes/javax/swing/SortingFocusTraversalPolicy.java @@ -59,7 +59,6 @@ * @see java.util.Comparator * @since 1.4 */ -@SuppressWarnings("removal") public class SortingFocusTraversalPolicy extends InternalFrameFocusTraversalPolicy { @@ -96,12 +95,10 @@ public class SortingFocusTraversalPolicy * When false, the default (tim-sort) algo is used, which may lead to an exception. * See: JDK-8048887 */ - private static final boolean legacySortingFTPEnabled; - - static { - legacySortingFTPEnabled = "true".equals(AccessController.doPrivileged( - new GetPropertyAction("swing.legacySortingFTPEnabled", "true"))); - } + @SuppressWarnings("removal") + private static final boolean legacySortingFTPEnabled = "true".equals( + AccessController.doPrivileged( + new GetPropertyAction("swing.legacySortingFTPEnabled", "true"))); /** * Constructs a SortingFocusTraversalPolicy without a Comparator. diff --git a/src/java.desktop/share/classes/javax/swing/plaf/basic/BasicHTML.java b/src/java.desktop/share/classes/javax/swing/plaf/basic/BasicHTML.java index 3e1108fb25bd2..112c04f00af8b 100644 --- a/src/java.desktop/share/classes/javax/swing/plaf/basic/BasicHTML.java +++ b/src/java.desktop/share/classes/javax/swing/plaf/basic/BasicHTML.java @@ -33,6 +33,7 @@ import javax.swing.text.*; import javax.swing.text.html.*; +import sun.swing.SwingAccessor; import sun.swing.SwingUtilities2; /** @@ -220,7 +221,7 @@ public static void updateRenderer(JComponent c, String text) { View value = null; View oldValue = (View)c.getClientProperty(BasicHTML.propertyKey); Boolean htmlDisabled = (Boolean) c.getClientProperty(htmlDisable); - if (htmlDisabled != Boolean.TRUE && BasicHTML.isHTMLString(text)) { + if (!(Boolean.TRUE.equals(htmlDisabled)) && BasicHTML.isHTMLString(text)) { value = BasicHTML.createHTMLView(c, text); } if (value != oldValue && oldValue != null) { @@ -376,15 +377,36 @@ public ViewFactory getViewFactory() { */ static class BasicHTMLViewFactory extends HTMLEditorKit.HTMLFactory { public View create(Element elem) { - View view = super.create(elem); + View view = null; + try { + setAllowHTMLObject(); + view = super.create(elem); + } finally { + clearAllowHTMLObject(); + } if (view instanceof ImageView) { ((ImageView)view).setLoadsSynchronously(true); } return view; } - } + private static Boolean useOV = null; + + @SuppressWarnings("removal") + private static void setAllowHTMLObject() { + if (useOV == null) { + useOV = java.security.AccessController.doPrivileged( + new sun.security.action.GetBooleanAction( + "swing.html.object")); + }; + SwingAccessor.setAllowHTMLObject(useOV); + } + + private static void clearAllowHTMLObject() { + SwingAccessor.setAllowHTMLObject(null); + } + } /** * The subclass of HTMLDocument that is used as the model. getForeground diff --git a/src/java.desktop/share/classes/javax/swing/plaf/synth/SynthLookAndFeel.java b/src/java.desktop/share/classes/javax/swing/plaf/synth/SynthLookAndFeel.java index 52c7ab078c38e..910d13668d2a3 100644 --- a/src/java.desktop/share/classes/javax/swing/plaf/synth/SynthLookAndFeel.java +++ b/src/java.desktop/share/classes/javax/swing/plaf/synth/SynthLookAndFeel.java @@ -615,6 +615,18 @@ public void load(InputStream input, Class resourceBase) throws * new URL(synthFile, path). Refer to * Synth File Format for more * information. + *

+ * Whilst this API may be safe for loading local resources that are + * delivered with a {@code LookAndFeel} or application, and so have an + * equal level of trust with application code, using it to load from + * remote resources, particularly any which may have a lower level of + * trust, is strongly discouraged. + * The alternative mechanisms to load styles from an {@code InputStream} + * {@linkplain #load(InputStream, Class)} + * using resources co-located with the application or by providing a + * {@code SynthStyleFactory} to + * {@linkplain #setStyleFactory setStyleFactory(SynthStyleFactory)} + * are preferred. * * @param url the URL to load the set of * SynthStyle from diff --git a/src/java.desktop/share/classes/javax/swing/plaf/synth/doc-files/synthFileFormat.html b/src/java.desktop/share/classes/javax/swing/plaf/synth/doc-files/synthFileFormat.html index 8eb1bbda9c193..7d63fa16dca2c 100644 --- a/src/java.desktop/share/classes/javax/swing/plaf/synth/doc-files/synthFileFormat.html +++ b/src/java.desktop/share/classes/javax/swing/plaf/synth/doc-files/synthFileFormat.html @@ -70,6 +70,8 @@

File Format

This example loads the look and feel from an input stream, using the specified class as the resource base to resolve paths. +

+

It is also possible to load a look and feel from an arbitrary URL as in the following example.

@@ -94,6 +96,11 @@

File Format

  • Remote JAR file, e.g. jar:http://host/synth-laf.jar!/laf.xml
  • +

    Note: Synth's file format allows for the definition of code to be executed. + Loading any code from a remote location should be used only + with extreme caution from a trusted source over a secure connection. + It is strongly discouraged for an application or a LookAndFeel to do so. +

    While the DTD for synth is specified, the parser is not validating. Parsing will fail only if a necessary attribute is not diff --git a/src/java.desktop/share/classes/javax/swing/text/html/HTMLEditorKit.java b/src/java.desktop/share/classes/javax/swing/text/html/HTMLEditorKit.java index 7ec09676d2b55..4ceb656a0a3bb 100644 --- a/src/java.desktop/share/classes/javax/swing/text/html/HTMLEditorKit.java +++ b/src/java.desktop/share/classes/javax/swing/text/html/HTMLEditorKit.java @@ -41,6 +41,7 @@ import java.security.AccessController; import java.security.PrivilegedAction; import javax.swing.text.html.parser.ParserDelegator; +import sun.swing.SwingAccessor; /** * The Swing JEditorPane text component supports different kinds @@ -1326,7 +1327,11 @@ public View create(Element elem) { (kind == HTML.Tag.TEXTAREA)) { return new FormView(elem); } else if (kind == HTML.Tag.OBJECT) { - return new ObjectView(elem); + if (SwingAccessor.getAllowHTMLObject()) { + return new ObjectView(elem); + } else { + return new ObjectView(elem, false); + } } else if (kind == HTML.Tag.FRAMESET) { if (elem.getAttributes().isDefined(HTML.Attribute.ROWS)) { return new FrameSetView(elem, View.Y_AXIS); diff --git a/src/java.desktop/share/classes/javax/swing/text/html/ObjectView.java b/src/java.desktop/share/classes/javax/swing/text/html/ObjectView.java index 175ca18bfb689..050a79fe85e79 100644 --- a/src/java.desktop/share/classes/javax/swing/text/html/ObjectView.java +++ b/src/java.desktop/share/classes/javax/swing/text/html/ObjectView.java @@ -71,6 +71,8 @@ */ public class ObjectView extends ComponentView { + private boolean createComp = true; // default + /** * Creates a new ObjectView object. * @@ -80,6 +82,11 @@ public ObjectView(Element elem) { super(elem); } + ObjectView(Element elem, boolean createComp) { + super(elem); + this.createComp = createComp; + } + /** * Create the component. The classid is used * as a specification of the classname, which @@ -87,6 +94,9 @@ public ObjectView(Element elem) { */ @SuppressWarnings("deprecation") protected Component createComponent() { + if (!createComp) { + return getUnloadableRepresentation(); + } AttributeSet attr = getElement().getAttributes(); String classname = (String) attr.getAttribute(HTML.Attribute.CLASSID); try { diff --git a/src/java.desktop/share/classes/sun/awt/FontConfiguration.java b/src/java.desktop/share/classes/sun/awt/FontConfiguration.java index 0363cced01353..48fb0b5b66748 100644 --- a/src/java.desktop/share/classes/sun/awt/FontConfiguration.java +++ b/src/java.desktop/share/classes/sun/awt/FontConfiguration.java @@ -1252,15 +1252,24 @@ public String getFileNameFromPlatformName(String platformName) { return filenamesMap.get(platformName); } + private static final String fontconfigErrorMessage = + "Fontconfig head is null, check your fonts or fonts configuration"; + /** * Returns a configuration specific path to be appended to the font * search path. */ public String getExtraFontPath() { + if (head == null) { + throw new RuntimeException(fontconfigErrorMessage); + } return getString(head[INDEX_appendedfontpath]); } public String getVersion() { + if (head == null) { + throw new RuntimeException(fontconfigErrorMessage); + } return getString(head[INDEX_version]); } diff --git a/src/java.desktop/share/classes/sun/awt/image/ImageRepresentation.java b/src/java.desktop/share/classes/sun/awt/image/ImageRepresentation.java index 1fa4fb4d571af..52a4378fbe220 100644 --- a/src/java.desktop/share/classes/sun/awt/image/ImageRepresentation.java +++ b/src/java.desktop/share/classes/sun/awt/image/ImageRepresentation.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1995, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1995, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,24 +27,20 @@ import java.awt.Color; import java.awt.Graphics; -import java.awt.Transparency; -import java.awt.AWTException; +import java.awt.Graphics2D; import java.awt.Rectangle; +import java.awt.Transparency; +import java.awt.geom.AffineTransform; import java.awt.image.BufferedImage; import java.awt.image.ColorModel; +import java.awt.image.DataBuffer; +import java.awt.image.DataBufferInt; import java.awt.image.DirectColorModel; -import java.awt.image.IndexColorModel; import java.awt.image.ImageConsumer; import java.awt.image.ImageObserver; -import sun.awt.image.ByteComponentRaster; -import sun.awt.image.IntegerComponentRaster; +import java.awt.image.IndexColorModel; import java.awt.image.Raster; import java.awt.image.WritableRaster; -import java.awt.image.DataBuffer; -import java.awt.image.DataBufferInt; -import java.awt.Graphics2D; -import java.awt.geom.AffineTransform; -import sun.awt.image.ImageWatched; import java.util.Hashtable; public class ImageRepresentation extends ImageWatched implements ImageConsumer @@ -114,8 +110,8 @@ public synchronized void reconstruct(int flags) { try { startProduction(); missinginfo = flags & ~availinfo; - while ((availinfo & ImageObserver.ERROR) == 0 && - missinginfo != 0) + while ((availinfo & (ImageObserver.ERROR | ImageObserver.FRAMEBITS)) == 0 + && missinginfo != 0) { try { wait(); diff --git a/src/java.desktop/share/classes/sun/awt/image/SurfaceManager.java b/src/java.desktop/share/classes/sun/awt/image/SurfaceManager.java index fe0fd6e787db6..bb327889a8672 100644 --- a/src/java.desktop/share/classes/sun/awt/image/SurfaceManager.java +++ b/src/java.desktop/share/classes/sun/awt/image/SurfaceManager.java @@ -89,7 +89,7 @@ public static void setManager(Image img, SurfaceManager mgr) { imgaccessor.setSurfaceManager(img, mgr); } - private ConcurrentHashMap cacheMap; + private volatile ConcurrentHashMap cacheMap; /** * Return an arbitrary cached object for an arbitrary cache key. diff --git a/src/java.desktop/share/classes/sun/font/FontUtilities.java b/src/java.desktop/share/classes/sun/font/FontUtilities.java index 79ea7de393c2f..550cdcce3cf3f 100644 --- a/src/java.desktop/share/classes/sun/font/FontUtilities.java +++ b/src/java.desktop/share/classes/sun/font/FontUtilities.java @@ -38,7 +38,6 @@ /** * A collection of utility methods. */ -@SuppressWarnings("removal") public final class FontUtilities { public static boolean isLinux; @@ -56,7 +55,11 @@ public final class FontUtilities { // This static initializer block figures out the OS constants. static { + initStatic(); + } + @SuppressWarnings("removal") + private static void initStatic() { AccessController.doPrivileged(new PrivilegedAction() { @SuppressWarnings("deprecation") // PlatformLogger.setLevel is deprecated. @Override diff --git a/src/java.desktop/share/classes/sun/font/StrikeCache.java b/src/java.desktop/share/classes/sun/font/StrikeCache.java index c550b63562b8b..866498df6ee7e 100644 --- a/src/java.desktop/share/classes/sun/font/StrikeCache.java +++ b/src/java.desktop/share/classes/sun/font/StrikeCache.java @@ -61,7 +61,6 @@ */ -@SuppressWarnings("removal") public final class StrikeCache { static final Unsafe unsafe = Unsafe.getUnsafe(); @@ -135,6 +134,11 @@ public final class StrikeCache { static native void getGlyphCacheDescription(long[] infoArray); static { + initStatic(); + } + + @SuppressWarnings("removal") + private static void initStatic() { long[] nativeInfo = new long[13]; getGlyphCacheDescription(nativeInfo); diff --git a/src/java.desktop/share/classes/sun/java2d/SunGraphicsEnvironment.java b/src/java.desktop/share/classes/sun/java2d/SunGraphicsEnvironment.java index 014281e081131..e7ef49b1fc9a8 100644 --- a/src/java.desktop/share/classes/sun/java2d/SunGraphicsEnvironment.java +++ b/src/java.desktop/share/classes/sun/java2d/SunGraphicsEnvironment.java @@ -59,21 +59,19 @@ * @see GraphicsDevice * @see GraphicsConfiguration */ -@SuppressWarnings("removal") public abstract class SunGraphicsEnvironment extends GraphicsEnvironment implements DisplayChangedListener { /** Establish the default font to be used by SG2D. */ private final Font defaultFont = new Font(Font.DIALOG, Font.PLAIN, 12); - private static final boolean uiScaleEnabled; - private static final double debugScale; + @SuppressWarnings("removal") + private static final boolean uiScaleEnabled + = "true".equals(AccessController.doPrivileged( + new GetPropertyAction("sun.java2d.uiScale.enabled", "true"))); - static { - uiScaleEnabled = "true".equals(AccessController.doPrivileged( - new GetPropertyAction("sun.java2d.uiScale.enabled", "true"))); - debugScale = uiScaleEnabled ? getScaleFactor("sun.java2d.uiScale") : -1; - } + private static final double debugScale = + uiScaleEnabled ? getScaleFactor("sun.java2d.uiScale") : -1; protected GraphicsDevice[] screens; @@ -299,6 +297,7 @@ public static double getDebugScale() { public static double getScaleFactor(String propertyName) { + @SuppressWarnings("removal") String scaleFactor = AccessController.doPrivileged( new GetPropertyAction(propertyName, "-1")); diff --git a/src/java.desktop/share/classes/sun/swing/JLightweightFrame.java b/src/java.desktop/share/classes/sun/swing/JLightweightFrame.java index 609ec8c7aa917..c1167b4d8ba26 100644 --- a/src/java.desktop/share/classes/sun/swing/JLightweightFrame.java +++ b/src/java.desktop/share/classes/sun/swing/JLightweightFrame.java @@ -68,7 +68,7 @@ * @author Artem Ananiev * @author Anton Tarasov */ -@SuppressWarnings({"removal","serial"}) // JDK-implementation class +@SuppressWarnings("serial") // JDK-implementation class public final class JLightweightFrame extends LightweightFrame implements RootPaneContainer { private final JRootPane rootPane = new JRootPane(); @@ -83,19 +83,6 @@ public final class JLightweightFrame extends LightweightFrame implements RootPan private volatile double scaleFactorX; private volatile double scaleFactorY; - /** - * {@code copyBufferEnabled}, true by default, defines the following strategy. - * A duplicating (copy) buffer is created for the original pixel buffer. - * The copy buffer is synchronized with the original buffer every time the - * latter changes. {@code JLightweightFrame} passes the copy buffer array - * to the {@link LightweightContent#imageBufferReset} method. The code spot - * which synchronizes two buffers becomes the only critical section guarded - * by the lock (managed with the {@link LightweightContent#paintLock()}, - * {@link LightweightContent#paintUnlock()} methods). - */ - private static boolean copyBufferEnabled; - private int[] copyBuffer; - private PropertyChangeListener layoutSizeListener; private RepaintListener repaintListener; @@ -106,14 +93,28 @@ public void updateCursor(JLightweightFrame frame) { frame.updateClientCursor(); } }); - copyBufferEnabled = "true".equals(AccessController. - doPrivileged(new GetPropertyAction("swing.jlf.copyBufferEnabled", "true"))); } + /** + * {@code copyBufferEnabled}, true by default, defines the following strategy. + * A duplicating (copy) buffer is created for the original pixel buffer. + * The copy buffer is synchronized with the original buffer every time the + * latter changes. {@code JLightweightFrame} passes the copy buffer array + * to the {@link LightweightContent#imageBufferReset} method. The code spot + * which synchronizes two buffers becomes the only critical section guarded + * by the lock (managed with the {@link LightweightContent#paintLock()}, + * {@link LightweightContent#paintUnlock()} methods). + */ + @SuppressWarnings("removal") + private static boolean copyBufferEnabled = "true".equals(AccessController. + doPrivileged(new GetPropertyAction("swing.jlf.copyBufferEnabled", "true"))); + private int[] copyBuffer; + /** * Constructs a new, initially invisible {@code JLightweightFrame} * instance. */ + @SuppressWarnings("removal") public JLightweightFrame() { super(); AffineTransform defaultTransform = @@ -330,7 +331,7 @@ private void notifyImageUpdated(int x, int y, int width, int height) { content.imageUpdated(x, y, width, height); } - @SuppressWarnings("serial") // anonymous class inside + @SuppressWarnings({"removal","serial"}) // anonymous class inside private void initInterior() { contentPane = new JPanel() { @Override diff --git a/src/java.desktop/share/classes/sun/swing/SwingAccessor.java b/src/java.desktop/share/classes/sun/swing/SwingAccessor.java index 7aedad035c491..e52b4cbb3268b 100644 --- a/src/java.desktop/share/classes/sun/swing/SwingAccessor.java +++ b/src/java.desktop/share/classes/sun/swing/SwingAccessor.java @@ -283,4 +283,19 @@ private static void ensureClassInitialized(Class c) { MethodHandles.lookup().ensureInitialized(c); } catch (IllegalAccessException e) {} } + + private static ThreadLocal tlObj = new ThreadLocal(); + + public static Boolean getAllowHTMLObject() { + Boolean b = tlObj.get(); + if (b == null) { + return Boolean.TRUE; + } else { + return b; + } + } + + public static void setAllowHTMLObject(Boolean val) { + tlObj.set(val); + } } diff --git a/src/java.desktop/share/legal/freetype.md b/src/java.desktop/share/legal/freetype.md index e74da8869d3b9..d602abbe5ae67 100644 --- a/src/java.desktop/share/legal/freetype.md +++ b/src/java.desktop/share/legal/freetype.md @@ -1,4 +1,4 @@ -## The FreeType Project: Freetype v2.12.1 +## The FreeType Project: Freetype v2.13.0 ### FreeType Notice @@ -21,27 +21,26 @@ which fits your needs best. ### FreeType License ``` -Copyright (C) 1996-2022 by David Turner, Robert Wilhelm, and Werner Lemberg. -Copyright (C) 2007-2022 by Dereg Clegg and Michael Toftdal. -Copyright (C) 1996-2022 by Just van Rossum, David Turner, Robert Wilhelm, and Werner Lemberg. -Copyright (C) 2004-2022 by Masatake YAMATO and Redhat K.K. -Copyright (C) 2007-2022 by Derek Clegg and Michael Toftdal. -Copyright (C) 2007-2022 by David Turner. -Copyright (C) 2022 by David Turner, Robert Wilhelm, Werner Lemberg, and Moazin Khatti. -Copyright (C) 2007-2022 by Rahul Bhalerao , . -Copyright (C) 2008-2022 by David Turner, Robert Wilhelm, Werner Lemberg, and suzuki toshiya. -Copyright (C) 2019-2022 by Nikhil Ramakrishnan, David Turner, Robert Wilhelm, and Werner Lemberg. -Copyright (C) 2009-2022 by Oran Agra and Mickey Gabel. -Copyright (C) 2004-2022 by David Turner, Robert Wilhelm, Werner Lemberg, and George Williams. -Copyright (C) 2004-2022 by Masatake YAMATO, Red Hat K.K., -Copyright (C) 2003-2022 by Masatake YAMATO, Redhat K.K., -Copyright (C) 2013-2022 by Google, Inc. -Copyright (C) 2018-2022 by David Turner, Robert Wilhelm, Dominik Röttsches, and Werner Lemberg. -Copyright (C) 2005-2022 by David Turner, Robert Wilhelm, and Werner Lemberg. -Copyright 2013 by Google, Inc. - - - The FreeType Project LICENSE +Copyright (C) 1996-2023 by David Turner, Robert Wilhelm, and Werner Lemberg. +Copyright (C) 2007-2023 by Dereg Clegg and Michael Toftdal. +Copyright (C) 1996-2023 by Just van Rossum, David Turner, Robert Wilhelm, and Werner Lemberg. +Copyright (C) 2022-2023 by David Turner, Robert Wilhelm, Werner Lemberg, George Williams, and +Copyright (C) 2004-2023 by Masatake YAMATO and Redhat K.K. +Copyright (C) 2007-2023 by Derek Clegg and Michael Toftdal. +Copyright (C) 2003-2023 by Masatake YAMATO, Red Hat K.K., +Copyright (C) 1996-2023 by David Turner, Robert Wilhelm, Werner Lemberg, and Dominik Röttsches. +Copyright (C) 2007-2023 by David Turner. +Copyright (C) 2022-2023 by David Turner, Robert Wilhelm, Werner Lemberg, and Moazin Khatti. +Copyright (C) 2007-2023 by Rahul Bhalerao , . +Copyright (C) 2008-2023 by David Turner, Robert Wilhelm, Werner Lemberg, and suzuki toshiya. +Copyright (C) 2013-2023 by Google, Inc. +Copyright (C) 2019-2023 by Nikhil Ramakrishnan, David Turner, Robert Wilhelm, and Werner Lemberg. +Copyright (C) 2009-2023 by Oran Agra and Mickey Gabel. +Copyright (C) 2018-2023 by David Turner, Robert Wilhelm, Dominik Röttsches, and Werner Lemberg. +Copyright (C) 2004-2023 by David Turner, Robert Wilhelm, Werner Lemberg, and George Williams. + + + The FreeType Project LICENSE ---------------------------- 2006-Jan-27 @@ -206,7 +205,7 @@ Legal Terms Our home page can be found at - http://www.freetype.org + https://www.freetype.org ``` diff --git a/src/java.desktop/share/legal/harfbuzz.md b/src/java.desktop/share/legal/harfbuzz.md index 9037354540bd1..e2ed76aa7c6aa 100644 --- a/src/java.desktop/share/legal/harfbuzz.md +++ b/src/java.desktop/share/legal/harfbuzz.md @@ -1,8 +1,8 @@ -## Harfbuzz v4.4.1 +## Harfbuzz v7.2.0 ### Harfbuzz License -https://github.com/harfbuzz/harfbuzz/blob/4.4.1/COPYING +https://github.com/harfbuzz/harfbuzz/blob/7.2.0/COPYING
     
    @@ -10,23 +10,24 @@ HarfBuzz is licensed under the so-called "Old MIT" license.  Details follow.
     For parts of HarfBuzz that are licensed under different licenses see individual
     files names COPYING in subdirectories where applicable.
     
    -Copyright © 2010-2022  Google, Inc.
    +Copyright © 2010-2023  Google, Inc.
     Copyright © 2018-2020  Ebrahim Byagowi
    -Copyright © 2019-2020  Facebook, Inc.
    -Copyright © 2012-2015  Mozilla Foundation.
    -Copyright © 2011  Codethink Limited
    -Copyright © 2008-2010  Nokia Corporation and/or its subsidiary(-ies)
    -Copyright © 2009  Keith Stribley
    -Copyright © 2009  Martin Hosken and SIL International
    -Copyright © 2007  Chris Wilson
    -Copyright © 2005-2022 Behdad Esfahbod
    -Copyright © 2005  David Turner
     Copyright © 2004-2013  Red Hat, Inc.
    -Copyright © 1998-2004  David Turner and Werner Lemberg
    -Copyright © 2016  Elie Roux 
    +Copyright © 2019  Facebook, Inc.
    +Copyright © 2007  Chris Wilson
     Copyright © 2018-2019 Adobe Inc.
    +Copyright © 2006-2023 Behdad Esfahbod
    +Copyright © 1998-2004  David Turner and Werner Lemberg
    +Copyright © 2009  Keith Stribley
     Copyright © 2018  Khaled Hosny
    +Copyright © 2016  Elie Roux 
     Copyright © 2016  Igalia S.L.
    +Copyright © 2015  Mozilla Foundation.
    +Copyright © 1999  David Turner
    +Copyright © 2005  Werner Lemberg
    +Copyright © 2013-2015  Alexei Podtelezhnikov
    +Copyright © 2022 Matthias Clasen
    +Copyright © 2011  Codethink Limited
     
     For full copyright notices consult the individual files in the package.
     
    @@ -72,3 +73,23 @@ ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
     OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
     
     
    + +### AUTHORS File Information +``` + +Behdad Esfahbod +David Corbett +David Turner +Ebrahim Byagowi +Garret Rieger +Jonathan Kew +Khaled Hosny +Lars Knoll +Martin Hosken +Owen Taylor +Roderick Sheeter +Roozbeh Pournader +Simon Hausmann +Werner Lemberg + +``` diff --git a/src/java.desktop/share/legal/lcms.md b/src/java.desktop/share/legal/lcms.md index 1576edb8db6a3..da86a9c47ca31 100644 --- a/src/java.desktop/share/legal/lcms.md +++ b/src/java.desktop/share/legal/lcms.md @@ -1,8 +1,7 @@ -## Little Color Management System (LCMS) v2.14 +## Little Color Management System (LCMS) v2.15 ### LCMS License
    -
     README.1ST file information
     
     LittleCMS core is released under MIT License
    @@ -10,7 +9,7 @@ LittleCMS core is released under MIT License
     ---------------------------------
     
     Little CMS
    -Copyright (c) 1998-2022 Marti Maria Saguer
    +Copyright (c) 1998-2023 Marti Maria Saguer
     
     Permission is hereby granted, free of charge, to any person obtaining
     a copy of this software and associated documentation files (the
    @@ -32,7 +31,6 @@ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
     SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
     
     ---------------------------------
    -
     The below license applies to the following files:
     liblcms/cmssm.c
     
    @@ -44,12 +42,12 @@ SoftSurfer makes no warranty for this code, and cannot be held
     liable for any real or imagined damage resulting from its use.
     Users of this code must verify correctness for their application.
     
    -
     
    ### AUTHORS File Information ``` + Main Author ------------ Marti Maria @@ -90,11 +88,15 @@ Mark Allen Noel Carboni Sergei Trofimovic Philipp Knechtges +Amyspark +Lovell Fuller +Eli Schwartz Special Thanks -------------- Artifex software AlienSkin software +libVIPS Jan Morovic Jos Vernon (WebSupergoo) Harald Schneider (Maxon) @@ -103,5 +105,4 @@ Dimitrios Anastassakis Lemke Software Tim Zaman - ``` diff --git a/src/java.desktop/share/legal/libpng.md b/src/java.desktop/share/legal/libpng.md index 4f69da5383a2e..f11cfe580ce97 100644 --- a/src/java.desktop/share/legal/libpng.md +++ b/src/java.desktop/share/legal/libpng.md @@ -1,4 +1,4 @@ -## libpng v1.6.38 +## libpng v1.6.39 ### libpng License
    @@ -188,9 +188,10 @@ Authors, for copyright and licensing purposes.
      * Arm Holdings
        - Richard Townsend
      * Google Inc.
    +   - Dan Field
    +   - Leon Scroggins III
        - Matt Sarett
        - Mike Klein
    -   - Dan Field
        - Sami Boukortt
     
     The build projects, the build scripts, the test scripts, and other
    diff --git a/src/java.desktop/share/native/libawt/java2d/loops/FourByteAbgrPre.c b/src/java.desktop/share/native/libawt/java2d/loops/FourByteAbgrPre.c
    index c95d6a985a133..af109a7d88031 100644
    --- a/src/java.desktop/share/native/libawt/java2d/loops/FourByteAbgrPre.c
    +++ b/src/java.desktop/share/native/libawt/java2d/loops/FourByteAbgrPre.c
    @@ -1,5 +1,5 @@
     /*
    - * Copyright (c) 2000, 2005, Oracle and/or its affiliates. All rights reserved.
    + * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved.
      * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
      *
      * This code is free software; you can redistribute it and/or modify it
    @@ -69,6 +69,8 @@ DECLARE_SRCOVER_MASKBLIT(IntArgb, FourByteAbgrPre);
     DECLARE_ALPHA_MASKBLIT(IntArgb, FourByteAbgrPre);
     DECLARE_SRCOVER_MASKBLIT(IntArgbPre, FourByteAbgrPre);
     DECLARE_ALPHA_MASKBLIT(IntArgbPre, FourByteAbgrPre);
    +DECLARE_SRCOVER_MASKBLIT(FourByteAbgrPre, IntArgbPre);
    +DECLARE_ALPHA_MASKBLIT(FourByteAbgrPre, IntArgbPre);
     DECLARE_ALPHA_MASKBLIT(IntRgb, FourByteAbgrPre);
     DECLARE_SOLID_DRAWGLYPHLISTAA(FourByteAbgrPre);
     DECLARE_SOLID_DRAWGLYPHLISTLCD(FourByteAbgrPre);
    @@ -103,6 +105,8 @@ NativePrimitive FourByteAbgrPrePrimitives[] = {
         REGISTER_ALPHA_MASKBLIT(IntArgb, FourByteAbgrPre),
         REGISTER_SRCOVER_MASKBLIT(IntArgbPre, FourByteAbgrPre),
         REGISTER_ALPHA_MASKBLIT(IntArgbPre, FourByteAbgrPre),
    +    REGISTER_SRCOVER_MASKBLIT(FourByteAbgrPre, IntArgbPre),
    +    REGISTER_ALPHA_MASKBLIT(FourByteAbgrPre, IntArgbPre),
         REGISTER_ALPHA_MASKBLIT(IntRgb, FourByteAbgrPre),
         REGISTER_SOLID_DRAWGLYPHLISTAA(FourByteAbgrPre),
         REGISTER_SOLID_DRAWGLYPHLISTLCD(FourByteAbgrPre),
    @@ -177,6 +181,10 @@ DEFINE_SRCOVER_MASKBLIT(IntArgbPre, FourByteAbgrPre, 4ByteArgb)
     
     DEFINE_ALPHA_MASKBLIT(IntArgbPre, FourByteAbgrPre, 4ByteArgb)
     
    +DEFINE_SRCOVER_MASKBLIT(FourByteAbgrPre, IntArgbPre, 4ByteArgb)
    +
    +DEFINE_ALPHA_MASKBLIT(FourByteAbgrPre, IntArgbPre, 4ByteArgb)
    +
     DEFINE_ALPHA_MASKBLIT(IntRgb, FourByteAbgrPre, 4ByteArgb)
     
     DEFINE_SOLID_DRAWGLYPHLISTAA(FourByteAbgrPre, 4ByteArgb)
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/config/ftconfig.h b/src/java.desktop/share/native/libfreetype/include/freetype/config/ftconfig.h
    index c696e900a67d6..a85151699d0e0 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/config/ftconfig.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/config/ftconfig.h
    @@ -4,7 +4,7 @@
      *
      *   ANSI-specific configuration file (specification only).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/config/ftheader.h b/src/java.desktop/share/native/libfreetype/include/freetype/config/ftheader.h
    index a8c6833df779b..e607bce15c576 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/config/ftheader.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/config/ftheader.h
    @@ -4,7 +4,7 @@
      *
      *   Build macros of the FreeType 2 library.
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/config/ftoption.h b/src/java.desktop/share/native/libfreetype/include/freetype/config/ftoption.h
    index 2b6b310aaf39c..c13a3ef42880c 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/config/ftoption.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/config/ftoption.h
    @@ -4,7 +4,7 @@
      *
      *   User-selectable configuration macros (specification only).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -461,9 +461,9 @@ FT_BEGIN_HEADER
        *   while compiling in 'release' mode):
        *
        *   ```
    -   *     _af_debug_disable_horz_hints
    -   *     _af_debug_disable_vert_hints
    -   *     _af_debug_disable_blue_hints
    +   *     af_debug_disable_horz_hints_
    +   *     af_debug_disable_vert_hints_
    +   *     af_debug_disable_blue_hints_
        *   ```
        *
        *   Additionally, the following functions provide dumps of various
    @@ -480,7 +480,7 @@ FT_BEGIN_HEADER
        *   As an argument, they use another global variable:
        *
        *   ```
    -   *     _af_debug_hints
    +   *     af_debug_hints_
        *   ```
        *
        *   Please have a look at the `ftgrid` demo program to see how those
    @@ -584,12 +584,12 @@ FT_BEGIN_HEADER
       /**************************************************************************
        *
        * Define `TT_CONFIG_OPTION_POSTSCRIPT_NAMES` if you want to be able to
    -   * load and enumerate the glyph Postscript names in a TrueType or OpenType
    +   * load and enumerate Postscript names of glyphs in a TrueType or OpenType
        * file.
        *
    -   * Note that when you do not compile the 'psnames' module by undefining the
    -   * above `FT_CONFIG_OPTION_POSTSCRIPT_NAMES`, the 'sfnt' module will
    -   * contain additional code used to read the PS Names table from a font.
    +   * Note that if you do not compile the 'psnames' module by undefining the
    +   * above `FT_CONFIG_OPTION_POSTSCRIPT_NAMES` macro, the 'sfnt' module will
    +   * contain additional code to read the PostScript name table from a font.
        *
        * (By default, the module uses 'psnames' to extract glyph names.)
        */
    @@ -739,6 +739,24 @@ FT_BEGIN_HEADER
     #define TT_CONFIG_OPTION_GX_VAR_SUPPORT
     
     
    +  /**************************************************************************
    +   *
    +   * Define `TT_CONFIG_OPTION_NO_BORING_EXPANSION` if you want to exclude
    +   * support for 'boring' OpenType specification expansions.
    +   *
    +   *   https://github.com/harfbuzz/boring-expansion-spec
    +   *
    +   * Right now, the following features are covered:
    +   *
    +   *   - 'avar' version 2.0
    +   *
    +   * Most likely, this is a temporary configuration option to be removed in
    +   * the near future, since it is assumed that eventually those features are
    +   * added to the OpenType standard.
    +   */
    +/* #define TT_CONFIG_OPTION_NO_BORING_EXPANSION */
    +
    +
       /**************************************************************************
        *
        * Define `TT_CONFIG_OPTION_BDF` if you want to include support for an
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/config/ftstdlib.h b/src/java.desktop/share/native/libfreetype/include/freetype/config/ftstdlib.h
    index 7958c2a5f750a..3c9d2ae59a414 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/config/ftstdlib.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/config/ftstdlib.h
    @@ -5,7 +5,7 @@
      *   ANSI-specific library and header configuration file (specification
      *   only).
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/config/integer-types.h b/src/java.desktop/share/native/libfreetype/include/freetype/config/integer-types.h
    index d9d2638d1e654..7258b50854150 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/config/integer-types.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/config/integer-types.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType integer types definitions.
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/config/mac-support.h b/src/java.desktop/share/native/libfreetype/include/freetype/config/mac-support.h
    index e42c9fe410d2e..b77b96d5db835 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/config/mac-support.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/config/mac-support.h
    @@ -4,7 +4,7 @@
      *
      *   Mac/OS X support configuration header.
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/config/public-macros.h b/src/java.desktop/share/native/libfreetype/include/freetype/config/public-macros.h
    index 0074134f1d199..23d0fa6a329c6 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/config/public-macros.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/config/public-macros.h
    @@ -4,7 +4,7 @@
      *
      *   Define a set of compiler macros used in public FreeType headers.
      *
    - * Copyright (C) 2020-2022 by
    + * Copyright (C) 2020-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/freetype.h b/src/java.desktop/share/native/libfreetype/include/freetype/freetype.h
    index aa1a4fe3891ac..efff74fe399e3 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/freetype.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/freetype.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType high-level API and common types (specification only).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -215,7 +215,6 @@ FT_BEGIN_HEADER
        *   FT_Get_Char_Index
        *   FT_Get_First_Char
        *   FT_Get_Next_Char
    -   *   FT_Get_Name_Index
        *   FT_Load_Char
        *
        *   FT_OPEN_MEMORY
    @@ -254,14 +253,15 @@ FT_BEGIN_HEADER
        *   FT_Get_Kerning
        *   FT_Kerning_Mode
        *   FT_Get_Track_Kerning
    -   *   FT_Get_Glyph_Name
    -   *   FT_Get_Postscript_Name
        *
        *   FT_CharMapRec
        *   FT_Select_Charmap
        *   FT_Set_Charmap
        *   FT_Get_Charmap_Index
        *
    +   *   FT_Get_Name_Index
    +   *   FT_Get_Glyph_Name
    +   *   FT_Get_Postscript_Name
        *   FT_Get_FSType_Flags
        *   FT_Get_SubGlyph_Info
        *
    @@ -646,7 +646,7 @@ FT_BEGIN_HEADER
        *
        * @note:
        *   Despite the name, this enumeration lists specific character
    -   *   repertories (i.e., charsets), and not text encoding methods (e.g.,
    +   *   repertoires (i.e., charsets), and not text encoding methods (e.g.,
        *   UTF-8, UTF-16, etc.).
        *
        *   Other encodings might be defined in the future.
    @@ -779,7 +779,7 @@ FT_BEGIN_HEADER
        *   `encoding_id`.  If, for example, `encoding_id` is `TT_MAC_ID_ROMAN`
        *   and the language ID (minus~1) is `TT_MAC_LANGID_GREEK`, it is the
        *   Greek encoding, not Roman.  `TT_MAC_ID_ARABIC` with
    -   *   `TT_MAC_LANGID_FARSI` means the Farsi variant the Arabic encoding.
    +   *   `TT_MAC_LANGID_FARSI` means the Farsi variant of the Arabic encoding.
        */
       typedef enum  FT_Encoding_
       {
    @@ -1167,9 +1167,9 @@ FT_BEGIN_HEADER
        *   FT_FACE_FLAG_KERNING ::
        *     The face contains kerning information.  If set, the kerning distance
        *     can be retrieved using the function @FT_Get_Kerning.  Otherwise the
    -   *     function always return the vector (0,0).  Note that FreeType doesn't
    -   *     handle kerning data from the SFNT 'GPOS' table (as present in many
    -   *     OpenType fonts).
    +   *     function always returns the vector (0,0).  Note that FreeType
    +   *     doesn't handle kerning data from the SFNT 'GPOS' table (as present
    +   *     in many OpenType fonts).
        *
        *   FT_FACE_FLAG_FAST_GLYPHS ::
        *     THIS FLAG IS DEPRECATED.  DO NOT USE OR TEST IT.
    @@ -1892,13 +1892,13 @@ FT_BEGIN_HEADER
        *     The advance width of the unhinted glyph.  Its value is expressed in
        *     16.16 fractional pixels, unless @FT_LOAD_LINEAR_DESIGN is set when
        *     loading the glyph.  This field can be important to perform correct
    -   *     WYSIWYG layout.  Only relevant for outline glyphs.
    +   *     WYSIWYG layout.  Only relevant for scalable glyphs.
        *
        *   linearVertAdvance ::
        *     The advance height of the unhinted glyph.  Its value is expressed in
        *     16.16 fractional pixels, unless @FT_LOAD_LINEAR_DESIGN is set when
        *     loading the glyph.  This field can be important to perform correct
    -   *     WYSIWYG layout.  Only relevant for outline glyphs.
    +   *     WYSIWYG layout.  Only relevant for scalable glyphs.
        *
        *   advance ::
        *     This shorthand is, depending on @FT_LOAD_IGNORE_TRANSFORM, the
    @@ -2593,8 +2593,8 @@ FT_BEGIN_HEADER
        *   stream attachments.
        */
       FT_EXPORT( FT_Error )
    -  FT_Attach_Stream( FT_Face        face,
    -                    FT_Open_Args*  parameters );
    +  FT_Attach_Stream( FT_Face              face,
    +                    const FT_Open_Args*  parameters );
     
     
       /**************************************************************************
    @@ -3077,7 +3077,7 @@ FT_BEGIN_HEADER
        *
        *   FT_LOAD_NO_HINTING ::
        *     Disable hinting.  This generally generates 'blurrier' bitmap glyphs
    -   *     when the glyph are rendered in any of the anti-aliased modes.  See
    +   *     when the glyphs are rendered in any of the anti-aliased modes.  See
        *     also the note below.
        *
        *     This flag is implied by @FT_LOAD_NO_SCALE.
    @@ -3434,7 +3434,7 @@ FT_BEGIN_HEADER
        *     are not interested in the value.
        *
        *   delta ::
    -   *     A pointer a translation vector.  Set this to NULL if you are not
    +   *     A pointer to a translation vector.  Set this to NULL if you are not
        *     interested in the value.
        *
        * @since:
    @@ -3559,9 +3559,10 @@ FT_BEGIN_HEADER
        *
        *   2. The `sdf` rasterizer has limited support for handling intersecting
        *      contours and *cannot* handle self-intersecting contours whatsoever.
    -   *      Self-intersection happens when a single connected contour intersect
    -   *      itself at some point; having these in your font definitely pose a
    -   *      problem to the rasterizer and cause artifacts, too.
    +   *      Self-intersection happens when a single connected contour
    +   *      intersects itself at some point; having these in your font
    +   *      definitely poses a problem to the rasterizer and cause artifacts,
    +   *      too.
        *
        *   3. Generating SDF for really small glyphs may result in undesirable
        *      output; the pixel grid (which stores distance information) becomes
    @@ -3840,89 +3841,6 @@ FT_BEGIN_HEADER
                             FT_Fixed*  akerning );
     
     
    -  /**************************************************************************
    -   *
    -   * @function:
    -   *   FT_Get_Glyph_Name
    -   *
    -   * @description:
    -   *   Retrieve the ASCII name of a given glyph in a face.  This only works
    -   *   for those faces where @FT_HAS_GLYPH_NAMES(face) returns~1.
    -   *
    -   * @input:
    -   *   face ::
    -   *     A handle to a source face object.
    -   *
    -   *   glyph_index ::
    -   *     The glyph index.
    -   *
    -   *   buffer_max ::
    -   *     The maximum number of bytes available in the buffer.
    -   *
    -   * @output:
    -   *   buffer ::
    -   *     A pointer to a target buffer where the name is copied to.
    -   *
    -   * @return:
    -   *   FreeType error code.  0~means success.
    -   *
    -   * @note:
    -   *   An error is returned if the face doesn't provide glyph names or if the
    -   *   glyph index is invalid.  In all cases of failure, the first byte of
    -   *   `buffer` is set to~0 to indicate an empty name.
    -   *
    -   *   The glyph name is truncated to fit within the buffer if it is too
    -   *   long.  The returned string is always zero-terminated.
    -   *
    -   *   Be aware that FreeType reorders glyph indices internally so that glyph
    -   *   index~0 always corresponds to the 'missing glyph' (called '.notdef').
    -   *
    -   *   This function always returns an error if the config macro
    -   *   `FT_CONFIG_OPTION_NO_GLYPH_NAMES` is not defined in `ftoption.h`.
    -   */
    -  FT_EXPORT( FT_Error )
    -  FT_Get_Glyph_Name( FT_Face     face,
    -                     FT_UInt     glyph_index,
    -                     FT_Pointer  buffer,
    -                     FT_UInt     buffer_max );
    -
    -
    -  /**************************************************************************
    -   *
    -   * @function:
    -   *   FT_Get_Postscript_Name
    -   *
    -   * @description:
    -   *   Retrieve the ASCII PostScript name of a given face, if available.
    -   *   This only works with PostScript, TrueType, and OpenType fonts.
    -   *
    -   * @input:
    -   *   face ::
    -   *     A handle to the source face object.
    -   *
    -   * @return:
    -   *   A pointer to the face's PostScript name.  `NULL` if unavailable.
    -   *
    -   * @note:
    -   *   The returned pointer is owned by the face and is destroyed with it.
    -   *
    -   *   For variation fonts, this string changes if you select a different
    -   *   instance, and you have to call `FT_Get_PostScript_Name` again to
    -   *   retrieve it.  FreeType follows Adobe TechNote #5902, 'Generating
    -   *   PostScript Names for Fonts Using OpenType Font Variations'.
    -   *
    -   *     https://download.macromedia.com/pub/developer/opentype/tech-notes/5902.AdobePSNameGeneration.html
    -   *
    -   *   [Since 2.9] Special PostScript names for named instances are only
    -   *   returned if the named instance is set with @FT_Set_Named_Instance (and
    -   *   the font has corresponding entries in its 'fvar' table).  If
    -   *   @FT_IS_VARIATION returns true, the algorithmically derived PostScript
    -   *   name is provided, not looking up special entries for named instances.
    -   */
    -  FT_EXPORT( const char* )
    -  FT_Get_Postscript_Name( FT_Face  face );
    -
    -
       /**************************************************************************
        *
        * @function:
    @@ -4243,7 +4161,8 @@ FT_BEGIN_HEADER
        *   FT_Get_Name_Index
        *
        * @description:
    -   *   Return the glyph index of a given glyph name.
    +   *   Return the glyph index of a given glyph name.  This only works
    +   *   for those faces where @FT_HAS_GLYPH_NAMES returns true.
        *
        * @input:
        *   face ::
    @@ -4254,12 +4173,107 @@ FT_BEGIN_HEADER
        *
        * @return:
        *   The glyph index.  0~means 'undefined character code'.
    +   *
    +   * @note:
    +   *   Acceptable glyph names might come from the [Adobe Glyph
    +   *   List](https://github.com/adobe-type-tools/agl-aglfn).  See
    +   *   @FT_Get_Glyph_Name for the inverse functionality.
    +   *
    +   *   This function has limited capabilities if the config macro
    +   *   `FT_CONFIG_OPTION_POSTSCRIPT_NAMES` is not defined in `ftoption.h`:
    +   *   It then works only for fonts that actually embed glyph names (which
    +   *   many recent OpenType fonts do not).
        */
       FT_EXPORT( FT_UInt )
       FT_Get_Name_Index( FT_Face           face,
                          const FT_String*  glyph_name );
     
     
    +  /**************************************************************************
    +   *
    +   * @function:
    +   *   FT_Get_Glyph_Name
    +   *
    +   * @description:
    +   *   Retrieve the ASCII name of a given glyph in a face.  This only works
    +   *   for those faces where @FT_HAS_GLYPH_NAMES returns true.
    +   *
    +   * @input:
    +   *   face ::
    +   *     A handle to a source face object.
    +   *
    +   *   glyph_index ::
    +   *     The glyph index.
    +   *
    +   *   buffer_max ::
    +   *     The maximum number of bytes available in the buffer.
    +   *
    +   * @output:
    +   *   buffer ::
    +   *     A pointer to a target buffer where the name is copied to.
    +   *
    +   * @return:
    +   *   FreeType error code.  0~means success.
    +   *
    +   * @note:
    +   *   An error is returned if the face doesn't provide glyph names or if the
    +   *   glyph index is invalid.  In all cases of failure, the first byte of
    +   *   `buffer` is set to~0 to indicate an empty name.
    +   *
    +   *   The glyph name is truncated to fit within the buffer if it is too
    +   *   long.  The returned string is always zero-terminated.
    +   *
    +   *   Be aware that FreeType reorders glyph indices internally so that glyph
    +   *   index~0 always corresponds to the 'missing glyph' (called '.notdef').
    +   *
    +   *   This function has limited capabilities if the config macro
    +   *   `FT_CONFIG_OPTION_POSTSCRIPT_NAMES` is not defined in `ftoption.h`:
    +   *   It then works only for fonts that actually embed glyph names (which
    +   *   many recent OpenType fonts do not).
    +   */
    +  FT_EXPORT( FT_Error )
    +  FT_Get_Glyph_Name( FT_Face     face,
    +                     FT_UInt     glyph_index,
    +                     FT_Pointer  buffer,
    +                     FT_UInt     buffer_max );
    +
    +
    +  /**************************************************************************
    +   *
    +   * @function:
    +   *   FT_Get_Postscript_Name
    +   *
    +   * @description:
    +   *   Retrieve the ASCII PostScript name of a given face, if available.
    +   *   This only works with PostScript, TrueType, and OpenType fonts.
    +   *
    +   * @input:
    +   *   face ::
    +   *     A handle to the source face object.
    +   *
    +   * @return:
    +   *   A pointer to the face's PostScript name.  `NULL` if unavailable.
    +   *
    +   * @note:
    +   *   The returned pointer is owned by the face and is destroyed with it.
    +   *
    +   *   For variation fonts, this string changes if you select a different
    +   *   instance, and you have to call `FT_Get_PostScript_Name` again to
    +   *   retrieve it.  FreeType follows Adobe TechNote #5902, 'Generating
    +   *   PostScript Names for Fonts Using OpenType Font Variations'.
    +   *
    +   *     https://download.macromedia.com/pub/developer/opentype/tech-notes/5902.AdobePSNameGeneration.html
    +   *
    +   *   [Since 2.9] Special PostScript names for named instances are only
    +   *   returned if the named instance is set with @FT_Set_Named_Instance (and
    +   *   the font has corresponding entries in its 'fvar' table).  If
    +   *   @FT_IS_VARIATION returns true, the algorithmically derived PostScript
    +   *   name is provided, not looking up special entries for named instances.
    +   */
    +  FT_EXPORT( const char* )
    +  FT_Get_Postscript_Name( FT_Face  face );
    +
    +
       /**************************************************************************
        *
        * @enum:
    @@ -4346,13 +4360,6 @@ FT_BEGIN_HEADER
                             FT_Matrix    *p_transform );
     
     
    -  /**************************************************************************
    -   *
    -   * @section:
    -   *   base_interface
    -   *
    -   */
    -
       /**************************************************************************
        *
        * @enum:
    @@ -4688,7 +4695,8 @@ FT_BEGIN_HEADER
        *
        * @description:
        *   This section contains various functions used to perform computations
    -   *   on 16.16 fixed-float numbers or 2d vectors.
    +   *   on 16.16 fixed-point numbers or 2D vectors.  FreeType does not use
    +   *   floating-point data types.
        *
        *   **Attention**: Most arithmetic functions take `FT_Long` as arguments.
        *   For historical reasons, FreeType was designed under the assumption
    @@ -4941,8 +4949,8 @@ FT_BEGIN_HEADER
        *
        */
     #define FREETYPE_MAJOR  2
    -#define FREETYPE_MINOR  12
    -#define FREETYPE_PATCH  1
    +#define FREETYPE_MINOR  13
    +#define FREETYPE_PATCH  0
     
     
       /**************************************************************************
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftadvanc.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftadvanc.h
    index 8ce4846668c64..4560ded6dcbd3 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftadvanc.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftadvanc.h
    @@ -4,7 +4,7 @@
      *
      *   Quick computation of advance widths (specification only).
      *
    - * Copyright (C) 2008-2022 by
    + * Copyright (C) 2008-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftbbox.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftbbox.h
    index 768478f399bc8..fc21740fc2bf8 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftbbox.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftbbox.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType exact bbox computation (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftbdf.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftbdf.h
    index 04d6094f753c5..e8ce6431285d2 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftbdf.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftbdf.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType API for accessing BDF-specific strings (specification).
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftbitmap.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftbitmap.h
    index c3462dadc51d9..eb6b4b1eebedf 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftbitmap.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftbitmap.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType utility functions for bitmaps (specification).
      *
    - * Copyright (C) 2004-2022 by
    + * Copyright (C) 2004-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftcid.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftcid.h
    index d80108387ac2c..ef22939022453 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftcid.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftcid.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType API for accessing CID font information (specification).
      *
    - * Copyright (C) 2007-2022 by
    + * Copyright (C) 2007-2023 by
      * Dereg Clegg and Michael Toftdal.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftcolor.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftcolor.h
    index 3edaee4ec1950..eae200fdf1485 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftcolor.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftcolor.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType's glyph color management (specification).
      *
    - * Copyright (C) 2018-2022 by
    + * Copyright (C) 2018-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -456,6 +456,9 @@ FT_BEGIN_HEADER
        *                                           &iterator ) );
        *     }
        *   ```
    +   *
    +   * @since:
    +   *   2.10
        */
       FT_EXPORT( FT_Bool )
       FT_Get_Color_Glyph_Layer( FT_Face            face,
    @@ -475,7 +478,7 @@ FT_BEGIN_HEADER
        *   extensions to the 'COLR' table, see
        *   'https://github.com/googlefonts/colr-gradients-spec'.
        *
    -   *   The enumeration values losely correspond with the format numbers of
    +   *   The enumeration values loosely correspond with the format numbers of
        *   the specification: FreeType always returns a fully specified 'Paint'
        *   structure for the 'Transform', 'Translate', 'Scale', 'Rotate', and
        *   'Skew' table types even though the specification has different formats
    @@ -489,9 +492,7 @@ FT_BEGIN_HEADER
        *   structures.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef enum  FT_PaintFormat_
       {
    @@ -521,9 +522,10 @@ FT_BEGIN_HEADER
        *
        * @description:
        *   This iterator object is needed for @FT_Get_Colorline_Stops.  It keeps
    -   *   state while iterating over the stops of an @FT_ColorLine,
    -   *   representing the `ColorLine` struct of the v1 extensions to 'COLR',
    -   *   see 'https://github.com/googlefonts/colr-gradients-spec'.
    +   *   state while iterating over the stops of an @FT_ColorLine, representing
    +   *   the `ColorLine` struct of the v1 extensions to 'COLR', see
    +   *   'https://github.com/googlefonts/colr-gradients-spec'.  Do not manually
    +   *   modify fields of this iterator.
        *
        * @fields:
        *   num_color_stops ::
    @@ -537,10 +539,12 @@ FT_BEGIN_HEADER
        *     An opaque pointer into 'COLR' table data.  Set by @FT_Get_Paint.
        *     Updated by @FT_Get_Colorline_Stops.
        *
    -   * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    +   *   read_variable ::
    +   *     A boolean keeping track of whether variable color lines are to be
    +   *     read.  Set by @FT_Get_Paint.
        *
    +   * @since:
    +   *   2.13
        */
       typedef struct  FT_ColorStopIterator_
       {
    @@ -549,6 +553,8 @@ FT_BEGIN_HEADER
     
         FT_Byte*  p;
     
    +    FT_Bool  read_variable;
    +
       } FT_ColorStopIterator;
     
     
    @@ -569,9 +575,7 @@ FT_BEGIN_HEADER
        *     Alpha transparency value multiplied with the value from 'CPAL'.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_ColorIndex_
       {
    @@ -592,19 +596,18 @@ FT_BEGIN_HEADER
        *
        * @fields:
        *   stop_offset ::
    -   *     The stop offset between 0 and 1 along the gradient.
    +   *     The stop offset along the gradient, expressed as a 16.16 fixed-point
    +   *     coordinate.
        *
        *   color ::
        *     The color information for this stop, see @FT_ColorIndex.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_ColorStop_
       {
    -    FT_F2Dot14     stop_offset;
    +    FT_Fixed       stop_offset;
         FT_ColorIndex  color;
     
       } FT_ColorStop;
    @@ -621,9 +624,7 @@ FT_BEGIN_HEADER
        *   It describes how the gradient fill continues at the other boundaries.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef enum  FT_PaintExtend_
       {
    @@ -653,9 +654,7 @@ FT_BEGIN_HEADER
        *     actual @FT_ColorStop's.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_ColorLine_
       {
    @@ -699,9 +698,7 @@ FT_BEGIN_HEADER
        *     y translation.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_Affine_23_
       {
    @@ -722,9 +719,7 @@ FT_BEGIN_HEADER
        *   'https://www.w3.org/TR/compositing-1/#porterduffcompositingoperators'.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef enum  FT_Composite_Mode_
       {
    @@ -786,9 +781,7 @@ FT_BEGIN_HEADER
        *     to be provided.  Do not set this value.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_Opaque_Paint_
       {
    @@ -815,9 +808,7 @@ FT_BEGIN_HEADER
        *     The layer iterator that describes the layers of this paint.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_PaintColrLayers_
       {
    @@ -842,9 +833,7 @@ FT_BEGIN_HEADER
        *     The color information for this solid paint, see @FT_ColorIndex.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_PaintSolid_
       {
    @@ -883,9 +872,7 @@ FT_BEGIN_HEADER
        *     Otherwise equal to~p0.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_PaintLinearGradient_
       {
    @@ -908,8 +895,7 @@ FT_BEGIN_HEADER
        *   A structure representing a `PaintRadialGradient` value of the 'COLR'
        *   v1 extensions, see
        *   'https://github.com/googlefonts/colr-gradients-spec'.  The glyph
    -   *   layer filled with this paint is drawn filled filled with a radial
    -   *   gradient.
    +   *   layer filled with this paint is drawn filled with a radial gradient.
        *
        * @fields:
        *   colorline ::
    @@ -933,9 +919,7 @@ FT_BEGIN_HEADER
        *     units represented as a 16.16 fixed-point value.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_PaintRadialGradient_
       {
    @@ -983,9 +967,7 @@ FT_BEGIN_HEADER
        *     given counter-clockwise, starting from the (positive) y~axis.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_PaintSweepGradient_
       {
    @@ -1016,9 +998,7 @@ FT_BEGIN_HEADER
        *     information that is filled with paint.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_PaintGlyph_
       {
    @@ -1042,9 +1022,7 @@ FT_BEGIN_HEADER
        *     this paint.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_PaintColrGlyph_
       {
    @@ -1070,9 +1048,7 @@ FT_BEGIN_HEADER
        *     16.16 fixed-point values.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_PaintTransform_
       {
    @@ -1105,9 +1081,7 @@ FT_BEGIN_HEADER
        *     16.16 fixed-point value.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_PaintTranslate_
       {
    @@ -1156,9 +1130,7 @@ FT_BEGIN_HEADER
        *     16.16 fixed-point value.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward-compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_PaintScale_
       {
    @@ -1194,16 +1166,14 @@ FT_BEGIN_HEADER
        *
        *   center_x ::
        *     The x~coordinate of the pivot point of the rotation in font
    -   *     units) represented as a 16.16 fixed-point value.
    +   *     units represented as a 16.16 fixed-point value.
        *
        *   center_y ::
        *     The y~coordinate of the pivot point of the rotation in font
        *     units represented as a 16.16 fixed-point value.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
     
       typedef struct  FT_PaintRotate_
    @@ -1252,9 +1222,7 @@ FT_BEGIN_HEADER
        *     represented as a 16.16 fixed-point value.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_PaintSkew_
       {
    @@ -1275,9 +1243,8 @@ FT_BEGIN_HEADER
        *   FT_PaintComposite
        *
        * @description:
    -   *   A structure representing a 'COLR'v1 `PaintComposite` paint table.
    -   *   Used for compositing two paints in a 'COLR' v1 directed acycling
    -   *   graph.
    +   *   A structure representing a 'COLR' v1 `PaintComposite` paint table.
    +   *   Used for compositing two paints in a 'COLR' v1 directed acyclic graph.
        *
        * @fields:
        *   source_paint ::
    @@ -1293,9 +1260,7 @@ FT_BEGIN_HEADER
        *     `source_paint` is composited onto.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_PaintComposite_
       {
    @@ -1339,9 +1304,7 @@ FT_BEGIN_HEADER
        *       * @FT_PaintColrGlyph
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_COLR_Paint_
       {
    @@ -1386,9 +1349,7 @@ FT_BEGIN_HEADER
        *     Do not output an initial root transform.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef enum  FT_Color_Root_Transform_
       {
    @@ -1429,9 +1390,7 @@ FT_BEGIN_HEADER
        *     fixed-point coordinates in 26.6 format.
        *
        * @since:
    -   *   2.12 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       typedef struct  FT_ClipBox_
       {
    @@ -1524,9 +1483,7 @@ FT_BEGIN_HEADER
        *   error, value~0 is returned also.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       FT_EXPORT( FT_Bool )
       FT_Get_Color_Glyph_Paint( FT_Face                  face,
    @@ -1568,9 +1525,7 @@ FT_BEGIN_HEADER
        *   and remove transforms configured using @FT_Set_Transform.
        *
        * @since:
    -   *   2.12 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       FT_EXPORT( FT_Bool )
       FT_Get_Color_Glyph_ClipBox( FT_Face      face,
    @@ -1617,9 +1572,7 @@ FT_BEGIN_HEADER
        *   object can not be retrieved or any other error occurs.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       FT_EXPORT( FT_Bool )
       FT_Get_Paint_Layers( FT_Face            face,
    @@ -1660,9 +1613,7 @@ FT_BEGIN_HEADER
        *   also.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       FT_EXPORT( FT_Bool )
       FT_Get_Colorline_Stops( FT_Face                face,
    @@ -1698,9 +1649,7 @@ FT_BEGIN_HEADER
        *   this paint or any other error occured.
        *
        * @since:
    -   *   2.11 -- **currently experimental only!**  There might be changes
    -   *   without retaining backward compatibility of both the API and ABI.
    -   *
    +   *   2.13
        */
       FT_EXPORT( FT_Bool )
       FT_Get_Paint( FT_Face         face,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftdriver.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftdriver.h
    index 0dc91e8b40489..f90946fd17d3c 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftdriver.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftdriver.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType API for controlling driver modules (specification only).
      *
    - * Copyright (C) 2017-2022 by
    + * Copyright (C) 2017-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -820,7 +820,6 @@ FT_BEGIN_HEADER
        *   2.5
        */
     
    -
       /**************************************************************************
        *
        * @property:
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/fterrdef.h b/src/java.desktop/share/native/libfreetype/include/freetype/fterrdef.h
    index a3acfce4304da..d59b3cc2da277 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/fterrdef.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/fterrdef.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType error codes (specification).
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/fterrors.h b/src/java.desktop/share/native/libfreetype/include/freetype/fterrors.h
    index ff1b375d7d5d1..15ef3f76b59fc 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/fterrors.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/fterrors.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType error code handling (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -29,7 +29,7 @@
        *
        * @description:
        *   The header file `fterrors.h` (which is automatically included by
    -   *   `freetype.h` defines the handling of FreeType's enumeration
    +   *   `freetype.h`) defines the handling of FreeType's enumeration
        *   constants.  It can also be used to generate error message strings
        *   with a small macro trick explained below.
        *
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftfntfmt.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftfntfmt.h
    index 77d553578ba6b..c0018fc830c25 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftfntfmt.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftfntfmt.h
    @@ -4,7 +4,7 @@
      *
      *   Support functions for font formats.
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftgasp.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftgasp.h
    index d4ab9b32dbd31..d5f19add8f209 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftgasp.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftgasp.h
    @@ -4,7 +4,7 @@
      *
      *   Access of TrueType's 'gasp' table (specification).
      *
    - * Copyright (C) 2007-2022 by
    + * Copyright (C) 2007-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftglyph.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftglyph.h
    index 6b77bd3d2a903..4658895f7a962 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftglyph.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftglyph.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType convenience functions to handle glyphs (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -355,7 +355,7 @@ FT_BEGIN_HEADER
        *
        * @output:
        *   aglyph ::
    -   *     A handle to the glyph object.
    +   *     A handle to the glyph object.  `NULL` in case of error.
        *
        * @return:
        *   FreeType error code.  0~means success.
    @@ -385,7 +385,7 @@ FT_BEGIN_HEADER
        *
        * @output:
        *   target ::
    -   *     A handle to the target glyph object.  0~in case of error.
    +   *     A handle to the target glyph object.  `NULL` in case of error.
        *
        * @return:
        *   FreeType error code.  0~means success.
    @@ -413,7 +413,7 @@ FT_BEGIN_HEADER
        *
        *   delta ::
        *     A pointer to a 2d vector to apply.  Coordinates are expressed in
    -   *     1/64th of a pixel.
    +   *     1/64 of a pixel.
        *
        * @return:
        *   FreeType error code (if not 0, the glyph format is not scalable).
    @@ -500,7 +500,7 @@ FT_BEGIN_HEADER
        * @output:
        *   acbox ::
        *     The glyph coordinate bounding box.  Coordinates are expressed in
    -   *     1/64th of pixels if it is grid-fitted.
    +   *     1/64 of pixels if it is grid-fitted.
        *
        * @note:
        *   Coordinates are relative to the glyph origin, using the y~upwards
    @@ -671,7 +671,7 @@ FT_BEGIN_HEADER
        *
        * @input:
        *   glyph ::
    -   *     A handle to the target glyph object.
    +   *     A handle to the target glyph object.  Can be `NULL`.
        */
       FT_EXPORT( void )
       FT_Done_Glyph( FT_Glyph  glyph );
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftgzip.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftgzip.h
    index 0880290f9e2f7..443ec29db1bd3 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftgzip.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftgzip.h
    @@ -4,7 +4,7 @@
      *
      *   Gzip-compressed stream support.
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftimage.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftimage.h
    index 7f2d721cdc2d9..2e8e6734cc048 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftimage.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftimage.h
    @@ -5,7 +5,7 @@
      *   FreeType glyph image formats and default raster interface
      *   (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftincrem.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftincrem.h
    index 3b3d93c2d311c..2d4f5def241b0 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftincrem.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftincrem.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType incremental loading (specification).
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftlcdfil.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftlcdfil.h
    index c767c6cb483ad..d3723e16f67dc 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftlcdfil.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftlcdfil.h
    @@ -5,7 +5,7 @@
      *   FreeType API for color filtering of subpixel bitmap glyphs
      *   (specification).
      *
    - * Copyright (C) 2006-2022 by
    + * Copyright (C) 2006-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -137,11 +137,11 @@ FT_BEGIN_HEADER
        *
        *   FT_LCD_FILTER_DEFAULT ::
        *     This is a beveled, normalized, and color-balanced five-tap filter
    -   *     with weights of [0x08 0x4D 0x56 0x4D 0x08] in 1/256th units.
    +   *     with weights of [0x08 0x4D 0x56 0x4D 0x08] in 1/256 units.
        *
        *   FT_LCD_FILTER_LIGHT ::
        *     this is a boxy, normalized, and color-balanced three-tap filter with
    -   *     weights of [0x00 0x55 0x56 0x55 0x00] in 1/256th units.
    +   *     weights of [0x00 0x55 0x56 0x55 0x00] in 1/256 units.
        *
        *   FT_LCD_FILTER_LEGACY ::
        *   FT_LCD_FILTER_LEGACY1 ::
    @@ -226,7 +226,7 @@ FT_BEGIN_HEADER
        *
        *   weights ::
        *     A pointer to an array; the function copies the first five bytes and
    -   *     uses them to specify the filter weights in 1/256th units.
    +   *     uses them to specify the filter weights in 1/256 units.
        *
        * @return:
        *   FreeType error code.  0~means success.
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftlist.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftlist.h
    index 4dca2bf163d00..b55313133593f 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftlist.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftlist.h
    @@ -4,7 +4,7 @@
      *
      *   Generic list support for FreeType (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftlogging.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftlogging.h
    index 7213dc30a8a15..2246dc8365193 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftlogging.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftlogging.h
    @@ -4,7 +4,7 @@
      *
      *   Additional debugging APIs.
      *
    - * Copyright (C) 2020-2022 by
    + * Copyright (C) 2020-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftmac.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftmac.h
    index 3dd61d0fe12a7..a91e38f9ea752 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftmac.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftmac.h
    @@ -4,7 +4,7 @@
      *
      *   Additional Mac-specific API.
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * Just van Rossum, David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftmm.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftmm.h
    index c74ce618cb492..e381ef3d30a17 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftmm.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftmm.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType Multiple Master font interface (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -398,6 +398,10 @@ FT_BEGIN_HEADER
        *   FreeType error code.  0~means success.
        *
        * @note:
    +   *   The design coordinates are 16.16 fractional values for TrueType GX and
    +   *   OpenType variation fonts.  For Adobe MM fonts, the values are
    +   *   integers.
    +   *
        *   [Since 2.8.1] To reset all axes to the default values, call the
        *   function with `num_coords` set to zero and `coords` set to `NULL`.
        *   [Since 2.9] 'Default values' means the currently selected named
    @@ -440,6 +444,11 @@ FT_BEGIN_HEADER
        * @return:
        *   FreeType error code.  0~means success.
        *
    +   * @note:
    +   *   The design coordinates are 16.16 fractional values for TrueType GX and
    +   *   OpenType variation fonts.  For Adobe MM fonts, the values are
    +   *   integers.
    +   *
        * @since:
        *   2.7.1
        */
    @@ -471,9 +480,9 @@ FT_BEGIN_HEADER
        *     the number of axes, use default values for the remaining axes.
        *
        *   coords ::
    -   *     The design coordinates array (each element must be between 0 and 1.0
    -   *     for Adobe MM fonts, and between -1.0 and 1.0 for TrueType GX and
    -   *     OpenType variation fonts).
    +   *     The design coordinates array.  Each element is a 16.16 fractional
    +   *     value and must be between 0 and 1.0 for Adobe MM fonts, and between
    +   *     -1.0 and 1.0 for TrueType GX and OpenType variation fonts.
        *
        * @return:
        *   FreeType error code.  0~means success.
    @@ -518,7 +527,7 @@ FT_BEGIN_HEADER
        *
        * @output:
        *   coords ::
    -   *     The normalized blend coordinates array.
    +   *     The normalized blend coordinates array (as 16.16 fractional values).
        *
        * @return:
        *   FreeType error code.  0~means success.
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftmodapi.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftmodapi.h
    index b78db724c7305..c8f0c2c2a45ec 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftmodapi.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftmodapi.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType modules public interface (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftmoderr.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftmoderr.h
    index 88d291777178f..c8c892dcce8b2 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftmoderr.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftmoderr.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType module error offsets (specification).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftoutln.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftoutln.h
    index 46ebf9371bb6d..54434b25f6f47 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftoutln.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftoutln.h
    @@ -5,7 +5,7 @@
      *   Support for the FT_Outline type used to store glyph shapes of
      *   most scalable font formats (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftparams.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftparams.h
    index 72080f396a3d9..6a9f243bc904c 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftparams.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftparams.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType API for possible FT_Parameter tags (specification only).
      *
    - * Copyright (C) 2017-2022 by
    + * Copyright (C) 2017-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftrender.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftrender.h
    index 0fab3f8c2a2ce..a8576dab00268 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftrender.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftrender.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType renderer modules public interface (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftsizes.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftsizes.h
    index e30938d86246f..7bfb1aed4c255 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftsizes.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftsizes.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType size objects management (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftsnames.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftsnames.h
    index 384096a585768..9d5d22bb25543 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftsnames.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftsnames.h
    @@ -7,7 +7,7 @@
      *
      *   This is _not_ used to retrieve glyph names!
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftstroke.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftstroke.h
    index 12c006d3fb823..b3d90802a56a1 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftstroke.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftstroke.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType path stroker (specification).
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -293,7 +293,7 @@ FT_BEGIN_HEADER
        *
        *   miter_limit ::
        *     The maximum reciprocal sine of half-angle at the miter join,
    -   *     expressed as 16.16 fixed point value.
    +   *     expressed as 16.16 fixed-point value.
        *
        * @note:
        *   The `radius` is expressed in the same units as the outline
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftsynth.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftsynth.h
    index afc40b1d84a4c..5d196976572df 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftsynth.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftsynth.h
    @@ -5,7 +5,7 @@
      *   FreeType synthesizing code for emboldening and slanting
      *   (specification).
      *
    - * Copyright (C) 2000-2022 by
    + * Copyright (C) 2000-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -68,10 +68,19 @@ FT_BEGIN_HEADER
       FT_EXPORT( void )
       FT_GlyphSlot_Embolden( FT_GlyphSlot  slot );
     
    -  /* Slant an outline glyph to the right by about 12 degrees. */
    +  /* Slant an outline glyph to the right by about 12 degrees.              */
       FT_EXPORT( void )
       FT_GlyphSlot_Oblique( FT_GlyphSlot  slot );
     
    +  /* Slant an outline glyph by a given sine of an angle.  You can apply    */
    +  /* slant along either x- or y-axis by choosing a corresponding non-zero  */
    +  /* argument.  If both slants are non-zero, some affine transformation    */
    +  /* will result.                                                          */
    +  FT_EXPORT( void )
    +  FT_GlyphSlot_Slant( FT_GlyphSlot  slot,
    +                      FT_Fixed      xslant,
    +                      FT_Fixed      yslant );
    +
       /* */
     
     
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ftsystem.h b/src/java.desktop/share/native/libfreetype/include/freetype/ftsystem.h
    index 5f8aec7b7cece..a995b078de5e9 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ftsystem.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ftsystem.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType low-level system interface definition (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -229,7 +229,8 @@ FT_BEGIN_HEADER
        *     A handle to the source stream.
        *
        *   offset ::
    -   *     The offset of read in stream (always from start).
    +   *     The offset from the start of the stream to seek to if this is a seek
    +   *     operation (see note).
        *
        *   buffer ::
        *     The address of the read buffer.
    @@ -241,8 +242,13 @@ FT_BEGIN_HEADER
        *   The number of bytes effectively read by the stream.
        *
        * @note:
    -   *   This function might be called to perform a seek or skip operation with
    -   *   a `count` of~0.  A non-zero return value then indicates an error.
    +   *   This function performs a seek *or* a read operation depending on the
    +   *   argument values.  If `count` is zero, the operation is a seek to
    +   *   `offset` bytes.  If `count` is >~0, the operation is a read of `count`
    +   *   bytes from the current position in the stream, and the `offset` value
    +   *   should be ignored.
    +   *
    +   *   For seek operations, a non-zero return value indicates an error.
        *
        */
       typedef unsigned long
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/fttrigon.h b/src/java.desktop/share/native/libfreetype/include/freetype/fttrigon.h
    index 4e8d871decc5e..294981a6f3127 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/fttrigon.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/fttrigon.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType trigonometric functions (specification).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/fttypes.h b/src/java.desktop/share/native/libfreetype/include/freetype/fttypes.h
    index 29f32fbb261ee..5b109f0c73c2f 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/fttypes.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/fttypes.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType simple types definitions (specification only).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -45,7 +45,10 @@ FT_BEGIN_HEADER
        * @description:
        *   This section contains the basic data types defined by FreeType~2,
        *   ranging from simple scalar types to bitmap descriptors.  More
    -   *   font-specific structures are defined in a different section.
    +   *   font-specific structures are defined in a different section.  Note
    +   *   that FreeType does not use floating-point data types.  Fractional
    +   *   values are represented by fixed-point integers, with lower bits
    +   *   storing the fractional part.
        *
        * @order:
        *   FT_Byte
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/autohint.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/autohint.h
    index aedf48984d433..bf9c8b7cf2a50 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/autohint.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/autohint.h
    @@ -4,7 +4,7 @@
      *
      *   High-level 'autohint' module-specific interface (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/cffotypes.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/cffotypes.h
    index 700f586c41e83..50d535384989d 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/cffotypes.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/cffotypes.h
    @@ -4,7 +4,7 @@
      *
      *   Basic OpenType/CFF object type definitions (specification).
      *
    - * Copyright (C) 2017-2022 by
    + * Copyright (C) 2017-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/cfftypes.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/cfftypes.h
    index 23d26c1b346b3..c2521764caa07 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/cfftypes.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/cfftypes.h
    @@ -5,7 +5,7 @@
      *   Basic OpenType/CFF type definitions and interface (specification
      *   only).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -315,7 +315,7 @@ FT_BEGIN_HEADER
         /* The normal stack then points to these values instead of the DICT   */
         /* because all other operators in Private DICT clear the stack.       */
         /* `blend_stack' could be cleared at each operator other than blend.  */
    -    /* Blended values are stored as 5-byte fixed point values.            */
    +    /* Blended values are stored as 5-byte fixed-point values.            */
     
         FT_Byte*  blend_stack;    /* base of stack allocation     */
         FT_Byte*  blend_top;      /* first empty slot             */
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/compiler-macros.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/compiler-macros.h
    index 66fa13c3c50da..7883317fed98b 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/compiler-macros.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/compiler-macros.h
    @@ -4,7 +4,7 @@
      *
      *   Compiler-specific macro definitions used internally by FreeType.
      *
    - * Copyright (C) 2020-2022 by
    + * Copyright (C) 2020-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -34,6 +34,19 @@ FT_BEGIN_HEADER
     #  if defined( _COMPILER_VERSION ) && ( _COMPILER_VERSION >= 730 )
     #    pragma set woff 3505
     #  endif
    +#endif
    +
    +  /* Newer compilers warn for fall-through case statements. */
    +#ifndef FALL_THROUGH
    +#  if ( defined( __STDC_VERSION__ ) && __STDC_VERSION__ > 201710L ) || \
    +      ( defined( __cplusplus ) && __cplusplus > 201402L )
    +#    define FALL_THROUGH  [[__fallthrough__]]
    +#  elif ( defined( __GNUC__ ) && __GNUC__ >= 7 )          || \
    +        ( defined( __clang__ ) && __clang_major__ >= 10 )
    +#    define FALL_THROUGH  __attribute__(( __fallthrough__ ))
    +#  else
    +#    define FALL_THROUGH  ( (void)0 )
    +#  endif
     #endif
     
       /*
    @@ -258,7 +271,7 @@ FT_BEGIN_HEADER
        * To export a variable, use `FT_EXPORT_VAR`.
        */
     
    -  /* See `freetype/config/compiler_macros.h` for the `FT_EXPORT` definition */
    +  /* See `freetype/config/public-macros.h` for the `FT_EXPORT` definition */
     #define FT_EXPORT_DEF( x )  FT_FUNCTION_DEFINITION( x )
     
       /*
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftcalc.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftcalc.h
    index e6a87db94ee86..d1baa392bd643 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftcalc.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftcalc.h
    @@ -4,7 +4,7 @@
      *
      *   Arithmetic computations (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -278,6 +278,40 @@ FT_BEGIN_HEADER
                           FT_Long  c );
     
     
    +  /**************************************************************************
    +   *
    +   * @function:
    +   *   FT_MulAddFix
    +   *
    +   * @description:
    +   *   Compute `(s[0] * f[0] + s[1] * f[1] + ...) / 0x10000`, where `s[n]` is
    +   *   usually a 16.16 scalar.
    +   *
    +   * @input:
    +   *   s ::
    +   *     The array of scalars.
    +   *   f ::
    +   *     The array of factors.
    +   *   count ::
    +   *     The number of entries in the array.
    +   *
    +   * @return:
    +   *   The result of `(s[0] * f[0] + s[1] * f[1] + ...) / 0x10000`.
    +   *
    +   * @note:
    +   *   This function is currently used for the scaled delta computation of
    +   *   variation stores.  It internally uses 64-bit data types when
    +   *   available, otherwise it emulates 64-bit math by using 32-bit
    +   *   operations, which produce a correct result but most likely at a slower
    +   *   performance in comparison to the implementation base on `int64_t`.
    +   *
    +   */
    +  FT_BASE( FT_Int32 )
    +  FT_MulAddFix( FT_Fixed*  s,
    +                FT_Int32*  f,
    +                FT_UInt    count );
    +
    +
       /*
        * A variant of FT_Matrix_Multiply which scales its result afterwards.  The
        * idea is that both `a' and `b' are scaled by factors of 10 so that the
    @@ -413,11 +447,11 @@ FT_BEGIN_HEADER
       extern __inline FT_Int32
       FT_MSB_i386( FT_UInt32  x );
     
    -#pragma aux FT_MSB_i386 =     \
    -  "bsr eax, eax"              \
    -  parm [eax] nomemory         \
    -  value [eax]                 \
    -  modify exact [eax] nomemory;
    +#pragma aux FT_MSB_i386 =             \
    +  "bsr eax, eax"                      \
    +  __parm [__eax] __nomemory           \
    +  __value [__eax]                     \
    +  __modify __exact [__eax] __nomemory;
     
     #define FT_MSB( x )  FT_MSB_i386( x )
     
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftdebug.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftdebug.h
    index f05b1395cb2c7..4e013ba1e2673 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftdebug.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftdebug.h
    @@ -4,7 +4,7 @@
      *
      *   Debugging and logging component (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftdrv.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftdrv.h
    index 9459a9a190173..f78912ca0c72a 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftdrv.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftdrv.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType internal font driver interface (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftgloadr.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftgloadr.h
    index f73b6631c8aeb..36e5509f9eab6 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftgloadr.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftgloadr.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType glyph loader (specification).
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftmemory.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftmemory.h
    index 10d753aa5e969..5eb1d21ff67c1 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftmemory.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftmemory.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType memory management macros (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -96,15 +96,15 @@ extern "C++"
     
     #ifdef FT_DEBUG_MEMORY
     
    -  FT_BASE( const char* )  _ft_debug_file;
    -  FT_BASE( long )         _ft_debug_lineno;
    +  FT_BASE( const char* )  ft_debug_file_;
    +  FT_BASE( long )         ft_debug_lineno_;
     
    -#define FT_DEBUG_INNER( exp )  ( _ft_debug_file   = __FILE__, \
    -                                 _ft_debug_lineno = __LINE__, \
    +#define FT_DEBUG_INNER( exp )  ( ft_debug_file_   = __FILE__, \
    +                                 ft_debug_lineno_ = __LINE__, \
                                      (exp) )
     
    -#define FT_ASSIGNP_INNER( p, exp )  ( _ft_debug_file   = __FILE__, \
    -                                      _ft_debug_lineno = __LINE__, \
    +#define FT_ASSIGNP_INNER( p, exp )  ( ft_debug_file_   = __FILE__, \
    +                                      ft_debug_lineno_ = __LINE__, \
                                           FT_ASSIGNP( p, exp ) )
     
     #else /* !FT_DEBUG_MEMORY */
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftmmtypes.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftmmtypes.h
    new file mode 100644
    index 0000000000000..b7c66c35deff8
    --- /dev/null
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftmmtypes.h
    @@ -0,0 +1,85 @@
    +/****************************************************************************
    + *
    + * ftmmtypes.h
    + *
    + *   OpenType Variations type definitions for internal use
    + *   with the multi-masters service (specification).
    + *
    + * Copyright (C) 2022-2023 by
    + * David Turner, Robert Wilhelm, Werner Lemberg, George Williams, and
    + * Dominik Röttsches.
    + *
    + * This file is part of the FreeType project, and may only be used,
    + * modified, and distributed under the terms of the FreeType project
    + * license, LICENSE.TXT.  By continuing to use, modify, or distribute
    + * this file you indicate that you have read the license and
    + * understand and accept it fully.
    + *
    + */
    +
    +
    +#ifndef FTMMTYPES_H_
    +#define FTMMTYPES_H_
    +
    +FT_BEGIN_HEADER
    +
    +
    +  typedef FT_Int32  FT_ItemVarDelta;
    +
    +  typedef struct  GX_ItemVarDataRec_
    +  {
    +    FT_UInt            itemCount;       /* number of delta sets per item    */
    +    FT_UInt            regionIdxCount;  /* number of region indices         */
    +    FT_UInt*           regionIndices;   /* array of `regionCount' indices;  */
    +                                        /* these index `varRegionList'      */
    +    FT_ItemVarDelta*   deltaSet;        /* array of `itemCount' deltas      */
    +                                        /* use `innerIndex' for this array  */
    +
    +  } GX_ItemVarDataRec, *GX_ItemVarData;
    +
    +
    +  /* contribution of one axis to a region */
    +  typedef struct  GX_AxisCoordsRec_
    +  {
    +    FT_Fixed  startCoord;
    +    FT_Fixed  peakCoord;      /* zero means no effect (factor = 1) */
    +    FT_Fixed  endCoord;
    +
    +  } GX_AxisCoordsRec, *GX_AxisCoords;
    +
    +
    +  typedef struct  GX_VarRegionRec_
    +  {
    +    GX_AxisCoords  axisList;               /* array of axisCount records */
    +
    +  } GX_VarRegionRec, *GX_VarRegion;
    +
    +
    +  /* item variation store */
    +  typedef struct  GX_ItemVarStoreRec_
    +  {
    +    FT_UInt         dataCount;
    +    GX_ItemVarData  varData;            /* array of dataCount records;     */
    +                                        /* use `outerIndex' for this array */
    +    FT_UShort     axisCount;
    +    FT_UInt       regionCount;          /* total number of regions defined */
    +    GX_VarRegion  varRegionList;
    +
    +  } GX_ItemVarStoreRec, *GX_ItemVarStore;
    +
    +
    +  typedef struct  GX_DeltaSetIdxMapRec_
    +  {
    +    FT_ULong  mapCount;
    +    FT_UInt*  outerIndex;               /* indices to item var data */
    +    FT_UInt*  innerIndex;               /* indices to delta set     */
    +
    +  } GX_DeltaSetIdxMapRec, *GX_DeltaSetIdxMap;
    +
    +
    +FT_END_HEADER
    +
    +#endif /* FTMMTYPES_H_ */
    +
    +
    +/* END */
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftobjs.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftobjs.h
    index 1c779ceaeb26e..28bc9b65f0584 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftobjs.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftobjs.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType private base classes (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftpsprop.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftpsprop.h
    index 47373211cb0db..1d5b287ad2081 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftpsprop.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftpsprop.h
    @@ -4,7 +4,7 @@
      *
      *   Get and set properties of PostScript drivers (specification).
      *
    - * Copyright (C) 2017-2022 by
    + * Copyright (C) 2017-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftrfork.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftrfork.h
    index 165e67f245b18..e96459921ef90 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftrfork.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftrfork.h
    @@ -4,7 +4,7 @@
      *
      *   Embedded resource forks accessor (specification).
      *
    - * Copyright (C) 2004-2022 by
    + * Copyright (C) 2004-2023 by
      * Masatake YAMATO and Redhat K.K.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftserv.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftserv.h
    index 78996d9c852fd..1e85d6d3856a9 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftserv.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftserv.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType services (specification only).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftstream.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftstream.h
    index aa51fe5a8731b..88e19287c8099 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftstream.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftstream.h
    @@ -4,7 +4,7 @@
      *
      *   Stream handling (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -238,42 +238,42 @@ FT_BEGIN_HEADER
     #define FT_NEXT_BYTE( buffer )         \
               ( (unsigned char)*buffer++ )
     
    -#define FT_NEXT_SHORT( buffer )                                   \
    -          ( (short)( buffer += 2, FT_PEEK_SHORT( buffer - 2 ) ) )
    +#define FT_NEXT_SHORT( buffer )                        \
    +          ( buffer += 2, FT_PEEK_SHORT( buffer - 2 ) )
     
    -#define FT_NEXT_USHORT( buffer )                                            \
    -          ( (unsigned short)( buffer += 2, FT_PEEK_USHORT( buffer - 2 ) ) )
    +#define FT_NEXT_USHORT( buffer )                        \
    +          ( buffer += 2, FT_PEEK_USHORT( buffer - 2 ) )
     
    -#define FT_NEXT_OFF3( buffer )                                  \
    -          ( (long)( buffer += 3, FT_PEEK_OFF3( buffer - 3 ) ) )
    +#define FT_NEXT_OFF3( buffer )                        \
    +          ( buffer += 3, FT_PEEK_OFF3( buffer - 3 ) )
     
    -#define FT_NEXT_UOFF3( buffer )                                           \
    -          ( (unsigned long)( buffer += 3, FT_PEEK_UOFF3( buffer - 3 ) ) )
    +#define FT_NEXT_UOFF3( buffer )                        \
    +          ( buffer += 3, FT_PEEK_UOFF3( buffer - 3 ) )
     
    -#define FT_NEXT_LONG( buffer )                                  \
    -          ( (long)( buffer += 4, FT_PEEK_LONG( buffer - 4 ) ) )
    +#define FT_NEXT_LONG( buffer )                        \
    +          ( buffer += 4, FT_PEEK_LONG( buffer - 4 ) )
     
    -#define FT_NEXT_ULONG( buffer )                                           \
    -          ( (unsigned long)( buffer += 4, FT_PEEK_ULONG( buffer - 4 ) ) )
    +#define FT_NEXT_ULONG( buffer )                        \
    +          ( buffer += 4, FT_PEEK_ULONG( buffer - 4 ) )
     
     
    -#define FT_NEXT_SHORT_LE( buffer )                                   \
    -          ( (short)( buffer += 2, FT_PEEK_SHORT_LE( buffer - 2 ) ) )
    +#define FT_NEXT_SHORT_LE( buffer )                        \
    +          ( buffer += 2, FT_PEEK_SHORT_LE( buffer - 2 ) )
     
    -#define FT_NEXT_USHORT_LE( buffer )                                            \
    -          ( (unsigned short)( buffer += 2, FT_PEEK_USHORT_LE( buffer - 2 ) ) )
    +#define FT_NEXT_USHORT_LE( buffer )                        \
    +          ( buffer += 2, FT_PEEK_USHORT_LE( buffer - 2 ) )
     
    -#define FT_NEXT_OFF3_LE( buffer )                                  \
    -          ( (long)( buffer += 3, FT_PEEK_OFF3_LE( buffer - 3 ) ) )
    +#define FT_NEXT_OFF3_LE( buffer )                        \
    +          ( buffer += 3, FT_PEEK_OFF3_LE( buffer - 3 ) )
     
    -#define FT_NEXT_UOFF3_LE( buffer )                                           \
    -          ( (unsigned long)( buffer += 3, FT_PEEK_UOFF3_LE( buffer - 3 ) ) )
    +#define FT_NEXT_UOFF3_LE( buffer )                        \
    +          ( buffer += 3, FT_PEEK_UOFF3_LE( buffer - 3 ) )
     
    -#define FT_NEXT_LONG_LE( buffer )                                  \
    -          ( (long)( buffer += 4, FT_PEEK_LONG_LE( buffer - 4 ) ) )
    +#define FT_NEXT_LONG_LE( buffer )                        \
    +          ( buffer += 4, FT_PEEK_LONG_LE( buffer - 4 ) )
     
    -#define FT_NEXT_ULONG_LE( buffer )                                           \
    -          ( (unsigned long)( buffer += 4, FT_PEEK_ULONG_LE( buffer - 4 ) ) )
    +#define FT_NEXT_ULONG_LE( buffer )                        \
    +          ( buffer += 4, FT_PEEK_ULONG_LE( buffer - 4 ) )
     
     
       /**************************************************************************
    @@ -307,17 +307,17 @@ FT_BEGIN_HEADER
     
     #define FT_GET_CHAR()       FT_GET_MACRO( FT_Stream_GetByte, FT_Char )
     #define FT_GET_BYTE()       FT_GET_MACRO( FT_Stream_GetByte, FT_Byte )
    -#define FT_GET_SHORT()      FT_GET_MACRO( FT_Stream_GetUShort, FT_Short )
    -#define FT_GET_USHORT()     FT_GET_MACRO( FT_Stream_GetUShort, FT_UShort )
    -#define FT_GET_UOFF3()      FT_GET_MACRO( FT_Stream_GetUOffset, FT_ULong )
    -#define FT_GET_LONG()       FT_GET_MACRO( FT_Stream_GetULong, FT_Long )
    -#define FT_GET_ULONG()      FT_GET_MACRO( FT_Stream_GetULong, FT_ULong )
    -#define FT_GET_TAG4()       FT_GET_MACRO( FT_Stream_GetULong, FT_ULong )
    -
    -#define FT_GET_SHORT_LE()   FT_GET_MACRO( FT_Stream_GetUShortLE, FT_Short )
    -#define FT_GET_USHORT_LE()  FT_GET_MACRO( FT_Stream_GetUShortLE, FT_UShort )
    -#define FT_GET_LONG_LE()    FT_GET_MACRO( FT_Stream_GetULongLE, FT_Long )
    -#define FT_GET_ULONG_LE()   FT_GET_MACRO( FT_Stream_GetULongLE, FT_ULong )
    +#define FT_GET_SHORT()      FT_GET_MACRO( FT_Stream_GetUShort, FT_Int16 )
    +#define FT_GET_USHORT()     FT_GET_MACRO( FT_Stream_GetUShort, FT_UInt16 )
    +#define FT_GET_UOFF3()      FT_GET_MACRO( FT_Stream_GetUOffset, FT_UInt32 )
    +#define FT_GET_LONG()       FT_GET_MACRO( FT_Stream_GetULong, FT_Int32 )
    +#define FT_GET_ULONG()      FT_GET_MACRO( FT_Stream_GetULong, FT_UInt32 )
    +#define FT_GET_TAG4()       FT_GET_MACRO( FT_Stream_GetULong, FT_UInt32 )
    +
    +#define FT_GET_SHORT_LE()   FT_GET_MACRO( FT_Stream_GetUShortLE, FT_Int16 )
    +#define FT_GET_USHORT_LE()  FT_GET_MACRO( FT_Stream_GetUShortLE, FT_UInt16 )
    +#define FT_GET_LONG_LE()    FT_GET_MACRO( FT_Stream_GetULongLE, FT_Int32 )
    +#define FT_GET_ULONG_LE()   FT_GET_MACRO( FT_Stream_GetULongLE, FT_UInt32 )
     #endif
     
     
    @@ -334,16 +334,16 @@ FT_BEGIN_HEADER
        */
     #define FT_READ_BYTE( var )       FT_READ_MACRO( FT_Stream_ReadByte, FT_Byte, var )
     #define FT_READ_CHAR( var )       FT_READ_MACRO( FT_Stream_ReadByte, FT_Char, var )
    -#define FT_READ_SHORT( var )      FT_READ_MACRO( FT_Stream_ReadUShort, FT_Short, var )
    -#define FT_READ_USHORT( var )     FT_READ_MACRO( FT_Stream_ReadUShort, FT_UShort, var )
    -#define FT_READ_UOFF3( var )      FT_READ_MACRO( FT_Stream_ReadUOffset, FT_ULong, var )
    -#define FT_READ_LONG( var )       FT_READ_MACRO( FT_Stream_ReadULong, FT_Long, var )
    -#define FT_READ_ULONG( var )      FT_READ_MACRO( FT_Stream_ReadULong, FT_ULong, var )
    +#define FT_READ_SHORT( var )      FT_READ_MACRO( FT_Stream_ReadUShort, FT_Int16, var )
    +#define FT_READ_USHORT( var )     FT_READ_MACRO( FT_Stream_ReadUShort, FT_UInt16, var )
    +#define FT_READ_UOFF3( var )      FT_READ_MACRO( FT_Stream_ReadUOffset, FT_UInt32, var )
    +#define FT_READ_LONG( var )       FT_READ_MACRO( FT_Stream_ReadULong, FT_Int32, var )
    +#define FT_READ_ULONG( var )      FT_READ_MACRO( FT_Stream_ReadULong, FT_UInt32, var )
     
    -#define FT_READ_SHORT_LE( var )   FT_READ_MACRO( FT_Stream_ReadUShortLE, FT_Short, var )
    -#define FT_READ_USHORT_LE( var )  FT_READ_MACRO( FT_Stream_ReadUShortLE, FT_UShort, var )
    -#define FT_READ_LONG_LE( var )    FT_READ_MACRO( FT_Stream_ReadULongLE, FT_Long, var )
    -#define FT_READ_ULONG_LE( var )   FT_READ_MACRO( FT_Stream_ReadULongLE, FT_ULong, var )
    +#define FT_READ_SHORT_LE( var )   FT_READ_MACRO( FT_Stream_ReadUShortLE, FT_Int16, var )
    +#define FT_READ_USHORT_LE( var )  FT_READ_MACRO( FT_Stream_ReadUShortLE, FT_UInt16, var )
    +#define FT_READ_LONG_LE( var )    FT_READ_MACRO( FT_Stream_ReadULongLE, FT_Int32, var )
    +#define FT_READ_ULONG_LE( var )   FT_READ_MACRO( FT_Stream_ReadULongLE, FT_UInt32, var )
     
     
     #ifndef FT_CONFIG_OPTION_NO_DEFAULT_SYSTEM
    @@ -459,23 +459,23 @@ FT_BEGIN_HEADER
       FT_Stream_GetByte( FT_Stream  stream );
     
       /* read a 16-bit big-endian unsigned integer from an entered frame */
    -  FT_BASE( FT_UShort )
    +  FT_BASE( FT_UInt16 )
       FT_Stream_GetUShort( FT_Stream  stream );
     
       /* read a 24-bit big-endian unsigned integer from an entered frame */
    -  FT_BASE( FT_ULong )
    +  FT_BASE( FT_UInt32 )
       FT_Stream_GetUOffset( FT_Stream  stream );
     
       /* read a 32-bit big-endian unsigned integer from an entered frame */
    -  FT_BASE( FT_ULong )
    +  FT_BASE( FT_UInt32 )
       FT_Stream_GetULong( FT_Stream  stream );
     
       /* read a 16-bit little-endian unsigned integer from an entered frame */
    -  FT_BASE( FT_UShort )
    +  FT_BASE( FT_UInt16 )
       FT_Stream_GetUShortLE( FT_Stream  stream );
     
       /* read a 32-bit little-endian unsigned integer from an entered frame */
    -  FT_BASE( FT_ULong )
    +  FT_BASE( FT_UInt32 )
       FT_Stream_GetULongLE( FT_Stream  stream );
     
     
    @@ -485,7 +485,7 @@ FT_BEGIN_HEADER
                           FT_Error*  error );
     
       /* read a 16-bit big-endian unsigned integer from a stream */
    -  FT_BASE( FT_UShort )
    +  FT_BASE( FT_UInt16 )
       FT_Stream_ReadUShort( FT_Stream  stream,
                             FT_Error*  error );
     
    @@ -495,17 +495,17 @@ FT_BEGIN_HEADER
                              FT_Error*  error );
     
       /* read a 32-bit big-endian integer from a stream */
    -  FT_BASE( FT_ULong )
    +  FT_BASE( FT_UInt32 )
       FT_Stream_ReadULong( FT_Stream  stream,
                            FT_Error*  error );
     
       /* read a 16-bit little-endian unsigned integer from a stream */
    -  FT_BASE( FT_UShort )
    +  FT_BASE( FT_UInt16 )
       FT_Stream_ReadUShortLE( FT_Stream  stream,
                               FT_Error*  error );
     
       /* read a 32-bit little-endian unsigned integer from a stream */
    -  FT_BASE( FT_ULong )
    +  FT_BASE( FT_UInt32 )
       FT_Stream_ReadULongLE( FT_Stream  stream,
                              FT_Error*  error );
     
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/fttrace.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/fttrace.h
    index 43c6a8713b9a3..319fe56fd2d37 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/fttrace.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/fttrace.h
    @@ -4,7 +4,7 @@
      *
      *   Tracing handling (specification only).
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftvalid.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftvalid.h
    index 171c2cb6f57e2..e98ee4e473738 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftvalid.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/ftvalid.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType validation support (specification).
      *
    - * Copyright (C) 2004-2022 by
    + * Copyright (C) 2004-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/psaux.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/psaux.h
    index 48ec1df963dec..dfb1987f86897 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/psaux.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/psaux.h
    @@ -5,7 +5,7 @@
      *   Auxiliary functions and data structures related to PostScript fonts
      *   (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -132,9 +132,6 @@ FT_BEGIN_HEADER
        *   max_elems ::
        *     The maximum number of elements in table.
        *
    -   *   num_elems ::
    -   *     The current number of elements in table.
    -   *
        *   elements ::
        *     A table of element addresses within the block.
        *
    @@ -155,7 +152,6 @@ FT_BEGIN_HEADER
         FT_ULong           init;
     
         FT_Int             max_elems;
    -    FT_Int             num_elems;
         FT_Byte**          elements;       /* addresses of table elements */
         FT_UInt*           lengths;        /* lengths of table elements   */
     
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/pshints.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/pshints.h
    index 5de83e45657ea..ededc4c72e7a1 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/pshints.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/pshints.h
    @@ -6,7 +6,7 @@
      *   recorders (specification only).  These are used to support native
      *   T1/T2 hints in the 'type1', 'cid', and 'cff' font drivers.
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -294,7 +294,7 @@ FT_BEGIN_HEADER
        *
        * @note:
        *   On input, all points within the outline are in font coordinates. On
    -   *   output, they are in 1/64th of pixels.
    +   *   output, they are in 1/64 of pixels.
        *
        *   The scaling transformation is taken from the 'globals' object which
        *   must correspond to the same font as the glyph.
    @@ -607,7 +607,7 @@ FT_BEGIN_HEADER
        *
        * @note:
        *   On input, all points within the outline are in font coordinates. On
    -   *   output, they are in 1/64th of pixels.
    +   *   output, they are in 1/64 of pixels.
        *
        *   The scaling transformation is taken from the 'globals' object which
        *   must correspond to the same font than the glyph.
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svbdf.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svbdf.h
    index 06e3b531c8748..bf0c1dcc71457 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svbdf.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svbdf.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType BDF services (specification).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svcfftl.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svcfftl.h
    index 1dea6bcda9716..4a20498ee0cfe 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svcfftl.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svcfftl.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType CFF tables loader service (specification).
      *
    - * Copyright (C) 2017-2022 by
    + * Copyright (C) 2017-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svcid.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svcid.h
    index acf9178d0a87c..06d0cb8fd62ea 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svcid.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svcid.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType CID font services (specification).
      *
    - * Copyright (C) 2007-2022 by
    + * Copyright (C) 2007-2023 by
      * Derek Clegg and Michael Toftdal.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svfntfmt.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svfntfmt.h
    index a7280319c5d46..bc45e80568fc7 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svfntfmt.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svfntfmt.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType font format service (specification only).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svgldict.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svgldict.h
    index 489021d897117..6437abfbf2e8b 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svgldict.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svgldict.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType glyph dictionary services (specification).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svgxval.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svgxval.h
    index 59ae411b55da1..31016afe0d040 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svgxval.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svgxval.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType API for validating TrueTypeGX/AAT tables (specification).
      *
    - * Copyright (C) 2004-2022 by
    + * Copyright (C) 2004-2023 by
      * Masatake YAMATO, Red Hat K.K.,
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svkern.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svkern.h
    index c567acad46d5d..bcabbc3e68fde 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svkern.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svkern.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType Kerning service (specification).
      *
    - * Copyright (C) 2006-2022 by
    + * Copyright (C) 2006-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svmetric.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svmetric.h
    index 7accdc46ff716..e588ea4872a14 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svmetric.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svmetric.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType services for metrics variations (specification).
      *
    - * Copyright (C) 2016-2022 by
    + * Copyright (C) 2016-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svmm.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svmm.h
    index c6394890ac66f..d94204232e1a3 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svmm.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svmm.h
    @@ -4,8 +4,8 @@
      *
      *   The FreeType Multiple Masters and GX var services (specification).
      *
    - * Copyright (C) 2003-2022 by
    - * David Turner, Robert Wilhelm, and Werner Lemberg.
    + * Copyright (C) 2003-2023 by
    + * David Turner, Robert Wilhelm, Werner Lemberg, and Dominik Röttsches.
      *
      * This file is part of the FreeType project, and may only be used,
      * modified, and distributed under the terms of the FreeType project
    @@ -19,7 +19,9 @@
     #ifndef SVMM_H_
     #define SVMM_H_
     
    +#include 
     #include 
    +#include 
     
     
     FT_BEGIN_HEADER
    @@ -96,53 +98,94 @@ FT_BEGIN_HEADER
                                       FT_UInt*   len,
                                       FT_Fixed*  weight_vector );
     
    +  typedef FT_Error
    +  (*FT_Var_Load_Delta_Set_Idx_Map_Func)( FT_Face            face,
    +                                         FT_ULong           offset,
    +                                         GX_DeltaSetIdxMap  map,
    +                                         GX_ItemVarStore    itemStore,
    +                                         FT_ULong           table_len );
    +
    +  typedef FT_Error
    +  (*FT_Var_Load_Item_Var_Store_Func)( FT_Face          face,
    +                                      FT_ULong         offset,
    +                                      GX_ItemVarStore  itemStore );
    +
    +  typedef FT_ItemVarDelta
    +  (*FT_Var_Get_Item_Delta_Func)( FT_Face          face,
    +                                 GX_ItemVarStore  itemStore,
    +                                 FT_UInt          outerIndex,
    +                                 FT_UInt          innerIndex );
    +
    +  typedef void
    +  (*FT_Var_Done_Item_Var_Store_Func)( FT_Face          face,
    +                                      GX_ItemVarStore  itemStore );
    +
    +  typedef void
    +  (*FT_Var_Done_Delta_Set_Idx_Map_Func)( FT_Face            face,
    +                                         GX_DeltaSetIdxMap  deltaSetIdxMap );
    +
     
       FT_DEFINE_SERVICE( MultiMasters )
       {
    -    FT_Get_MM_Func               get_mm;
    -    FT_Set_MM_Design_Func        set_mm_design;
    -    FT_Set_MM_Blend_Func         set_mm_blend;
    -    FT_Get_MM_Blend_Func         get_mm_blend;
    -    FT_Get_MM_Var_Func           get_mm_var;
    -    FT_Set_Var_Design_Func       set_var_design;
    -    FT_Get_Var_Design_Func       get_var_design;
    -    FT_Set_Instance_Func         set_instance;
    -    FT_Set_MM_WeightVector_Func  set_mm_weightvector;
    -    FT_Get_MM_WeightVector_Func  get_mm_weightvector;
    +    FT_Get_MM_Func                        get_mm;
    +    FT_Set_MM_Design_Func                 set_mm_design;
    +    FT_Set_MM_Blend_Func                  set_mm_blend;
    +    FT_Get_MM_Blend_Func                  get_mm_blend;
    +    FT_Get_MM_Var_Func                    get_mm_var;
    +    FT_Set_Var_Design_Func                set_var_design;
    +    FT_Get_Var_Design_Func                get_var_design;
    +    FT_Set_Instance_Func                  set_instance;
    +    FT_Set_MM_WeightVector_Func           set_mm_weightvector;
    +    FT_Get_MM_WeightVector_Func           get_mm_weightvector;
     
         /* for internal use; only needed for code sharing between modules */
    -    FT_Get_Var_Blend_Func  get_var_blend;
    -    FT_Done_Blend_Func     done_blend;
    +    FT_Var_Load_Delta_Set_Idx_Map_Func    load_delta_set_idx_map;
    +    FT_Var_Load_Item_Var_Store_Func       load_item_var_store;
    +    FT_Var_Get_Item_Delta_Func            get_item_delta;
    +    FT_Var_Done_Item_Var_Store_Func       done_item_var_store;
    +    FT_Var_Done_Delta_Set_Idx_Map_Func    done_delta_set_idx_map;
    +    FT_Get_Var_Blend_Func                 get_var_blend;
    +    FT_Done_Blend_Func                    done_blend;
       };
     
     
    -#define FT_DEFINE_SERVICE_MULTIMASTERSREC( class_,            \
    -                                           get_mm_,           \
    -                                           set_mm_design_,    \
    -                                           set_mm_blend_,     \
    -                                           get_mm_blend_,     \
    -                                           get_mm_var_,       \
    -                                           set_var_design_,   \
    -                                           get_var_design_,   \
    -                                           set_instance_,     \
    -                                           set_weightvector_, \
    -                                           get_weightvector_, \
    -                                           get_var_blend_,    \
    -                                           done_blend_ )      \
    -  static const FT_Service_MultiMastersRec  class_ =           \
    -  {                                                           \
    -    get_mm_,                                                  \
    -    set_mm_design_,                                           \
    -    set_mm_blend_,                                            \
    -    get_mm_blend_,                                            \
    -    get_mm_var_,                                              \
    -    set_var_design_,                                          \
    -    get_var_design_,                                          \
    -    set_instance_,                                            \
    -    set_weightvector_,                                        \
    -    get_weightvector_,                                        \
    -    get_var_blend_,                                           \
    -    done_blend_                                               \
    +#define FT_DEFINE_SERVICE_MULTIMASTERSREC( class_,                  \
    +                                           get_mm_,                 \
    +                                           set_mm_design_,          \
    +                                           set_mm_blend_,           \
    +                                           get_mm_blend_,           \
    +                                           get_mm_var_,             \
    +                                           set_var_design_,         \
    +                                           get_var_design_,         \
    +                                           set_instance_,           \
    +                                           set_weightvector_,       \
    +                                           get_weightvector_,       \
    +                                           load_delta_set_idx_map_, \
    +                                           load_item_var_store_,    \
    +                                           get_item_delta_,         \
    +                                           done_item_var_store_,    \
    +                                           done_delta_set_idx_map_, \
    +                                           get_var_blend_,          \
    +                                           done_blend_ )            \
    +  static const FT_Service_MultiMastersRec  class_ =                 \
    +  {                                                                 \
    +    get_mm_,                                                        \
    +    set_mm_design_,                                                 \
    +    set_mm_blend_,                                                  \
    +    get_mm_blend_,                                                  \
    +    get_mm_var_,                                                    \
    +    set_var_design_,                                                \
    +    get_var_design_,                                                \
    +    set_instance_,                                                  \
    +    set_weightvector_,                                              \
    +    get_weightvector_,                                              \
    +    load_delta_set_idx_map_,                                        \
    +    load_item_var_store_,                                           \
    +    get_item_delta_,                                                \
    +    done_item_var_store_,                                           \
    +    done_delta_set_idx_map_,                                        \
    +    get_var_blend_,                                                 \
    +    done_blend_                                                     \
       };
     
       /* */
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svotval.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svotval.h
    index 3c72d1f85574e..a4683cd5fb648 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svotval.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svotval.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType OpenType validation service (specification).
      *
    - * Copyright (C) 2004-2022 by
    + * Copyright (C) 2004-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svpfr.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svpfr.h
    index bde0ed3545b2d..fd189c7de773b 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svpfr.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svpfr.h
    @@ -4,7 +4,7 @@
      *
      *   Internal PFR service functions (specification).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svpostnm.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svpostnm.h
    index 05f6291e138ab..2b8f6dfecfb02 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svpostnm.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svpostnm.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType PostScript name services (specification).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svprop.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svprop.h
    index 29c568640bc0c..932ce32e03d78 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svprop.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svprop.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType property service (specification).
      *
    - * Copyright (C) 2012-2022 by
    + * Copyright (C) 2012-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svpscmap.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svpscmap.h
    index 7d586587a59c5..fd99d857e47d3 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svpscmap.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svpscmap.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType PostScript charmap service (specification).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svpsinfo.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svpsinfo.h
    index 6e45f3272d399..09c4cdccc5342 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svpsinfo.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svpsinfo.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType PostScript info service (specification).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svsfnt.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svsfnt.h
    index 03938a562b607..f98df2ef5fe70 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svsfnt.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svsfnt.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType SFNT table loading service (specification).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svttcmap.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svttcmap.h
    index a0b1bbd2f3d17..5f9eb02d66558 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svttcmap.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svttcmap.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType TrueType/sfnt cmap extra information service.
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * Masatake YAMATO, Redhat K.K.,
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svtteng.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svtteng.h
    index f8396eb08cf0e..ad577cb2904d5 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svtteng.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svtteng.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType TrueType engine query service (specification).
      *
    - * Copyright (C) 2006-2022 by
    + * Copyright (C) 2006-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svttglyf.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svttglyf.h
    index 982630c0aab75..ca6fff74444b3 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svttglyf.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svttglyf.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType TrueType glyph service.
      *
    - * Copyright (C) 2007-2022 by
    + * Copyright (C) 2007-2023 by
      * David Turner.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svwinfnt.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svwinfnt.h
    index 950f4a8824cab..002923f8c914a 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svwinfnt.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/services/svwinfnt.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType Windows FNT/FONT service (specification).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/sfnt.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/sfnt.h
    index c67b47e86060a..a2d4e15baafde 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/sfnt.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/sfnt.h
    @@ -4,7 +4,7 @@
      *
      *   High-level 'sfnt' driver interface (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/svginterface.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/svginterface.h
    index 1b325e5e9d1f5..f464b2c0583a8 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/svginterface.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/svginterface.h
    @@ -4,7 +4,7 @@
      *
      *   Interface of ot-svg module (specification only).
      *
    - * Copyright (C) 2022 by
    + * Copyright (C) 2022-2023 by
      * David Turner, Robert Wilhelm, Werner Lemberg, and Moazin Khatti.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/t1types.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/t1types.h
    index b6a3de14d0d56..5a105c5879534 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/t1types.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/t1types.h
    @@ -5,7 +5,7 @@
      *   Basic Type1/Type2 type definitions and interface (specification
      *   only).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -172,8 +172,8 @@ FT_BEGIN_HEADER
       {
         FT_Bool        IsCIDFont;
         FT_BBox        FontBBox;
    -    FT_Fixed       Ascender;
    -    FT_Fixed       Descender;
    +    FT_Fixed       Ascender;     /* optional, mind the zero */
    +    FT_Fixed       Descender;    /* optional, mind the zero */
         AFM_TrackKern  TrackKerns;   /* free if non-NULL */
         FT_UInt        NumTrackKern;
         AFM_KernPair   KernPairs;    /* free if non-NULL */
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/tttypes.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/tttypes.h
    index df719387b5807..3b521924ca5b5 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/tttypes.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/tttypes.h
    @@ -5,7 +5,7 @@
      *   Basic SFNT/TrueType type definitions and interface (specification
      *   only).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/internal/wofftypes.h b/src/java.desktop/share/native/libfreetype/include/freetype/internal/wofftypes.h
    index 94804fa72ffc1..0c1d8eeaf8c8b 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/internal/wofftypes.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/internal/wofftypes.h
    @@ -5,7 +5,7 @@
      *   Basic WOFF/WOFF2 type definitions and interface (specification
      *   only).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/otsvg.h b/src/java.desktop/share/native/libfreetype/include/freetype/otsvg.h
    index 2caadfdeeb64c..bfe9a6ab74e47 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/otsvg.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/otsvg.h
    @@ -4,7 +4,7 @@
      *
      *   Interface for OT-SVG support related things (specification).
      *
    - * Copyright (C) 2022 by
    + * Copyright (C) 2022-2023 by
      * David Turner, Robert Wilhelm, Werner Lemberg, and Moazin Khatti.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/t1tables.h b/src/java.desktop/share/native/libfreetype/include/freetype/t1tables.h
    index 4068b204a930d..1aecfbbd902f2 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/t1tables.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/t1tables.h
    @@ -5,7 +5,7 @@
      *   Basic Type 1/Type 2 tables definitions and interface (specification
      *   only).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/ttnameid.h b/src/java.desktop/share/native/libfreetype/include/freetype/ttnameid.h
    index 37b505a05bdc5..e31c68b9baf34 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/ttnameid.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/ttnameid.h
    @@ -4,7 +4,7 @@
      *
      *   TrueType name ID definitions (specification only).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/tttables.h b/src/java.desktop/share/native/libfreetype/include/freetype/tttables.h
    index 21664df7b34ef..a9f60e76201b1 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/tttables.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/tttables.h
    @@ -5,7 +5,7 @@
      *   Basic SFNT/TrueType tables definitions and interface
      *   (specification only).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -424,8 +424,8 @@ FT_BEGIN_HEADER
     
         /* only version 5 and higher: */
     
    -    FT_UShort  usLowerOpticalPointSize;       /* in twips (1/20th points) */
    -    FT_UShort  usUpperOpticalPointSize;       /* in twips (1/20th points) */
    +    FT_UShort  usLowerOpticalPointSize;       /* in twips (1/20 points) */
    +    FT_UShort  usUpperOpticalPointSize;       /* in twips (1/20 points) */
     
       } TT_OS2;
     
    diff --git a/src/java.desktop/share/native/libfreetype/include/freetype/tttags.h b/src/java.desktop/share/native/libfreetype/include/freetype/tttags.h
    index 8b807641b8ba9..9bf4fca23fb3a 100644
    --- a/src/java.desktop/share/native/libfreetype/include/freetype/tttags.h
    +++ b/src/java.desktop/share/native/libfreetype/include/freetype/tttags.h
    @@ -4,7 +4,7 @@
      *
      *   Tags for TrueType and OpenType tables (specification only).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/include/ft2build.h b/src/java.desktop/share/native/libfreetype/include/ft2build.h
    index 2543ac435a621..58491ceea1fb8 100644
    --- a/src/java.desktop/share/native/libfreetype/include/ft2build.h
    +++ b/src/java.desktop/share/native/libfreetype/include/ft2build.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType 2 build and setup macros.
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afblue.c b/src/java.desktop/share/native/libfreetype/src/autofit/afblue.c
    index b986eb4a13260..d7655b9b99e16 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afblue.c
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afblue.c
    @@ -7,7 +7,7 @@
      *
      *   Auto-fitter data for blue strings (body).
      *
    - * Copyright (C) 2013-2022 by
    + * Copyright (C) 2013-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afblue.cin b/src/java.desktop/share/native/libfreetype/src/autofit/afblue.cin
    index f7e27ad8e5a10..d561c5093b78b 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afblue.cin
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afblue.cin
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter data for blue strings (body).
      *
    - * Copyright (C) 2013-2022 by
    + * Copyright (C) 2013-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afblue.dat b/src/java.desktop/share/native/libfreetype/src/autofit/afblue.dat
    index 201acc4f6fb4e..b7efe8be6ce71 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afblue.dat
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afblue.dat
    @@ -2,7 +2,7 @@
     //
     //   Auto-fitter data for blue strings.
     //
    -// Copyright (C) 2013-2022 by
    +// Copyright (C) 2013-2023 by
     // David Turner, Robert Wilhelm, and Werner Lemberg.
     //
     // This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afblue.h b/src/java.desktop/share/native/libfreetype/src/autofit/afblue.h
    index 0e56abb94d78d..76f2f47cb0083 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afblue.h
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afblue.h
    @@ -7,7 +7,7 @@
      *
      *   Auto-fitter data for blue strings (specification).
      *
    - * Copyright (C) 2013-2022 by
    + * Copyright (C) 2013-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afblue.hin b/src/java.desktop/share/native/libfreetype/src/autofit/afblue.hin
    index f9fd5aa3b4442..6a31298e65fde 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afblue.hin
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afblue.hin
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter data for blue strings (specification).
      *
    - * Copyright (C) 2013-2022 by
    + * Copyright (C) 2013-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afcjk.c b/src/java.desktop/share/native/libfreetype/src/autofit/afcjk.c
    index 1853a17f5c36d..5daefff359cb7 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afcjk.c
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afcjk.c
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter hinting routines for CJK writing system (body).
      *
    - * Copyright (C) 2006-2022 by
    + * Copyright (C) 2006-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -650,7 +650,7 @@
           af_cjk_metrics_check_digits( metrics, face );
         }
     
    -    FT_Set_Charmap( face, oldmap );
    +    face->charmap = oldmap;
         return FT_Err_Ok;
       }
     
    @@ -741,9 +741,11 @@
                         ( dim == AF_DIMENSION_HORZ ) ? 'H' : 'V',
                         nn, blue->ref.org, blue->shoot.org ));
             FT_TRACE5(( "     ref:   cur=%.2f fit=%.2f\n",
    -                    blue->ref.cur / 64.0, blue->ref.fit / 64.0 ));
    +                    (double)blue->ref.cur / 64,
    +                    (double)blue->ref.fit / 64 ));
             FT_TRACE5(( "     shoot: cur=%.2f fit=%.2f\n",
    -                    blue->shoot.cur / 64.0, blue->shoot.fit / 64.0 ));
    +                    (double)blue->shoot.cur / 64,
    +                    (double)blue->shoot.fit / 64 ));
     
             blue->flags |= AF_CJK_BLUE_ACTIVE;
           }
    @@ -1044,7 +1046,7 @@
         {
           AF_Edge  found = NULL;
           FT_Pos   best  = 0xFFFFU;
    -      FT_Int   ee;
    +      FT_UInt  ee;
     
     
           /* look for an edge corresponding to the segment */
    @@ -1629,8 +1631,10 @@
         FT_TRACE5(( "  CJKLINK: edge %ld @%d (opos=%.2f) linked to %.2f,"
                     " dist was %.2f, now %.2f\n",
                     stem_edge - hints->axis[dim].edges, stem_edge->fpos,
    -                stem_edge->opos / 64.0, stem_edge->pos / 64.0,
    -                dist / 64.0, fitted_width / 64.0 ));
    +                (double)stem_edge->opos / 64,
    +                (double)stem_edge->pos / 64,
    +                (double)dist / 64,
    +                (double)fitted_width / 64 ));
       }
     
     
    @@ -1850,8 +1854,8 @@
     #ifdef FT_DEBUG_LEVEL_TRACE
             FT_TRACE5(( "  CJKBLUE: edge %ld @%d (opos=%.2f) snapped to %.2f,"
                         " was %.2f\n",
    -                    edge1 - edges, edge1->fpos, edge1->opos / 64.0,
    -                    blue->fit / 64.0, edge1->pos / 64.0 ));
    +                    edge1 - edges, edge1->fpos, (double)edge1->opos / 64,
    +                    (double)blue->fit / 64, (double)edge1->pos / 64 ));
     
             num_actions++;
     #endif
    @@ -2024,8 +2028,8 @@
     #if 0
           printf( "stem (%d,%d) adjusted (%.1f,%.1f)\n",
                    edge - edges, edge2 - edges,
    -               ( edge->pos - edge->opos ) / 64.0,
    -               ( edge2->pos - edge2->opos ) / 64.0 );
    +               (double)( edge->pos - edge->opos ) / 64,
    +               (double)( edge2->pos - edge2->opos ) / 64 );
     #endif
     
           anchor = edge;
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afcjk.h b/src/java.desktop/share/native/libfreetype/src/autofit/afcjk.h
    index bf948bcec0cae..bd7b81b3e24b4 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afcjk.h
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afcjk.h
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter hinting routines for CJK writing system (specification).
      *
    - * Copyright (C) 2006-2022 by
    + * Copyright (C) 2006-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afcover.h b/src/java.desktop/share/native/libfreetype/src/autofit/afcover.h
    index be71fe39deb03..102ed42782872 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afcover.h
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afcover.h
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter coverages (specification only).
      *
    - * Copyright (C) 2013-2022 by
    + * Copyright (C) 2013-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afdummy.c b/src/java.desktop/share/native/libfreetype/src/autofit/afdummy.c
    index 5fdbfcfd42983..a4629b528dc63 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afdummy.c
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afdummy.c
    @@ -5,7 +5,7 @@
      *   Auto-fitter dummy routines to be used if no hinting should be
      *   performed (body).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afdummy.h b/src/java.desktop/share/native/libfreetype/src/autofit/afdummy.h
    index 4dddbd5215da5..a7af3f62c9eca 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afdummy.h
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afdummy.h
    @@ -5,7 +5,7 @@
      *   Auto-fitter dummy routines to be used if no hinting should be
      *   performed (specification).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/aferrors.h b/src/java.desktop/share/native/libfreetype/src/autofit/aferrors.h
    index d31b1a9c8825b..88faf05c95005 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/aferrors.h
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/aferrors.h
    @@ -4,7 +4,7 @@
      *
      *   Autofitter error codes (specification only).
      *
    - * Copyright (C) 2005-2022 by
    + * Copyright (C) 2005-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afglobal.c b/src/java.desktop/share/native/libfreetype/src/autofit/afglobal.c
    index 87a3fbfb0fe85..ede27eb16609f 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afglobal.c
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afglobal.c
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter routines to compute global hinting values (body).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -129,13 +129,13 @@
         FT_Face     face        = globals->face;
         FT_CharMap  old_charmap = face->charmap;
         FT_UShort*  gstyles     = globals->glyph_styles;
    -    FT_UInt     ss;
    +    FT_UShort   ss;
    +    FT_UShort   dflt        = 0xFFFFU; /* a non-valid value */
         FT_UInt     i;
    -    FT_UInt     dflt        = ~0U; /* a non-valid value */
     
     
         /* the value AF_STYLE_UNASSIGNED means `uncovered glyph' */
    -    for ( i = 0; i < (FT_UInt)globals->glyph_count; i++ )
    +    for ( i = 0; i < globals->glyph_count; i++ )
           gstyles[i] = AF_STYLE_UNASSIGNED;
     
         error = FT_Select_Charmap( face, FT_ENCODING_UNICODE );
    @@ -168,8 +168,7 @@
            */
           if ( style_class->coverage == AF_COVERAGE_DEFAULT )
           {
    -        if ( (FT_UInt)style_class->script ==
    -             globals->module->default_script )
    +        if ( style_class->script == globals->module->default_script )
               dflt = ss;
     
             for ( range = script_class->script_uni_ranges;
    @@ -183,9 +182,9 @@
               gindex = FT_Get_Char_Index( face, charcode );
     
               if ( gindex != 0                                                &&
    -               gindex < (FT_ULong)globals->glyph_count                    &&
    +               gindex < globals->glyph_count                              &&
                    ( gstyles[gindex] & AF_STYLE_MASK ) == AF_STYLE_UNASSIGNED )
    -            gstyles[gindex] = (FT_UShort)ss;
    +            gstyles[gindex] = ss;
     
               for (;;)
               {
    @@ -194,9 +193,9 @@
                 if ( gindex == 0 || charcode > range->last )
                   break;
     
    -            if ( gindex < (FT_ULong)globals->glyph_count                    &&
    +            if ( gindex < globals->glyph_count                              &&
                      ( gstyles[gindex] & AF_STYLE_MASK ) == AF_STYLE_UNASSIGNED )
    -              gstyles[gindex] = (FT_UShort)ss;
    +              gstyles[gindex] = ss;
               }
             }
     
    @@ -211,9 +210,9 @@
     
               gindex = FT_Get_Char_Index( face, charcode );
     
    -          if ( gindex != 0                                          &&
    -               gindex < (FT_ULong)globals->glyph_count              &&
    -               ( gstyles[gindex] & AF_STYLE_MASK ) == (FT_UShort)ss )
    +          if ( gindex != 0                               &&
    +               gindex < globals->glyph_count             &&
    +               ( gstyles[gindex] & AF_STYLE_MASK ) == ss )
                 gstyles[gindex] |= AF_NONBASE;
     
               for (;;)
    @@ -223,8 +222,8 @@
                 if ( gindex == 0 || charcode > range->last )
                   break;
     
    -            if ( gindex < (FT_ULong)globals->glyph_count              &&
    -                 ( gstyles[gindex] & AF_STYLE_MASK ) == (FT_UShort)ss )
    +            if ( gindex < globals->glyph_count             &&
    +                 ( gstyles[gindex] & AF_STYLE_MASK ) == ss )
                   gstyles[gindex] |= AF_NONBASE;
               }
             }
    @@ -255,7 +254,7 @@
           FT_UInt  gindex = FT_Get_Char_Index( face, i );
     
     
    -      if ( gindex != 0 && gindex < (FT_ULong)globals->glyph_count )
    +      if ( gindex != 0 && gindex < globals->glyph_count )
             gstyles[gindex] |= AF_DIGIT;
         }
     
    @@ -266,7 +265,7 @@
          */
         if ( globals->module->fallback_style != AF_STYLE_UNASSIGNED )
         {
    -      FT_Long  nn;
    +      FT_UInt  nn;
     
     
           for ( nn = 0; nn < globals->glyph_count; nn++ )
    @@ -290,7 +289,7 @@
         {
           AF_StyleClass  style_class = af_style_classes[ss];
           FT_UInt        count       = 0;
    -      FT_Long        idx;
    +      FT_UInt        idx;
     
     
           FT_TRACE4(( "%s:\n", af_style_names[style_class->style] ));
    @@ -302,7 +301,7 @@
               if ( !( count % 10 ) )
                 FT_TRACE4(( " " ));
     
    -          FT_TRACE4(( " %ld", idx ));
    +          FT_TRACE4(( " %d", idx ));
               count++;
     
               if ( !( count % 10 ) )
    @@ -318,7 +317,7 @@
     
     #endif /* FT_DEBUG_LEVEL_TRACE */
     
    -    FT_Set_Charmap( face, old_charmap );
    +    face->charmap = old_charmap;
         return error;
       }
     
    @@ -345,7 +344,7 @@
         FT_ZERO( &globals->metrics );
     
         globals->face                      = face;
    -    globals->glyph_count               = face->num_glyphs;
    +    globals->glyph_count               = (FT_UInt)face->num_glyphs;
         /* right after the globals structure come the glyph styles */
         globals->glyph_styles              = (FT_UShort*)( globals + 1 );
         globals->module                    = module;
    @@ -357,7 +356,7 @@
         globals->scale_down_factor         = 0;
     
     #ifdef FT_CONFIG_OPTION_USE_HARFBUZZ
    -    globals->hb_font = hb_ft_font_create( face, NULL );
    +    globals->hb_font = hb_ft_font_create_( face, NULL );
         globals->hb_buf  = hb_buffer_create();
     #endif
     
    @@ -429,7 +428,7 @@
         FT_Error  error = FT_Err_Ok;
     
     
    -    if ( gindex >= (FT_ULong)globals->glyph_count )
    +    if ( gindex >= globals->glyph_count )
         {
           error = FT_THROW( Invalid_Argument );
           goto Exit;
    @@ -501,7 +500,7 @@
       af_face_globals_is_digit( AF_FaceGlobals  globals,
                                 FT_UInt         gindex )
       {
    -    if ( gindex < (FT_ULong)globals->glyph_count )
    +    if ( gindex < globals->glyph_count )
           return FT_BOOL( globals->glyph_styles[gindex] & AF_DIGIT );
     
         return FT_BOOL( 0 );
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afglobal.h b/src/java.desktop/share/native/libfreetype/src/autofit/afglobal.h
    index f7ebf8d57a266..83a7c2ff15bc0 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afglobal.h
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afglobal.h
    @@ -5,7 +5,7 @@
      *   Auto-fitter routines to compute global hinting values
      *   (specification).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -105,7 +105,7 @@ FT_BEGIN_HEADER
       typedef struct  AF_FaceGlobalsRec_
       {
         FT_Face          face;
    -    FT_Long          glyph_count;    /* same as face->num_glyphs */
    +    FT_UInt          glyph_count;    /* unsigned face->num_glyphs */
         FT_UShort*       glyph_styles;
     
     #ifdef FT_CONFIG_OPTION_USE_HARFBUZZ
    @@ -158,7 +158,7 @@ FT_BEGIN_HEADER
       FT_LOCAL( void )
       af_face_globals_free( AF_FaceGlobals  globals );
     
    -  FT_LOCAL_DEF( FT_Bool )
    +  FT_LOCAL( FT_Bool )
       af_face_globals_is_digit( AF_FaceGlobals  globals,
                                 FT_UInt         gindex );
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afhints.c b/src/java.desktop/share/native/libfreetype/src/autofit/afhints.c
    index ae7d10528da75..6515af9f04ead 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afhints.c
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afhints.c
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter hinting routines (body).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -151,9 +151,9 @@
         }
         else if ( axis->num_segments >= axis->max_segments )
         {
    -      FT_Int  old_max = axis->max_segments;
    -      FT_Int  new_max = old_max;
    -      FT_Int  big_max = (FT_Int)( FT_INT_MAX / sizeof ( *segment ) );
    +      FT_UInt  old_max = axis->max_segments;
    +      FT_UInt  new_max = old_max;
    +      FT_UInt  big_max = FT_INT_MAX / sizeof ( *segment );
     
     
           if ( old_max >= big_max )
    @@ -193,7 +193,7 @@
       /* Get new edge for given axis, direction, and position, */
       /* without initializing the edge itself.                 */
     
    -  FT_LOCAL( FT_Error )
    +  FT_LOCAL_DEF( FT_Error )
       af_axis_hints_new_edge( AF_AxisHints  axis,
                               FT_Int        fpos,
                               AF_Direction  dir,
    @@ -216,9 +216,9 @@
         }
         else if ( axis->num_edges >= axis->max_edges )
         {
    -      FT_Int  old_max = axis->max_edges;
    -      FT_Int  new_max = old_max;
    -      FT_Int  big_max = (FT_Int)( FT_INT_MAX / sizeof ( *edge ) );
    +      FT_UInt  old_max = axis->max_edges;
    +      FT_UInt  new_max = old_max;
    +      FT_UInt  big_max = FT_INT_MAX / sizeof ( *edge );
     
     
           if ( old_max >= big_max )
    @@ -471,10 +471,10 @@
     
                     point->fx,
                     point->fy,
    -                point->ox / 64.0,
    -                point->oy / 64.0,
    -                point->x / 64.0,
    -                point->y / 64.0,
    +                (double)point->ox / 64,
    +                (double)point->oy / 64,
    +                (double)point->x / 64,
    +                (double)point->y / 64,
     
                     af_print_idx( buf5, af_get_strong_edge_index( hints,
                                                                   point->before,
    @@ -597,7 +597,7 @@
       FT_Error
       af_glyph_hints_get_num_segments( AF_GlyphHints  hints,
                                        FT_Int         dimension,
    -                                   FT_Int*        num_segments )
    +                                   FT_UInt*       num_segments )
       {
         AF_Dimension  dim;
         AF_AxisHints  axis;
    @@ -623,7 +623,7 @@
       FT_Error
       af_glyph_hints_get_segment_offset( AF_GlyphHints  hints,
                                          FT_Int         dimension,
    -                                     FT_Int         idx,
    +                                     FT_UInt        idx,
                                          FT_Pos        *offset,
                                          FT_Bool       *is_blue,
                                          FT_Pos        *blue_offset )
    @@ -640,7 +640,7 @@
     
         axis = &hints->axis[dim];
     
    -    if ( idx < 0 || idx >= axis->num_segments )
    +    if ( idx >= axis->num_segments )
           return FT_THROW( Invalid_Argument );
     
         seg      = &axis->segments[idx];
    @@ -692,13 +692,13 @@
           if ( dimension == AF_DIMENSION_HORZ )
             AF_DUMP(( "Table of %s edges (1px=%.2fu, 10u=%.2fpx):\n",
                       "vertical",
    -                  65536.0 * 64.0 / hints->x_scale,
    -                  10.0 * hints->x_scale / 65536.0 / 64.0 ));
    +                  65536 * 64 / (double)hints->x_scale,
    +                  10 * (double)hints->x_scale / 65536 / 64 ));
           else
             AF_DUMP(( "Table of %s edges (1px=%.2fu, 10u=%.2fpx):\n",
                       "horizontal",
    -                  65536.0 * 64.0 / hints->y_scale,
    -                  10.0 * hints->y_scale / 65536.0 / 64.0 ));
    +                  65536 * 64 / (double)hints->y_scale,
    +                  10 * (double)hints->y_scale / 65536 / 64 ));
     
           if ( axis->num_edges )
           {
    @@ -714,14 +714,14 @@
             AF_DUMP(( "  %5d  %7.2f  %5s  %4s  %5s"
                       "    %c   %7.2f  %7.2f  %11s\n",
                       AF_INDEX_NUM( edge, edges ),
    -                  (int)edge->opos / 64.0,
    +                  (double)(int)edge->opos / 64,
                       af_dir_str( (AF_Direction)edge->dir ),
                       af_print_idx( buf1, AF_INDEX_NUM( edge->link, edges ) ),
                       af_print_idx( buf2, AF_INDEX_NUM( edge->serif, edges ) ),
     
                       edge->blue_edge ? 'y' : 'n',
    -                  edge->opos / 64.0,
    -                  edge->pos / 64.0,
    +                  (double)edge->opos / 64,
    +                  (double)edge->pos / 64,
                       af_edge_flags_to_string( edge->flags ) ));
           AF_DUMP(( "\n" ));
         }
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afhints.h b/src/java.desktop/share/native/libfreetype/src/autofit/afhints.h
    index 96001cd80dadf..d1cf9529bf12d 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afhints.h
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afhints.h
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter hinting routines (specification).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -21,8 +21,6 @@
     
     #include "aftypes.h"
     
    -#define xxAF_SORT_SEGMENTS
    -
     FT_BEGIN_HEADER
     
       /*
    @@ -310,15 +308,12 @@ FT_BEGIN_HEADER
     
       typedef struct  AF_AxisHintsRec_
       {
    -    FT_Int        num_segments; /* number of used segments      */
    -    FT_Int        max_segments; /* number of allocated segments */
    +    FT_UInt       num_segments; /* number of used segments      */
    +    FT_UInt       max_segments; /* number of allocated segments */
         AF_Segment    segments;     /* segments array               */
    -#ifdef AF_SORT_SEGMENTS
    -    FT_Int        mid_segments;
    -#endif
     
    -    FT_Int        num_edges;    /* number of used edges      */
    -    FT_Int        max_edges;    /* number of allocated edges */
    +    FT_UInt       num_edges;    /* number of used edges      */
    +    FT_UInt       max_edges;    /* number of allocated edges */
         AF_Edge       edges;        /* edges array               */
     
         AF_Direction  major_dir;    /* either vertical or horizontal */
    @@ -380,14 +375,14 @@ FT_BEGIN_HEADER
     #ifdef FT_DEBUG_AUTOFIT
     
     #define AF_HINTS_DO_HORIZONTAL( h )                                     \
    -          ( !_af_debug_disable_horz_hints                            && \
    +          ( !af_debug_disable_horz_hints_                            && \
                 !AF_HINTS_TEST_SCALER( h, AF_SCALER_FLAG_NO_HORIZONTAL ) )
     
     #define AF_HINTS_DO_VERTICAL( h )                                     \
    -          ( !_af_debug_disable_vert_hints                          && \
    +          ( !af_debug_disable_vert_hints_                          && \
                 !AF_HINTS_TEST_SCALER( h, AF_SCALER_FLAG_NO_VERTICAL ) )
     
    -#define AF_HINTS_DO_BLUES( h )  ( !_af_debug_disable_blue_hints )
    +#define AF_HINTS_DO_BLUES( h )  ( !af_debug_disable_blue_hints_ )
     
     #else /* !FT_DEBUG_AUTOFIT */
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afindic.c b/src/java.desktop/share/native/libfreetype/src/autofit/afindic.c
    index 5bf0b5f9451ae..289a09d71d8a5 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afindic.c
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afindic.c
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter hinting routines for Indic writing system (body).
      *
    - * Copyright (C) 2007-2022 by
    + * Copyright (C) 2007-2023 by
      * Rahul Bhalerao , .
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -49,8 +49,7 @@
           af_cjk_metrics_check_digits( metrics, face );
         }
     
    -    FT_Set_Charmap( face, oldmap );
    -
    +    face->charmap = oldmap;
         return FT_Err_Ok;
       }
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afindic.h b/src/java.desktop/share/native/libfreetype/src/autofit/afindic.h
    index 59ae11a677326..3eb67f63b0013 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afindic.h
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afindic.h
    @@ -5,7 +5,7 @@
      *   Auto-fitter hinting routines for Indic writing system
      *   (specification).
      *
    - * Copyright (C) 2007-2022 by
    + * Copyright (C) 2007-2023 by
      * Rahul Bhalerao , .
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/aflatin.c b/src/java.desktop/share/native/libfreetype/src/autofit/aflatin.c
    index bed0ccee0808f..4b3c59b3c3117 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/aflatin.c
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/aflatin.c
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter hinting routines for latin writing system (body).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -1043,7 +1043,7 @@
           AF_FaceGlobals  globals = metrics->root.globals;
           FT_UShort*      gstyles = globals->glyph_styles;
     
    -      FT_Long  i;
    +      FT_UInt  i;
     
     
           FT_TRACE5(( "no blue zones found:"
    @@ -1157,7 +1157,7 @@
         }
     
       Exit:
    -    FT_Set_Charmap( face, oldmap );
    +    face->charmap = oldmap;
         return error;
       }
     
    @@ -1275,8 +1275,8 @@
                   FT_TRACE5(( "                           "
                               " vertical scaling changed"
                               " from %.5f to %.5f (by %ld%%)\n",
    -                          scale / 65536.0,
    -                          new_scale / 65536.0,
    +                          (double)scale / 65536,
    +                          (double)new_scale / 65536,
                               ( fitted - scaled ) * 100 / scaled ));
                   FT_TRACE5(( "\n" ));
     
    @@ -1327,7 +1327,7 @@
     
           FT_TRACE5(( "  %ld scaled to %.2f\n",
                       width->org,
    -                  width->cur / 64.0 ));
    +                  (double)width->cur / 64 ));
         }
     
         FT_TRACE5(( "\n" ));
    @@ -1471,13 +1471,13 @@
             FT_TRACE5(( "  reference %d: %ld scaled to %.2f%s\n",
                         nn,
                         blue->ref.org,
    -                    blue->ref.fit / 64.0,
    +                    (double)blue->ref.fit / 64,
                         ( blue->flags & AF_LATIN_BLUE_ACTIVE ) ? ""
                                                                : " (inactive)" ));
             FT_TRACE5(( "  overshoot %d: %ld scaled to %.2f%s\n",
                         nn,
                         blue->shoot.org,
    -                    blue->shoot.fit / 64.0,
    +                    (double)blue->shoot.fit / 64,
                         ( blue->flags & AF_LATIN_BLUE_ACTIVE ) ? ""
                                                                : " (inactive)" ));
           }
    @@ -2203,7 +2203,7 @@
         for ( seg = segments; seg < segment_limit; seg++ )
         {
           AF_Edge  found = NULL;
    -      FT_Int   ee;
    +      FT_UInt  ee;
     
     
           /* ignore too short segments, too wide ones, and, in this loop, */
    @@ -2277,7 +2277,7 @@
         for ( seg = segments; seg < segment_limit; seg++ )
         {
           AF_Edge  found = NULL;
    -      FT_Int   ee;
    +      FT_UInt  ee;
     
     
           if ( seg->dir != AF_DIR_NONE )
    @@ -2955,8 +2955,9 @@
     
         FT_TRACE5(( "  LINK: edge %ld (opos=%.2f) linked to %.2f,"
                     " dist was %.2f, now %.2f\n",
    -                stem_edge - hints->axis[dim].edges, stem_edge->opos / 64.0,
    -                stem_edge->pos / 64.0, dist / 64.0, fitted_width / 64.0 ));
    +                stem_edge - hints->axis[dim].edges,
    +                (double)stem_edge->opos / 64, (double)stem_edge->pos / 64,
    +                (double)dist / 64, (double)fitted_width / 64 ));
       }
     
     
    @@ -3079,13 +3080,15 @@
             if ( !anchor )
               FT_TRACE5(( "  BLUE_ANCHOR: edge %ld (opos=%.2f) snapped to %.2f,"
                           " was %.2f (anchor=edge %ld)\n",
    -                      edge1 - edges, edge1->opos / 64.0, blue->fit / 64.0,
    -                      edge1->pos / 64.0, edge - edges ));
    +                      edge1 - edges,
    +                      (double)edge1->opos / 64, (double)blue->fit / 64,
    +                      (double)edge1->pos / 64, edge - edges ));
             else
               FT_TRACE5(( "  BLUE: edge %ld (opos=%.2f) snapped to %.2f,"
                           " was %.2f\n",
    -                      edge1 - edges, edge1->opos / 64.0, blue->fit / 64.0,
    -                      edge1->pos / 64.0 ));
    +                      edge1 - edges,
    +                      (double)edge1->opos / 64, (double)blue->fit / 64,
    +                      (double)edge1->pos / 64 ));
     
             num_actions++;
     #endif
    @@ -3201,9 +3204,9 @@
     
             FT_TRACE5(( "  ANCHOR: edge %ld (opos=%.2f) and %ld (opos=%.2f)"
                         " snapped to %.2f and %.2f\n",
    -                    edge - edges, edge->opos / 64.0,
    -                    edge2 - edges, edge2->opos / 64.0,
    -                    edge->pos / 64.0, edge2->pos / 64.0 ));
    +                    edge - edges, (double)edge->opos / 64,
    +                    edge2 - edges, (double)edge2->opos / 64,
    +                    (double)edge->pos / 64, (double)edge2->pos / 64 ));
     
             af_latin_align_linked_edge( hints, dim, edge, edge2 );
     
    @@ -3229,8 +3232,8 @@
             if ( edge2->flags & AF_EDGE_DONE )
             {
               FT_TRACE5(( "  ADJUST: edge %ld (pos=%.2f) moved to %.2f\n",
    -                      edge - edges, edge->pos / 64.0,
    -                      ( edge2->pos - cur_len ) / 64.0 ));
    +                      edge - edges, (double)edge->pos / 64,
    +                      (double)( edge2->pos - cur_len ) / 64 ));
     
               edge->pos = edge2->pos - cur_len;
             }
    @@ -3271,9 +3274,9 @@
     
               FT_TRACE5(( "  STEM: edge %ld (opos=%.2f) linked to %ld (opos=%.2f)"
                           " snapped to %.2f and %.2f\n",
    -                      edge - edges, edge->opos / 64.0,
    -                      edge2 - edges, edge2->opos / 64.0,
    -                      edge->pos / 64.0, edge2->pos / 64.0 ));
    +                      edge - edges, (double)edge->opos / 64,
    +                      edge2 - edges, (double)edge2->opos / 64,
    +                      (double)edge->pos / 64, (double)edge2->pos / 64 ));
             }
     
             else
    @@ -3302,9 +3305,9 @@
     
               FT_TRACE5(( "  STEM: edge %ld (opos=%.2f) linked to %ld (opos=%.2f)"
                           " snapped to %.2f and %.2f\n",
    -                      edge - edges, edge->opos / 64.0,
    -                      edge2 - edges, edge2->opos / 64.0,
    -                      edge->pos / 64.0, edge2->pos / 64.0 ));
    +                      edge - edges, (double)edge->opos / 64,
    +                      edge2 - edges, (double)edge2->opos / 64,
    +                      (double)edge->pos / 64, (double)edge2->pos / 64 ));
             }
     
     #ifdef FT_DEBUG_LEVEL_TRACE
    @@ -3325,8 +3328,8 @@
     #ifdef FT_DEBUG_LEVEL_TRACE
                 FT_TRACE5(( "  BOUND: edge %ld (pos=%.2f) moved to %.2f\n",
                             edge - edges,
    -                        edge->pos / 64.0,
    -                        edge[-1].pos / 64.0 ));
    +                        (double)edge->pos / 64,
    +                        (double)edge[-1].pos / 64 ));
     
                 num_actions++;
     #endif
    @@ -3427,9 +3430,9 @@
               af_latin_align_serif_edge( hints, edge->serif, edge );
               FT_TRACE5(( "  SERIF: edge %ld (opos=%.2f) serif to %ld (opos=%.2f)"
                           " aligned to %.2f\n",
    -                      edge - edges, edge->opos / 64.0,
    -                      edge->serif - edges, edge->serif->opos / 64.0,
    -                      edge->pos / 64.0 ));
    +                      edge - edges, (double)edge->opos / 64,
    +                      edge->serif - edges, (double)edge->serif->opos / 64,
    +                      (double)edge->pos / 64 ));
             }
             else if ( !anchor )
             {
    @@ -3437,7 +3440,8 @@
               anchor    = edge;
               FT_TRACE5(( "  SERIF_ANCHOR: edge %ld (opos=%.2f)"
                           " snapped to %.2f\n",
    -                      edge-edges, edge->opos / 64.0, edge->pos / 64.0 ));
    +                      edge-edges,
    +                      (double)edge->opos / 64, (double)edge->pos / 64 ));
             }
             else
             {
    @@ -3465,9 +3469,9 @@
     
                 FT_TRACE5(( "  SERIF_LINK1: edge %ld (opos=%.2f) snapped to %.2f"
                             " from %ld (opos=%.2f)\n",
    -                        edge - edges, edge->opos / 64.0,
    -                        edge->pos / 64.0,
    -                        before - edges, before->opos / 64.0 ));
    +                        edge - edges, (double)edge->opos / 64,
    +                        (double)edge->pos / 64,
    +                        before - edges, (double)before->opos / 64 ));
               }
               else
               {
    @@ -3475,7 +3479,8 @@
                             ( ( edge->opos - anchor->opos + 16 ) & ~31 );
                 FT_TRACE5(( "  SERIF_LINK2: edge %ld (opos=%.2f)"
                             " snapped to %.2f\n",
    -                        edge - edges, edge->opos / 64.0, edge->pos / 64.0 ));
    +                        edge - edges,
    +                        (double)edge->opos / 64, (double)edge->pos / 64 ));
               }
             }
     
    @@ -3495,8 +3500,8 @@
     #ifdef FT_DEBUG_LEVEL_TRACE
                 FT_TRACE5(( "  BOUND: edge %ld (pos=%.2f) moved to %.2f\n",
                             edge - edges,
    -                        edge->pos / 64.0,
    -                        edge[-1].pos / 64.0 ));
    +                        (double)edge->pos / 64,
    +                        (double)edge[-1].pos / 64 ));
     
                 num_actions++;
     #endif
    @@ -3516,8 +3521,8 @@
     #ifdef FT_DEBUG_LEVEL_TRACE
                 FT_TRACE5(( "  BOUND: edge %ld (pos=%.2f) moved to %.2f\n",
                             edge - edges,
    -                        edge->pos / 64.0,
    -                        edge[1].pos / 64.0 ));
    +                        (double)edge->pos / 64,
    +                        (double)edge[1].pos / 64 ));
     
                 num_actions++;
     #endif
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/aflatin.h b/src/java.desktop/share/native/libfreetype/src/autofit/aflatin.h
    index facc663450d2b..3c6a7ee4f62ac 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/aflatin.h
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/aflatin.h
    @@ -5,7 +5,7 @@
      *   Auto-fitter hinting routines for latin writing system
      *   (specification).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afloader.c b/src/java.desktop/share/native/libfreetype/src/autofit/afloader.c
    index e55183a509f3c..c8082796fe881 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afloader.c
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afloader.c
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter glyph loading routines (body).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -229,9 +229,6 @@
         AF_WritingSystemClass  writing_system_class;
     
     
    -    if ( !size )
    -      return FT_THROW( Invalid_Size_Handle );
    -
         FT_ZERO( &scaler );
     
         if ( !size_internal->autohint_metrics.x_scale                          ||
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afloader.h b/src/java.desktop/share/native/libfreetype/src/autofit/afloader.h
    index b345e46395336..e4e197e374fd3 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afloader.h
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afloader.h
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter glyph loading routines (specification).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -75,7 +75,7 @@ FT_BEGIN_HEADER
                             FT_UInt    gindex,
                             FT_Int32   load_flags );
     
    -  FT_LOCAL_DEF( FT_Fixed )
    +  FT_LOCAL( FT_Fixed )
       af_loader_compute_darkening( AF_Loader  loader,
                                    FT_Face    face,
                                    FT_Pos     standard_width );
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afmodule.c b/src/java.desktop/share/native/libfreetype/src/autofit/afmodule.c
    index 1b14ae682eb36..92e5156ab2dc7 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afmodule.c
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afmodule.c
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter module implementation (body).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -43,14 +43,14 @@
     
     #endif
     
    -  int  _af_debug_disable_horz_hints;
    -  int  _af_debug_disable_vert_hints;
    -  int  _af_debug_disable_blue_hints;
    +  int  af_debug_disable_horz_hints_;
    +  int  af_debug_disable_vert_hints_;
    +  int  af_debug_disable_blue_hints_;
     
       /* we use a global object instead of a local one for debugging */
    -  static AF_GlyphHintsRec  _af_debug_hints_rec[1];
    +  static AF_GlyphHintsRec  af_debug_hints_rec_[1];
     
    -  void*  _af_debug_hints = _af_debug_hints_rec;
    +  void*  af_debug_hints_ = af_debug_hints_rec_;
     #endif
     
     #include 
    @@ -119,8 +119,8 @@
     
         if ( !ft_strcmp( property_name, "fallback-script" ) )
         {
    -      FT_UInt*  fallback_script;
    -      FT_UInt   ss;
    +      AF_Script*  fallback_script;
    +      FT_UInt     ss;
     
     
     #ifdef FT_CONFIG_OPTION_ENVIRONMENT_PROPERTIES
    @@ -128,7 +128,7 @@
             return FT_THROW( Invalid_Argument );
     #endif
     
    -      fallback_script = (FT_UInt*)value;
    +      fallback_script = (AF_Script*)value;
     
           /* We translate the fallback script to a fallback style that uses */
           /* `fallback-script' as its script and `AF_COVERAGE_NONE' as its  */
    @@ -138,8 +138,8 @@
             AF_StyleClass  style_class = af_style_classes[ss];
     
     
    -        if ( (FT_UInt)style_class->script == *fallback_script &&
    -             style_class->coverage == AF_COVERAGE_DEFAULT     )
    +        if ( style_class->script   == *fallback_script    &&
    +             style_class->coverage == AF_COVERAGE_DEFAULT )
             {
               module->fallback_style = ss;
               break;
    @@ -157,7 +157,7 @@
         }
         else if ( !ft_strcmp( property_name, "default-script" ) )
         {
    -      FT_UInt*  default_script;
    +      AF_Script*  default_script;
     
     
     #ifdef FT_CONFIG_OPTION_ENVIRONMENT_PROPERTIES
    @@ -165,7 +165,7 @@
             return FT_THROW( Invalid_Argument );
     #endif
     
    -      default_script = (FT_UInt*)value;
    +      default_script = (AF_Script*)value;
     
           module->default_script = *default_script;
     
    @@ -291,8 +291,6 @@
       {
         FT_Error   error          = FT_Err_Ok;
         AF_Module  module         = (AF_Module)ft_module;
    -    FT_UInt    fallback_style = module->fallback_style;
    -    FT_UInt    default_script = module->default_script;
     
     
         if ( !ft_strcmp( property_name, "glyph-to-script-map" ) )
    @@ -309,9 +307,9 @@
         }
         else if ( !ft_strcmp( property_name, "fallback-script" ) )
         {
    -      FT_UInt*  val = (FT_UInt*)value;
    +      AF_Script*  val = (AF_Script*)value;
     
    -      AF_StyleClass  style_class = af_style_classes[fallback_style];
    +      AF_StyleClass  style_class = af_style_classes[module->fallback_style];
     
     
           *val = style_class->script;
    @@ -320,10 +318,10 @@
         }
         else if ( !ft_strcmp( property_name, "default-script" ) )
         {
    -      FT_UInt*  val = (FT_UInt*)value;
    +      AF_Script*  val = (AF_Script*)value;
     
     
    -      *val = default_script;
    +      *val = module->default_script;
     
           return error;
         }
    @@ -425,8 +423,8 @@
         FT_UNUSED( ft_module );
     
     #ifdef FT_DEBUG_AUTOFIT
    -    if ( _af_debug_hints_rec->memory )
    -      af_glyph_hints_done( _af_debug_hints_rec );
    +    if ( af_debug_hints_rec_->memory )
    +      af_glyph_hints_done( af_debug_hints_rec_ );
     #endif
       }
     
    @@ -445,7 +443,7 @@
     
         /* in debug mode, we use a global object that survives this routine */
     
    -    AF_GlyphHints  hints = _af_debug_hints_rec;
    +    AF_GlyphHints  hints = af_debug_hints_rec_;
         AF_LoaderRec   loader[1];
     
         FT_UNUSED( size );
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afmodule.h b/src/java.desktop/share/native/libfreetype/src/autofit/afmodule.h
    index 1d1bfaf5447f4..4b8b4562c67c7 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afmodule.h
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afmodule.h
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter module implementation (specification).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -36,7 +36,7 @@ FT_BEGIN_HEADER
         FT_ModuleRec  root;
     
         FT_UInt       fallback_style;
    -    FT_UInt       default_script;
    +    AF_Script     default_script;
         FT_Bool       no_stem_darkening;
         FT_Int        darken_params[8];
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afranges.c b/src/java.desktop/share/native/libfreetype/src/autofit/afranges.c
    index 2de1991a57ad4..cfcaf340a79e6 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afranges.c
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afranges.c
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter Unicode script ranges (body).
      *
    - * Copyright (C) 2013-2022 by
    + * Copyright (C) 2013-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afranges.h b/src/java.desktop/share/native/libfreetype/src/autofit/afranges.h
    index acd01faf6874b..5775738bc0bb0 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afranges.h
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afranges.h
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter Unicode script ranges (specification).
      *
    - * Copyright (C) 2013-2022 by
    + * Copyright (C) 2013-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afscript.h b/src/java.desktop/share/native/libfreetype/src/autofit/afscript.h
    index 172b598069600..3a101937d7032 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afscript.h
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afscript.h
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter scripts (specification only).
      *
    - * Copyright (C) 2013-2022 by
    + * Copyright (C) 2013-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afshaper.c b/src/java.desktop/share/native/libfreetype/src/autofit/afshaper.c
    index 298480d864ffa..1b8b870e89dda 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afshaper.c
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afshaper.c
    @@ -4,7 +4,7 @@
      *
      *   HarfBuzz interface for accessing OpenType features (body).
      *
    - * Copyright (C) 2013-2022 by
    + * Copyright (C) 2013-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afshaper.h b/src/java.desktop/share/native/libfreetype/src/autofit/afshaper.h
    index 558f03bdef0be..054a18ffbc2d3 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afshaper.h
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afshaper.h
    @@ -4,7 +4,7 @@
      *
      *   HarfBuzz interface for accessing OpenType features (specification).
      *
    - * Copyright (C) 2013-2022 by
    + * Copyright (C) 2013-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -27,7 +27,7 @@
     
     #include 
     #include 
    -#include 
    +#include "ft-hb.h"
     
     #endif
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afstyles.h b/src/java.desktop/share/native/libfreetype/src/autofit/afstyles.h
    index 9080b9fb657ee..73ebef01716c3 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afstyles.h
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afstyles.h
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter styles (specification only).
      *
    - * Copyright (C) 2013-2022 by
    + * Copyright (C) 2013-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/aftypes.h b/src/java.desktop/share/native/libfreetype/src/autofit/aftypes.h
    index 754aad7ba4bc6..661519449653d 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/aftypes.h
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/aftypes.h
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter types (specification only).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -57,10 +57,10 @@ FT_BEGIN_HEADER
     
     #ifdef FT_DEBUG_AUTOFIT
     
    -extern int    _af_debug_disable_horz_hints;
    -extern int    _af_debug_disable_vert_hints;
    -extern int    _af_debug_disable_blue_hints;
    -extern void*  _af_debug_hints;
    +extern int    af_debug_disable_horz_hints_;
    +extern int    af_debug_disable_vert_hints_;
    +extern int    af_debug_disable_blue_hints_;
    +extern void*  af_debug_hints_;
     
     #endif /* FT_DEBUG_AUTOFIT */
     
    @@ -119,13 +119,13 @@ extern void*  _af_debug_hints;
     
       typedef struct  AF_ScalerRec_
       {
    -    FT_Face         face;        /* source font face                        */
    -    FT_Fixed        x_scale;     /* from font units to 1/64th device pixels */
    -    FT_Fixed        y_scale;     /* from font units to 1/64th device pixels */
    -    FT_Pos          x_delta;     /* in 1/64th device pixels                 */
    -    FT_Pos          y_delta;     /* in 1/64th device pixels                 */
    -    FT_Render_Mode  render_mode; /* monochrome, anti-aliased, LCD, etc.     */
    -    FT_UInt32       flags;       /* additional control flags, see above     */
    +    FT_Face         face;        /* source font face                      */
    +    FT_Fixed        x_scale;     /* from font units to 1/64 device pixels */
    +    FT_Fixed        y_scale;     /* from font units to 1/64 device pixels */
    +    FT_Pos          x_delta;     /* in 1/64 device pixels                 */
    +    FT_Pos          y_delta;     /* in 1/64 device pixels                 */
    +    FT_Render_Mode  render_mode; /* monochrome, anti-aliased, LCD, etc.   */
    +    FT_UInt32       flags;       /* additional control flags, see above   */
     
       } AF_ScalerRec, *AF_Scaler;
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afws-decl.h b/src/java.desktop/share/native/libfreetype/src/autofit/afws-decl.h
    index c93845ef9535a..48c888afed88e 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afws-decl.h
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afws-decl.h
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter writing system declarations (specification only).
      *
    - * Copyright (C) 2013-2022 by
    + * Copyright (C) 2013-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/autofit/afws-iter.h b/src/java.desktop/share/native/libfreetype/src/autofit/afws-iter.h
    index 9cda3509bc129..a0a686f8cee7f 100644
    --- a/src/java.desktop/share/native/libfreetype/src/autofit/afws-iter.h
    +++ b/src/java.desktop/share/native/libfreetype/src/autofit/afws-iter.h
    @@ -4,7 +4,7 @@
      *
      *   Auto-fitter writing systems iterator (specification only).
      *
    - * Copyright (C) 2013-2022 by
    + * Copyright (C) 2013-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftadvanc.c b/src/java.desktop/share/native/libfreetype/src/base/ftadvanc.c
    index fc6b4288174fd..de25476fe9287 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftadvanc.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftadvanc.c
    @@ -4,7 +4,7 @@
      *
      *   Quick computation of advance widths (body).
      *
    - * Copyright (C) 2008-2022 by
    + * Copyright (C) 2008-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -23,7 +23,7 @@
     
     
       static FT_Error
    -  _ft_face_scale_advances( FT_Face    face,
    +  ft_face_scale_advances_( FT_Face    face,
                                FT_Fixed*  advances,
                                FT_UInt    count,
                                FT_Int32   flags )
    @@ -96,7 +96,7 @@
     
           error = func( face, gindex, 1, flags, padvance );
           if ( !error )
    -        return _ft_face_scale_advances( face, padvance, 1, flags );
    +        return ft_face_scale_advances_( face, padvance, 1, flags );
     
           if ( FT_ERR_NEQ( error, Unimplemented_Feature ) )
             return error;
    @@ -142,7 +142,7 @@
         {
           error = func( face, start, count, flags, padvances );
           if ( !error )
    -        return _ft_face_scale_advances( face, padvances, count, flags );
    +        return ft_face_scale_advances_( face, padvances, count, flags );
     
           if ( FT_ERR_NEQ( error, Unimplemented_Feature ) )
             return error;
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftbase.h b/src/java.desktop/share/native/libfreetype/src/base/ftbase.h
    index f873566f22236..00790d3b226f6 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftbase.h
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftbase.h
    @@ -4,7 +4,7 @@
      *
      *   Private functions used in the `base' module (specification).
      *
    - * Copyright (C) 2008-2022 by
    + * Copyright (C) 2008-2023 by
      * David Turner, Robert Wilhelm, Werner Lemberg, and suzuki toshiya.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftbbox.c b/src/java.desktop/share/native/libfreetype/src/base/ftbbox.c
    index 30aedf780cde1..7dd71882ea5da 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftbbox.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftbbox.c
    @@ -4,7 +4,7 @@
      *
      *   FreeType bbox computation (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftbitmap.c b/src/java.desktop/share/native/libfreetype/src/base/ftbitmap.c
    index 7825895ad6a76..1c93648dcbc88 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftbitmap.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftbitmap.c
    @@ -4,7 +4,7 @@
      *
      *   FreeType utility functions for bitmaps (body).
      *
    - * Copyright (C) 2004-2022 by
    + * Copyright (C) 2004-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -66,11 +66,8 @@
       {
         FT_Memory  memory;
         FT_Error   error  = FT_Err_Ok;
    -
    -    FT_Int    pitch;
    -    FT_ULong  size;
    -
    -    FT_Int  source_pitch_sign, target_pitch_sign;
    +    FT_Int     pitch;
    +    FT_Int     flip;
     
     
         if ( !library )
    @@ -82,53 +79,29 @@
         if ( source == target )
           return FT_Err_Ok;
     
    -    source_pitch_sign = source->pitch < 0 ? -1 : 1;
    -    target_pitch_sign = target->pitch < 0 ? -1 : 1;
    +    flip = ( source->pitch < 0 && target->pitch > 0 ) ||
    +           ( source->pitch > 0 && target->pitch < 0 );
     
    -    if ( !source->buffer )
    -    {
    -      *target = *source;
    -      if ( source_pitch_sign != target_pitch_sign )
    -        target->pitch = -target->pitch;
    +    memory = library->memory;
    +    FT_FREE( target->buffer );
    +
    +    *target = *source;
    +
    +    if ( flip )
    +      target->pitch = -target->pitch;
     
    +    if ( !source->buffer )
           return FT_Err_Ok;
    -    }
     
    -    memory = library->memory;
         pitch  = source->pitch;
    -
         if ( pitch < 0 )
           pitch = -pitch;
    -    size = (FT_ULong)pitch * source->rows;
    -
    -    if ( target->buffer )
    -    {
    -      FT_Int    target_pitch = target->pitch;
    -      FT_ULong  target_size;
     
    -
    -      if ( target_pitch < 0 )
    -        target_pitch = -target_pitch;
    -      target_size = (FT_ULong)target_pitch * target->rows;
    -
    -      if ( target_size != size )
    -        FT_MEM_QREALLOC( target->buffer, target_size, size );
    -    }
    -    else
    -      FT_MEM_QALLOC( target->buffer, size );
    +    FT_MEM_QALLOC_MULT( target->buffer, target->rows, pitch );
     
         if ( !error )
         {
    -      unsigned char *p;
    -
    -
    -      p = target->buffer;
    -      *target = *source;
    -      target->buffer = p;
    -
    -      if ( source_pitch_sign == target_pitch_sign )
    -        FT_MEM_COPY( target->buffer, source->buffer, size );
    -      else
    +      if ( flip )
           {
             /* take care of bitmap flow */
             FT_UInt   i;
    @@ -146,6 +119,9 @@
               t -= pitch;
             }
           }
    +      else
    +        FT_MEM_COPY( target->buffer, source->buffer,
    +                     (FT_Long)source->rows * pitch );
         }
     
         return error;
    @@ -542,39 +518,31 @@
         case FT_PIXEL_MODE_LCD_V:
         case FT_PIXEL_MODE_BGRA:
           {
    -        FT_Int    pad, old_target_pitch, target_pitch;
    -        FT_ULong  old_size;
    +        FT_Int  width = (FT_Int)source->width;
    +        FT_Int  neg   = ( target->pitch == 0 && source->pitch < 0 ) ||
    +                          target->pitch  < 0;
     
     
    -        old_target_pitch = target->pitch;
    -        if ( old_target_pitch < 0 )
    -          old_target_pitch = -old_target_pitch;
    -
    -        old_size = target->rows * (FT_UInt)old_target_pitch;
    +        FT_Bitmap_Done( library, target );
     
             target->pixel_mode = FT_PIXEL_MODE_GRAY;
             target->rows       = source->rows;
             target->width      = source->width;
     
    -        pad = 0;
    -        if ( alignment > 0 )
    +        if ( alignment )
             {
    -          pad = (FT_Int)source->width % alignment;
    -          if ( pad != 0 )
    -            pad = alignment - pad;
    -        }
    +          FT_Int  rem = width % alignment;
     
    -        target_pitch = (FT_Int)source->width + pad;
     
    -        if ( target_pitch > 0                                               &&
    -             (FT_ULong)target->rows > FT_ULONG_MAX / (FT_ULong)target_pitch )
    -          return FT_THROW( Invalid_Argument );
    +          if ( rem )
    +            width = alignment > 0 ? width - rem + alignment
    +                                  : width - rem - alignment;
    +        }
     
    -        if ( FT_QREALLOC( target->buffer,
    -                          old_size, target->rows * (FT_UInt)target_pitch ) )
    +        if ( FT_QALLOC_MULT( target->buffer, target->rows, width ) )
               return error;
     
    -        target->pitch = target->pitch < 0 ? -target_pitch : target_pitch;
    +        target->pitch = neg ? -width : width;
           }
           break;
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftcalc.c b/src/java.desktop/share/native/libfreetype/src/base/ftcalc.c
    index 6c1e7fbd45a0a..13e74f3353bf3 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftcalc.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftcalc.c
    @@ -4,7 +4,7 @@
      *
      *   Arithmetic computations (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -1085,4 +1085,71 @@
       }
     
     
    +  FT_BASE_DEF( FT_Int32 )
    +  FT_MulAddFix( FT_Fixed*  s,
    +                FT_Int32*  f,
    +                FT_UInt    count )
    +  {
    +    FT_UInt   i;
    +    FT_Int64  temp;
    +#ifndef FT_INT64
    +    FT_Int64  halfUnit;
    +#endif
    +
    +
    +#ifdef FT_INT64
    +    temp = 0;
    +
    +    for ( i = 0; i < count; ++i )
    +      temp += (FT_Int64)s[i] * f[i];
    +
    +    return ( temp + 0x8000 ) >> 16;
    +#else
    +    temp.hi = 0;
    +    temp.lo = 0;
    +
    +    for ( i = 0; i < count; ++i )
    +    {
    +      FT_Int64  multResult;
    +
    +      FT_Int     sign  = 1;
    +      FT_UInt32  carry = 0;
    +
    +      FT_UInt32  scalar;
    +      FT_UInt32  factor;
    +
    +
    +      scalar = (FT_UInt32)s[i];
    +      factor = (FT_UInt32)f[i];
    +
    +      FT_MOVE_SIGN( s[i], scalar, sign );
    +      FT_MOVE_SIGN( f[i], factor, sign );
    +
    +      ft_multo64( scalar, factor, &multResult );
    +
    +      if ( sign < 0 )
    +      {
    +        /* Emulated `FT_Int64` negation. */
    +        carry = ( multResult.lo == 0 );
    +
    +        multResult.lo = ~multResult.lo + 1;
    +        multResult.hi = ~multResult.hi + carry;
    +      }
    +
    +      FT_Add64( &temp, &multResult, &temp );
    +    }
    +
    +    /* Round value. */
    +    halfUnit.hi = 0;
    +    halfUnit.lo = 0x8000;
    +    FT_Add64( &temp, &halfUnit, &temp );
    +
    +    return (FT_Int32)( ( (FT_Int32)( temp.hi & 0xFFFF ) << 16 ) |
    +                                   ( temp.lo >> 16 )            );
    +
    +#endif /* !FT_INT64 */
    +
    +  }
    +
    +
     /* END */
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftcid.c b/src/java.desktop/share/native/libfreetype/src/base/ftcid.c
    index b882ca3de0aea..866cd23e91b59 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftcid.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftcid.c
    @@ -4,7 +4,7 @@
      *
      *   FreeType API for accessing CID font information.
      *
    - * Copyright (C) 2007-2022 by
    + * Copyright (C) 2007-2023 by
      * Derek Clegg and Michael Toftdal.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftcolor.c b/src/java.desktop/share/native/libfreetype/src/base/ftcolor.c
    new file mode 100644
    index 0000000000000..bcd6e893d4a57
    --- /dev/null
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftcolor.c
    @@ -0,0 +1,156 @@
    +/****************************************************************************
    + *
    + * ftcolor.c
    + *
    + *   FreeType's glyph color management (body).
    + *
    + * Copyright (C) 2018-2023 by
    + * David Turner, Robert Wilhelm, and Werner Lemberg.
    + *
    + * This file is part of the FreeType project, and may only be used,
    + * modified, and distributed under the terms of the FreeType project
    + * license, LICENSE.TXT.  By continuing to use, modify, or distribute
    + * this file you indicate that you have read the license and
    + * understand and accept it fully.
    + *
    + */
    +
    +
    +#include 
    +#include 
    +#include 
    +#include 
    +
    +
    +#ifdef TT_CONFIG_OPTION_COLOR_LAYERS
    +
    +  static
    +  const FT_Palette_Data  null_palette_data = { 0, NULL, NULL, 0, NULL };
    +
    +
    +  /* documentation is in ftcolor.h */
    +
    +  FT_EXPORT_DEF( FT_Error )
    +  FT_Palette_Data_Get( FT_Face           face,
    +                       FT_Palette_Data  *apalette_data )
    +  {
    +    if ( !face )
    +      return FT_THROW( Invalid_Face_Handle );
    +    if ( !apalette_data)
    +      return FT_THROW( Invalid_Argument );
    +
    +    if ( FT_IS_SFNT( face ) )
    +      *apalette_data = ( (TT_Face)face )->palette_data;
    +    else
    +      *apalette_data = null_palette_data;
    +
    +    return FT_Err_Ok;
    +  }
    +
    +
    +  /* documentation is in ftcolor.h */
    +
    +  FT_EXPORT_DEF( FT_Error )
    +  FT_Palette_Select( FT_Face     face,
    +                     FT_UShort   palette_index,
    +                     FT_Color*  *apalette )
    +  {
    +    FT_Error  error;
    +
    +    TT_Face       ttface;
    +    SFNT_Service  sfnt;
    +
    +
    +    if ( !face )
    +      return FT_THROW( Invalid_Face_Handle );
    +
    +    if ( !FT_IS_SFNT( face ) )
    +    {
    +      if ( apalette )
    +        *apalette = NULL;
    +
    +      return FT_Err_Ok;
    +    }
    +
    +    ttface = (TT_Face)face;
    +    sfnt   = (SFNT_Service)ttface->sfnt;
    +
    +    error = sfnt->set_palette( ttface, palette_index );
    +    if ( error )
    +      return error;
    +
    +    ttface->palette_index = palette_index;
    +
    +    if ( apalette )
    +      *apalette = ttface->palette;
    +
    +    return FT_Err_Ok;
    +  }
    +
    +
    +  /* documentation is in ftcolor.h */
    +
    +  FT_EXPORT_DEF( FT_Error )
    +  FT_Palette_Set_Foreground_Color( FT_Face   face,
    +                                   FT_Color  foreground_color )
    +  {
    +    TT_Face  ttface;
    +
    +
    +    if ( !face )
    +      return FT_THROW( Invalid_Face_Handle );
    +
    +    if ( !FT_IS_SFNT( face ) )
    +      return FT_Err_Ok;
    +
    +    ttface = (TT_Face)face;
    +
    +    ttface->foreground_color      = foreground_color;
    +    ttface->have_foreground_color = 1;
    +
    +    return FT_Err_Ok;
    +  }
    +
    +#else /* !TT_CONFIG_OPTION_COLOR_LAYERS */
    +
    +  FT_EXPORT_DEF( FT_Error )
    +  FT_Palette_Data_Get( FT_Face           face,
    +                       FT_Palette_Data  *apalette_data )
    +  {
    +    FT_UNUSED( face );
    +    FT_UNUSED( apalette_data );
    +
    +
    +    return FT_THROW( Unimplemented_Feature );
    +  }
    +
    +
    +  FT_EXPORT_DEF( FT_Error )
    +  FT_Palette_Select( FT_Face     face,
    +                     FT_UShort   palette_index,
    +                     FT_Color*  *apalette )
    +  {
    +    FT_UNUSED( face );
    +    FT_UNUSED( palette_index );
    +    FT_UNUSED( apalette );
    +
    +
    +    return FT_THROW( Unimplemented_Feature );
    +  }
    +
    +
    +  FT_EXPORT_DEF( FT_Error )
    +  FT_Palette_Set_Foreground_Color( FT_Face   face,
    +                                   FT_Color  foreground_color )
    +  {
    +    FT_UNUSED( face );
    +    FT_UNUSED( foreground_color );
    +
    +
    +    return FT_THROW( Unimplemented_Feature );
    +  }
    +
    +#endif /* !TT_CONFIG_OPTION_COLOR_LAYERS */
    +
    +
    +/* END */
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftdbgmem.c b/src/java.desktop/share/native/libfreetype/src/base/ftdbgmem.c
    index 1df83c404dbd8..6730c4c8d3802 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftdbgmem.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftdbgmem.c
    @@ -4,7 +4,7 @@
      *
      *   Memory debugger (body).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -35,8 +35,8 @@
     
     #include FT_CONFIG_STANDARD_LIBRARY_H
     
    -  FT_BASE_DEF( const char* )  _ft_debug_file   = NULL;
    -  FT_BASE_DEF( long )         _ft_debug_lineno = 0;
    +  FT_BASE_DEF( const char* )  ft_debug_file_   = NULL;
    +  FT_BASE_DEF( long )         ft_debug_lineno_ = 0;
     
       extern void
       FT_DumpMemory( FT_Memory  memory );
    @@ -415,8 +415,8 @@
     
         /* cast to FT_PtrDist first since void* can be larger */
         /* than FT_UInt32 and GCC 4.1.1 emits a warning       */
    -    hash  = (FT_UInt32)(FT_PtrDist)(void*)_ft_debug_file +
    -              (FT_UInt32)( 5 * _ft_debug_lineno );
    +    hash  = (FT_UInt32)(FT_PtrDist)(void*)ft_debug_file_ +
    +              (FT_UInt32)( 5 * ft_debug_lineno_ );
         pnode = &table->sources[hash % FT_MEM_SOURCE_BUCKETS];
     
         for (;;)
    @@ -425,8 +425,8 @@
           if ( !node )
             break;
     
    -      if ( node->file_name == _ft_debug_file   &&
    -           node->line_no   == _ft_debug_lineno )
    +      if ( node->file_name == ft_debug_file_   &&
    +           node->line_no   == ft_debug_lineno_ )
             goto Exit;
     
           pnode = &node->link;
    @@ -437,8 +437,8 @@
           ft_mem_debug_panic(
             "not enough memory to perform memory debugging\n" );
     
    -    node->file_name = _ft_debug_file;
    -    node->line_no   = _ft_debug_lineno;
    +    node->file_name = ft_debug_file_;
    +    node->line_no   = ft_debug_lineno_;
     
         node->cur_blocks = 0;
         node->max_blocks = 0;
    @@ -495,7 +495,7 @@
                 "org=%s:%d new=%s:%d\n",
                 node->address, node->size,
                 FT_FILENAME( node->source->file_name ), node->source->line_no,
    -            FT_FILENAME( _ft_debug_file ), _ft_debug_lineno );
    +            FT_FILENAME( ft_debug_file_ ), ft_debug_lineno_ );
             }
           }
     
    @@ -582,7 +582,7 @@
                 "  Block was allocated at (%s:%ld)\n"
                 "  and released at (%s:%ld).",
                 address,
    -            FT_FILENAME( _ft_debug_file ), _ft_debug_lineno,
    +            FT_FILENAME( ft_debug_file_ ), ft_debug_lineno_,
                 FT_FILENAME( node->source->file_name ), node->source->line_no,
                 FT_FILENAME( node->free_file_name ), node->free_line_no );
     
    @@ -604,8 +604,8 @@
               /* we simply invert the node's size to indicate that the node */
               /* was freed.                                                 */
               node->size           = -node->size;
    -          node->free_file_name = _ft_debug_file;
    -          node->free_line_no   = _ft_debug_lineno;
    +          node->free_file_name = ft_debug_file_;
    +          node->free_line_no   = ft_debug_lineno_;
             }
             else
             {
    @@ -627,7 +627,7 @@
             ft_mem_debug_panic(
               "trying to free unknown block at %p in (%s:%ld)\n",
               address,
    -          FT_FILENAME( _ft_debug_file ), _ft_debug_lineno );
    +          FT_FILENAME( ft_debug_file_ ), ft_debug_lineno_ );
         }
       }
     
    @@ -661,8 +661,8 @@
           table->alloc_count++;
         }
     
    -    _ft_debug_file   = "";
    -    _ft_debug_lineno = 0;
    +    ft_debug_file_   = "";
    +    ft_debug_lineno_ = 0;
     
         return (FT_Pointer)block;
       }
    @@ -677,8 +677,8 @@
     
         if ( !block )
           ft_mem_debug_panic( "trying to free NULL in (%s:%ld)",
    -                          FT_FILENAME( _ft_debug_file ),
    -                          _ft_debug_lineno );
    +                          FT_FILENAME( ft_debug_file_ ),
    +                          ft_debug_lineno_ );
     
         ft_mem_table_remove( table, (FT_Byte*)block, 0 );
     
    @@ -687,8 +687,8 @@
     
         table->alloc_count--;
     
    -    _ft_debug_file   = "";
    -    _ft_debug_lineno = 0;
    +    ft_debug_file_   = "";
    +    ft_debug_lineno_ = 0;
       }
     
     
    @@ -703,8 +703,8 @@
         FT_Pointer   new_block;
         FT_Long      delta;
     
    -    const char*  file_name = FT_FILENAME( _ft_debug_file );
    -    FT_Long      line_no   = _ft_debug_lineno;
    +    const char*  file_name = FT_FILENAME( ft_debug_file_ );
    +    FT_Long      line_no   = ft_debug_lineno_;
     
     
         /* unlikely, but possible */
    @@ -767,8 +767,8 @@
     
         ft_mem_table_remove( table, (FT_Byte*)block, delta );
     
    -    _ft_debug_file   = "";
    -    _ft_debug_lineno = 0;
    +    ft_debug_file_   = "";
    +    ft_debug_lineno_ = 0;
     
         if ( !table->keep_alive )
           ft_mem_table_free( table, block );
    @@ -874,7 +874,7 @@
       }
     
     
    -  static int
    +  FT_COMPARE_DEF( int )
       ft_mem_source_compare( const void*  p1,
                              const void*  p2 )
       {
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftdebug.c b/src/java.desktop/share/native/libfreetype/src/base/ftdebug.c
    index 648fff44edb3d..61c4563b0c433 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftdebug.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftdebug.c
    @@ -4,7 +4,7 @@
      *
      *   Debugging and logging component (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftfntfmt.c b/src/java.desktop/share/native/libfreetype/src/base/ftfntfmt.c
    index e69c1e06842c2..0b41f7cc83d00 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftfntfmt.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftfntfmt.c
    @@ -4,7 +4,7 @@
      *
      *   FreeType utility file for font formats (body).
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftfstype.c b/src/java.desktop/share/native/libfreetype/src/base/ftfstype.c
    index 009d58c57d5f2..ea24e64c6ea92 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftfstype.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftfstype.c
    @@ -4,7 +4,7 @@
      *
      *   FreeType utility file to access FSType data (body).
      *
    - * Copyright (C) 2008-2022 by
    + * Copyright (C) 2008-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftgasp.c b/src/java.desktop/share/native/libfreetype/src/base/ftgasp.c
    index 7567e3077ae88..29b7b08b787c3 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftgasp.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftgasp.c
    @@ -4,7 +4,7 @@
      *
      *   Access of TrueType's `gasp' table (body).
      *
    - * Copyright (C) 2007-2022 by
    + * Copyright (C) 2007-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftgloadr.c b/src/java.desktop/share/native/libfreetype/src/base/ftgloadr.c
    index f05abdee81820..9823d09e41afa 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftgloadr.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftgloadr.c
    @@ -4,7 +4,7 @@
      *
      *   The FreeType glyph loader (body).
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -212,12 +212,12 @@
         FT_Outline*  current = &loader->current.outline;
         FT_Bool      adjust  = 0;
     
    -    FT_UInt      new_max, old_max;
    +    FT_UInt  new_max, old_max, min_new_max;
     
     
         error = FT_GlyphLoader_CreateExtra( loader );
         if ( error )
    -      return error;
    +      goto Exit;
     
         /* check points & tags */
         new_max = (FT_UInt)base->n_points + (FT_UInt)current->n_points +
    @@ -226,10 +226,18 @@
     
         if ( new_max > old_max )
         {
    -      new_max = FT_PAD_CEIL( new_max, 8 );
    +      if ( new_max > FT_OUTLINE_POINTS_MAX )
    +      {
    +        error = FT_THROW( Array_Too_Large );
    +        goto Exit;
    +      }
     
    +      min_new_max = old_max + ( old_max >> 1 );
    +      if ( new_max < min_new_max )
    +        new_max = min_new_max;
    +      new_max = FT_PAD_CEIL( new_max, 8 );
           if ( new_max > FT_OUTLINE_POINTS_MAX )
    -        return FT_THROW( Array_Too_Large );
    +        new_max = FT_OUTLINE_POINTS_MAX;
     
           if ( FT_RENEW_ARRAY( base->points, old_max, new_max ) ||
                FT_RENEW_ARRAY( base->tags,   old_max, new_max ) )
    @@ -254,7 +262,7 @@
     
         error = FT_GlyphLoader_CreateExtra( loader );
         if ( error )
    -      return error;
    +      goto Exit;
     
         /* check contours */
         old_max = loader->max_contours;
    @@ -262,10 +270,18 @@
                   n_contours;
         if ( new_max > old_max )
         {
    -      new_max = FT_PAD_CEIL( new_max, 4 );
    +      if ( new_max > FT_OUTLINE_CONTOURS_MAX )
    +      {
    +        error = FT_THROW( Array_Too_Large );
    +        goto Exit;
    +      }
     
    +      min_new_max = old_max + ( old_max >> 1 );
    +      if ( new_max < min_new_max )
    +        new_max = min_new_max;
    +      new_max = FT_PAD_CEIL( new_max, 4 );
           if ( new_max > FT_OUTLINE_CONTOURS_MAX )
    -        return FT_THROW( Array_Too_Large );
    +        new_max = FT_OUTLINE_CONTOURS_MAX;
     
           if ( FT_RENEW_ARRAY( base->contours, old_max, new_max ) )
             goto Exit;
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftglyph.c b/src/java.desktop/share/native/libfreetype/src/base/ftglyph.c
    index 571dca1a965ef..393d4949f849d 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftglyph.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftglyph.c
    @@ -4,7 +4,7 @@
      *
      *   FreeType convenience functions to handle glyphs (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -682,7 +682,10 @@
       Exit2:
         /* if an error occurred, destroy the glyph */
         if ( error )
    +    {
           FT_Done_Glyph( glyph );
    +      *aglyph = NULL;
    +    }
         else
           *aglyph = glyph;
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftinit.c b/src/java.desktop/share/native/libfreetype/src/base/ftinit.c
    index 0f29a6017e507..c9c71d24bf961 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftinit.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftinit.c
    @@ -4,7 +4,7 @@
      *
      *   FreeType initialization layer (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftlcdfil.c b/src/java.desktop/share/native/libfreetype/src/base/ftlcdfil.c
    index e72f6d668d7d9..6c3fd66e0bb15 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftlcdfil.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftlcdfil.c
    @@ -4,7 +4,7 @@
      *
      *   FreeType API for color filtering of subpixel bitmap glyphs (body).
      *
    - * Copyright (C) 2006-2022 by
    + * Copyright (C) 2006-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftmac.c b/src/java.desktop/share/native/libfreetype/src/base/ftmac.c
    index 21f1894ad3808..de34e834f255e 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftmac.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftmac.c
    @@ -8,7 +8,7 @@
      * This file is for Mac OS X only; see builds/mac/ftoldmac.c for
      * classic platforms built by MPW.
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * Just van Rossum, David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -67,6 +67,7 @@
     
     #include 
     #include 
    +#include 
     #include 
     #include "ftbase.h"
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftmm.c b/src/java.desktop/share/native/libfreetype/src/base/ftmm.c
    index dbbd87c9b9cea..a2b4bd03d78cf 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftmm.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftmm.c
    @@ -4,7 +4,7 @@
      *
      *   Multiple Master font support (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftobjs.c b/src/java.desktop/share/native/libfreetype/src/base/ftobjs.c
    index eeda69c3ed84a..ad6ef0ae168e6 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftobjs.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftobjs.c
    @@ -4,7 +4,7 @@
      *
      *   The FreeType private base classes (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -508,7 +508,7 @@
     
         case FT_PIXEL_MODE_LCD_V:
           height *= 3;
    -      /* fall through */
    +      FALL_THROUGH;
     
         case FT_PIXEL_MODE_GRAY:
         default:
    @@ -605,7 +605,7 @@
     
     
             FT_FREE( doc->svg_document );
    -        slot->internal->load_flags &= ~FT_GLYPH_OWN_GZIP_SVG;
    +        slot->internal->flags &= ~FT_GLYPH_OWN_GZIP_SVG;
           }
         }
     #endif
    @@ -631,8 +631,9 @@
     #ifdef FT_CONFIG_OPTION_SVG
         if ( slot->face->face_flags & FT_FACE_FLAG_SVG )
         {
    -      /* free memory in case SVG was there */
    -      if ( slot->internal->flags & FT_GLYPH_OWN_GZIP_SVG )
    +      /* Free memory in case SVG was there.                          */
    +      /* `slot->internal` might be NULL in out-of-memory situations. */
    +      if ( slot->internal && slot->internal->flags & FT_GLYPH_OWN_GZIP_SVG )
           {
             FT_SVG_Document  doc = (FT_SVG_Document)slot->other;
     
    @@ -1184,28 +1185,34 @@
                     pixel_modes[slot->bitmap.pixel_mode],
                     slot->bitmap.pixel_mode ));
         FT_TRACE5(( "\n" ));
    -    FT_TRACE5(( "  x advance: %f\n", slot->advance.x / 64.0 ));
    -    FT_TRACE5(( "  y advance: %f\n", slot->advance.y / 64.0 ));
    +    FT_TRACE5(( "  x advance: %f\n", (double)slot->advance.x / 64 ));
    +    FT_TRACE5(( "  y advance: %f\n", (double)slot->advance.y / 64 ));
         FT_TRACE5(( "  linear x advance: %f\n",
    -                slot->linearHoriAdvance / 65536.0 ));
    +                (double)slot->linearHoriAdvance / 65536 ));
         FT_TRACE5(( "  linear y advance: %f\n",
    -                slot->linearVertAdvance / 65536.0 ));
    +                (double)slot->linearVertAdvance / 65536 ));
     
         {
           FT_Glyph_Metrics*  metrics = &slot->metrics;
     
     
           FT_TRACE5(( "  metrics:\n" ));
    -      FT_TRACE5(( "    width:  %f\n", metrics->width  / 64.0 ));
    -      FT_TRACE5(( "    height: %f\n", metrics->height / 64.0 ));
    +      FT_TRACE5(( "    width:  %f\n", (double)metrics->width / 64 ));
    +      FT_TRACE5(( "    height: %f\n", (double)metrics->height / 64 ));
           FT_TRACE5(( "\n" ));
    -      FT_TRACE5(( "    horiBearingX: %f\n", metrics->horiBearingX / 64.0 ));
    -      FT_TRACE5(( "    horiBearingY: %f\n", metrics->horiBearingY / 64.0 ));
    -      FT_TRACE5(( "    horiAdvance:  %f\n", metrics->horiAdvance  / 64.0 ));
    +      FT_TRACE5(( "    horiBearingX: %f\n",
    +                  (double)metrics->horiBearingX / 64 ));
    +      FT_TRACE5(( "    horiBearingY: %f\n",
    +                  (double)metrics->horiBearingY / 64 ));
    +      FT_TRACE5(( "    horiAdvance:  %f\n",
    +                  (double)metrics->horiAdvance / 64 ));
           FT_TRACE5(( "\n" ));
    -      FT_TRACE5(( "    vertBearingX: %f\n", metrics->vertBearingX / 64.0 ));
    -      FT_TRACE5(( "    vertBearingY: %f\n", metrics->vertBearingY / 64.0 ));
    -      FT_TRACE5(( "    vertAdvance:  %f\n", metrics->vertAdvance  / 64.0 ));
    +      FT_TRACE5(( "    vertBearingX: %f\n",
    +                  (double)metrics->vertBearingX / 64 ));
    +      FT_TRACE5(( "    vertBearingY: %f\n",
    +                  (double)metrics->vertBearingY / 64 ));
    +      FT_TRACE5(( "    vertAdvance:  %f\n",
    +                  (double)metrics->vertAdvance / 64 ));
         }
     #endif
     
    @@ -1488,7 +1495,7 @@
       static FT_Error
       open_face( FT_Driver      driver,
                  FT_Stream      *astream,
    -             FT_Bool        external_stream,
    +             FT_Bool        *anexternal_stream,
                  FT_Long        face_index,
                  FT_Int         num_params,
                  FT_Parameter*  params,
    @@ -1514,7 +1521,7 @@
         face->stream = *astream;
     
         /* set the FT_FACE_FLAG_EXTERNAL_STREAM bit for FT_Done_Face */
    -    if ( external_stream )
    +    if ( *anexternal_stream )
           face->face_flags |= FT_FACE_FLAG_EXTERNAL_STREAM;
     
         if ( FT_NEW( internal ) )
    @@ -1544,7 +1551,10 @@
                                     (FT_Int)face_index,
                                     num_params,
                                     params );
    -    *astream = face->stream; /* Stream may have been changed. */
    +    /* Stream may have been changed. */
    +    *astream = face->stream;
    +    *anexternal_stream =
    +      ( face->face_flags & FT_FACE_FLAG_EXTERNAL_STREAM ) != 0;
         if ( error )
           goto Fail;
     
    @@ -1668,13 +1678,13 @@
       static void
       memory_stream_close( FT_Stream  stream )
       {
    -    FT_Memory  memory = stream->memory;
    +    FT_Memory  memory = (FT_Memory)stream->descriptor.pointer;
     
     
         FT_FREE( stream->base );
    -
         stream->size  = 0;
         stream->close = NULL;
    +    FT_FREE( stream );
       }
     
     
    @@ -1705,7 +1715,8 @@
     
         FT_Stream_OpenMemory( stream, base, size );
     
    -    stream->close = close;
    +    stream->descriptor.pointer = memory;
    +    stream->close              = close;
     
         *astream = stream;
     
    @@ -1726,28 +1737,36 @@
       {
         FT_Open_Args  args;
         FT_Error      error;
    -    FT_Stream     stream = NULL;
         FT_Memory     memory = library->memory;
     
     
    +    args.flags = 0;
    +
    +    if ( driver_name )
    +    {
    +      args.driver = FT_Get_Module( library, driver_name );
    +      if ( !args.driver )
    +      {
    +        FT_FREE( base );
    +        return FT_THROW( Missing_Module );
    +      }
    +
    +      args.flags = args.flags | FT_OPEN_DRIVER;
    +    }
    +
    +    /* `memory_stream_close` also frees the stream object. */
         error = new_memory_stream( library,
                                    base,
                                    size,
                                    memory_stream_close,
    -                               &stream );
    +                               &args.stream );
         if ( error )
         {
           FT_FREE( base );
           return error;
         }
     
    -    args.flags  = FT_OPEN_STREAM;
    -    args.stream = stream;
    -    if ( driver_name )
    -    {
    -      args.flags  = args.flags | FT_OPEN_DRIVER;
    -      args.driver = FT_Get_Module( library, driver_name );
    -    }
    +    args.flags |= FT_OPEN_STREAM;
     
     #ifdef FT_MACINTOSH
         /* At this point, the face index has served its purpose;  */
    @@ -1759,21 +1778,7 @@
           face_index &= 0x7FFF0000L; /* retain GX data */
     #endif
     
    -    error = ft_open_face_internal( library, &args, face_index, aface, 0 );
    -
    -    if ( !error )
    -      (*aface)->face_flags &= ~FT_FACE_FLAG_EXTERNAL_STREAM;
    -    else
    -#ifdef FT_MACINTOSH
    -      FT_Stream_Free( stream, 0 );
    -#else
    -    {
    -      FT_Stream_Close( stream );
    -      FT_FREE( stream );
    -    }
    -#endif
    -
    -    return error;
    +    return ft_open_face_internal( library, &args, face_index, aface, 0 );
       }
     
     
    @@ -1916,7 +1921,7 @@
                                        sfnt_ps,
                                        length,
                                        FT_MIN( face_index, 0 ),
    -                                   is_sfnt_cid ? "cid" : "type1",
    +                                   is_sfnt_cid ? "t1cid" : "type1",
                                        aface );
       Exit:
         {
    @@ -2177,7 +2182,7 @@
         FT_Byte*   sfnt_data = NULL;
         FT_Error   error;
         FT_ULong   flag_offset;
    -    FT_Long    rlen;
    +    FT_ULong   rlen;
         int        is_cff;
         FT_Long    face_index_in_resource = 0;
     
    @@ -2192,11 +2197,11 @@
         if ( error )
           goto Exit;
     
    -    if ( FT_READ_LONG( rlen ) )
    +    if ( FT_READ_ULONG( rlen ) )
           goto Exit;
    -    if ( rlen < 1 )
    +    if ( !rlen )
           return FT_THROW( Cannot_Open_Resource );
    -    if ( (FT_ULong)rlen > FT_MAC_RFORK_MAX_LEN )
    +    if ( rlen > FT_MAC_RFORK_MAX_LEN )
           return FT_THROW( Invalid_Offset );
     
         error = open_face_PS_from_sfnt_stream( library,
    @@ -2214,8 +2219,9 @@
     
         if ( FT_QALLOC( sfnt_data, rlen ) )
           return error;
    -    error = FT_Stream_Read( stream, (FT_Byte *)sfnt_data, (FT_ULong)rlen );
    -    if ( error ) {
    +    error = FT_Stream_Read( stream, (FT_Byte *)sfnt_data, rlen );
    +    if ( error )
    +    {
           FT_FREE( sfnt_data );
           goto Exit;
         }
    @@ -2223,7 +2229,7 @@
         is_cff = rlen > 4 && !ft_memcmp( sfnt_data, "OTTO", 4 );
         error = open_face_from_buffer( library,
                                        sfnt_data,
    -                                   (FT_ULong)rlen,
    +                                   rlen,
                                        face_index_in_resource,
                                        is_cff ? "cff" : "truetype",
                                        aface );
    @@ -2552,7 +2558,7 @@
     
         /* test for valid `library' delayed to `FT_Stream_New' */
     
    -    if ( ( !aface && face_index >= 0 ) || !args )
    +    if ( !args )
           return FT_THROW( Invalid_Argument );
     
         external_stream = FT_BOOL( ( args->flags & FT_OPEN_STREAM ) &&
    @@ -2563,6 +2569,14 @@
         if ( error )
           goto Fail3;
     
    +    /* Do this error check after `FT_Stream_New` to ensure that the */
    +    /* 'close' callback is called.                                  */
    +    if ( !aface && face_index >= 0 )
    +    {
    +      error = FT_THROW( Invalid_Argument );
    +      goto Fail3;
    +    }
    +
         memory = library->memory;
     
         /* If the font driver is specified in the `args' structure, use */
    @@ -2584,7 +2598,7 @@
               params     = args->params;
             }
     
    -        error = open_face( driver, &stream, external_stream, face_index,
    +        error = open_face( driver, &stream, &external_stream, face_index,
                                num_params, params, &face );
             if ( !error )
               goto Success;
    @@ -2620,7 +2634,7 @@
                 params     = args->params;
               }
     
    -          error = open_face( driver, &stream, external_stream, face_index,
    +          error = open_face( driver, &stream, &external_stream, face_index,
                                  num_params, params, &face );
               if ( !error )
                 goto Success;
    @@ -2852,8 +2866,8 @@
       /* documentation is in freetype.h */
     
       FT_EXPORT_DEF( FT_Error )
    -  FT_Attach_Stream( FT_Face        face,
    -                    FT_Open_Args*  parameters )
    +  FT_Attach_Stream( FT_Face              face,
    +                    const FT_Open_Args*  parameters )
       {
         FT_Stream  stream;
         FT_Error   error;
    @@ -3278,34 +3292,49 @@
           scaled_h = FT_REQUEST_HEIGHT( req );
     
           /* determine scales */
    -      if ( req->width )
    +      if ( req->height || !req->width )
           {
    -        metrics->x_scale = FT_DivFix( scaled_w, w );
    -
    -        if ( req->height )
    +        if ( h == 0 )
             {
    -          metrics->y_scale = FT_DivFix( scaled_h, h );
    -
    -          if ( req->type == FT_SIZE_REQUEST_TYPE_CELL )
    -          {
    -            if ( metrics->y_scale > metrics->x_scale )
    -              metrics->y_scale = metrics->x_scale;
    -            else
    -              metrics->x_scale = metrics->y_scale;
    -          }
    +          FT_ERROR(( "FT_Request_Metrics: Divide by zero\n" ));
    +          error = FT_ERR( Divide_By_Zero );
    +          goto Exit;
             }
    -        else
    +
    +        metrics->y_scale = FT_DivFix( scaled_h, h );
    +      }
    +
    +      if ( req->width )
    +      {
    +        if ( w == 0 )
             {
    -          metrics->y_scale = metrics->x_scale;
    -          scaled_h = FT_MulDiv( scaled_w, h, w );
    +          FT_ERROR(( "FT_Request_Metrics: Divide by zero\n" ));
    +          error = FT_ERR( Divide_By_Zero );
    +          goto Exit;
             }
    +
    +        metrics->x_scale = FT_DivFix( scaled_w, w );
           }
           else
           {
    -        metrics->x_scale = metrics->y_scale = FT_DivFix( scaled_h, h );
    +        metrics->x_scale = metrics->y_scale;
             scaled_w = FT_MulDiv( scaled_h, w, h );
           }
     
    +      if ( !req->height )
    +      {
    +        metrics->y_scale = metrics->x_scale;
    +        scaled_h = FT_MulDiv( scaled_w, h, w );
    +      }
    +
    +      if ( req->type == FT_SIZE_REQUEST_TYPE_CELL )
    +      {
    +        if ( metrics->y_scale > metrics->x_scale )
    +          metrics->y_scale = metrics->x_scale;
    +        else
    +          metrics->x_scale = metrics->y_scale;
    +      }
    +
       Calculate_Ppem:
           /* calculate the ppems */
           if ( req->type != FT_SIZE_REQUEST_TYPE_NOMINAL )
    @@ -3379,15 +3408,19 @@
     
     
           FT_TRACE5(( "  x scale: %ld (%f)\n",
    -                  metrics->x_scale, metrics->x_scale / 65536.0 ));
    +                  metrics->x_scale, (double)metrics->x_scale / 65536 ));
           FT_TRACE5(( "  y scale: %ld (%f)\n",
    -                  metrics->y_scale, metrics->y_scale / 65536.0 ));
    -      FT_TRACE5(( "  ascender: %f\n",    metrics->ascender / 64.0 ));
    -      FT_TRACE5(( "  descender: %f\n",   metrics->descender / 64.0 ));
    -      FT_TRACE5(( "  height: %f\n",      metrics->height / 64.0 ));
    -      FT_TRACE5(( "  max advance: %f\n", metrics->max_advance / 64.0 ));
    -      FT_TRACE5(( "  x ppem: %d\n",      metrics->x_ppem ));
    -      FT_TRACE5(( "  y ppem: %d\n",      metrics->y_ppem ));
    +                  metrics->y_scale, (double)metrics->y_scale / 65536 ));
    +      FT_TRACE5(( "  ascender: %f\n",
    +                  (double)metrics->ascender / 64 ));
    +      FT_TRACE5(( "  descender: %f\n",
    +                  (double)metrics->descender / 64 ));
    +      FT_TRACE5(( "  height: %f\n",
    +                  (double)metrics->height / 64 ));
    +      FT_TRACE5(( "  max advance: %f\n",
    +                  (double)metrics->max_advance / 64 ));
    +      FT_TRACE5(( "  x ppem: %d\n", metrics->x_ppem ));
    +      FT_TRACE5(( "  y ppem: %d\n", metrics->y_ppem ));
         }
     #endif
     
    @@ -3459,15 +3492,19 @@
     
     
           FT_TRACE5(( "  x scale: %ld (%f)\n",
    -                  metrics->x_scale, metrics->x_scale / 65536.0 ));
    +                  metrics->x_scale, (double)metrics->x_scale / 65536 ));
           FT_TRACE5(( "  y scale: %ld (%f)\n",
    -                  metrics->y_scale, metrics->y_scale / 65536.0 ));
    -      FT_TRACE5(( "  ascender: %f\n",    metrics->ascender / 64.0 ));
    -      FT_TRACE5(( "  descender: %f\n",   metrics->descender / 64.0 ));
    -      FT_TRACE5(( "  height: %f\n",      metrics->height / 64.0 ));
    -      FT_TRACE5(( "  max advance: %f\n", metrics->max_advance / 64.0 ));
    -      FT_TRACE5(( "  x ppem: %d\n",      metrics->x_ppem ));
    -      FT_TRACE5(( "  y ppem: %d\n",      metrics->y_ppem ));
    +                  metrics->y_scale, (double)metrics->y_scale / 65536 ));
    +      FT_TRACE5(( "  ascender: %f\n",
    +                  (double)metrics->ascender / 64 ));
    +      FT_TRACE5(( "  descender: %f\n",
    +                  (double)metrics->descender / 64 ));
    +      FT_TRACE5(( "  height: %f\n",
    +                  (double)metrics->height / 64 ));
    +      FT_TRACE5(( "  max advance: %f\n",
    +                  (double)metrics->max_advance / 64 ));
    +      FT_TRACE5(( "  x ppem: %d\n", metrics->x_ppem ));
    +      FT_TRACE5(( "  y ppem: %d\n", metrics->y_ppem ));
         }
     #endif
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftoutln.c b/src/java.desktop/share/native/libfreetype/src/base/ftoutln.c
    index 624df03ad8d16..30ff21ff39e6f 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftoutln.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftoutln.c
    @@ -4,7 +4,7 @@
      *
      *   FreeType outline management (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -130,7 +130,7 @@
           }
     
           FT_TRACE5(( "  move to (%.2f, %.2f)\n",
    -                  v_start.x / 64.0, v_start.y / 64.0 ));
    +                  (double)v_start.x / 64, (double)v_start.y / 64 ));
           error = func_interface->move_to( &v_start, user );
           if ( error )
             goto Exit;
    @@ -152,7 +152,7 @@
                 vec.y = SCALED( point->y );
     
                 FT_TRACE5(( "  line to (%.2f, %.2f)\n",
    -                        vec.x / 64.0, vec.y / 64.0 ));
    +                        (double)vec.x / 64, (double)vec.y / 64 ));
                 error = func_interface->line_to( &vec, user );
                 if ( error )
                   goto Exit;
    @@ -181,8 +181,10 @@
                 {
                   FT_TRACE5(( "  conic to (%.2f, %.2f)"
                               " with control (%.2f, %.2f)\n",
    -                          vec.x / 64.0, vec.y / 64.0,
    -                          v_control.x / 64.0, v_control.y / 64.0 ));
    +                          (double)vec.x / 64,
    +                          (double)vec.y / 64,
    +                          (double)v_control.x / 64,
    +                          (double)v_control.y / 64 ));
                   error = func_interface->conic_to( &v_control, &vec, user );
                   if ( error )
                     goto Exit;
    @@ -197,8 +199,10 @@
     
                 FT_TRACE5(( "  conic to (%.2f, %.2f)"
                             " with control (%.2f, %.2f)\n",
    -                        v_middle.x / 64.0, v_middle.y / 64.0,
    -                        v_control.x / 64.0, v_control.y / 64.0 ));
    +                        (double)v_middle.x / 64,
    +                        (double)v_middle.y / 64,
    +                        (double)v_control.x / 64,
    +                        (double)v_control.y / 64 ));
                 error = func_interface->conic_to( &v_control, &v_middle, user );
                 if ( error )
                   goto Exit;
    @@ -209,8 +213,10 @@
     
               FT_TRACE5(( "  conic to (%.2f, %.2f)"
                           " with control (%.2f, %.2f)\n",
    -                      v_start.x / 64.0, v_start.y / 64.0,
    -                      v_control.x / 64.0, v_control.y / 64.0 ));
    +                      (double)v_start.x / 64,
    +                      (double)v_start.y / 64,
    +                      (double)v_control.x / 64,
    +                      (double)v_control.y / 64 ));
               error = func_interface->conic_to( &v_control, &v_start, user );
               goto Close;
     
    @@ -242,9 +248,12 @@
     
                   FT_TRACE5(( "  cubic to (%.2f, %.2f)"
                               " with controls (%.2f, %.2f) and (%.2f, %.2f)\n",
    -                          vec.x / 64.0, vec.y / 64.0,
    -                          vec1.x / 64.0, vec1.y / 64.0,
    -                          vec2.x / 64.0, vec2.y / 64.0 ));
    +                          (double)vec.x / 64,
    +                          (double)vec.y / 64,
    +                          (double)vec1.x / 64,
    +                          (double)vec1.y / 64,
    +                          (double)vec2.x / 64,
    +                          (double)vec2.y / 64 ));
                   error = func_interface->cubic_to( &vec1, &vec2, &vec, user );
                   if ( error )
                     goto Exit;
    @@ -253,9 +262,12 @@
     
                 FT_TRACE5(( "  cubic to (%.2f, %.2f)"
                             " with controls (%.2f, %.2f) and (%.2f, %.2f)\n",
    -                        v_start.x / 64.0, v_start.y / 64.0,
    -                        vec1.x / 64.0, vec1.y / 64.0,
    -                        vec2.x / 64.0, vec2.y / 64.0 ));
    +                        (double)v_start.x / 64,
    +                        (double)v_start.y / 64,
    +                        (double)vec1.x / 64,
    +                        (double)vec1.y / 64,
    +                        (double)vec2.x / 64,
    +                        (double)vec2.y / 64 ));
                 error = func_interface->cubic_to( &vec1, &vec2, &v_start, user );
                 goto Close;
               }
    @@ -264,7 +276,7 @@
     
           /* close the contour with a line segment */
           FT_TRACE5(( "  line to (%.2f, %.2f)\n",
    -                  v_start.x / 64.0, v_start.y / 64.0 ));
    +                  (double)v_start.x / 64, (double)v_start.y / 64 ));
           error = func_interface->line_to( &v_start, user );
     
         Close:
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftpatent.c b/src/java.desktop/share/native/libfreetype/src/base/ftpatent.c
    index 353ed2b5317c2..cb5efadffb1df 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftpatent.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftpatent.c
    @@ -5,7 +5,7 @@
      *   FreeType API for checking patented TrueType bytecode instructions
      *   (body).  Obsolete, retained for backward compatibility.
      *
    - * Copyright (C) 2007-2022 by
    + * Copyright (C) 2007-2023 by
      * David Turner.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftpsprop.c b/src/java.desktop/share/native/libfreetype/src/base/ftpsprop.c
    index 81fcee08f6f8d..cefdf489d7f68 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftpsprop.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftpsprop.c
    @@ -5,7 +5,7 @@
      *   Get and set properties of PostScript drivers (body).
      *   See `ftdriver.h' for available properties.
      *
    - * Copyright (C) 2017-2022 by
    + * Copyright (C) 2017-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftrfork.c b/src/java.desktop/share/native/libfreetype/src/base/ftrfork.c
    index 356998d3fa14e..2ab430195f20b 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftrfork.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftrfork.c
    @@ -4,7 +4,7 @@
      *
      *   Embedded resource forks accessor (body).
      *
    - * Copyright (C) 2004-2022 by
    + * Copyright (C) 2004-2023 by
      * Masatake YAMATO and Redhat K.K.
      *
      * FT_Raccess_Get_HeaderInfo() and raccess_guess_darwin_hfsplus() are
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftsnames.c b/src/java.desktop/share/native/libfreetype/src/base/ftsnames.c
    index 3bf20c389b387..1917a3f1dffbd 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftsnames.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftsnames.c
    @@ -7,7 +7,7 @@
      *
      *   This is _not_ used to retrieve glyph names!
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftstream.c b/src/java.desktop/share/native/libfreetype/src/base/ftstream.c
    index cc926565c32a7..05c5637578be1 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftstream.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftstream.c
    @@ -4,7 +4,7 @@
      *
      *   I/O stream support (body).
      *
    - * Copyright (C) 2000-2022 by
    + * Copyright (C) 2000-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -261,7 +261,7 @@
           }
     
     #ifdef FT_DEBUG_MEMORY
    -      /* assume _ft_debug_file and _ft_debug_lineno are already set */
    +      /* assume `ft_debug_file_` and `ft_debug_lineno_` are already set */
           stream->base = (unsigned char*)ft_mem_qalloc( memory,
                                                         (FT_Long)count,
                                                         &error );
    @@ -363,11 +363,11 @@
       }
     
     
    -  FT_BASE_DEF( FT_UShort )
    +  FT_BASE_DEF( FT_UInt16 )
       FT_Stream_GetUShort( FT_Stream  stream )
       {
         FT_Byte*   p;
    -    FT_UShort  result;
    +    FT_UInt16  result;
     
     
         FT_ASSERT( stream && stream->cursor );
    @@ -382,11 +382,11 @@
       }
     
     
    -  FT_BASE_DEF( FT_UShort )
    +  FT_BASE_DEF( FT_UInt16 )
       FT_Stream_GetUShortLE( FT_Stream  stream )
       {
         FT_Byte*   p;
    -    FT_UShort  result;
    +    FT_UInt16  result;
     
     
         FT_ASSERT( stream && stream->cursor );
    @@ -401,11 +401,11 @@
       }
     
     
    -  FT_BASE_DEF( FT_ULong )
    +  FT_BASE_DEF( FT_UInt32 )
       FT_Stream_GetUOffset( FT_Stream  stream )
       {
         FT_Byte*  p;
    -    FT_ULong  result;
    +    FT_UInt32 result;
     
     
         FT_ASSERT( stream && stream->cursor );
    @@ -419,11 +419,11 @@
       }
     
     
    -  FT_BASE_DEF( FT_ULong )
    +  FT_BASE_DEF( FT_UInt32 )
       FT_Stream_GetULong( FT_Stream  stream )
       {
         FT_Byte*  p;
    -    FT_ULong  result;
    +    FT_UInt32 result;
     
     
         FT_ASSERT( stream && stream->cursor );
    @@ -437,11 +437,11 @@
       }
     
     
    -  FT_BASE_DEF( FT_ULong )
    +  FT_BASE_DEF( FT_UInt32 )
       FT_Stream_GetULongLE( FT_Stream  stream )
       {
         FT_Byte*  p;
    -    FT_ULong  result;
    +    FT_UInt32 result;
     
     
         FT_ASSERT( stream && stream->cursor );
    @@ -493,13 +493,13 @@
       }
     
     
    -  FT_BASE_DEF( FT_UShort )
    +  FT_BASE_DEF( FT_UInt16 )
       FT_Stream_ReadUShort( FT_Stream  stream,
                             FT_Error*  error )
       {
         FT_Byte    reads[2];
         FT_Byte*   p;
    -    FT_UShort  result = 0;
    +    FT_UInt16  result = 0;
     
     
         FT_ASSERT( stream );
    @@ -538,13 +538,13 @@
       }
     
     
    -  FT_BASE_DEF( FT_UShort )
    +  FT_BASE_DEF( FT_UInt16 )
       FT_Stream_ReadUShortLE( FT_Stream  stream,
                               FT_Error*  error )
       {
         FT_Byte    reads[2];
         FT_Byte*   p;
    -    FT_UShort  result = 0;
    +    FT_UInt16  result = 0;
     
     
         FT_ASSERT( stream );
    @@ -628,13 +628,13 @@
       }
     
     
    -  FT_BASE_DEF( FT_ULong )
    +  FT_BASE_DEF( FT_UInt32 )
       FT_Stream_ReadULong( FT_Stream  stream,
                            FT_Error*  error )
       {
         FT_Byte   reads[4];
         FT_Byte*  p;
    -    FT_ULong  result = 0;
    +    FT_UInt32 result = 0;
     
     
         FT_ASSERT( stream );
    @@ -673,13 +673,13 @@
       }
     
     
    -  FT_BASE_DEF( FT_ULong )
    +  FT_BASE_DEF( FT_UInt32 )
       FT_Stream_ReadULongLE( FT_Stream  stream,
                              FT_Error*  error )
       {
         FT_Byte   reads[4];
         FT_Byte*  p;
    -    FT_ULong  result = 0;
    +    FT_UInt32 result = 0;
     
     
         FT_ASSERT( stream );
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftstroke.c b/src/java.desktop/share/native/libfreetype/src/base/ftstroke.c
    index aa983f940f261..db358e772ed1a 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftstroke.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftstroke.c
    @@ -4,7 +4,7 @@
      *
      *   FreeType path stroker (body).
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftsynth.c b/src/java.desktop/share/native/libfreetype/src/base/ftsynth.c
    index 10bbe0dfdaf1e..6ec25e13e47f2 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftsynth.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftsynth.c
    @@ -4,7 +4,7 @@
      *
      *   FreeType synthesizing code for emboldening and slanting (body).
      *
    - * Copyright (C) 2000-2022 by
    + * Copyright (C) 2000-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -45,6 +45,18 @@
     
       FT_EXPORT_DEF( void )
       FT_GlyphSlot_Oblique( FT_GlyphSlot  slot )
    +  {
    +    /* Value '0x0366A' corresponds to a shear angle of about 12 degrees. */
    +    FT_GlyphSlot_Slant( slot, 0x0366A, 0 );
    +  }
    +
    +
    +  /* documentation is in ftsynth.h */
    +
    +  FT_EXPORT_DEF( void )
    +  FT_GlyphSlot_Slant( FT_GlyphSlot  slot,
    +                      FT_Fixed      xslant,
    +                      FT_Fixed      yslant )
       {
         FT_Matrix    transform;
         FT_Outline*  outline;
    @@ -61,13 +73,11 @@
     
         /* we don't touch the advance width */
     
    -    /* For italic, simply apply a shear transform, with an angle */
    -    /* of about 12 degrees.                                      */
    -
    +    /* For italic, simply apply a shear transform */
         transform.xx = 0x10000L;
    -    transform.yx = 0x00000L;
    +    transform.yx = -yslant;
     
    -    transform.xy = 0x0366AL;
    +    transform.xy = xslant;
         transform.yy = 0x10000L;
     
         FT_Outline_Transform( outline, &transform );
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftsystem.c b/src/java.desktop/share/native/libfreetype/src/base/ftsystem.c
    index d8826b23671f9..fcd289d19f4bb 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftsystem.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftsystem.c
    @@ -4,7 +4,7 @@
      *
      *   ANSI-specific FreeType low-level system interface (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/fttrigon.c b/src/java.desktop/share/native/libfreetype/src/base/fttrigon.c
    index 6964edb0f5934..2dd2c3459e5fd 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/fttrigon.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/fttrigon.c
    @@ -4,7 +4,7 @@
      *
      *   FreeType trigonometric functions (body).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/fttype1.c b/src/java.desktop/share/native/libfreetype/src/base/fttype1.c
    index de3d5a48bdc54..637c5cf775e6c 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/fttype1.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/fttype1.c
    @@ -4,7 +4,7 @@
      *
      *   FreeType utility file for PS names support (body).
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/base/ftutil.c b/src/java.desktop/share/native/libfreetype/src/base/ftutil.c
    index 5a913825802db..6120846d2ca0c 100644
    --- a/src/java.desktop/share/native/libfreetype/src/base/ftutil.c
    +++ b/src/java.desktop/share/native/libfreetype/src/base/ftutil.c
    @@ -4,7 +4,7 @@
      *
      *   FreeType utility file for memory and list management (body).
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cff/cffcmap.c b/src/java.desktop/share/native/libfreetype/src/cff/cffcmap.c
    index 2d667a7248f6e..6ed3143222766 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cff/cffcmap.c
    +++ b/src/java.desktop/share/native/libfreetype/src/cff/cffcmap.c
    @@ -4,7 +4,7 @@
      *
      *   CFF character mapping table (cmap) support (body).
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cff/cffcmap.h b/src/java.desktop/share/native/libfreetype/src/cff/cffcmap.h
    index 2818d3c6feda8..b2afc2fab622c 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cff/cffcmap.h
    +++ b/src/java.desktop/share/native/libfreetype/src/cff/cffcmap.h
    @@ -4,7 +4,7 @@
      *
      *   CFF character mapping table (cmap) support (specification).
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cff/cffdrivr.c b/src/java.desktop/share/native/libfreetype/src/cff/cffdrivr.c
    index d945afdfe8259..4e2e0e00deba8 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cff/cffdrivr.c
    +++ b/src/java.desktop/share/native/libfreetype/src/cff/cffdrivr.c
    @@ -4,8 +4,8 @@
      *
      *   OpenType font driver implementation (body).
      *
    - * Copyright (C) 1996-2022 by
    - * David Turner, Robert Wilhelm, and Werner Lemberg.
    + * Copyright (C) 1996-2023 by
    + * David Turner, Robert Wilhelm, Werner Lemberg, and Dominik Röttsches.
      *
      * This file is part of the FreeType project, and may only be used,
      * modified, and distributed under the terms of the FreeType project
    @@ -936,22 +936,103 @@
       }
     
     
    +  static FT_Error
    +  cff_load_item_variation_store( CFF_Face         face,
    +                                 FT_ULong         offset,
    +                                 GX_ItemVarStore  itemStore )
    +  {
    +    FT_Service_MultiMasters  mm = (FT_Service_MultiMasters)face->mm;
    +
    +
    +    return mm->load_item_var_store( FT_FACE(face), offset, itemStore );
    +  }
    +
    +
    +  static FT_Error
    +  cff_load_delta_set_index_mapping( CFF_Face           face,
    +                                    FT_ULong           offset,
    +                                    GX_DeltaSetIdxMap  map,
    +                                    GX_ItemVarStore    itemStore,
    +                                    FT_ULong           table_len )
    +  {
    +    FT_Service_MultiMasters  mm = (FT_Service_MultiMasters)face->mm;
    +
    +
    +    return mm->load_delta_set_idx_map( FT_FACE( face ), offset, map,
    +                                       itemStore, table_len );
    +  }
    +
    +
    +  static FT_Int
    +  cff_get_item_delta( CFF_Face         face,
    +                      GX_ItemVarStore  itemStore,
    +                      FT_UInt          outerIndex,
    +                      FT_UInt          innerIndex )
    +  {
    +    FT_Service_MultiMasters  mm = (FT_Service_MultiMasters)face->mm;
    +
    +
    +    return mm->get_item_delta( FT_FACE( face ), itemStore,
    +                               outerIndex, innerIndex );
    +  }
    +
    +
    +  static void
    +  cff_done_item_variation_store( CFF_Face          face,
    +                                 GX_ItemVarStore  itemStore )
    +  {
    +    FT_Service_MultiMasters  mm = (FT_Service_MultiMasters)face->mm;
    +
    +
    +    mm->done_item_var_store( FT_FACE( face ), itemStore );
    +  }
    +
    +
    +  static void
    +  cff_done_delta_set_index_map( CFF_Face           face,
    +                                GX_DeltaSetIdxMap  deltaSetIdxMap )
    +  {
    +    FT_Service_MultiMasters  mm = (FT_Service_MultiMasters)face->mm;
    +
    +
    +    mm->done_delta_set_idx_map( FT_FACE ( face ), deltaSetIdxMap );
    +  }
    +
    +
    +
       FT_DEFINE_SERVICE_MULTIMASTERSREC(
         cff_service_multi_masters,
     
    -    (FT_Get_MM_Func)             NULL,                    /* get_mm              */
    -    (FT_Set_MM_Design_Func)      NULL,                    /* set_mm_design       */
    -    (FT_Set_MM_Blend_Func)       cff_set_mm_blend,        /* set_mm_blend        */
    -    (FT_Get_MM_Blend_Func)       cff_get_mm_blend,        /* get_mm_blend        */
    -    (FT_Get_MM_Var_Func)         cff_get_mm_var,          /* get_mm_var          */
    -    (FT_Set_Var_Design_Func)     cff_set_var_design,      /* set_var_design      */
    -    (FT_Get_Var_Design_Func)     cff_get_var_design,      /* get_var_design      */
    -    (FT_Set_Instance_Func)       cff_set_instance,        /* set_instance        */
    -    (FT_Set_MM_WeightVector_Func)cff_set_mm_weightvector, /* set_mm_weightvector */
    -    (FT_Get_MM_WeightVector_Func)cff_get_mm_weightvector, /* get_mm_weightvector */
    -
    -    (FT_Get_Var_Blend_Func)      cff_get_var_blend,       /* get_var_blend       */
    -    (FT_Done_Blend_Func)         cff_done_blend           /* done_blend          */
    +    (FT_Get_MM_Func)        NULL,               /* get_mm                    */
    +    (FT_Set_MM_Design_Func) NULL,               /* set_mm_design             */
    +    (FT_Set_MM_Blend_Func)  cff_set_mm_blend,   /* set_mm_blend              */
    +    (FT_Get_MM_Blend_Func)  cff_get_mm_blend,   /* get_mm_blend              */
    +    (FT_Get_MM_Var_Func)    cff_get_mm_var,     /* get_mm_var                */
    +    (FT_Set_Var_Design_Func)cff_set_var_design, /* set_var_design            */
    +    (FT_Get_Var_Design_Func)cff_get_var_design, /* get_var_design            */
    +    (FT_Set_Instance_Func)  cff_set_instance,   /* set_instance              */
    +    (FT_Set_MM_WeightVector_Func)
    +                            cff_set_mm_weightvector,
    +                                                /* set_mm_weightvector       */
    +    (FT_Get_MM_WeightVector_Func)
    +                            cff_get_mm_weightvector,
    +                                                /* get_mm_weightvector       */
    +    (FT_Var_Load_Delta_Set_Idx_Map_Func)
    +                            cff_load_delta_set_index_mapping,
    +                                                /* load_delta_set_idx_map    */
    +    (FT_Var_Load_Item_Var_Store_Func)
    +                            cff_load_item_variation_store,
    +                                                /* load_item_variation_store */
    +    (FT_Var_Get_Item_Delta_Func)
    +                            cff_get_item_delta, /* get_item_delta            */
    +    (FT_Var_Done_Item_Var_Store_Func)
    +                            cff_done_item_variation_store,
    +                                                /* done_item_variation_store */
    +    (FT_Var_Done_Delta_Set_Idx_Map_Func)
    +                            cff_done_delta_set_index_map,
    +                                                /* done_delta_set_index_map  */
    +    (FT_Get_Var_Blend_Func) cff_get_var_blend,  /* get_var_blend             */
    +    (FT_Done_Blend_Func)    cff_done_blend      /* done_blend                */
       )
     
     
    @@ -1027,8 +1108,7 @@
       /*************************************************************************/
       /*************************************************************************/
     
    -#if !defined FT_CONFIG_OPTION_NO_GLYPH_NAMES && \
    -     defined TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +#if defined TT_CONFIG_OPTION_GX_VAR_SUPPORT
       FT_DEFINE_SERVICEDESCREC10(
         cff_services,
     
    @@ -1043,7 +1123,7 @@
         FT_SERVICE_ID_PROPERTIES,           &cff_service_properties,
         FT_SERVICE_ID_CFF_LOAD,             &cff_service_cff_load
       )
    -#elif !defined FT_CONFIG_OPTION_NO_GLYPH_NAMES
    +#else
       FT_DEFINE_SERVICEDESCREC8(
         cff_services,
     
    @@ -1056,32 +1136,6 @@
         FT_SERVICE_ID_PROPERTIES,           &cff_service_properties,
         FT_SERVICE_ID_CFF_LOAD,             &cff_service_cff_load
       )
    -#elif defined TT_CONFIG_OPTION_GX_VAR_SUPPORT
    -  FT_DEFINE_SERVICEDESCREC9(
    -    cff_services,
    -
    -    FT_SERVICE_ID_FONT_FORMAT,          FT_FONT_FORMAT_CFF,
    -    FT_SERVICE_ID_MULTI_MASTERS,        &cff_service_multi_masters,
    -    FT_SERVICE_ID_METRICS_VARIATIONS,   &cff_service_metrics_var,
    -    FT_SERVICE_ID_POSTSCRIPT_INFO,      &cff_service_ps_info,
    -    FT_SERVICE_ID_POSTSCRIPT_FONT_NAME, &cff_service_ps_name,
    -    FT_SERVICE_ID_TT_CMAP,              &cff_service_get_cmap_info,
    -    FT_SERVICE_ID_CID,                  &cff_service_cid_info,
    -    FT_SERVICE_ID_PROPERTIES,           &cff_service_properties,
    -    FT_SERVICE_ID_CFF_LOAD,             &cff_service_cff_load
    -  )
    -#else
    -  FT_DEFINE_SERVICEDESCREC7(
    -    cff_services,
    -
    -    FT_SERVICE_ID_FONT_FORMAT,          FT_FONT_FORMAT_CFF,
    -    FT_SERVICE_ID_POSTSCRIPT_INFO,      &cff_service_ps_info,
    -    FT_SERVICE_ID_POSTSCRIPT_FONT_NAME, &cff_service_ps_name,
    -    FT_SERVICE_ID_TT_CMAP,              &cff_service_get_cmap_info,
    -    FT_SERVICE_ID_CID,                  &cff_service_cid_info,
    -    FT_SERVICE_ID_PROPERTIES,           &cff_service_properties,
    -    FT_SERVICE_ID_CFF_LOAD,             &cff_service_cff_load
    -  )
     #endif
     
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/cff/cffdrivr.h b/src/java.desktop/share/native/libfreetype/src/cff/cffdrivr.h
    index a312003be738a..ab1f147bb2a6a 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cff/cffdrivr.h
    +++ b/src/java.desktop/share/native/libfreetype/src/cff/cffdrivr.h
    @@ -4,7 +4,7 @@
      *
      *   High-level OpenType driver interface (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cff/cfferrs.h b/src/java.desktop/share/native/libfreetype/src/cff/cfferrs.h
    index 90d32897c7864..bc9a3043fcff7 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cff/cfferrs.h
    +++ b/src/java.desktop/share/native/libfreetype/src/cff/cfferrs.h
    @@ -4,7 +4,7 @@
      *
      *   CFF error codes (specification only).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cff/cffgload.c b/src/java.desktop/share/native/libfreetype/src/cff/cffgload.c
    index 7586b886f10b8..cfa0aaf2b696b 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cff/cffgload.c
    +++ b/src/java.desktop/share/native/libfreetype/src/cff/cffgload.c
    @@ -4,7 +4,7 @@
      *
      *   OpenType Glyph Loader (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -356,18 +356,14 @@
     
     #ifdef FT_CONFIG_OPTION_SVG
         /* check for OT-SVG */
    -    if ( ( load_flags & FT_LOAD_COLOR )     &&
    -         ( (TT_Face)glyph->root.face )->svg )
    +    if ( ( load_flags & FT_LOAD_COLOR ) && face->svg )
         {
           /*
            * We load the SVG document and try to grab the advances from the
            * table.  For the bearings we rely on the presetting hook to do that.
            */
     
    -      FT_Short      dummy;
    -      FT_UShort     advanceX;
    -      FT_UShort     advanceY;
    -      SFNT_Service  sfnt;
    +      SFNT_Service  sfnt  = (SFNT_Service)face->sfnt;
     
     
           if ( size && (size->root.metrics.x_ppem < 1 ||
    @@ -379,10 +375,17 @@
     
           FT_TRACE3(( "Trying to load SVG glyph\n" ));
     
    -      sfnt  = (SFNT_Service)((TT_Face)glyph->root.face)->sfnt;
           error = sfnt->load_svg_doc( (FT_GlyphSlot)glyph, glyph_index );
           if ( !error )
           {
    +        FT_Fixed  x_scale = size->root.metrics.x_scale;
    +        FT_Fixed  y_scale = size->root.metrics.y_scale;
    +
    +        FT_Short   dummy;
    +        FT_UShort  advanceX;
    +        FT_UShort  advanceY;
    +
    +
             FT_TRACE3(( "Successfully loaded SVG glyph\n" ));
     
             glyph->root.format = FT_GLYPH_FORMAT_SVG;
    @@ -404,17 +407,11 @@
                                &dummy,
                                &advanceY );
     
    -        advanceX =
    -          (FT_UShort)FT_MulDiv( advanceX,
    -                                glyph->root.face->size->metrics.x_ppem,
    -                                glyph->root.face->units_per_EM );
    -        advanceY =
    -          (FT_UShort)FT_MulDiv( advanceY,
    -                                glyph->root.face->size->metrics.y_ppem,
    -                                glyph->root.face->units_per_EM );
    +        glyph->root.linearHoriAdvance = advanceX;
    +        glyph->root.linearVertAdvance = advanceY;
     
    -        glyph->root.metrics.horiAdvance = advanceX << 6;
    -        glyph->root.metrics.vertAdvance = advanceY << 6;
    +        glyph->root.metrics.horiAdvance = FT_MulFix( advanceX, x_scale );
    +        glyph->root.metrics.vertAdvance = FT_MulFix( advanceY, y_scale );
     
             return error;
           }
    @@ -491,13 +488,14 @@
           decoder.builder.no_recurse =
             FT_BOOL( load_flags & FT_LOAD_NO_RECURSE );
     
    -      /* now load the unscaled outline */
    -      error = cff_get_glyph_data( face, glyph_index,
    -                                  &charstring, &charstring_len );
    +      /* this function also checks for a valid subfont index */
    +      error = decoder_funcs->prepare( &decoder, size, glyph_index );
           if ( error )
             goto Glyph_Build_Finished;
     
    -      error = decoder_funcs->prepare( &decoder, size, glyph_index );
    +      /* now load the unscaled outline */
    +      error = cff_get_glyph_data( face, glyph_index,
    +                                  &charstring, &charstring_len );
           if ( error )
             goto Glyph_Build_Finished;
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/cff/cffgload.h b/src/java.desktop/share/native/libfreetype/src/cff/cffgload.h
    index 33616b9684673..3b8cf236ddc21 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cff/cffgload.h
    +++ b/src/java.desktop/share/native/libfreetype/src/cff/cffgload.h
    @@ -4,7 +4,7 @@
      *
      *   OpenType Glyph Loader (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cff/cffload.c b/src/java.desktop/share/native/libfreetype/src/cff/cffload.c
    index d6f8a1013d877..4b8c6e16c58eb 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cff/cffload.c
    +++ b/src/java.desktop/share/native/libfreetype/src/cff/cffload.c
    @@ -4,7 +4,7 @@
      *
      *   OpenType and CFF data/program tables loader (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -1288,7 +1288,7 @@
       /* Blended values are written to a different buffer,     */
       /* using reserved operator 255.                          */
       /*                                                       */
    -  /* Blend calculation is done in 16.16 fixed point.       */
    +  /* Blend calculation is done in 16.16 fixed-point.       */
       FT_LOCAL_DEF( FT_Error )
       cff_blend_doBlend( CFF_SubFont  subFont,
                          CFF_Parser   parser,
    @@ -1364,7 +1364,7 @@
           FT_UInt32        sum;
     
     
    -      /* convert inputs to 16.16 fixed point */
    +      /* convert inputs to 16.16 fixed-point */
           sum = cff_parse_num( parser, &parser->stack[i + base] ) * 0x10000;
     
           for ( j = 1; j < blend->lenBV; j++ )
    @@ -1373,7 +1373,7 @@
           /* point parser stack to new value on blend_stack */
           parser->stack[i + base] = subFont->blend_top;
     
    -      /* Push blended result as Type 2 5-byte fixed point number.  This */
    +      /* Push blended result as Type 2 5-byte fixed-point number.  This */
           /* will not conflict with actual DICTs because 255 is a reserved  */
           /* opcode in both CFF and CFF2 DICTs.  See `cff_parse_num' for    */
           /* decode of this, which rounds to an integer.                    */
    diff --git a/src/java.desktop/share/native/libfreetype/src/cff/cffload.h b/src/java.desktop/share/native/libfreetype/src/cff/cffload.h
    index a3cc642b7775a..5a41cdebc8e74 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cff/cffload.h
    +++ b/src/java.desktop/share/native/libfreetype/src/cff/cffload.h
    @@ -4,7 +4,7 @@
      *
      *   OpenType & CFF data/program tables loader (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cff/cffobjs.c b/src/java.desktop/share/native/libfreetype/src/cff/cffobjs.c
    index fa42accb65609..40cd9bf91733c 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cff/cffobjs.c
    +++ b/src/java.desktop/share/native/libfreetype/src/cff/cffobjs.c
    @@ -4,7 +4,7 @@
      *
      *   OpenType objects manager (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -1031,12 +1031,10 @@
             cffface->style_flags = flags;
           }
     
    -#ifndef FT_CONFIG_OPTION_NO_GLYPH_NAMES
           /* CID-keyed CFF or CFF2 fonts don't have glyph names -- the SFNT */
           /* loader has unset this flag because of the 3.0 `post' table.    */
           if ( dict->cid_registry == 0xFFFFU && !cff2 )
             cffface->face_flags |= FT_FACE_FLAG_GLYPH_NAMES;
    -#endif
     
           if ( dict->cid_registry != 0xFFFFU && pure_cff )
             cffface->face_flags |= FT_FACE_FLAG_CID_KEYED;
    diff --git a/src/java.desktop/share/native/libfreetype/src/cff/cffobjs.h b/src/java.desktop/share/native/libfreetype/src/cff/cffobjs.h
    index d48c1cded9f54..8f05f6132bc01 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cff/cffobjs.h
    +++ b/src/java.desktop/share/native/libfreetype/src/cff/cffobjs.h
    @@ -4,7 +4,7 @@
      *
      *   OpenType objects manager (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cff/cffparse.c b/src/java.desktop/share/native/libfreetype/src/cff/cffparse.c
    index 2536a21866b88..e16206fd553ae 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cff/cffparse.c
    +++ b/src/java.desktop/share/native/libfreetype/src/cff/cffparse.c
    @@ -4,7 +4,7 @@
      *
      *   CFF token stream parser (body)
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -530,7 +530,7 @@
     
         else if ( **d == 255 )
         {
    -      /* 16.16 fixed point is used internally for CFF2 blend results. */
    +      /* 16.16 fixed-point is used internally for CFF2 blend results. */
           /* Since these are trusted values, a limit check is not needed. */
     
           /* After the 255, 4 bytes give the number.                 */
    @@ -758,12 +758,12 @@
           *upm = (FT_ULong)power_tens[-max_scaling];
     
           FT_TRACE4(( " [%f %f %f %f %f %f]\n",
    -                  (double)matrix->xx / *upm / 65536,
    -                  (double)matrix->xy / *upm / 65536,
    -                  (double)matrix->yx / *upm / 65536,
    -                  (double)matrix->yy / *upm / 65536,
    -                  (double)offset->x  / *upm / 65536,
    -                  (double)offset->y  / *upm / 65536 ));
    +                  (double)matrix->xx / (double)*upm / 65536,
    +                  (double)matrix->xy / (double)*upm / 65536,
    +                  (double)matrix->yx / (double)*upm / 65536,
    +                  (double)matrix->yy / (double)*upm / 65536,
    +                  (double)offset->x  / (double)*upm / 65536,
    +                  (double)offset->y  / (double)*upm / 65536 ));
     
           if ( !FT_Matrix_Check( matrix ) )
           {
    diff --git a/src/java.desktop/share/native/libfreetype/src/cff/cffparse.h b/src/java.desktop/share/native/libfreetype/src/cff/cffparse.h
    index 55b6fe6e7cfd5..58d59fa4ac5e5 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cff/cffparse.h
    +++ b/src/java.desktop/share/native/libfreetype/src/cff/cffparse.h
    @@ -4,7 +4,7 @@
      *
      *   CFF token stream parser (specification)
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cff/cfftoken.h b/src/java.desktop/share/native/libfreetype/src/cff/cfftoken.h
    index 15237de9e5fbd..b61cb0e66e8d5 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cff/cfftoken.h
    +++ b/src/java.desktop/share/native/libfreetype/src/cff/cfftoken.h
    @@ -4,7 +4,7 @@
      *
      *   CFF token definitions (specification only).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cid/ciderrs.h b/src/java.desktop/share/native/libfreetype/src/cid/ciderrs.h
    index d07da5a01d8f3..40a1097d0ac7a 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cid/ciderrs.h
    +++ b/src/java.desktop/share/native/libfreetype/src/cid/ciderrs.h
    @@ -4,7 +4,7 @@
      *
      *   CID error codes (specification only).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cid/cidgload.c b/src/java.desktop/share/native/libfreetype/src/cid/cidgload.c
    index 24d37d3295240..ba4b7565d5479 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cid/cidgload.c
    +++ b/src/java.desktop/share/native/libfreetype/src/cid/cidgload.c
    @@ -4,7 +4,7 @@
      *
      *   CID-keyed Type1 Glyph Loader (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cid/cidgload.h b/src/java.desktop/share/native/libfreetype/src/cid/cidgload.h
    index c06bb29d3d344..97954d418ffac 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cid/cidgload.h
    +++ b/src/java.desktop/share/native/libfreetype/src/cid/cidgload.h
    @@ -4,7 +4,7 @@
      *
      *   OpenType Glyph Loader (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cid/cidload.c b/src/java.desktop/share/native/libfreetype/src/cid/cidload.c
    index fe8fa1abffe4c..26daa5da7f646 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cid/cidload.c
    +++ b/src/java.desktop/share/native/libfreetype/src/cid/cidload.c
    @@ -4,7 +4,7 @@
      *
      *   CID-keyed Type1 font loader (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cid/cidload.h b/src/java.desktop/share/native/libfreetype/src/cid/cidload.h
    index 90ced9280b182..d12d2962a6866 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cid/cidload.h
    +++ b/src/java.desktop/share/native/libfreetype/src/cid/cidload.h
    @@ -4,7 +4,7 @@
      *
      *   CID-keyed Type1 font loader (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cid/cidobjs.c b/src/java.desktop/share/native/libfreetype/src/cid/cidobjs.c
    index c39de6369cd54..06b2139a93dc1 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cid/cidobjs.c
    +++ b/src/java.desktop/share/native/libfreetype/src/cid/cidobjs.c
    @@ -4,7 +4,7 @@
      *
      *   CID objects manager (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -153,7 +153,7 @@
       }
     
     
    -  FT_LOCAL( FT_Error )
    +  FT_LOCAL_DEF( FT_Error )
       cid_size_request( FT_Size          size,
                         FT_Size_Request  req )
       {
    diff --git a/src/java.desktop/share/native/libfreetype/src/cid/cidobjs.h b/src/java.desktop/share/native/libfreetype/src/cid/cidobjs.h
    index fd76a1cba5dec..83c0c61c3ca63 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cid/cidobjs.h
    +++ b/src/java.desktop/share/native/libfreetype/src/cid/cidobjs.h
    @@ -4,7 +4,7 @@
      *
      *   CID objects manager (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cid/cidparse.c b/src/java.desktop/share/native/libfreetype/src/cid/cidparse.c
    index cfc820561f3f1..16889db9b6fb2 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cid/cidparse.c
    +++ b/src/java.desktop/share/native/libfreetype/src/cid/cidparse.c
    @@ -4,7 +4,7 @@
      *
      *   CID-keyed Type1 parser (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cid/cidparse.h b/src/java.desktop/share/native/libfreetype/src/cid/cidparse.h
    index ba363f7803a9e..2fd4e7a9310b1 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cid/cidparse.h
    +++ b/src/java.desktop/share/native/libfreetype/src/cid/cidparse.h
    @@ -4,7 +4,7 @@
      *
      *   CID-keyed Type1 parser (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cid/cidriver.c b/src/java.desktop/share/native/libfreetype/src/cid/cidriver.c
    index a63c01064adc8..f7499237d73c2 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cid/cidriver.c
    +++ b/src/java.desktop/share/native/libfreetype/src/cid/cidriver.c
    @@ -4,7 +4,7 @@
      *
      *   CID driver interface (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cid/cidriver.h b/src/java.desktop/share/native/libfreetype/src/cid/cidriver.h
    index 5073b7a8eb573..a6249385c8116 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cid/cidriver.h
    +++ b/src/java.desktop/share/native/libfreetype/src/cid/cidriver.h
    @@ -4,7 +4,7 @@
      *
      *   High-level CID driver interface (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/cid/cidtoken.h b/src/java.desktop/share/native/libfreetype/src/cid/cidtoken.h
    index 7640137eac0af..925951acdbd28 100644
    --- a/src/java.desktop/share/native/libfreetype/src/cid/cidtoken.h
    +++ b/src/java.desktop/share/native/libfreetype/src/cid/cidtoken.h
    @@ -4,7 +4,7 @@
      *
      *   CID token definitions (specification only).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/afmparse.c b/src/java.desktop/share/native/libfreetype/src/psaux/afmparse.c
    index bd86129f7e280..68f95698e6570 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/afmparse.c
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/afmparse.c
    @@ -4,7 +4,7 @@
      *
      *   AFM parser (body).
      *
    - * Copyright (C) 2006-2022 by
    + * Copyright (C) 2006-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -563,7 +563,7 @@
       }
     
     
    -  FT_LOCAL( void )
    +  FT_LOCAL_DEF( void )
       afm_parser_done( AFM_Parser  parser )
       {
         FT_Memory  memory = parser->memory;
    @@ -1061,7 +1061,7 @@
             if ( error )
               goto Fail;
             /* we only support kern data, so ... */
    -        /* fall through                      */
    +        FALL_THROUGH;
     
           case AFM_TOKEN_ENDFONTMETRICS:
             return FT_Err_Ok;
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/afmparse.h b/src/java.desktop/share/native/libfreetype/src/psaux/afmparse.h
    index eee49e3601551..2d3b6e6e1695f 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/afmparse.h
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/afmparse.h
    @@ -4,7 +4,7 @@
      *
      *   AFM parser (specification).
      *
    - * Copyright (C) 2006-2022 by
    + * Copyright (C) 2006-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/cffdecode.c b/src/java.desktop/share/native/libfreetype/src/psaux/cffdecode.c
    index 92139c93ad821..2cd91c96f357d 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/cffdecode.c
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/cffdecode.c
    @@ -4,7 +4,7 @@
      *
      *   PostScript CFF (Type 2) decoding routines (body).
      *
    - * Copyright (C) 2017-2022 by
    + * Copyright (C) 2017-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/cffdecode.h b/src/java.desktop/share/native/libfreetype/src/psaux/cffdecode.h
    index a9f6761824b27..e8bb4001cba23 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/cffdecode.h
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/cffdecode.h
    @@ -4,7 +4,7 @@
      *
      *   PostScript CFF (Type 2) decoding routines (specification).
      *
    - * Copyright (C) 2017-2022 by
    + * Copyright (C) 2017-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/psauxerr.h b/src/java.desktop/share/native/libfreetype/src/psaux/psauxerr.h
    index 1504b34ee56dd..895ffa48c2c31 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/psauxerr.h
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/psauxerr.h
    @@ -4,7 +4,7 @@
      *
      *   PS auxiliary module error codes (specification only).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/psauxmod.c b/src/java.desktop/share/native/libfreetype/src/psaux/psauxmod.c
    index 113490abcd2d7..45e35aa53c4cb 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/psauxmod.c
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/psauxmod.c
    @@ -4,7 +4,7 @@
      *
      *   FreeType auxiliary PostScript module implementation (body).
      *
    - * Copyright (C) 2000-2022 by
    + * Copyright (C) 2000-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/psauxmod.h b/src/java.desktop/share/native/libfreetype/src/psaux/psauxmod.h
    index 2d508edc2acae..94dbf48813c16 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/psauxmod.h
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/psauxmod.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType auxiliary PostScript module implementation (specification).
      *
    - * Copyright (C) 2000-2022 by
    + * Copyright (C) 2000-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/psconv.c b/src/java.desktop/share/native/libfreetype/src/psaux/psconv.c
    index 9b8c0d90c3053..b9c7138d84637 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/psconv.c
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/psconv.c
    @@ -4,7 +4,7 @@
      *
      *   Some convenience conversions (body).
      *
    - * Copyright (C) 2006-2022 by
    + * Copyright (C) 2006-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/psconv.h b/src/java.desktop/share/native/libfreetype/src/psaux/psconv.h
    index 650d7c93b27e2..b7c3ee00be872 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/psconv.h
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/psconv.h
    @@ -4,7 +4,7 @@
      *
      *   Some convenience conversions (specification).
      *
    - * Copyright (C) 2006-2022 by
    + * Copyright (C) 2006-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/psfixed.h b/src/java.desktop/share/native/libfreetype/src/psaux/psfixed.h
    index 7dff9ef1bd5b0..299d076370504 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/psfixed.h
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/psfixed.h
    @@ -2,7 +2,7 @@
      *
      * psfixed.h
      *
    - *   Adobe's code for Fixed Point Mathematics (specification only).
    + *   Adobe's code for Fixed-Point Mathematics (specification only).
      *
      * Copyright 2007-2013 Adobe Systems Incorporated.
      *
    @@ -43,10 +43,10 @@
     FT_BEGIN_HEADER
     
     
    -  /* rasterizer integer and fixed point arithmetic must be 32-bit */
    +  /* rasterizer integer and fixed-point arithmetic must be 32-bit */
     
     #define   CF2_Fixed  CF2_F16Dot16
    -  typedef FT_Int32   CF2_Frac;   /* 2.30 fixed point */
    +  typedef FT_Int32   CF2_Frac;   /* 2.30 fixed-point */
     
     
     #define CF2_FIXED_MAX      ( (CF2_Fixed)0x7FFFFFFFL )
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/psft.c b/src/java.desktop/share/native/libfreetype/src/psaux/psft.c
    index ac72d8259c4bc..618864e6e0728 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/psft.c
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/psft.c
    @@ -68,11 +68,10 @@
         CF2_Fixed  maxScale;
     
     
    -    FT_ASSERT( unitsPerEm > 0 );
    -
         if ( transform->a <= 0 || transform->d <= 0 )
           return FT_THROW( Invalid_Size_Handle );
     
    +    FT_ASSERT( unitsPerEm > 0 );
         FT_ASSERT( transform->b == 0 && transform->c == 0 );
         FT_ASSERT( transform->tx == 0 && transform->ty == 0 );
     
    @@ -297,7 +296,6 @@
       cf2_getUnitsPerEm( PS_Decoder*  decoder )
       {
         FT_ASSERT( decoder && decoder->builder.face );
    -    FT_ASSERT( decoder->builder.face->units_per_EM );
     
         return decoder->builder.face->units_per_EM;
       }
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/psglue.h b/src/java.desktop/share/native/libfreetype/src/psaux/psglue.h
    index 022aafbfcacc2..63085d71cf508 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/psglue.h
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/psglue.h
    @@ -72,7 +72,7 @@ FT_BEGIN_HEADER
       } CF2_PathOp;
     
     
    -  /* a matrix of fixed point values */
    +  /* a matrix of fixed-point values */
       typedef struct  CF2_Matrix_
       {
         CF2_F16Dot16  a;
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/pshints.c b/src/java.desktop/share/native/libfreetype/src/psaux/pshints.c
    index ad472c98df5c3..6f44d0adbb728 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/pshints.c
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/pshints.c
    @@ -693,8 +693,10 @@
             CF2_Fixed  midpoint =
                          cf2_hintmap_map(
                            hintmap->initialHintMap,
    -                       ADD_INT32( secondHintEdge->csCoord,
    -                                  firstHintEdge->csCoord ) / 2 );
    +                       ADD_INT32(
    +                         firstHintEdge->csCoord,
    +                         SUB_INT32 ( secondHintEdge->csCoord,
    +                                     firstHintEdge->csCoord ) / 2 ) );
             CF2_Fixed  halfWidth =
                          FT_MulFix( SUB_INT32( secondHintEdge->csCoord,
                                                firstHintEdge->csCoord ) / 2,
    @@ -1034,10 +1036,10 @@
         {
           FT_TRACE6(( "flags: [p]air [g]host [t]op"
                       " [b]ottom [L]ocked [S]ynthetic\n" ));
    -      FT_TRACE6(( "Initial hintmap" ));
    +      FT_TRACE6(( "Initial hintmap:\n" ));
         }
         else
    -      FT_TRACE6(( "Hints:" ));
    +      FT_TRACE6(( "Hints:\n" ));
     #endif
     
         cf2_hintmap_dump( hintmap );
    @@ -1054,7 +1056,7 @@
         /* adjust positions of hint edges that are not locked to blue zones */
         cf2_hintmap_adjustHints( hintmap );
     
    -    FT_TRACE6(( "(adjusted)\n" ));
    +    FT_TRACE6(( "Hints adjusted:\n" ));
         cf2_hintmap_dump( hintmap );
     
         /* save the position of all hints that were used in this hint map; */
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/psobjs.c b/src/java.desktop/share/native/libfreetype/src/psaux/psobjs.c
    index 371e5380207df..8da755d0e5769 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/psobjs.c
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/psobjs.c
    @@ -4,7 +4,7 @@
      *
      *   Auxiliary functions for PostScript fonts (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -84,7 +84,6 @@
     
         table->max_elems = count;
         table->init      = 0xDEADBEEFUL;
    -    table->num_elems = 0;
         table->block     = NULL;
         table->capacity  = 0;
         table->cursor    = 0;
    @@ -235,7 +234,7 @@
         FT_Memory  memory = table->memory;
     
     
    -    if ( (FT_ULong)table->init == 0xDEADBEEFUL )
    +    if ( table->init == 0xDEADBEEFUL )
         {
           FT_FREE( table->block );
           FT_FREE( table->elements );
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/psobjs.h b/src/java.desktop/share/native/libfreetype/src/psaux/psobjs.h
    index f01d4bd503a60..d5bce541082fd 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/psobjs.h
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/psobjs.h
    @@ -4,7 +4,7 @@
      *
      *   Auxiliary functions for PostScript fonts (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/psstack.h b/src/java.desktop/share/native/libfreetype/src/psaux/psstack.h
    index b9ef9edf1b2d5..907b42400074c 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/psstack.h
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/psstack.h
    @@ -49,8 +49,8 @@ FT_BEGIN_HEADER
       {
         union
         {
    -      CF2_Fixed  r;      /* 16.16 fixed point */
    -      CF2_Frac   f;      /* 2.30 fixed point (for font matrix) */
    +      CF2_Fixed  r;      /* 16.16 fixed-point */
    +      CF2_Frac   f;      /* 2.30 fixed-point (for font matrix) */
           CF2_Int    i;
         } u;
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/t1cmap.c b/src/java.desktop/share/native/libfreetype/src/psaux/t1cmap.c
    index f297ce75e1f4a..bf0a393b456e8 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/t1cmap.c
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/t1cmap.c
    @@ -4,7 +4,7 @@
      *
      *   Type 1 character map support (body).
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/t1cmap.h b/src/java.desktop/share/native/libfreetype/src/psaux/t1cmap.h
    index 460d91f5904c3..b3702498a5537 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/t1cmap.h
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/t1cmap.h
    @@ -4,7 +4,7 @@
      *
      *   Type 1 character map support (specification).
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/t1decode.c b/src/java.desktop/share/native/libfreetype/src/psaux/t1decode.c
    index 1cdf436fa72e0..bfed45b53a3a2 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/t1decode.c
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/t1decode.c
    @@ -4,7 +4,7 @@
      *
      *   PostScript Type 1 decoding routines (body).
      *
    - * Copyright (C) 2000-2022 by
    + * Copyright (C) 2000-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/psaux/t1decode.h b/src/java.desktop/share/native/libfreetype/src/psaux/t1decode.h
    index d60d61c9043b2..0970def960b22 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psaux/t1decode.h
    +++ b/src/java.desktop/share/native/libfreetype/src/psaux/t1decode.h
    @@ -4,7 +4,7 @@
      *
      *   PostScript Type 1 decoding routines (specification).
      *
    - * Copyright (C) 2000-2022 by
    + * Copyright (C) 2000-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/pshinter/pshalgo.c b/src/java.desktop/share/native/libfreetype/src/pshinter/pshalgo.c
    index dca539766f23d..a7f321291a94a 100644
    --- a/src/java.desktop/share/native/libfreetype/src/pshinter/pshalgo.c
    +++ b/src/java.desktop/share/native/libfreetype/src/pshinter/pshalgo.c
    @@ -4,7 +4,7 @@
      *
      *   PostScript hinting algorithm (body).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used
    diff --git a/src/java.desktop/share/native/libfreetype/src/pshinter/pshalgo.h b/src/java.desktop/share/native/libfreetype/src/pshinter/pshalgo.h
    index ab978bf6d0bf5..3f0ba28a6930b 100644
    --- a/src/java.desktop/share/native/libfreetype/src/pshinter/pshalgo.h
    +++ b/src/java.desktop/share/native/libfreetype/src/pshinter/pshalgo.h
    @@ -4,7 +4,7 @@
      *
      *   PostScript hinting algorithm (specification).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/pshinter/pshglob.c b/src/java.desktop/share/native/libfreetype/src/pshinter/pshglob.c
    index 887ea8d9c1828..d4c5eb32b1c57 100644
    --- a/src/java.desktop/share/native/libfreetype/src/pshinter/pshglob.c
    +++ b/src/java.desktop/share/native/libfreetype/src/pshinter/pshglob.c
    @@ -5,7 +5,7 @@
      *   PostScript hinter global hinting management (body).
      *   Inspired by the new auto-hinter module.
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used
    diff --git a/src/java.desktop/share/native/libfreetype/src/pshinter/pshglob.h b/src/java.desktop/share/native/libfreetype/src/pshinter/pshglob.h
    index 47247f969ebd6..579eb2148a572 100644
    --- a/src/java.desktop/share/native/libfreetype/src/pshinter/pshglob.h
    +++ b/src/java.desktop/share/native/libfreetype/src/pshinter/pshglob.h
    @@ -4,7 +4,7 @@
      *
      *   PostScript hinter global hinting management.
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/pshinter/pshmod.c b/src/java.desktop/share/native/libfreetype/src/pshinter/pshmod.c
    index a74a4fe99f260..a12e485660153 100644
    --- a/src/java.desktop/share/native/libfreetype/src/pshinter/pshmod.c
    +++ b/src/java.desktop/share/native/libfreetype/src/pshinter/pshmod.c
    @@ -4,7 +4,7 @@
      *
      *   FreeType PostScript hinter module implementation (body).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/pshinter/pshmod.h b/src/java.desktop/share/native/libfreetype/src/pshinter/pshmod.h
    index cdf95b7c203e1..4bd781a35d76b 100644
    --- a/src/java.desktop/share/native/libfreetype/src/pshinter/pshmod.h
    +++ b/src/java.desktop/share/native/libfreetype/src/pshinter/pshmod.h
    @@ -4,7 +4,7 @@
      *
      *   PostScript hinter module interface (specification).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/pshinter/pshnterr.h b/src/java.desktop/share/native/libfreetype/src/pshinter/pshnterr.h
    index 789afb599018a..97624952d8caf 100644
    --- a/src/java.desktop/share/native/libfreetype/src/pshinter/pshnterr.h
    +++ b/src/java.desktop/share/native/libfreetype/src/pshinter/pshnterr.h
    @@ -4,7 +4,7 @@
      *
      *   PS Hinter error codes (specification only).
      *
    - * Copyright (C) 2003-2022 by
    + * Copyright (C) 2003-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/pshinter/pshrec.c b/src/java.desktop/share/native/libfreetype/src/pshinter/pshrec.c
    index 2a5cffbadae1f..58c8cf1b4861b 100644
    --- a/src/java.desktop/share/native/libfreetype/src/pshinter/pshrec.c
    +++ b/src/java.desktop/share/native/libfreetype/src/pshinter/pshrec.c
    @@ -4,7 +4,7 @@
      *
      *   FreeType PostScript hints recorder (body).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -765,7 +765,7 @@
     
     
       /* destroy hints */
    -  FT_LOCAL( void )
    +  FT_LOCAL_DEF( void )
       ps_hints_done( PS_Hints  hints )
       {
         FT_Memory  memory = hints->memory;
    @@ -779,7 +779,7 @@
       }
     
     
    -  FT_LOCAL( void )
    +  FT_LOCAL_DEF( void )
       ps_hints_init( PS_Hints   hints,
                      FT_Memory  memory )
       {
    diff --git a/src/java.desktop/share/native/libfreetype/src/pshinter/pshrec.h b/src/java.desktop/share/native/libfreetype/src/pshinter/pshrec.h
    index a0d37979cc067..0b2484af12100 100644
    --- a/src/java.desktop/share/native/libfreetype/src/pshinter/pshrec.h
    +++ b/src/java.desktop/share/native/libfreetype/src/pshinter/pshrec.h
    @@ -4,7 +4,7 @@
      *
      *   Postscript (Type1/Type2) hints recorder (specification).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/psnames/psmodule.c b/src/java.desktop/share/native/libfreetype/src/psnames/psmodule.c
    index e7d51e950eb8a..db454e558ebb6 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psnames/psmodule.c
    +++ b/src/java.desktop/share/native/libfreetype/src/psnames/psmodule.c
    @@ -4,7 +4,7 @@
      *
      *   psnames module implementation (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -412,21 +412,18 @@
       ps_unicodes_char_index( PS_Unicodes  table,
                               FT_UInt32    unicode )
       {
    -    PS_UniMap  *min, *max, *mid, *result = NULL;
    +    PS_UniMap  *result = NULL;
    +    PS_UniMap  *min = table->maps;
    +    PS_UniMap  *max = min + table->num_maps;
    +    PS_UniMap  *mid = min + ( ( max - min ) >> 1 );
     
     
         /* Perform a binary search on the table. */
    -
    -    min = table->maps;
    -    max = min + table->num_maps - 1;
    -
    -    while ( min <= max )
    +    while ( min < max )
         {
           FT_UInt32  base_glyph;
     
     
    -      mid = min + ( ( max - min ) >> 1 );
    -
           if ( mid->unicode == unicode )
           {
             result = mid;
    @@ -438,13 +435,15 @@
           if ( base_glyph == unicode )
             result = mid; /* remember match but continue search for base glyph */
     
    -      if ( min == max )
    -        break;
    -
           if ( base_glyph < unicode )
             min = mid + 1;
           else
    -        max = mid - 1;
    +        max = mid;
    +
    +      /* reasonable prediction in a continuous block */
    +      mid += unicode - base_glyph;
    +      if ( mid >= max || mid < min )
    +        mid = min + ( ( max - min ) >> 1 );
         }
     
         if ( result )
    @@ -465,14 +464,13 @@
         {
           FT_UInt     min = 0;
           FT_UInt     max = table->num_maps;
    -      FT_UInt     mid;
    +      FT_UInt     mid = min + ( ( max - min ) >> 1 );
           PS_UniMap*  map;
           FT_UInt32   base_glyph;
     
     
           while ( min < max )
           {
    -        mid = min + ( ( max - min ) >> 1 );
             map = table->maps + mid;
     
             if ( map->unicode == char_code )
    @@ -490,6 +488,11 @@
               min = mid + 1;
             else
               max = mid;
    +
    +        /* reasonable prediction in a continuous block */
    +        mid += char_code - base_glyph;
    +        if ( mid >= max || mid < min )
    +          mid = min + ( max - min ) / 2;
           }
     
           if ( result )
    diff --git a/src/java.desktop/share/native/libfreetype/src/psnames/psmodule.h b/src/java.desktop/share/native/libfreetype/src/psnames/psmodule.h
    index ff3eda564c711..0904700bfb8d1 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psnames/psmodule.h
    +++ b/src/java.desktop/share/native/libfreetype/src/psnames/psmodule.h
    @@ -4,7 +4,7 @@
      *
      *   High-level psnames module interface (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/psnames/psnamerr.h b/src/java.desktop/share/native/libfreetype/src/psnames/psnamerr.h
    index ae56620275f0e..0073f8228486b 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psnames/psnamerr.h
    +++ b/src/java.desktop/share/native/libfreetype/src/psnames/psnamerr.h
    @@ -4,7 +4,7 @@
      *
      *   PS names module error codes (specification only).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/psnames/pstables.h b/src/java.desktop/share/native/libfreetype/src/psnames/pstables.h
    index d28d580b9ca56..7f92cce603960 100644
    --- a/src/java.desktop/share/native/libfreetype/src/psnames/pstables.h
    +++ b/src/java.desktop/share/native/libfreetype/src/psnames/pstables.h
    @@ -4,7 +4,7 @@
      *
      *   PostScript glyph names.
      *
    - * Copyright (C) 2005-2022 by
    + * Copyright (C) 2005-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/raster/ftmisc.h b/src/java.desktop/share/native/libfreetype/src/raster/ftmisc.h
    index 75fb5f8437d8a..33dbfd631e92e 100644
    --- a/src/java.desktop/share/native/libfreetype/src/raster/ftmisc.h
    +++ b/src/java.desktop/share/native/libfreetype/src/raster/ftmisc.h
    @@ -5,7 +5,7 @@
      *   Miscellaneous macros for stand-alone rasterizer (specification
      *   only).
      *
    - * Copyright (C) 2005-2022 by
    + * Copyright (C) 2005-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used
    diff --git a/src/java.desktop/share/native/libfreetype/src/raster/ftraster.c b/src/java.desktop/share/native/libfreetype/src/raster/ftraster.c
    index 68b0e1fdd95a7..67cbfd5d9b7eb 100644
    --- a/src/java.desktop/share/native/libfreetype/src/raster/ftraster.c
    +++ b/src/java.desktop/share/native/libfreetype/src/raster/ftraster.c
    @@ -4,7 +4,7 @@
      *
      *   The FreeType glyph rasterizer (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -2219,8 +2219,8 @@
         /* represent multiples of 1/(1<<12) = 1/4096                    */
         FT_TRACE7(( "  y=%d x=[% .12f;% .12f]",
                     y,
    -                x1 / (double)ras.precision,
    -                x2 / (double)ras.precision ));
    +                (double)x1 / (double)ras.precision,
    +                (double)x2 / (double)ras.precision ));
     
         /* Drop-out control */
     
    @@ -2294,8 +2294,8 @@
     
         FT_TRACE7(( "  y=%d x=[% .12f;% .12f]",
                     y,
    -                x1 / (double)ras.precision,
    -                x2 / (double)ras.precision ));
    +                (double)x1 / (double)ras.precision,
    +                (double)x2 / (double)ras.precision ));
     
         /* Drop-out control */
     
    @@ -2477,8 +2477,8 @@
     
         FT_TRACE7(( "  x=%d y=[% .12f;% .12f]",
                     y,
    -                x1 / (double)ras.precision,
    -                x2 / (double)ras.precision ));
    +                (double)x1 / (double)ras.precision,
    +                (double)x2 / (double)ras.precision ));
     
         /* We should not need this procedure but the vertical sweep   */
         /* mishandles horizontal lines through pixel centers.  So we  */
    @@ -2548,8 +2548,8 @@
     
         FT_TRACE7(( "  x=%d y=[% .12f;% .12f]",
                     y,
    -                x1 / (double)ras.precision,
    -                x2 / (double)ras.precision ));
    +                (double)x1 / (double)ras.precision,
    +                (double)x2 / (double)ras.precision ));
     
         /* During the horizontal sweep, we only take care of drop-outs */
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/raster/ftraster.h b/src/java.desktop/share/native/libfreetype/src/raster/ftraster.h
    index e9ece8cf0bf00..b511b3a99e9c5 100644
    --- a/src/java.desktop/share/native/libfreetype/src/raster/ftraster.h
    +++ b/src/java.desktop/share/native/libfreetype/src/raster/ftraster.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType glyph rasterizer (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used
    diff --git a/src/java.desktop/share/native/libfreetype/src/raster/ftrend1.c b/src/java.desktop/share/native/libfreetype/src/raster/ftrend1.c
    index f319f03d9c690..0b5d867147847 100644
    --- a/src/java.desktop/share/native/libfreetype/src/raster/ftrend1.c
    +++ b/src/java.desktop/share/native/libfreetype/src/raster/ftrend1.c
    @@ -4,7 +4,7 @@
      *
      *   The FreeType glyph rasterizer interface (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/raster/ftrend1.h b/src/java.desktop/share/native/libfreetype/src/raster/ftrend1.h
    index 14ec336b1114f..cec35c8528ac3 100644
    --- a/src/java.desktop/share/native/libfreetype/src/raster/ftrend1.h
    +++ b/src/java.desktop/share/native/libfreetype/src/raster/ftrend1.h
    @@ -4,7 +4,7 @@
      *
      *   The FreeType glyph rasterizer interface (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/raster/rasterrs.h b/src/java.desktop/share/native/libfreetype/src/raster/rasterrs.h
    index 8b1ebf07a3c81..989d8b44be157 100644
    --- a/src/java.desktop/share/native/libfreetype/src/raster/rasterrs.h
    +++ b/src/java.desktop/share/native/libfreetype/src/raster/rasterrs.h
    @@ -4,7 +4,7 @@
      *
      *   monochrome renderer error codes (specification only).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/pngshim.c b/src/java.desktop/share/native/libfreetype/src/sfnt/pngshim.c
    index 0ce4bdb6b5ca2..423b07b02a574 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/pngshim.c
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/pngshim.c
    @@ -4,7 +4,7 @@
      *
      *   PNG Bitmap glyph support.
      *
    - * Copyright (C) 2013-2022 by
    + * Copyright (C) 2013-2023 by
      * Google, Inc.
      * Written by Stuart Gill and Behdad Esfahbod.
      *
    @@ -239,7 +239,7 @@
           *e = FT_THROW( Invalid_Stream_Read );
           png_error( png, NULL );
     
    -      return;
    +      /* return; (never reached) */
         }
     
         ft_memcpy( data, stream->cursor, length );
    @@ -407,7 +407,8 @@
         switch ( color_type )
         {
         default:
    -      /* Shouldn't happen, but fall through. */
    +      /* Shouldn't happen, but ... */
    +      FALL_THROUGH;
     
         case PNG_COLOR_TYPE_RGB_ALPHA:
           png_set_read_user_transform_fn( png, premultiply_data );
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/pngshim.h b/src/java.desktop/share/native/libfreetype/src/sfnt/pngshim.h
    index 36d749c3c3589..903bd2bc3482d 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/pngshim.h
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/pngshim.h
    @@ -4,7 +4,7 @@
      *
      *   PNG Bitmap glyph support.
      *
    - * Copyright (C) 2013-2022 by
    + * Copyright (C) 2013-2023 by
      * Google, Inc.
      * Written by Stuart Gill and Behdad Esfahbod.
      *
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/sfdriver.c b/src/java.desktop/share/native/libfreetype/src/sfnt/sfdriver.c
    index cc121e57907e0..762883db54200 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/sfdriver.c
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/sfdriver.c
    @@ -4,7 +4,7 @@
      *
      *   High-level SFNT driver interface (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -378,61 +378,61 @@
           {
           case 15:
             k4 ^= (FT_UInt32)tail[14] << 16;
    -        /* fall through */
    +        FALL_THROUGH;
           case 14:
             k4 ^= (FT_UInt32)tail[13] << 8;
    -        /* fall through */
    +        FALL_THROUGH;
           case 13:
             k4 ^= (FT_UInt32)tail[12];
             k4 *= c4;
             k4  = ROTL32( k4, 18 );
             k4 *= c1;
             h4 ^= k4;
    -        /* fall through */
    +        FALL_THROUGH;
     
           case 12:
             k3 ^= (FT_UInt32)tail[11] << 24;
    -        /* fall through */
    +        FALL_THROUGH;
           case 11:
             k3 ^= (FT_UInt32)tail[10] << 16;
    -        /* fall through */
    +        FALL_THROUGH;
           case 10:
             k3 ^= (FT_UInt32)tail[9] << 8;
    -        /* fall through */
    +        FALL_THROUGH;
           case 9:
             k3 ^= (FT_UInt32)tail[8];
             k3 *= c3;
             k3  = ROTL32( k3, 17 );
             k3 *= c4;
             h3 ^= k3;
    -        /* fall through */
    +        FALL_THROUGH;
     
           case 8:
             k2 ^= (FT_UInt32)tail[7] << 24;
    -        /* fall through */
    +        FALL_THROUGH;
           case 7:
             k2 ^= (FT_UInt32)tail[6] << 16;
    -        /* fall through */
    +        FALL_THROUGH;
           case 6:
             k2 ^= (FT_UInt32)tail[5] << 8;
    -        /* fall through */
    +        FALL_THROUGH;
           case 5:
             k2 ^= (FT_UInt32)tail[4];
             k2 *= c2;
             k2  = ROTL32( k2, 16 );
             k2 *= c3;
             h2 ^= k2;
    -        /* fall through */
    +        FALL_THROUGH;
     
           case 4:
             k1 ^= (FT_UInt32)tail[3] << 24;
    -        /* fall through */
    +        FALL_THROUGH;
           case 3:
             k1 ^= (FT_UInt32)tail[2] << 16;
    -        /* fall through */
    +        FALL_THROUGH;
           case 2:
             k1 ^= (FT_UInt32)tail[1] << 8;
    -        /* fall through */
    +        FALL_THROUGH;
           case 1:
             k1 ^= (FT_UInt32)tail[0];
             k1 *= c1;
    @@ -657,7 +657,7 @@
     
     
       /*
    -   * Find the shortest decimal representation of a 16.16 fixed point
    +   * Find the shortest decimal representation of a 16.16 fixed-point
        * number.  The function fills `buf' with the result, returning a pointer
        * to the position after the representation's last byte.
        */
    @@ -733,7 +733,7 @@
             an equivalent representation of `fixed'.
     
             The above FOR loop always finds the larger of the two values; I
    -        verified this by iterating over all possible fixed point numbers.
    +        verified this by iterating over all possible fixed-point numbers.
     
             If the remainder is 17232*10, both values are equally good, and we
             take the next even number (following IEEE 754's `round to nearest,
    @@ -741,7 +741,7 @@
     
             If the remainder is smaller than 17232*10, the lower of the two
             numbers is nearer to the exact result (values 17232 and 34480 were
    -        also found by testing all possible fixed point values).
    +        also found by testing all possible fixed-point values).
     
             We use this to find a shorter decimal representation.  If not ending
             with digit zero, we take the representation with less error.
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/sfdriver.h b/src/java.desktop/share/native/libfreetype/src/sfnt/sfdriver.h
    index 6a2e3e9c7b0aa..2445958b69f42 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/sfdriver.h
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/sfdriver.h
    @@ -4,7 +4,7 @@
      *
      *   High-level SFNT driver interface (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/sferrors.h b/src/java.desktop/share/native/libfreetype/src/sfnt/sferrors.h
    index 99ef3f9fce95b..e7a8eb04bb883 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/sferrors.h
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/sferrors.h
    @@ -4,7 +4,7 @@
      *
      *   SFNT error codes (specification only).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/sfobjs.c b/src/java.desktop/share/native/libfreetype/src/sfnt/sfobjs.c
    index a0da984e7ac28..e018934ccaa1b 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/sfobjs.c
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/sfobjs.c
    @@ -4,7 +4,7 @@
      *
      *   SFNT object management (base).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -1107,13 +1107,7 @@
           /* Don't bother to load the tables unless somebody asks for them. */
           /* No need to do work which will (probably) not be used.          */
           if ( face->variation_support & TT_FACE_FLAG_VAR_FVAR )
    -      {
    -        if ( tt_face_lookup_table( face, TTAG_glyf ) != 0 &&
    -             tt_face_lookup_table( face, TTAG_gvar ) != 0 )
    -          flags |= FT_FACE_FLAG_MULTIPLE_MASTERS;
    -        if ( tt_face_lookup_table( face, TTAG_CFF2 ) != 0 )
    -          flags |= FT_FACE_FLAG_MULTIPLE_MASTERS;
    -      }
    +        flags |= FT_FACE_FLAG_MULTIPLE_MASTERS;
     #endif
     
           root->face_flags = flags;
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/sfobjs.h b/src/java.desktop/share/native/libfreetype/src/sfnt/sfobjs.h
    index 1d99bfede47b0..906aebbf904fe 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/sfobjs.h
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/sfobjs.h
    @@ -4,7 +4,7 @@
      *
      *   SFNT object management (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/sfwoff.c b/src/java.desktop/share/native/libfreetype/src/sfnt/sfwoff.c
    index 0e8ec3fa9321e..9559bf3421431 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/sfwoff.c
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/sfwoff.c
    @@ -4,7 +4,7 @@
      *
      *   WOFF format management (base).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -162,8 +162,7 @@
         }
     
         /* Don't trust `totalSfntSize' before thorough checks. */
    -    if ( FT_QALLOC( sfnt, 12 + woff.num_tables * 16UL ) ||
    -         FT_NEW( sfnt_stream )                          )
    +    if ( FT_QALLOC( sfnt, 12 ) || FT_NEW( sfnt_stream ) )
           goto Exit;
     
         sfnt_header = sfnt;
    @@ -196,8 +195,8 @@
         /* tag value, the tables themselves are not.  We thus have to */
         /* sort them by offset and check that they don't overlap.     */
     
    -    if ( FT_NEW_ARRAY( tables, woff.num_tables )  ||
    -         FT_NEW_ARRAY( indices, woff.num_tables ) )
    +    if ( FT_QNEW_ARRAY( tables, woff.num_tables )  ||
    +         FT_QNEW_ARRAY( indices, woff.num_tables ) )
           goto Exit;
     
         FT_TRACE2(( "\n" ));
    @@ -328,9 +327,7 @@
         }
     
         /* Now use `totalSfntSize'. */
    -    if ( FT_REALLOC( sfnt,
    -                     12 + woff.num_tables * 16UL,
    -                     woff.totalSfntSize ) )
    +    if ( FT_QREALLOC( sfnt, 12, woff.totalSfntSize ) )
           goto Exit;
     
         sfnt_header = sfnt + 12;
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/sfwoff.h b/src/java.desktop/share/native/libfreetype/src/sfnt/sfwoff.h
    index 5866a16194e41..d4384227376b5 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/sfwoff.h
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/sfwoff.h
    @@ -4,7 +4,7 @@
      *
      *   WOFFF format management (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/sfwoff2.c b/src/java.desktop/share/native/libfreetype/src/sfnt/sfwoff2.c
    index b2855b8e72b1e..7a01977f8667c 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/sfwoff2.c
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/sfwoff2.c
    @@ -4,7 +4,7 @@
      *
      *   WOFF2 format management (base).
      *
    - * Copyright (C) 2019-2022 by
    + * Copyright (C) 2019-2023 by
      * Nikhil Ramakrishnan, David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -229,9 +229,9 @@
         {
           FT_TRACE6(( "Reallocating %lu to %lu.\n",
                       *dst_size, (*offset + size) ));
    -      if ( FT_REALLOC( dst,
    -                       (FT_ULong)( *dst_size ),
    -                       (FT_ULong)( *offset + size ) ) )
    +      if ( FT_QREALLOC( dst,
    +                        (FT_ULong)( *dst_size ),
    +                        (FT_ULong)( *offset + size ) ) )
             goto Exit;
     
           *dst_size = *offset + size;
    @@ -784,7 +784,7 @@
           goto Fail;
     
         loca_buf_size = loca_values_size * offset_size;
    -    if ( FT_QNEW_ARRAY( loca_buf, loca_buf_size ) )
    +    if ( FT_QALLOC( loca_buf, loca_buf_size ) )
           goto Fail;
     
         dst = loca_buf;
    @@ -863,7 +863,7 @@
         WOFF2_Point  points       = NULL;
     
     
    -    if ( FT_NEW_ARRAY( substreams, num_substreams ) )
    +    if ( FT_QNEW_ARRAY( substreams, num_substreams ) )
           goto Fail;
     
         if ( FT_STREAM_SKIP( 2 ) )
    @@ -926,7 +926,7 @@
           offset += overlap_bitmap_length;
         }
     
    -    if ( FT_NEW_ARRAY( loca_values, num_glyphs + 1 ) )
    +    if ( FT_QNEW_ARRAY( loca_values, num_glyphs + 1 ) )
           goto Fail;
     
         points_size        = 0;
    @@ -938,10 +938,10 @@
         substreams[BBOX_STREAM].offset += bbox_bitmap_length;
     
         glyph_buf_size = WOFF2_DEFAULT_GLYPH_BUF;
    -    if ( FT_NEW_ARRAY( glyph_buf, glyph_buf_size ) )
    +    if ( FT_QALLOC( glyph_buf, glyph_buf_size ) )
           goto Fail;
     
    -    if ( FT_NEW_ARRAY( info->x_mins, num_glyphs ) )
    +    if ( FT_QNEW_ARRAY( info->x_mins, num_glyphs ) )
           goto Fail;
     
         for ( i = 0; i < num_glyphs; ++i )
    @@ -999,7 +999,7 @@
             size_needed = 12 + composite_size + instruction_size;
             if ( glyph_buf_size < size_needed )
             {
    -          if ( FT_RENEW_ARRAY( glyph_buf, glyph_buf_size, size_needed ) )
    +          if ( FT_QREALLOC( glyph_buf, glyph_buf_size, size_needed ) )
                 goto Fail;
               glyph_buf_size = size_needed;
             }
    @@ -1075,7 +1075,7 @@
                 have_overlap = TRUE;
             }
     
    -        if ( FT_NEW_ARRAY( n_points_arr, n_contours ) )
    +        if ( FT_QNEW_ARRAY( n_points_arr, n_contours ) )
               goto Fail;
     
             if ( FT_STREAM_SEEK( substreams[N_POINTS_STREAM].offset ) )
    @@ -1112,7 +1112,7 @@
     
             /* Create array to store point information. */
             points_size = total_n_points;
    -        if ( FT_NEW_ARRAY( points, points_size ) )
    +        if ( FT_QNEW_ARRAY( points, points_size ) )
               goto Fail;
     
             if ( triplet_decode( flags_buf,
    @@ -1141,7 +1141,7 @@
                           instruction_size;
             if ( glyph_buf_size < size_needed )
             {
    -          if ( FT_RENEW_ARRAY( glyph_buf, glyph_buf_size, size_needed ) )
    +          if ( FT_QREALLOC( glyph_buf, glyph_buf_size, size_needed ) )
                 goto Fail;
               glyph_buf_size = size_needed;
             }
    @@ -1226,8 +1226,7 @@
           *glyf_checksum += compute_ULong_sum( glyph_buf, glyph_size );
     
           /* Store x_mins, may be required to reconstruct `hmtx'. */
    -      if ( n_contours > 0 )
    -        info->x_mins[i] = (FT_Short)x_min;
    +      info->x_mins[i] = (FT_Short)x_min;
         }
     
         info->glyf_table->dst_length = dest_offset - info->glyf_table->dst_offset;
    @@ -1344,7 +1343,7 @@
         offset_size = index_format ? 4 : 2;
     
         /* Create `x_mins' array. */
    -    if ( FT_NEW_ARRAY( info->x_mins, num_glyphs ) )
    +    if ( FT_QNEW_ARRAY( info->x_mins, num_glyphs ) )
           return error;
     
         loca_offset = info->loca_table->src_offset;
    @@ -1432,8 +1431,8 @@
         if ( num_hmetrics < 1 )
           goto Fail;
     
    -    if ( FT_NEW_ARRAY( advance_widths, num_hmetrics ) ||
    -         FT_NEW_ARRAY( lsbs, num_glyphs )             )
    +    if ( FT_QNEW_ARRAY( advance_widths, num_hmetrics ) ||
    +         FT_QNEW_ARRAY( lsbs, num_glyphs )             )
           goto Fail;
     
         /* Read `advanceWidth' stream.  Always present. */
    @@ -1484,7 +1483,7 @@
     
         /* Build the hmtx table. */
         hmtx_table_size = 2 * num_hmetrics + 2 * num_glyphs;
    -    if ( FT_NEW_ARRAY( hmtx_table, hmtx_table_size ) )
    +    if ( FT_QALLOC( hmtx_table, hmtx_table_size ) )
           goto Fail;
     
         dst = hmtx_table;
    @@ -1541,10 +1540,10 @@
       {
         /* Memory management of `transformed_buf' is handled by the caller. */
     
    -    FT_Error   error       = FT_Err_Ok;
    -    FT_Stream  stream      = NULL;
    -    FT_Byte*   buf_cursor  = NULL;
    -    FT_Byte*   table_entry = NULL;
    +    FT_Error   error      = FT_Err_Ok;
    +    FT_Stream  stream     = NULL;
    +    FT_Byte*   buf_cursor = NULL;
    +    FT_Byte    table_entry[16];
     
         /* We are reallocating memory for `sfnt', so its pointer may change. */
         FT_Byte*   sfnt = *sfnt_bytes;
    @@ -1585,10 +1584,6 @@
           }
         }
     
    -    /* Create buffer for table entries. */
    -    if ( FT_NEW_ARRAY( table_entry, 16 ) )
    -      goto Fail;
    -
         /* Create a stream for the uncompressed buffer. */
         if ( FT_NEW( stream ) )
           goto Fail;
    @@ -1751,7 +1746,6 @@
         /* Set pointer of sfnt stream to its correct value. */
         *sfnt_bytes = sfnt;
     
    -    FT_FREE( table_entry );
         FT_Stream_Close( stream );
         FT_FREE( stream );
     
    @@ -1764,7 +1758,6 @@
         /* Set pointer of sfnt stream to its correct value. */
         *sfnt_bytes = sfnt;
     
    -    FT_FREE( table_entry );
         FT_Stream_Close( stream );
         FT_FREE( stream );
     
    @@ -1877,8 +1870,8 @@
         woff2.ttc_fonts = NULL;
     
         /* Read table directory. */
    -    if ( FT_NEW_ARRAY( tables, woff2.num_tables )  ||
    -         FT_NEW_ARRAY( indices, woff2.num_tables ) )
    +    if ( FT_QNEW_ARRAY( tables, woff2.num_tables )  ||
    +         FT_QNEW_ARRAY( indices, woff2.num_tables ) )
           goto Exit;
     
         FT_TRACE2(( "\n" ));
    @@ -1949,10 +1942,11 @@
             goto Exit;
           }
     
    +      table->flags      = flags;
           table->src_offset = src_offset;
           table->src_length = table->TransformLength;
           src_offset       += table->TransformLength;
    -      table->flags      = flags;
    +      table->dst_offset = 0;
     
           FT_TRACE2(( "  %c%c%c%c  %08d  %08d   %08ld    %08ld    %08ld\n",
                       (FT_Char)( table->Tag >> 24 ),
    @@ -2010,6 +2004,7 @@
     
           FT_TRACE4(( "Number of fonts in TTC: %d\n", woff2.num_fonts ));
     
    +      /* pre-zero pointers within in case of failure */
           if ( FT_NEW_ARRAY( woff2.ttc_fonts, woff2.num_fonts ) )
             goto Exit;
     
    @@ -2023,7 +2018,7 @@
             if ( FT_READ_ULONG( ttc_font->flavor ) )
               goto Exit;
     
    -        if ( FT_NEW_ARRAY( ttc_font->table_indices, ttc_font->num_tables ) )
    +        if ( FT_QNEW_ARRAY( ttc_font->table_indices, ttc_font->num_tables ) )
               goto Exit;
     
             FT_TRACE5(( "Number of tables in font %d: %d\n",
    @@ -2302,9 +2297,9 @@
         {
           FT_TRACE5(( "Trimming sfnt stream from %lu to %lu.\n",
                       sfnt_size, woff2.actual_sfnt_size ));
    -      if ( FT_REALLOC( sfnt,
    -                       (FT_ULong)( sfnt_size ),
    -                       (FT_ULong)( woff2.actual_sfnt_size ) ) )
    +      if ( FT_QREALLOC( sfnt,
    +                        (FT_ULong)( sfnt_size ),
    +                        (FT_ULong)( woff2.actual_sfnt_size ) ) )
             goto Exit;
         }
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/sfwoff2.h b/src/java.desktop/share/native/libfreetype/src/sfnt/sfwoff2.h
    index e84982ed9c341..4901286ee085b 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/sfwoff2.h
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/sfwoff2.h
    @@ -4,7 +4,7 @@
      *
      *   WOFFF2 format management (specification).
      *
    - * Copyright (C) 2019-2022 by
    + * Copyright (C) 2019-2023 by
      * Nikhil Ramakrishnan, David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/ttcmap.c b/src/java.desktop/share/native/libfreetype/src/sfnt/ttcmap.c
    index bfeabacb7d2b7..820cd08e6d51a 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/ttcmap.c
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/ttcmap.c
    @@ -4,7 +4,7 @@
      *
      *   TrueType character mapping table (cmap) support (body).
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -3879,13 +3879,14 @@
       }
     
     
    -  FT_LOCAL( FT_Error )
    +  FT_LOCAL_DEF( FT_Error )
       tt_get_cmap_info( FT_CharMap    charmap,
                         TT_CMapInfo  *cmap_info )
       {
         FT_CMap        cmap  = (FT_CMap)charmap;
         TT_CMap_Class  clazz = (TT_CMap_Class)cmap->clazz;
     
    +
         if ( clazz->get_cmap_info )
           return clazz->get_cmap_info( charmap, cmap_info );
         else
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/ttcmap.h b/src/java.desktop/share/native/libfreetype/src/sfnt/ttcmap.h
    index b10860b345e68..ff52917ed5bc0 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/ttcmap.h
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/ttcmap.h
    @@ -4,7 +4,7 @@
      *
      *   TrueType character mapping table (cmap) support (specification).
      *
    - * Copyright (C) 2002-2022 by
    + * Copyright (C) 2002-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/ttcmapc.h b/src/java.desktop/share/native/libfreetype/src/sfnt/ttcmapc.h
    index 6822a9cd6b924..0af48c2478af0 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/ttcmapc.h
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/ttcmapc.h
    @@ -4,7 +4,7 @@
      *
      *   TT CMAP classes definitions (specification only).
      *
    - * Copyright (C) 2009-2022 by
    + * Copyright (C) 2009-2023 by
      * Oran Agra and Mickey Gabel.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/ttcolr.c b/src/java.desktop/share/native/libfreetype/src/sfnt/ttcolr.c
    index d54231fd64750..5d98dcab8ffcf 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/ttcolr.c
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/ttcolr.c
    @@ -4,7 +4,7 @@
      *
      *   TrueType and OpenType colored glyph layer support (body).
      *
    - * Copyright (C) 2018-2022 by
    + * Copyright (C) 2018-2023 by
      * David Turner, Robert Wilhelm, Dominik Röttsches, and Werner Lemberg.
      *
      * Originally written by Shao Yu Zhang .
    @@ -34,6 +34,9 @@
     #include 
     #include 
     
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +#include 
    +#endif
     
     #ifdef TT_CONFIG_OPTION_COLOR_LAYERS
     
    @@ -46,17 +49,42 @@
     #define LAYER_V1_LIST_PAINT_OFFSET_SIZE   4U
     #define LAYER_V1_LIST_NUM_LAYERS_SIZE     4U
     #define COLOR_STOP_SIZE                   6U
    +#define VAR_IDX_BASE_SIZE                 4U
     #define LAYER_SIZE                        4U
    -#define COLR_HEADER_SIZE                 14U
    +/* https://docs.microsoft.com/en-us/typography/opentype/spec/colr#colr-header */
    +/* 3 * uint16 + 2 * Offset32 */
    +#define COLRV0_HEADER_SIZE               14U
    +/* COLRV0_HEADER_SIZE + 5 * Offset32 */
    +#define COLRV1_HEADER_SIZE               34U
    +
    +
    +#define ENSURE_READ_BYTES( byte_size )                             \
    +  if ( p < colr->paints_start_v1                                || \
    +       p > (FT_Byte*)colr->table + colr->table_size - byte_size )  \
    +    return 0
     
     
       typedef enum  FT_PaintFormat_Internal_
       {
    -    FT_COLR_PAINTFORMAT_INTERNAL_SCALE_CENTER         = 18,
    -    FT_COLR_PAINTFORMAT_INTERNAL_SCALE_UNIFORM        = 20,
    -    FT_COLR_PAINTFORMAT_INTERNAL_SCALE_UNIFORM_CENTER = 22,
    -    FT_COLR_PAINTFORMAT_INTERNAL_ROTATE_CENTER        = 26,
    -    FT_COLR_PAINTFORMAT_INTERNAL_SKEW_CENTER          = 30
    +    FT_COLR_PAINTFORMAT_INTERNAL_VAR_SOLID                = 3,
    +    FT_COLR_PAINTFORMAT_INTERNAL_VAR_LINEAR_GRADIENT      = 5,
    +    FT_COLR_PAINTFORMAT_INTERNAL_VAR_RADIAL_GRADIENT      = 7,
    +    FT_COLR_PAINTFORMAT_INTERNAL_VAR_SWEEP_GRADIENT       = 9,
    +    FT_COLR_PAINTFORMAT_INTERNAL_VAR_TRANSFORM            = 13,
    +    FT_COLR_PAINTFORMAT_INTERNAL_VAR_TRANSLATE            = 15,
    +    FT_COLR_PAINTFORMAT_INTERNAL_VAR_SCALE                = 17,
    +    FT_COLR_PAINTFORMAT_INTERNAL_SCALE_CENTER             = 18,
    +    FT_COLR_PAINTFORMAT_INTERNAL_VAR_SCALE_CENTER         = 19,
    +    FT_COLR_PAINTFORMAT_INTERNAL_SCALE_UNIFORM            = 20,
    +    FT_COLR_PAINTFORMAT_INTERNAL_VAR_SCALE_UNIFORM        = 21,
    +    FT_COLR_PAINTFORMAT_INTERNAL_SCALE_UNIFORM_CENTER     = 22,
    +    FT_COLR_PAINTFORMAT_INTERNAL_VAR_SCALE_UNIFORM_CENTER = 23,
    +    FT_COLR_PAINTFORMAT_INTERNAL_VAR_ROTATE               = 25,
    +    FT_COLR_PAINTFORMAT_INTERNAL_ROTATE_CENTER            = 26,
    +    FT_COLR_PAINTFORMAT_INTERNAL_VAR_ROTATE_CENTER        = 27,
    +    FT_COLR_PAINTFORMAT_INTERNAL_VAR_SKEW                 = 29,
    +    FT_COLR_PAINTFORMAT_INTERNAL_SKEW_CENTER              = 30,
    +    FT_COLR_PAINTFORMAT_INTERNAL_VAR_SKEW_CENTER          = 31,
     
       } FT_PaintFormat_Internal;
     
    @@ -104,6 +132,12 @@
          */
         FT_Byte*  paints_start_v1;
     
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +    /* Item Variation Store for variable 'COLR' v1. */
    +    GX_ItemVarStoreRec    var_store;
    +    GX_DeltaSetIdxMapRec  delta_set_idx_map;
    +#endif
    +
         /* The memory that backs up the `COLR' table. */
         void*     table;
         FT_ULong  table_size;
    @@ -139,6 +173,9 @@
         FT_ULong  base_glyphs_offset_v1, num_base_glyphs_v1;
         FT_ULong  layer_offset_v1, num_layers_v1, clip_list_offset;
         FT_ULong  table_size;
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +    FT_ULong  colr_offset_in_stream;
    +#endif
     
     
         /* `COLR' always needs `CPAL' */
    @@ -149,8 +186,12 @@
         if ( error )
           goto NoColr;
     
    -    if ( table_size < COLR_HEADER_SIZE )
    -      goto InvalidTable;
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +    colr_offset_in_stream = FT_STREAM_POS();
    +#endif
    +
    +    if ( table_size < COLRV0_HEADER_SIZE )
    +      goto NoColr;
     
         if ( FT_FRAME_EXTRACT( table_size, table ) )
           goto NoColr;
    @@ -183,9 +224,12 @@
     
         if ( colr->version == 1 )
         {
    +      if ( table_size < COLRV1_HEADER_SIZE )
    +        goto InvalidTable;
    +
           base_glyphs_offset_v1 = FT_NEXT_ULONG( p );
     
    -      if ( base_glyphs_offset_v1 >= table_size )
    +      if ( base_glyphs_offset_v1 + 4 >= table_size )
             goto InvalidTable;
     
           p1                 = (FT_Byte*)( table + base_glyphs_offset_v1 );
    @@ -205,6 +249,9 @@
     
           if ( layer_offset_v1 )
           {
    +        if ( layer_offset_v1 + 4 >= table_size )
    +          goto InvalidTable;
    +
             p1            = (FT_Byte*)( table + layer_offset_v1 );
             num_layers_v1 = FT_PEEK_ULONG( p1 );
     
    @@ -239,6 +286,65 @@
             colr->clip_list = (FT_Byte*)( table + clip_list_offset );
           else
             colr->clip_list = 0;
    +
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +      colr->var_store.dataCount     = 0;
    +      colr->var_store.varData       = NULL;
    +      colr->var_store.axisCount     = 0;
    +      colr->var_store.regionCount   = 0;
    +      colr->var_store.varRegionList = 0;
    +
    +      colr->delta_set_idx_map.mapCount   = 0;
    +      colr->delta_set_idx_map.outerIndex = NULL;
    +      colr->delta_set_idx_map.innerIndex = NULL;
    +
    +      if ( face->variation_support & TT_FACE_FLAG_VAR_FVAR )
    +      {
    +        FT_ULong  var_idx_map_offset, var_store_offset;
    +
    +        FT_Service_MultiMasters  mm = (FT_Service_MultiMasters)face->mm;
    +
    +
    +        var_idx_map_offset = FT_NEXT_ULONG( p );
    +
    +        if ( var_idx_map_offset >= table_size )
    +          goto InvalidTable;
    +
    +        var_store_offset = FT_NEXT_ULONG( p );
    +        if ( var_store_offset >= table_size )
    +          goto InvalidTable;
    +
    +        if ( var_store_offset )
    +        {
    +          /* If variation info has not been initialized yet, try doing so, */
    +          /* otherwise loading the variation store will fail as it         */
    +          /* requires access to `blend` for checking the number of axes.   */
    +          if ( !face->blend )
    +            if ( mm->get_mm_var( FT_FACE( face ), NULL ) )
    +              goto InvalidTable;
    +
    +          /* Try loading `VarIdxMap` and `VarStore`. */
    +          error = mm->load_item_var_store(
    +                    FT_FACE( face ),
    +                    colr_offset_in_stream + var_store_offset,
    +                    &colr->var_store );
    +          if ( error != FT_Err_Ok )
    +            goto InvalidTable;
    +        }
    +
    +        if ( colr->var_store.axisCount && var_idx_map_offset )
    +        {
    +          error = mm->load_delta_set_idx_map(
    +                    FT_FACE( face ),
    +                    colr_offset_in_stream + var_idx_map_offset,
    +                    &colr->delta_set_idx_map,
    +                    &colr->var_store,
    +                    table_size );
    +          if ( error != FT_Err_Ok )
    +            goto InvalidTable;
    +        }
    +      }
    +#endif /* TT_CONFIG_OPTION_GX_VAR_SUPPORT */
         }
     
         colr->base_glyphs = (FT_Byte*)( table + base_glyph_offset );
    @@ -251,6 +357,18 @@
         return FT_Err_Ok;
     
       InvalidTable:
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +    {
    +      FT_Service_MultiMasters  mm = (FT_Service_MultiMasters)face->mm;
    +
    +
    +      mm->done_delta_set_idx_map( FT_FACE( face ),
    +                                  &colr->delta_set_idx_map );
    +      mm->done_item_var_store( FT_FACE( face ),
    +                               &colr->var_store );
    +    }
    +#endif
    +
         error = FT_THROW( Invalid_Table );
     
       NoColr:
    @@ -272,6 +390,17 @@
     
         if ( colr )
         {
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +      {
    +        FT_Service_MultiMasters  mm = (FT_Service_MultiMasters)face->mm;
    +
    +
    +        mm->done_delta_set_idx_map( FT_FACE( face ),
    +                                    &colr->delta_set_idx_map );
    +        mm->done_item_var_store( FT_FACE( face ),
    +                                 &colr->var_store );
    +      }
    +#endif
           FT_FRAME_RELEASE( colr->table );
           FT_FREE( colr );
         }
    @@ -354,7 +483,9 @@
           iterator->p = colr->layers + offset;
         }
     
    -    if ( iterator->layer >= iterator->num_layers )
    +    if ( iterator->layer >= iterator->num_layers                     ||
    +         iterator->p < colr->layers                                  ||
    +         iterator->p >= ( (FT_Byte*)colr->table + colr->table_size ) )
           return 0;
     
         *aglyph_index = FT_NEXT_USHORT( iterator->p );
    @@ -372,13 +503,17 @@
     
     
       static FT_Bool
    -  read_color_line( FT_Byte*      color_line_p,
    -                   FT_ColorLine  *colorline )
    +  read_color_line( Colr*          colr,
    +                   FT_Byte*       color_line_p,
    +                   FT_ColorLine*  colorline,
    +                   FT_Bool        read_variable )
       {
         FT_Byte*        p = color_line_p;
         FT_PaintExtend  paint_extend;
     
     
    +    ENSURE_READ_BYTES( 3 );
    +
         paint_extend = (FT_PaintExtend)FT_NEXT_BYTE( p );
         if ( paint_extend > FT_COLR_PAINT_EXTEND_REFLECT )
           return 0;
    @@ -388,6 +523,7 @@
         colorline->color_stop_iterator.num_color_stops    = FT_NEXT_USHORT( p );
         colorline->color_stop_iterator.p                  = p;
         colorline->color_stop_iterator.current_color_stop = 0;
    +    colorline->color_stop_iterator.read_variable      = read_variable;
     
         return 1;
       }
    @@ -413,6 +549,10 @@
         if ( !child_table_pointer )
           return 0;
     
    +    if ( *p < colr->paints_start_v1                            ||
    +         *p > (FT_Byte*)colr->table + colr->table_size - 1 - 3 )
    +      return 0;
    +
         paint_offset = FT_NEXT_UOFF3( *p );
         if ( !paint_offset )
           return 0;
    @@ -428,20 +568,85 @@
       }
     
     
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +
       static FT_Bool
    -  read_paint( Colr*           colr,
    +  get_deltas_for_var_index_base ( TT_Face           face,
    +                                  Colr*             colr,
    +                                  FT_ULong          var_index_base,
    +                                  FT_UInt           num_deltas,
    +                                  FT_ItemVarDelta*  deltas )
    +  {
    +    FT_UInt   outer_index    = 0;
    +    FT_UInt   inner_index    = 0;
    +    FT_ULong  loop_var_index = var_index_base;
    +
    +    FT_Service_MultiMasters  mm = (FT_Service_MultiMasters)face->mm;
    +
    +    FT_UInt  i = 0;
    +
    +
    +    if ( var_index_base == 0xFFFFFFFF )
    +    {
    +      for ( i = 0; i < num_deltas; ++i )
    +        deltas[i] = 0;
    +      return 1;
    +    }
    +
    +    for ( i = 0; i < num_deltas; ++i )
    +    {
    +      loop_var_index = var_index_base + i;
    +
    +      if ( colr->delta_set_idx_map.innerIndex )
    +      {
    +        if ( loop_var_index >= colr->delta_set_idx_map.mapCount )
    +          loop_var_index = colr->delta_set_idx_map.mapCount - 1;
    +
    +        outer_index = colr->delta_set_idx_map.outerIndex[loop_var_index];
    +        inner_index = colr->delta_set_idx_map.innerIndex[loop_var_index];
    +      }
    +      else
    +      {
    +        outer_index = 0;
    +        inner_index = loop_var_index;
    +      }
    +
    +      deltas[i] = mm->get_item_delta( FT_FACE( face ), &colr->var_store,
    +                                      outer_index, inner_index );
    +    }
    +
    +    return 1;
    +  }
    +
    +#endif /* TT_CONFIG_OPTION_GX_VAR_SUPPORT */
    +
    +
    +  static FT_Bool
    +  read_paint( TT_Face         face,
    +              Colr*           colr,
                   FT_Byte*        p,
                   FT_COLR_Paint*  apaint )
       {
    -    FT_Byte*  paint_base     = p;
    -    FT_Byte*  child_table_p  = NULL;
    +    FT_Byte*  paint_base    = p;
    +    FT_Byte*  child_table_p = NULL;
    +    FT_Bool   do_read_var   = FALSE;
    +
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +    FT_ULong         var_index_base = 0;
    +    /* Longest varIndexBase offset is 5 in the spec. */
    +    FT_ItemVarDelta  item_deltas[6] = { 0, 0, 0, 0, 0, 0 };
    +#else
    +    FT_UNUSED( face );
    +#endif
     
     
         if ( !p || !colr || !colr->table )
           return 0;
     
    -    if ( p < colr->paints_start_v1                         ||
    -         p >= ( (FT_Byte*)colr->table + colr->table_size ) )
    +    /* The last byte of the 'COLR' table is at 'size-1'; subtract 1 of    */
    +    /* that to account for the expected format byte we are going to read. */
    +    if ( p < colr->paints_start_v1                        ||
    +         p > (FT_Byte*)colr->table + colr->table_size - 2 )
           return 0;
     
         apaint->format = (FT_PaintFormat)FT_NEXT_BYTE( p );
    @@ -475,16 +680,37 @@
           return 1;
         }
     
    -    else if ( apaint->format == FT_COLR_PAINTFORMAT_SOLID )
    +    else if ( apaint->format == FT_COLR_PAINTFORMAT_SOLID ||
    +              (FT_PaintFormat_Internal)apaint->format ==
    +                 FT_COLR_PAINTFORMAT_INTERNAL_VAR_SOLID   )
         {
    +      ENSURE_READ_BYTES( 4 );
           apaint->u.solid.color.palette_index = FT_NEXT_USHORT( p );
           apaint->u.solid.color.alpha         = FT_NEXT_SHORT( p );
     
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +      if ( (FT_PaintFormat_Internal)apaint->format ==
    +              FT_COLR_PAINTFORMAT_INTERNAL_VAR_SOLID )
    +      {
    +        ENSURE_READ_BYTES( 4 );
    +        var_index_base = FT_NEXT_ULONG( p );
    +
    +        if ( !get_deltas_for_var_index_base( face, colr, var_index_base, 1,
    +                                             item_deltas ) )
    +          return 0;
    +
    +        apaint->u.solid.color.alpha += item_deltas[0];
    +      }
    +#endif
    +
    +      apaint->format = FT_COLR_PAINTFORMAT_SOLID;
    +
           return 1;
         }
     
         else if ( apaint->format == FT_COLR_PAINTFORMAT_COLR_GLYPH )
         {
    +      ENSURE_READ_BYTES(2);
           apaint->u.colr_glyph.glyphID = FT_NEXT_USHORT( p );
     
           return 1;
    @@ -500,16 +726,23 @@
         if ( !get_child_table_pointer( colr, paint_base, &p, &child_table_p ) )
           return 0;
     
    -    if ( apaint->format == FT_COLR_PAINTFORMAT_LINEAR_GRADIENT )
    +    if ( apaint->format == FT_COLR_PAINTFORMAT_LINEAR_GRADIENT      ||
    +         ( do_read_var =
    +             ( (FT_PaintFormat_Internal)apaint->format ==
    +               FT_COLR_PAINTFORMAT_INTERNAL_VAR_LINEAR_GRADIENT ) ) )
         {
    -      if ( !read_color_line( child_table_p,
    -                             &apaint->u.linear_gradient.colorline ) )
    +      if ( !read_color_line( colr,
    +                             child_table_p,
    +                             &apaint->u.linear_gradient.colorline,
    +                             do_read_var ) )
             return 0;
     
           /*
    -       * In order to support variations expose these as FT_Fixed 16.16 values so
    -       * that we can support fractional values after interpolation.
    +       * In order to support variations expose these as FT_Fixed 16.16
    +       * values so that we can support fractional values after
    +       * interpolation.
            */
    +      ENSURE_READ_BYTES( 12 );
           apaint->u.linear_gradient.p0.x = INT_TO_FIXED( FT_NEXT_SHORT( p ) );
           apaint->u.linear_gradient.p0.y = INT_TO_FIXED( FT_NEXT_SHORT( p ) );
           apaint->u.linear_gradient.p1.x = INT_TO_FIXED( FT_NEXT_SHORT( p ) );
    @@ -517,23 +750,52 @@
           apaint->u.linear_gradient.p2.x = INT_TO_FIXED( FT_NEXT_SHORT( p ) );
           apaint->u.linear_gradient.p2.y = INT_TO_FIXED( FT_NEXT_SHORT( p ) );
     
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +      if ( do_read_var )
    +      {
    +        ENSURE_READ_BYTES( 4 );
    +        var_index_base = FT_NEXT_ULONG ( p );
    +
    +        if ( !get_deltas_for_var_index_base( face, colr, var_index_base, 6,
    +                                             item_deltas ) )
    +          return 0;
    +
    +        apaint->u.linear_gradient.p0.x += INT_TO_FIXED( item_deltas[0] );
    +        apaint->u.linear_gradient.p0.y += INT_TO_FIXED( item_deltas[1] );
    +        apaint->u.linear_gradient.p1.x += INT_TO_FIXED( item_deltas[2] );
    +        apaint->u.linear_gradient.p1.y += INT_TO_FIXED( item_deltas[3] );
    +        apaint->u.linear_gradient.p2.x += INT_TO_FIXED( item_deltas[4] );
    +        apaint->u.linear_gradient.p2.y += INT_TO_FIXED( item_deltas[5] );
    +      }
    +#endif
    +
    +      apaint->format = FT_COLR_PAINTFORMAT_LINEAR_GRADIENT;
    +
           return 1;
         }
     
    -    else if ( apaint->format == FT_COLR_PAINTFORMAT_RADIAL_GRADIENT )
    +    else if ( apaint->format == FT_COLR_PAINTFORMAT_RADIAL_GRADIENT      ||
    +              ( do_read_var =
    +                  ( (FT_PaintFormat_Internal)apaint->format ==
    +                    FT_COLR_PAINTFORMAT_INTERNAL_VAR_RADIAL_GRADIENT ) ) )
         {
           FT_Pos  tmp;
     
     
    -      if ( !read_color_line( child_table_p,
    -                             &apaint->u.radial_gradient.colorline ) )
    +      if ( !read_color_line( colr,
    +                             child_table_p,
    +                             &apaint->u.radial_gradient.colorline,
    +                             do_read_var ) )
             return 0;
     
    +
           /* In the OpenType specification, `r0` and `r1` are defined as   */
           /* `UFWORD`.  Since FreeType doesn't have a corresponding 16.16  */
           /* format we convert to `FWORD` and replace negative values with */
           /* (32bit) `FT_INT_MAX`.                                         */
     
    +      ENSURE_READ_BYTES( 12 );
    +
           apaint->u.radial_gradient.c0.x = INT_TO_FIXED( FT_NEXT_SHORT( p ) );
           apaint->u.radial_gradient.c0.y = INT_TO_FIXED( FT_NEXT_SHORT( p ) );
     
    @@ -546,15 +808,47 @@
           tmp                          = INT_TO_FIXED( FT_NEXT_SHORT( p ) );
           apaint->u.radial_gradient.r1 = tmp < 0 ? FT_INT_MAX : tmp;
     
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +      if ( do_read_var )
    +      {
    +        ENSURE_READ_BYTES( 4 );
    +        var_index_base = FT_NEXT_ULONG ( p );
    +
    +        if ( !get_deltas_for_var_index_base( face, colr, var_index_base, 6,
    +                                             item_deltas ) )
    +          return 0;
    +
    +        apaint->u.radial_gradient.c0.x += INT_TO_FIXED( item_deltas[0] );
    +        apaint->u.radial_gradient.c0.y += INT_TO_FIXED( item_deltas[1] );
    +
    +        // TODO: Anything to be done about UFWORD deltas here?
    +        apaint->u.radial_gradient.r0 += INT_TO_FIXED( item_deltas[2] );
    +
    +        apaint->u.radial_gradient.c1.x += INT_TO_FIXED( item_deltas[3] );
    +        apaint->u.radial_gradient.c1.y += INT_TO_FIXED( item_deltas[4] );
    +
    +        apaint->u.radial_gradient.r1 += INT_TO_FIXED( item_deltas[5] );
    +      }
    +#endif
    +
    +      apaint->format = FT_COLR_PAINTFORMAT_RADIAL_GRADIENT;
    +
           return 1;
         }
     
    -    else if ( apaint->format == FT_COLR_PAINTFORMAT_SWEEP_GRADIENT )
    +    else if ( apaint->format == FT_COLR_PAINTFORMAT_SWEEP_GRADIENT      ||
    +              ( do_read_var =
    +                  ( (FT_PaintFormat_Internal)apaint->format ==
    +                    FT_COLR_PAINTFORMAT_INTERNAL_VAR_SWEEP_GRADIENT ) ) )
         {
    -      if ( !read_color_line( child_table_p,
    -                             &apaint->u.sweep_gradient.colorline ) )
    +      if ( !read_color_line( colr,
    +                             child_table_p,
    +                             &apaint->u.sweep_gradient.colorline,
    +                             do_read_var) )
             return 0;
     
    +      ENSURE_READ_BYTES( 8 );
    +
           apaint->u.sweep_gradient.center.x =
               INT_TO_FIXED( FT_NEXT_SHORT( p ) );
           apaint->u.sweep_gradient.center.y =
    @@ -565,11 +859,34 @@
           apaint->u.sweep_gradient.end_angle =
               F2DOT14_TO_FIXED( FT_NEXT_SHORT( p ) );
     
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +      if ( do_read_var )
    +      {
    +        ENSURE_READ_BYTES( 4 );
    +        var_index_base = FT_NEXT_ULONG ( p );
    +
    +        if ( !get_deltas_for_var_index_base( face, colr, var_index_base, 4,
    +                                             item_deltas ) )
    +          return 0;
    +
    +        // TODO: Handle overflow?
    +        apaint->u.sweep_gradient.center.x += INT_TO_FIXED( item_deltas[0] );
    +        apaint->u.sweep_gradient.center.y += INT_TO_FIXED( item_deltas[1] );
    +
    +        apaint->u.sweep_gradient.start_angle +=
    +          F2DOT14_TO_FIXED( item_deltas[2] );
    +        apaint->u.sweep_gradient.end_angle +=
    +          F2DOT14_TO_FIXED( item_deltas[3] );
    +      }
    +#endif
    +      apaint->format = FT_COLR_PAINTFORMAT_SWEEP_GRADIENT;
    +
           return 1;
         }
     
         if ( apaint->format == FT_COLR_PAINTFORMAT_GLYPH )
         {
    +      ENSURE_READ_BYTES( 2 );
           apaint->u.glyph.paint.p                     = child_table_p;
           apaint->u.glyph.paint.insert_root_transform = 0;
           apaint->u.glyph.glyphID                     = FT_NEXT_USHORT( p );
    @@ -577,7 +894,9 @@
           return 1;
         }
     
    -    else if ( apaint->format == FT_COLR_PAINTFORMAT_TRANSFORM )
    +    else if ( apaint->format == FT_COLR_PAINTFORMAT_TRANSFORM ||
    +              (FT_PaintFormat_Internal)apaint->format ==
    +                FT_COLR_PAINTFORMAT_INTERNAL_VAR_TRANSFORM    )
         {
           apaint->u.transform.paint.p                     = child_table_p;
           apaint->u.transform.paint.insert_root_transform = 0;
    @@ -591,6 +910,7 @@
            * The following matrix coefficients are encoded as
            * OpenType 16.16 fixed-point values.
            */
    +      ENSURE_READ_BYTES( 24 );
           apaint->u.transform.affine.xx = FT_NEXT_LONG( p );
           apaint->u.transform.affine.yx = FT_NEXT_LONG( p );
           apaint->u.transform.affine.xy = FT_NEXT_LONG( p );
    @@ -598,51 +918,101 @@
           apaint->u.transform.affine.dx = FT_NEXT_LONG( p );
           apaint->u.transform.affine.dy = FT_NEXT_LONG( p );
     
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +      if ( (FT_PaintFormat_Internal)apaint->format ==
    +             FT_COLR_PAINTFORMAT_INTERNAL_VAR_TRANSFORM )
    +      {
    +        ENSURE_READ_BYTES( 4 );
    +        var_index_base = FT_NEXT_ULONG( p );
    +
    +        if ( !get_deltas_for_var_index_base( face, colr, var_index_base, 6,
    +                                             item_deltas ) )
    +          return 0;
    +
    +        apaint->u.transform.affine.xx += (FT_Fixed)item_deltas[0];
    +        apaint->u.transform.affine.yx += (FT_Fixed)item_deltas[1];
    +        apaint->u.transform.affine.xy += (FT_Fixed)item_deltas[2];
    +        apaint->u.transform.affine.yy += (FT_Fixed)item_deltas[3];
    +        apaint->u.transform.affine.dx += (FT_Fixed)item_deltas[4];
    +        apaint->u.transform.affine.dy += (FT_Fixed)item_deltas[5];
    +      }
    +#endif
    +
    +      apaint->format = FT_COLR_PAINTFORMAT_TRANSFORM;
    +
           return 1;
         }
     
    -    else if ( apaint->format == FT_COLR_PAINTFORMAT_TRANSLATE )
    +    else if ( apaint->format == FT_COLR_PAINTFORMAT_TRANSLATE ||
    +              (FT_PaintFormat_Internal)apaint->format ==
    +                FT_COLR_PAINTFORMAT_INTERNAL_VAR_TRANSLATE    )
         {
           apaint->u.translate.paint.p                     = child_table_p;
           apaint->u.translate.paint.insert_root_transform = 0;
     
    +      ENSURE_READ_BYTES( 4 );
           apaint->u.translate.dx = INT_TO_FIXED( FT_NEXT_SHORT( p ) );
           apaint->u.translate.dy = INT_TO_FIXED( FT_NEXT_SHORT( p ) );
     
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +      if ( (FT_PaintFormat_Internal)apaint->format ==
    +             FT_COLR_PAINTFORMAT_INTERNAL_VAR_TRANSLATE )
    +      {
    +        ENSURE_READ_BYTES( 4 );
    +        var_index_base = FT_NEXT_ULONG( p );
    +
    +        if ( !get_deltas_for_var_index_base( face, colr, var_index_base, 2,
    +                                             item_deltas ) )
    +          return 0;
    +
    +        apaint->u.translate.dx += INT_TO_FIXED( item_deltas[0] );
    +        apaint->u.translate.dy += INT_TO_FIXED( item_deltas[1] );
    +      }
    +#endif
    +
    +      apaint->format = FT_COLR_PAINTFORMAT_TRANSLATE;
    +
           return 1;
         }
     
    -    else if ( apaint->format ==
    -                FT_COLR_PAINTFORMAT_SCALE                         ||
    -              (FT_PaintFormat_Internal)apaint->format ==
    -                FT_COLR_PAINTFORMAT_INTERNAL_SCALE_CENTER         ||
    -              (FT_PaintFormat_Internal)apaint->format ==
    -                FT_COLR_PAINTFORMAT_INTERNAL_SCALE_UNIFORM        ||
    -              (FT_PaintFormat_Internal)apaint->format ==
    -                FT_COLR_PAINTFORMAT_INTERNAL_SCALE_UNIFORM_CENTER )
    +    else if ( apaint->format >= FT_COLR_PAINTFORMAT_SCALE             &&
    +              (FT_PaintFormat_Internal)apaint->format <=
    +                FT_COLR_PAINTFORMAT_INTERNAL_VAR_SCALE_UNIFORM_CENTER )
         {
           apaint->u.scale.paint.p                     = child_table_p;
           apaint->u.scale.paint.insert_root_transform = 0;
     
           /* All scale paints get at least one scale value. */
    +      ENSURE_READ_BYTES( 2 );
           apaint->u.scale.scale_x = F2DOT14_TO_FIXED( FT_NEXT_SHORT( p ) );
     
           /* Non-uniform ones read an extra y value. */
    -      if ( apaint->format ==
    -             FT_COLR_PAINTFORMAT_SCALE                 ||
    +      if ( apaint->format == FT_COLR_PAINTFORMAT_SCALE     ||
    +           (FT_PaintFormat_Internal)apaint->format ==
    +             FT_COLR_PAINTFORMAT_INTERNAL_VAR_SCALE        ||
    +           (FT_PaintFormat_Internal)apaint->format ==
    +             FT_COLR_PAINTFORMAT_INTERNAL_SCALE_CENTER     ||
                (FT_PaintFormat_Internal)apaint->format ==
    -             FT_COLR_PAINTFORMAT_INTERNAL_SCALE_CENTER )
    +             FT_COLR_PAINTFORMAT_INTERNAL_VAR_SCALE_CENTER )
    +      {
    +        ENSURE_READ_BYTES( 2 );
             apaint->u.scale.scale_y = F2DOT14_TO_FIXED( FT_NEXT_SHORT( p ) );
    +      }
           else
             apaint->u.scale.scale_y = apaint->u.scale.scale_x;
     
           /* Scale paints that have a center read center coordinates, */
           /* otherwise the center is (0,0).                           */
           if ( (FT_PaintFormat_Internal)apaint->format ==
    -             FT_COLR_PAINTFORMAT_INTERNAL_SCALE_CENTER         ||
    +             FT_COLR_PAINTFORMAT_INTERNAL_SCALE_CENTER             ||
    +           (FT_PaintFormat_Internal)apaint->format ==
    +             FT_COLR_PAINTFORMAT_INTERNAL_VAR_SCALE_CENTER         ||
                (FT_PaintFormat_Internal)apaint->format ==
    -             FT_COLR_PAINTFORMAT_INTERNAL_SCALE_UNIFORM_CENTER )
    +             FT_COLR_PAINTFORMAT_INTERNAL_SCALE_UNIFORM_CENTER     ||
    +           (FT_PaintFormat_Internal)apaint->format ==
    +             FT_COLR_PAINTFORMAT_INTERNAL_VAR_SCALE_UNIFORM_CENTER )
           {
    +        ENSURE_READ_BYTES( 4 );
             apaint->u.scale.center_x = INT_TO_FIXED( FT_NEXT_SHORT ( p ) );
             apaint->u.scale.center_y = INT_TO_FIXED( FT_NEXT_SHORT ( p ) );
           }
    @@ -652,6 +1022,71 @@
             apaint->u.scale.center_y = 0;
           }
     
    +      /* Base values set, now handle variations. */
    +
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +      if ( (FT_PaintFormat_Internal)apaint->format ==
    +             FT_COLR_PAINTFORMAT_INTERNAL_VAR_SCALE                ||
    +           (FT_PaintFormat_Internal)apaint->format ==
    +             FT_COLR_PAINTFORMAT_INTERNAL_VAR_SCALE_CENTER         ||
    +           (FT_PaintFormat_Internal)apaint->format ==
    +             FT_COLR_PAINTFORMAT_INTERNAL_VAR_SCALE_UNIFORM        ||
    +           (FT_PaintFormat_Internal)apaint->format ==
    +             FT_COLR_PAINTFORMAT_INTERNAL_VAR_SCALE_UNIFORM_CENTER )
    +      {
    +        ENSURE_READ_BYTES( 4 );
    +        var_index_base = FT_NEXT_ULONG( p );
    +
    +        if ( (FT_PaintFormat_Internal)apaint->format ==
    +               FT_COLR_PAINTFORMAT_INTERNAL_VAR_SCALE )
    +        {
    +          if ( !get_deltas_for_var_index_base( face, colr, var_index_base, 2,
    +                                               item_deltas ) )
    +            return 0;
    +
    +          apaint->u.scale.scale_x += F2DOT14_TO_FIXED( item_deltas[0] );
    +          apaint->u.scale.scale_y += F2DOT14_TO_FIXED( item_deltas[1] );
    +        }
    +
    +        if ( (FT_PaintFormat_Internal)apaint->format ==
    +               FT_COLR_PAINTFORMAT_INTERNAL_VAR_SCALE_CENTER )
    +        {
    +          if ( !get_deltas_for_var_index_base( face, colr, var_index_base, 4,
    +                                               item_deltas ) )
    +            return 0;
    +
    +          apaint->u.scale.scale_x  += F2DOT14_TO_FIXED( item_deltas[0] );
    +          apaint->u.scale.scale_y  += F2DOT14_TO_FIXED( item_deltas[1] );
    +          apaint->u.scale.center_x += INT_TO_FIXED( item_deltas[2] );
    +          apaint->u.scale.center_y += INT_TO_FIXED( item_deltas[3] );
    +        }
    +
    +        if ( (FT_PaintFormat_Internal)apaint->format ==
    +               FT_COLR_PAINTFORMAT_INTERNAL_VAR_SCALE_UNIFORM )
    +        {
    +          if ( !get_deltas_for_var_index_base( face, colr, var_index_base, 1,
    +                                               item_deltas ) )
    +            return 0;
    +
    +          apaint->u.scale.scale_x += F2DOT14_TO_FIXED( item_deltas[0] );
    +          apaint->u.scale.scale_y += F2DOT14_TO_FIXED( item_deltas[0] );
    +        }
    +
    +        if ( (FT_PaintFormat_Internal)apaint->format ==
    +               FT_COLR_PAINTFORMAT_INTERNAL_VAR_SCALE_UNIFORM_CENTER )
    +        {
    +          if ( !get_deltas_for_var_index_base( face, colr, var_index_base, 3,
    +                                               item_deltas ) )
    +            return 0;
    +
    +          apaint->u.scale.scale_x  += F2DOT14_TO_FIXED( item_deltas[0] );
    +          apaint->u.scale.scale_y  += F2DOT14_TO_FIXED( item_deltas[0] );
    +          apaint->u.scale.center_x += INT_TO_FIXED( item_deltas[1] );
    +          apaint->u.scale.center_y += INT_TO_FIXED( item_deltas[2] );
    +        }
    +      }
    +#endif
    +
           /* FT 'COLR' v1 API output format always returns fully defined */
           /* structs; we thus set the format to the public API value.    */
           apaint->format = FT_COLR_PAINTFORMAT_SCALE;
    @@ -659,18 +1094,26 @@
           return 1;
         }
     
    -    else if ( apaint->format == FT_COLR_PAINTFORMAT_ROTATE ||
    +    else if ( apaint->format == FT_COLR_PAINTFORMAT_ROTATE     ||
    +              (FT_PaintFormat_Internal)apaint->format ==
    +                FT_COLR_PAINTFORMAT_INTERNAL_ROTATE_CENTER     ||
                   (FT_PaintFormat_Internal)apaint->format ==
    -                FT_COLR_PAINTFORMAT_INTERNAL_ROTATE_CENTER )
    +                FT_COLR_PAINTFORMAT_INTERNAL_VAR_ROTATE        ||
    +              (FT_PaintFormat_Internal)apaint->format ==
    +                FT_COLR_PAINTFORMAT_INTERNAL_VAR_ROTATE_CENTER )
         {
           apaint->u.rotate.paint.p                     = child_table_p;
           apaint->u.rotate.paint.insert_root_transform = 0;
     
    +      ENSURE_READ_BYTES( 2 );
           apaint->u.rotate.angle = F2DOT14_TO_FIXED( FT_NEXT_SHORT( p ) );
     
           if ( (FT_PaintFormat_Internal)apaint->format ==
    -           FT_COLR_PAINTFORMAT_INTERNAL_ROTATE_CENTER )
    +             FT_COLR_PAINTFORMAT_INTERNAL_ROTATE_CENTER     ||
    +           (FT_PaintFormat_Internal)apaint->format ==
    +             FT_COLR_PAINTFORMAT_INTERNAL_VAR_ROTATE_CENTER )
           {
    +        ENSURE_READ_BYTES( 4 );
             apaint->u.rotate.center_x = INT_TO_FIXED( FT_NEXT_SHORT( p ) );
             apaint->u.rotate.center_y = INT_TO_FIXED( FT_NEXT_SHORT( p ) );
           }
    @@ -680,24 +1123,69 @@
             apaint->u.rotate.center_y = 0;
           }
     
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +      if ( (FT_PaintFormat_Internal)apaint->format ==
    +             FT_COLR_PAINTFORMAT_INTERNAL_VAR_ROTATE        ||
    +           (FT_PaintFormat_Internal)apaint->format ==
    +             FT_COLR_PAINTFORMAT_INTERNAL_VAR_ROTATE_CENTER )
    +      {
    +        FT_UInt  num_deltas = 0;
    +
    +
    +        ENSURE_READ_BYTES( 4 );
    +        var_index_base = FT_NEXT_ULONG( p );
    +
    +        if ( (FT_PaintFormat_Internal)apaint->format ==
    +               FT_COLR_PAINTFORMAT_INTERNAL_VAR_ROTATE_CENTER )
    +          num_deltas = 3;
    +        if ( (FT_PaintFormat_Internal)apaint->format ==
    +               FT_COLR_PAINTFORMAT_INTERNAL_VAR_ROTATE )
    +          num_deltas = 1;
    +
    +        if ( num_deltas > 0 )
    +        {
    +          if ( !get_deltas_for_var_index_base( face, colr, var_index_base,
    +                                               num_deltas, item_deltas ) )
    +            return 0;
    +
    +          apaint->u.rotate.angle += F2DOT14_TO_FIXED( item_deltas[0] );
    +
    +          if ( num_deltas == 3 )
    +          {
    +            apaint->u.rotate.center_x += INT_TO_FIXED( item_deltas[1] );
    +            apaint->u.rotate.center_y += INT_TO_FIXED( item_deltas[2] );
    +          }
    +        }
    +      }
    +#endif
    +
           apaint->format = FT_COLR_PAINTFORMAT_ROTATE;
     
    +
           return 1;
         }
     
    -    else if ( apaint->format == FT_COLR_PAINTFORMAT_SKEW ||
    +    else if ( apaint->format == FT_COLR_PAINTFORMAT_SKEW     ||
    +              (FT_PaintFormat_Internal)apaint->format ==
    +                FT_COLR_PAINTFORMAT_INTERNAL_VAR_SKEW        ||
    +              (FT_PaintFormat_Internal)apaint->format ==
    +                FT_COLR_PAINTFORMAT_INTERNAL_SKEW_CENTER     ||
                   (FT_PaintFormat_Internal)apaint->format ==
    -                FT_COLR_PAINTFORMAT_INTERNAL_SKEW_CENTER )
    +                FT_COLR_PAINTFORMAT_INTERNAL_VAR_SKEW_CENTER )
         {
           apaint->u.skew.paint.p                     = child_table_p;
           apaint->u.skew.paint.insert_root_transform = 0;
     
    +      ENSURE_READ_BYTES( 4 );
           apaint->u.skew.x_skew_angle = F2DOT14_TO_FIXED( FT_NEXT_SHORT( p ) );
           apaint->u.skew.y_skew_angle = F2DOT14_TO_FIXED( FT_NEXT_SHORT( p ) );
     
           if ( (FT_PaintFormat_Internal)apaint->format ==
    -           FT_COLR_PAINTFORMAT_INTERNAL_SKEW_CENTER )
    +             FT_COLR_PAINTFORMAT_INTERNAL_SKEW_CENTER     ||
    +           (FT_PaintFormat_Internal)apaint->format ==
    +             FT_COLR_PAINTFORMAT_INTERNAL_VAR_SKEW_CENTER )
           {
    +        ENSURE_READ_BYTES( 4 );
             apaint->u.skew.center_x = INT_TO_FIXED( FT_NEXT_SHORT( p ) );
             apaint->u.skew.center_y = INT_TO_FIXED( FT_NEXT_SHORT( p ) );
           }
    @@ -707,6 +1195,42 @@
             apaint->u.skew.center_y = 0;
           }
     
    +
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +      if ( (FT_PaintFormat_Internal)apaint->format ==
    +             FT_COLR_PAINTFORMAT_INTERNAL_VAR_SKEW        ||
    +           (FT_PaintFormat_Internal)apaint->format ==
    +             FT_COLR_PAINTFORMAT_INTERNAL_VAR_SKEW_CENTER )
    +      {
    +        ENSURE_READ_BYTES( 4 );
    +        var_index_base = FT_NEXT_ULONG( p );
    +
    +        if ( (FT_PaintFormat_Internal)apaint->format ==
    +               FT_COLR_PAINTFORMAT_INTERNAL_VAR_SKEW )
    +        {
    +          if ( !get_deltas_for_var_index_base( face, colr, var_index_base, 2,
    +                                               item_deltas ) )
    +            return 0;
    +
    +          apaint->u.skew.x_skew_angle += F2DOT14_TO_FIXED( item_deltas[0] );
    +          apaint->u.skew.y_skew_angle += F2DOT14_TO_FIXED( item_deltas[1] );
    +        }
    +
    +        if ( (FT_PaintFormat_Internal)apaint->format ==
    +               FT_COLR_PAINTFORMAT_INTERNAL_VAR_SKEW_CENTER )
    +        {
    +          if ( !get_deltas_for_var_index_base( face, colr, var_index_base, 4,
    +                                               item_deltas ) )
    +            return 0;
    +
    +          apaint->u.skew.x_skew_angle += F2DOT14_TO_FIXED( item_deltas[0] );
    +          apaint->u.skew.y_skew_angle += F2DOT14_TO_FIXED( item_deltas[1] );
    +          apaint->u.skew.center_x     += INT_TO_FIXED( item_deltas[2] );
    +          apaint->u.skew.center_y     += INT_TO_FIXED( item_deltas[3] );
    +        }
    +      }
    +#endif
    +
           apaint->format = FT_COLR_PAINTFORMAT_SKEW;
     
           return 1;
    @@ -720,6 +1244,7 @@
           apaint->u.composite.source_paint.p                     = child_table_p;
           apaint->u.composite.source_paint.insert_root_transform = 0;
     
    +      ENSURE_READ_BYTES( 1 );
           composite_mode = FT_NEXT_BYTE( p );
           if ( composite_mode >= FT_COLR_COMPOSITE_MAX )
             return 0;
    @@ -871,7 +1396,7 @@
         clip_list_format = FT_NEXT_BYTE ( p );
     
         /* Format byte used here to be able to upgrade ClipList for >16bit */
    -    /* glyph ids; for now we can expect it to be 0.                    */
    +    /* glyph ids; for now we can expect it to be 1.                    */
         if ( !( clip_list_format == 1 ) )
           return 0;
     
    @@ -899,7 +1424,7 @@
     
             format = FT_NEXT_BYTE( p1 );
     
    -        if ( format > 1 )
    +        if ( format > 2 )
               return 0;
     
             /* Check whether we can extract four `FWORD`. */
    @@ -913,11 +1438,40 @@
             font_clip_box.xMin = FT_MulFix( FT_NEXT_SHORT( p1 ),
                                             face->root.size->metrics.x_scale );
             font_clip_box.yMin = FT_MulFix( FT_NEXT_SHORT( p1 ),
    -                                        face->root.size->metrics.x_scale );
    +                                        face->root.size->metrics.y_scale );
             font_clip_box.xMax = FT_MulFix( FT_NEXT_SHORT( p1 ),
                                             face->root.size->metrics.x_scale );
             font_clip_box.yMax = FT_MulFix( FT_NEXT_SHORT( p1 ),
    -                                        face->root.size->metrics.x_scale );
    +                                        face->root.size->metrics.y_scale );
    +
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +        if ( format == 2 )
    +        {
    +          FT_ULong         var_index_base = 0;
    +          /* varIndexBase offset for clipbox is 3 at most. */
    +          FT_ItemVarDelta  item_deltas[4] = { 0, 0, 0, 0 };
    +
    +
    +          /* Check whether we can extract a 32-bit varIndexBase now. */
    +          if ( p1 > limit - 4 )
    +            return 0;
    +
    +          var_index_base = FT_NEXT_ULONG( p1 );
    +
    +          if ( !get_deltas_for_var_index_base( face, colr, var_index_base, 4,
    +                                               item_deltas ) )
    +            return 0;
    +
    +          font_clip_box.xMin +=
    +            FT_MulFix( item_deltas[0], face->root.size->metrics.x_scale );
    +          font_clip_box.yMin +=
    +            FT_MulFix( item_deltas[1], face->root.size->metrics.y_scale );
    +          font_clip_box.xMax +=
    +            FT_MulFix( item_deltas[2], face->root.size->metrics.x_scale );
    +          font_clip_box.yMax +=
    +            FT_MulFix( item_deltas[3], face->root.size->metrics.y_scale );
    +        }
    +#endif
     
             /* Make 4 corner points (xMin, yMin), (xMax, yMax) and transform */
             /* them.  If we we would only transform two corner points and    */
    @@ -985,13 +1539,6 @@
          */
         p = iterator->p;
     
    -    /*
    -     * First ensure that p is within COLRv1.
    -     */
    -    if ( p < colr->layers_v1                               ||
    -         p >= ( (FT_Byte*)colr->table + colr->table_size ) )
    -      return 0;
    -
         /*
          * Do a cursor sanity check of the iterator.  Counting backwards from
          * where it stands, we need to end up at a position after the beginning
    @@ -1008,6 +1555,14 @@
                colr->num_layers_v1 * LAYER_V1_LIST_PAINT_OFFSET_SIZE ) )
           return 0;
     
    +    /*
    +     * Before reading, ensure that `p` is within 'COLR' v1 and we can read a
    +     * 4-byte ULONG.
    +     */
    +    if ( p < colr->layers_v1                              ||
    +         p > (FT_Byte*)colr->table + colr->table_size - 4 )
    +      return 0;
    +
         paint_offset =
           FT_NEXT_ULONG( p );
         opaque_paint->insert_root_transform =
    @@ -1037,29 +1592,67 @@
         Colr*  colr = (Colr*)face->colr;
     
         FT_Byte*  p;
    +    FT_ULong  var_index_base;
    +    FT_Byte*  last_entry_p = NULL;
    +    FT_UInt   entry_size   = COLOR_STOP_SIZE;
     
     
    -    if ( !colr || !colr->table )
    +    if ( !colr || !colr->table || !iterator )
           return 0;
     
         if ( iterator->current_color_stop >= iterator->num_color_stops )
           return 0;
     
    -    if ( iterator->p +
    -           ( ( iterator->num_color_stops - iterator->current_color_stop ) *
    -             COLOR_STOP_SIZE ) >
    -         ( (FT_Byte *)colr->table + colr->table_size ) )
    +    if ( iterator->read_variable )
    +      entry_size += VAR_IDX_BASE_SIZE;
    +
    +    /* Calculate the start pointer for the last to-be-read (Var)ColorStop */
    +    /* and check whether we can read a full (Var)ColorStop at that        */
    +    /* position by comparing it to the position that is the size of one   */
    +    /* (Var)ColorStop before the end of the 'COLR' table.                 */
    +    last_entry_p =
    +      iterator->p + ( iterator->num_color_stops - 1 -
    +                      iterator->current_color_stop ) * entry_size;
    +    if ( iterator->p < colr->paints_start_v1          ||
    +         last_entry_p > (FT_Byte*)colr->table +
    +                        colr->table_size - entry_size )
           return 0;
     
         /* Iterator points at first `ColorStop` of `ColorLine`. */
         p = iterator->p;
     
    -    color_stop->stop_offset = FT_NEXT_SHORT( p );
    +    color_stop->stop_offset = F2DOT14_TO_FIXED( FT_NEXT_SHORT( p ) );
     
         color_stop->color.palette_index = FT_NEXT_USHORT( p );
     
         color_stop->color.alpha = FT_NEXT_SHORT( p );
     
    +    if ( iterator->read_variable )
    +    {
    +      /* Pointer p needs to be advanced independently of whether we intend */
    +      /* to take variable deltas into account or not.  Otherwise iteration */
    +      /* would fail due to wrong offsets.                                  */
    +      var_index_base = FT_NEXT_ULONG( p );
    +
    +#ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    +      {
    +        FT_Int  item_deltas[2];
    +
    +
    +        if ( !get_deltas_for_var_index_base( face, colr,
    +                                             var_index_base,
    +                                             2,
    +                                             item_deltas ) )
    +          return 0;
    +
    +        color_stop->stop_offset += F2DOT14_TO_FIXED( item_deltas[0] );
    +        color_stop->color.alpha += item_deltas[1];
    +      }
    +#else
    +      FT_UNUSED( var_index_base );
    +#endif
    +    }
    +
         iterator->p = p;
         iterator->current_color_stop++;
     
    @@ -1139,7 +1732,7 @@
           return 1;
         }
     
    -    return read_paint( colr, opaque_paint.p, paint );
    +    return read_paint( face, colr, opaque_paint.p, paint );
       }
     
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/ttcolr.h b/src/java.desktop/share/native/libfreetype/src/sfnt/ttcolr.h
    index 4200cb2976554..20c85f0359fe7 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/ttcolr.h
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/ttcolr.h
    @@ -4,7 +4,7 @@
      *
      *   TrueType and OpenType colored glyph layer support (specification).
      *
    - * Copyright (C) 2018-2022 by
    + * Copyright (C) 2018-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * Originally written by Shao Yu Zhang .
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/ttcpal.c b/src/java.desktop/share/native/libfreetype/src/sfnt/ttcpal.c
    index 9ae535cbda4a9..4279bc0bd1033 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/ttcpal.c
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/ttcpal.c
    @@ -4,7 +4,7 @@
      *
      *   TrueType and OpenType color palette support (body).
      *
    - * Copyright (C) 2018-2022 by
    + * Copyright (C) 2018-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * Originally written by Shao Yu Zhang .
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/ttcpal.h b/src/java.desktop/share/native/libfreetype/src/sfnt/ttcpal.h
    index 4717d224fc831..8e9913f0ccd4a 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/ttcpal.h
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/ttcpal.h
    @@ -4,7 +4,7 @@
      *
      *   TrueType and OpenType color palette support (specification).
      *
    - * Copyright (C) 2018-2022 by
    + * Copyright (C) 2018-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * Originally written by Shao Yu Zhang .
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/ttkern.c b/src/java.desktop/share/native/libfreetype/src/sfnt/ttkern.c
    index ca1c5094065d0..a47d08bd6de66 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/ttkern.c
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/ttkern.c
    @@ -5,7 +5,7 @@
      *   Load the basic TrueType kerning table.  This doesn't handle
      *   kerning data within the GPOS table at the moment.
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/ttkern.h b/src/java.desktop/share/native/libfreetype/src/sfnt/ttkern.h
    index f063558313e72..960c7da4946d7 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/ttkern.h
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/ttkern.h
    @@ -5,7 +5,7 @@
      *   Load the basic TrueType kerning table.  This doesn't handle
      *   kerning data within the GPOS table at the moment.
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/ttload.c b/src/java.desktop/share/native/libfreetype/src/sfnt/ttload.c
    index c83bd197fe79c..14f625c824352 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/ttload.c
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/ttload.c
    @@ -5,7 +5,7 @@
      *   Load the basic TrueType tables, i.e., tables that can be either in
      *   TTF or OTF fonts (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -206,7 +206,7 @@
           if ( FT_STREAM_READ_FIELDS( table_dir_entry_fields, &table ) )
           {
             FT_TRACE2(( "check_table_dir:"
    -                    " can read only %d table%s in font (instead of %d)\n",
    +                    " can read only %hu table%s in font (instead of %hu)\n",
                         nn, nn == 1 ? "" : "s", sfnt->num_tables ));
             sfnt->num_tables = nn;
             break;
    @@ -216,7 +216,7 @@
     
           if ( table.Offset > stream->size )
           {
    -        FT_TRACE2(( "check_table_dir: table entry %d invalid\n", nn ));
    +        FT_TRACE2(( "check_table_dir: table entry %hu invalid\n", nn ));
             continue;
           }
           else if ( table.Length > stream->size - table.Offset )
    @@ -231,7 +231,7 @@
               valid_entries++;
             else
             {
    -          FT_TRACE2(( "check_table_dir: table entry %d invalid\n", nn ));
    +          FT_TRACE2(( "check_table_dir: table entry %hu invalid\n", nn ));
               continue;
             }
           }
    @@ -380,7 +380,7 @@
     
         /* load the table directory */
     
    -    FT_TRACE2(( "-- Number of tables: %10u\n",    sfnt.num_tables ));
    +    FT_TRACE2(( "-- Number of tables: %10hu\n",   sfnt.num_tables ));
         FT_TRACE2(( "-- Format version:   0x%08lx\n", sfnt.format_tag ));
     
         if ( sfnt.format_tag != TTAG_OTTO )
    @@ -671,8 +671,8 @@
         if ( FT_STREAM_READ_FIELDS( header_fields, header ) )
           goto Exit;
     
    -    FT_TRACE3(( "Units per EM: %4u\n", header->Units_Per_EM ));
    -    FT_TRACE3(( "IndexToLoc:   %4d\n", header->Index_To_Loc_Format ));
    +    FT_TRACE3(( "Units per EM: %4hu\n", header->Units_Per_EM ));
    +    FT_TRACE3(( "IndexToLoc:   %4hd\n", header->Index_To_Loc_Format ));
     
       Exit:
         return error;
    @@ -802,7 +802,7 @@
           }
         }
     
    -    FT_TRACE3(( "numGlyphs: %u\n", maxProfile->numGlyphs ));
    +    FT_TRACE3(( "numGlyphs: %hu\n", maxProfile->numGlyphs ));
     
       Exit:
         return error;
    @@ -1265,11 +1265,11 @@
           }
         }
     
    -    FT_TRACE3(( "sTypoAscender:  %4d\n",   os2->sTypoAscender ));
    -    FT_TRACE3(( "sTypoDescender: %4d\n",   os2->sTypoDescender ));
    -    FT_TRACE3(( "usWinAscent:    %4u\n",   os2->usWinAscent ));
    -    FT_TRACE3(( "usWinDescent:   %4u\n",   os2->usWinDescent ));
    -    FT_TRACE3(( "fsSelection:    0x%2x\n", os2->fsSelection ));
    +    FT_TRACE3(( "sTypoAscender:  %4hd\n",   os2->sTypoAscender ));
    +    FT_TRACE3(( "sTypoDescender: %4hd\n",   os2->sTypoDescender ));
    +    FT_TRACE3(( "usWinAscent:    %4hu\n",   os2->usWinAscent ));
    +    FT_TRACE3(( "usWinDescent:   %4hu\n",   os2->usWinDescent ));
    +    FT_TRACE3(( "fsSelection:    0x%2hx\n", os2->fsSelection ));
     
       Exit:
         return error;
    @@ -1468,7 +1468,7 @@
           gasp_ranges[j].maxPPEM  = FT_GET_USHORT();
           gasp_ranges[j].gaspFlag = FT_GET_USHORT();
     
    -      FT_TRACE3(( "gaspRange %d: rangeMaxPPEM %5d, rangeGaspBehavior 0x%x\n",
    +      FT_TRACE3(( "gaspRange %hu: rangeMaxPPEM %5hu, rangeGaspBehavior 0x%hx\n",
                       j,
                       gasp_ranges[j].maxPPEM,
                       gasp_ranges[j].gaspFlag ));
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/ttload.h b/src/java.desktop/share/native/libfreetype/src/sfnt/ttload.h
    index 5368971c3164c..1499dd5735f58 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/ttload.h
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/ttload.h
    @@ -5,7 +5,7 @@
      *   Load the basic TrueType tables, i.e., tables that can be either in
      *   TTF or OTF fonts (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/ttmtx.c b/src/java.desktop/share/native/libfreetype/src/sfnt/ttmtx.c
    index 88377327c61a6..5e53e6dd4a317 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/ttmtx.c
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/ttmtx.c
    @@ -4,7 +4,7 @@
      *
      *   Load the metrics tables common to TTF and OTF fonts (body).
      *
    - * Copyright (C) 2006-2022 by
    + * Copyright (C) 2006-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -306,7 +306,7 @@
         }
     
     #ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    -    if ( var )
    +    if ( var && face->blend )
         {
           FT_Face  f = FT_FACE( face );
           FT_Int   a = (FT_Int)*aadvance;
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/ttmtx.h b/src/java.desktop/share/native/libfreetype/src/sfnt/ttmtx.h
    index 1e45b949a5561..56d2b62766164 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/ttmtx.h
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/ttmtx.h
    @@ -4,7 +4,7 @@
      *
      *   Load the metrics tables common to TTF and OTF fonts (specification).
      *
    - * Copyright (C) 2006-2022 by
    + * Copyright (C) 2006-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/ttpost.c b/src/java.desktop/share/native/libfreetype/src/sfnt/ttpost.c
    index 1a885a15c53ad..0e17c6f34aeb5 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/ttpost.c
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/ttpost.c
    @@ -5,7 +5,7 @@
      *   PostScript name table processing for TrueType and OpenType fonts
      *   (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -58,7 +58,7 @@
     #define MAC_NAME( x )  (FT_String*)psnames->macintosh_name( (FT_UInt)(x) )
     
     
    -#else /* FT_CONFIG_OPTION_POSTSCRIPT_NAMES */
    +#else /* !FT_CONFIG_OPTION_POSTSCRIPT_NAMES */
     
     
        /* Otherwise, we ignore the `psnames' module, and provide our own  */
    @@ -152,7 +152,7 @@
       };
     
     
    -#endif /* FT_CONFIG_OPTION_POSTSCRIPT_NAMES */
    +#endif /* !FT_CONFIG_OPTION_POSTSCRIPT_NAMES */
     
     
       static FT_Error
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/ttpost.h b/src/java.desktop/share/native/libfreetype/src/sfnt/ttpost.h
    index bf9342a9f5e88..528f1c5f2f207 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/ttpost.h
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/ttpost.h
    @@ -5,7 +5,7 @@
      *   PostScript name table processing for TrueType and OpenType fonts
      *   (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/ttsbit.c b/src/java.desktop/share/native/libfreetype/src/sfnt/ttsbit.c
    index bf73d04e54018..3c06955131578 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/ttsbit.c
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/ttsbit.c
    @@ -4,7 +4,7 @@
      *
      *   TrueType and OpenType embedded bitmap support (body).
      *
    - * Copyright (C) 2005-2022 by
    + * Copyright (C) 2005-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * Copyright 2013 by Google, Inc.
    @@ -385,11 +385,9 @@
     
             /* set the scale values (in 16.16 units) so advances */
             /* from the hmtx and vmtx table are scaled correctly */
    -        metrics->x_scale = FT_MulDiv( metrics->x_ppem,
    -                                      64 * 0x10000,
    +        metrics->x_scale = FT_DivFix( metrics->x_ppem * 64,
                                           face->header.Units_Per_EM );
    -        metrics->y_scale = FT_MulDiv( metrics->y_ppem,
    -                                      64 * 0x10000,
    +        metrics->y_scale = FT_DivFix( metrics->y_ppem * 64,
                                           face->header.Units_Per_EM );
     
             return FT_Err_Ok;
    @@ -399,9 +397,9 @@
           {
             FT_Stream       stream = face->root.stream;
             FT_UInt         offset;
    -        FT_UShort       upem, ppem, resolution;
    +        FT_UShort       ppem, resolution;
             TT_HoriHeader  *hori;
    -        FT_Pos          ppem_; /* to reduce casts */
    +        FT_Fixed        scale;
     
             FT_Error  error;
             FT_Byte*  p;
    @@ -424,32 +422,23 @@
     
             FT_FRAME_EXIT();
     
    -        upem = face->header.Units_Per_EM;
    -        hori = &face->horizontal;
    -
             metrics->x_ppem = ppem;
             metrics->y_ppem = ppem;
     
    -        ppem_ = (FT_Pos)ppem;
    +        scale = FT_DivFix( ppem * 64, face->header.Units_Per_EM );
    +        hori  = &face->horizontal;
     
    -        metrics->ascender =
    -          FT_MulDiv( hori->Ascender, ppem_ * 64, upem );
    -        metrics->descender =
    -          FT_MulDiv( hori->Descender, ppem_ * 64, upem );
    -        metrics->height =
    -          FT_MulDiv( hori->Ascender - hori->Descender + hori->Line_Gap,
    -                     ppem_ * 64, upem );
    -        metrics->max_advance =
    -          FT_MulDiv( hori->advance_Width_Max, ppem_ * 64, upem );
    +        metrics->ascender    = FT_MulFix( hori->Ascender, scale );
    +        metrics->descender   = FT_MulFix( hori->Descender, scale );
    +        metrics->height      =
    +          FT_MulFix( hori->Ascender - hori->Descender + hori->Line_Gap,
    +                     scale );
    +        metrics->max_advance = FT_MulFix( hori->advance_Width_Max, scale );
     
             /* set the scale values (in 16.16 units) so advances */
             /* from the hmtx and vmtx table are scaled correctly */
    -        metrics->x_scale = FT_MulDiv( metrics->x_ppem,
    -                                      64 * 0x10000,
    -                                      face->header.Units_Per_EM );
    -        metrics->y_scale = FT_MulDiv( metrics->y_ppem,
    -                                      64 * 0x10000,
    -                                      face->header.Units_Per_EM );
    +        metrics->x_scale = scale;
    +        metrics->y_scale = scale;
     
             return error;
           }
    @@ -1204,7 +1193,7 @@
               goto Fail;
     
             p += 1;  /* skip padding */
    -        /* fall-through */
    +        FALL_THROUGH;
     
           case 9:
             loader = tt_sbit_decoder_load_compound;
    @@ -1604,7 +1593,7 @@
         return error;
       }
     
    -  FT_LOCAL( FT_Error )
    +  FT_LOCAL_DEF( FT_Error )
       tt_face_load_sbit_image( TT_Face              face,
                                FT_ULong             strike_index,
                                FT_UInt              glyph_index,
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/ttsbit.h b/src/java.desktop/share/native/libfreetype/src/sfnt/ttsbit.h
    index c967bffba3ebb..07e2db461a5c9 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/ttsbit.h
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/ttsbit.h
    @@ -4,7 +4,7 @@
      *
      *   TrueType and OpenType embedded bitmap support (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/woff2tags.c b/src/java.desktop/share/native/libfreetype/src/sfnt/woff2tags.c
    index 7d79fef39a863..7a0a351f06c03 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/woff2tags.c
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/woff2tags.c
    @@ -4,7 +4,7 @@
      *
      *   WOFF2 Font table tags (base).
      *
    - * Copyright (C) 2019-2022 by
    + * Copyright (C) 2019-2023 by
      * Nikhil Ramakrishnan, David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/sfnt/woff2tags.h b/src/java.desktop/share/native/libfreetype/src/sfnt/woff2tags.h
    index 05df85aba02df..1201848e5ecd9 100644
    --- a/src/java.desktop/share/native/libfreetype/src/sfnt/woff2tags.h
    +++ b/src/java.desktop/share/native/libfreetype/src/sfnt/woff2tags.h
    @@ -4,7 +4,7 @@
      *
      *   WOFF2 Font table tags (specification).
      *
    - * Copyright (C) 2019-2022 by
    + * Copyright (C) 2019-2023 by
      * Nikhil Ramakrishnan, David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/smooth/ftgrays.c b/src/java.desktop/share/native/libfreetype/src/smooth/ftgrays.c
    index 622035aa797c8..d9f20eef131d2 100644
    --- a/src/java.desktop/share/native/libfreetype/src/smooth/ftgrays.c
    +++ b/src/java.desktop/share/native/libfreetype/src/smooth/ftgrays.c
    @@ -4,7 +4,7 @@
      *
      *   A new `perfect' anti-aliasing renderer (body).
      *
    - * Copyright (C) 2000-2022 by
    + * Copyright (C) 2000-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -418,21 +418,21 @@ typedef ptrdiff_t  FT_PtrDist;
     
       /* It is faster to write small spans byte-by-byte than calling     */
       /* `memset'.  This is mainly due to the cost of the function call. */
    -#define FT_GRAY_SET( d, s, count )                          \
    -  FT_BEGIN_STMNT                                            \
    -    unsigned char* q = d;                                   \
    -    switch ( count )                                        \
    -    {                                                       \
    -      case 7: *q++ = (unsigned char)s; /* fall through */   \
    -      case 6: *q++ = (unsigned char)s; /* fall through */   \
    -      case 5: *q++ = (unsigned char)s; /* fall through */   \
    -      case 4: *q++ = (unsigned char)s; /* fall through */   \
    -      case 3: *q++ = (unsigned char)s; /* fall through */   \
    -      case 2: *q++ = (unsigned char)s; /* fall through */   \
    -      case 1: *q   = (unsigned char)s; /* fall through */   \
    -      case 0: break;                                        \
    -      default: FT_MEM_SET( d, s, count );                   \
    -    }                                                       \
    +#define FT_GRAY_SET( d, s, count )                   \
    +  FT_BEGIN_STMNT                                     \
    +    unsigned char* q = d;                            \
    +    switch ( count )                                 \
    +    {                                                \
    +      case 7: *q++ = (unsigned char)s; FALL_THROUGH; \
    +      case 6: *q++ = (unsigned char)s; FALL_THROUGH; \
    +      case 5: *q++ = (unsigned char)s; FALL_THROUGH; \
    +      case 4: *q++ = (unsigned char)s; FALL_THROUGH; \
    +      case 3: *q++ = (unsigned char)s; FALL_THROUGH; \
    +      case 2: *q++ = (unsigned char)s; FALL_THROUGH; \
    +      case 1: *q   = (unsigned char)s; FALL_THROUGH; \
    +      case 0: break;                                 \
    +      default: FT_MEM_SET( d, s, count );            \
    +    }                                                \
       FT_END_STMNT
     
     
    @@ -1909,10 +1909,10 @@ typedef ptrdiff_t  FT_PtrDist;
     
     
       static int
    -  gray_convert_glyph_inner( RAS_ARG,
    +  gray_convert_glyph_inner( RAS_ARG_
                                 int  continued )
       {
    -    int  error;
    +    volatile int  error;
     
     
         if ( ft_setjmp( ras.jump_buffer ) == 0 )
    @@ -2004,7 +2004,7 @@ typedef ptrdiff_t  FT_PtrDist;
             ras.max_ey    = band[0];
             ras.count_ey  = width;
     
    -        error     = gray_convert_glyph_inner( RAS_VAR, continued );
    +        error     = gray_convert_glyph_inner( RAS_VAR_ continued );
             continued = 1;
     
             if ( !error )
    diff --git a/src/java.desktop/share/native/libfreetype/src/smooth/ftgrays.h b/src/java.desktop/share/native/libfreetype/src/smooth/ftgrays.h
    index 13bf2baaa2be5..a5001bf40d325 100644
    --- a/src/java.desktop/share/native/libfreetype/src/smooth/ftgrays.h
    +++ b/src/java.desktop/share/native/libfreetype/src/smooth/ftgrays.h
    @@ -4,7 +4,7 @@
      *
      *   FreeType smooth renderer declaration
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/smooth/ftsmerrs.h b/src/java.desktop/share/native/libfreetype/src/smooth/ftsmerrs.h
    index 7bc607798817f..f4ac93dc410f3 100644
    --- a/src/java.desktop/share/native/libfreetype/src/smooth/ftsmerrs.h
    +++ b/src/java.desktop/share/native/libfreetype/src/smooth/ftsmerrs.h
    @@ -4,7 +4,7 @@
      *
      *   smooth renderer error codes (specification only).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/smooth/ftsmooth.c b/src/java.desktop/share/native/libfreetype/src/smooth/ftsmooth.c
    index df227c3758a74..cdbc78c3e538d 100644
    --- a/src/java.desktop/share/native/libfreetype/src/smooth/ftsmooth.c
    +++ b/src/java.desktop/share/native/libfreetype/src/smooth/ftsmooth.c
    @@ -4,7 +4,7 @@
      *
      *   Anti-aliasing renderer interface (body).
      *
    - * Copyright (C) 2000-2022 by
    + * Copyright (C) 2000-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/smooth/ftsmooth.h b/src/java.desktop/share/native/libfreetype/src/smooth/ftsmooth.h
    index 87f09faea41b2..f8bdc9938b32c 100644
    --- a/src/java.desktop/share/native/libfreetype/src/smooth/ftsmooth.h
    +++ b/src/java.desktop/share/native/libfreetype/src/smooth/ftsmooth.h
    @@ -4,7 +4,7 @@
      *
      *   Anti-aliasing renderer interface (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/truetype/ttdriver.c b/src/java.desktop/share/native/libfreetype/src/truetype/ttdriver.c
    index 245d97cb58502..4bea63ef84391 100644
    --- a/src/java.desktop/share/native/libfreetype/src/truetype/ttdriver.c
    +++ b/src/java.desktop/share/native/libfreetype/src/truetype/ttdriver.c
    @@ -4,7 +4,7 @@
      *
      *   TrueType font driver implementation (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -507,19 +507,34 @@
       FT_DEFINE_SERVICE_MULTIMASTERSREC(
         tt_service_gx_multi_masters,
     
    -    (FT_Get_MM_Func)             NULL,                  /* get_mm              */
    -    (FT_Set_MM_Design_Func)      NULL,                  /* set_mm_design       */
    -    (FT_Set_MM_Blend_Func)       TT_Set_MM_Blend,       /* set_mm_blend        */
    -    (FT_Get_MM_Blend_Func)       TT_Get_MM_Blend,       /* get_mm_blend        */
    -    (FT_Get_MM_Var_Func)         TT_Get_MM_Var,         /* get_mm_var          */
    -    (FT_Set_Var_Design_Func)     TT_Set_Var_Design,     /* set_var_design      */
    -    (FT_Get_Var_Design_Func)     TT_Get_Var_Design,     /* get_var_design      */
    -    (FT_Set_Instance_Func)       TT_Set_Named_Instance, /* set_instance        */
    -    (FT_Set_MM_WeightVector_Func)NULL,                  /* set_mm_weightvector */
    -    (FT_Get_MM_WeightVector_Func)NULL,                  /* get_mm_weightvector */
    -
    -    (FT_Get_Var_Blend_Func)      tt_get_var_blend,      /* get_var_blend       */
    -    (FT_Done_Blend_Func)         tt_done_blend          /* done_blend          */
    +    (FT_Get_MM_Func)        NULL,                  /* get_mm                    */
    +    (FT_Set_MM_Design_Func) NULL,                  /* set_mm_design             */
    +    (FT_Set_MM_Blend_Func)  TT_Set_MM_Blend,       /* set_mm_blend              */
    +    (FT_Get_MM_Blend_Func)  TT_Get_MM_Blend,       /* get_mm_blend              */
    +    (FT_Get_MM_Var_Func)    TT_Get_MM_Var,         /* get_mm_var                */
    +    (FT_Set_Var_Design_Func)TT_Set_Var_Design,     /* set_var_design            */
    +    (FT_Get_Var_Design_Func)TT_Get_Var_Design,     /* get_var_design            */
    +    (FT_Set_Instance_Func)  TT_Set_Named_Instance, /* set_instance              */
    +    (FT_Set_MM_WeightVector_Func)
    +                            NULL,                  /* set_mm_weightvector       */
    +    (FT_Get_MM_WeightVector_Func)
    +                            NULL,                  /* get_mm_weightvector       */
    +    (FT_Var_Load_Delta_Set_Idx_Map_Func)
    +                            tt_var_load_delta_set_index_mapping,
    +                                                   /* load_delta_set_idx_map    */
    +    (FT_Var_Load_Item_Var_Store_Func)
    +                            tt_var_load_item_variation_store,
    +                                                   /* load_item_variation_store */
    +    (FT_Var_Get_Item_Delta_Func)
    +                            tt_var_get_item_delta, /* get_item_delta            */
    +    (FT_Var_Done_Item_Var_Store_Func)
    +                            tt_var_done_item_variation_store,
    +                                                   /* done_item_variation_store */
    +    (FT_Var_Done_Delta_Set_Idx_Map_Func)
    +                            tt_var_done_delta_set_index_map,
    +                                                   /* done_delta_set_index_map  */
    +    (FT_Get_Var_Blend_Func) tt_get_var_blend,      /* get_var_blend             */
    +    (FT_Done_Blend_Func)    tt_done_blend          /* done_blend                */
       )
     
       FT_DEFINE_SERVICE_METRICSVARIATIONSREC(
    diff --git a/src/java.desktop/share/native/libfreetype/src/truetype/ttdriver.h b/src/java.desktop/share/native/libfreetype/src/truetype/ttdriver.h
    index c477c0b1dd854..757a66f425d5b 100644
    --- a/src/java.desktop/share/native/libfreetype/src/truetype/ttdriver.h
    +++ b/src/java.desktop/share/native/libfreetype/src/truetype/ttdriver.h
    @@ -4,7 +4,7 @@
      *
      *   High-level TrueType driver interface (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/truetype/tterrors.h b/src/java.desktop/share/native/libfreetype/src/truetype/tterrors.h
    index 2c95ea17b2fce..008ee99853cbb 100644
    --- a/src/java.desktop/share/native/libfreetype/src/truetype/tterrors.h
    +++ b/src/java.desktop/share/native/libfreetype/src/truetype/tterrors.h
    @@ -4,7 +4,7 @@
      *
      *   TrueType error codes (specification only).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/truetype/ttgload.c b/src/java.desktop/share/native/libfreetype/src/truetype/ttgload.c
    index 2ca63d65a3a37..d33bdad642793 100644
    --- a/src/java.desktop/share/native/libfreetype/src/truetype/ttgload.c
    +++ b/src/java.desktop/share/native/libfreetype/src/truetype/ttgload.c
    @@ -4,7 +4,7 @@
      *
      *   TrueType Glyph Loader (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -737,19 +737,19 @@
     
             if ( subglyph->flags & WE_HAVE_A_SCALE )
               FT_TRACE7(( "      scaling: %f\n",
    -                      subglyph->transform.xx / 65536.0 ));
    +                      (double)subglyph->transform.xx / 65536 ));
             else if ( subglyph->flags & WE_HAVE_AN_XY_SCALE )
               FT_TRACE7(( "      scaling: x=%f, y=%f\n",
    -                      subglyph->transform.xx / 65536.0,
    -                      subglyph->transform.yy / 65536.0 ));
    +                      (double)subglyph->transform.xx / 65536,
    +                      (double)subglyph->transform.yy / 65536 ));
             else if ( subglyph->flags & WE_HAVE_A_2X2 )
             {
               FT_TRACE7(( "      scaling: xx=%f, yx=%f\n",
    -                      subglyph->transform.xx / 65536.0,
    -                      subglyph->transform.yx / 65536.0 ));
    +                      (double)subglyph->transform.xx / 65536,
    +                      (double)subglyph->transform.yx / 65536 ));
               FT_TRACE7(( "               xy=%f, yy=%f\n",
    -                      subglyph->transform.xy / 65536.0,
    -                      subglyph->transform.yy / 65536.0 ));
    +                      (double)subglyph->transform.xy / 65536,
    +                      (double)subglyph->transform.yy / 65536 ));
             }
     
             subglyph++;
    @@ -801,7 +801,7 @@
                        FT_UInt       start_point,
                        FT_UInt       start_contour )
       {
    -    zone->n_points    = (FT_UShort)load->outline.n_points -
    +    zone->n_points    = (FT_UShort)load->outline.n_points + 4 -
                               (FT_UShort)start_point;
         zone->n_contours  = load->outline.n_contours -
                               (FT_Short)start_contour;
    @@ -970,11 +970,6 @@
         outline->points[n_points + 2] = loader->pp3;
         outline->points[n_points + 3] = loader->pp4;
     
    -    outline->tags[n_points    ] = 0;
    -    outline->tags[n_points + 1] = 0;
    -    outline->tags[n_points + 2] = 0;
    -    outline->tags[n_points + 3] = 0;
    -
         n_points += 4;
     
     #ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
    @@ -985,24 +980,9 @@
             goto Exit;
     
           /* Deltas apply to the unscaled data. */
    -      error = TT_Vary_Apply_Glyph_Deltas( loader->face,
    -                                          loader->glyph_index,
    +      error = TT_Vary_Apply_Glyph_Deltas( loader,
                                               outline,
    -                                          unrounded,
    -                                          (FT_UInt)n_points );
    -
    -      /* recalculate linear horizontal and vertical advances */
    -      /* if we don't have HVAR and VVAR, respectively        */
    -
    -      /* XXX: change all FreeType modules to store `linear' and `vadvance' */
    -      /*      in 26.6 format before the `base' module scales them to 16.16 */
    -      if ( !( loader->face->variation_support & TT_FACE_FLAG_VAR_HADVANCE ) )
    -        loader->linear = FT_PIX_ROUND( unrounded[n_points - 3].x -
    -                                       unrounded[n_points - 4].x ) / 64;
    -      if ( !( loader->face->variation_support & TT_FACE_FLAG_VAR_VADVANCE ) )
    -        loader->vadvance = FT_PIX_ROUND( unrounded[n_points - 1].x -
    -                                         unrounded[n_points - 2].x ) / 64;
    -
    +                                          unrounded );
           if ( error )
             goto Exit;
         }
    @@ -1014,7 +994,7 @@
           tt_prepare_zone( &loader->zone, &gloader->current, 0, 0 );
     
           FT_ARRAY_COPY( loader->zone.orus, loader->zone.cur,
    -                     loader->zone.n_points + 4 );
    +                     loader->zone.n_points );
         }
     
         {
    @@ -1156,11 +1136,7 @@
         }
     
         if ( IS_HINTED( loader->load_flags ) )
    -    {
    -      loader->zone.n_points += 4;
    -
           error = TT_Hint_Glyph( loader, 0 );
    -    }
     
     #ifdef TT_CONFIG_OPTION_GX_VAR_SUPPORT
       Exit:
    @@ -1373,11 +1349,6 @@
         outline->points[outline->n_points + 2] = loader->pp3;
         outline->points[outline->n_points + 3] = loader->pp4;
     
    -    outline->tags[outline->n_points    ] = 0;
    -    outline->tags[outline->n_points + 1] = 0;
    -    outline->tags[outline->n_points + 2] = 0;
    -    outline->tags[outline->n_points + 3] = 0;
    -
     #ifdef TT_USE_BYTECODE_INTERPRETER
     
         {
    @@ -1436,11 +1407,9 @@
     
         /* Some points are likely touched during execution of  */
         /* instructions on components.  So let's untouch them. */
    -    for ( i = 0; i < loader->zone.n_points; i++ )
    +    for ( i = 0; i < loader->zone.n_points - 4U; i++ )
           loader->zone.tags[i] &= ~FT_CURVE_TAG_TOUCH_BOTH;
     
    -    loader->zone.n_points += 4;
    -
         return TT_Hint_Glyph( loader, 1 );
       }
     
    @@ -1761,57 +1730,29 @@
             /* a small outline structure with four elements for */
             /* communication with `TT_Vary_Apply_Glyph_Deltas'  */
             FT_Vector   points[4];
    -        char        tags[4]     = { 1, 1, 1, 1 };
    -        short       contours[4] = { 0, 1, 2, 3 };
             FT_Outline  outline;
     
             /* unrounded values */
             FT_Vector  unrounded[4] = { {0, 0}, {0, 0}, {0, 0}, {0, 0} };
     
     
    -        points[0].x = loader->pp1.x;
    -        points[0].y = loader->pp1.y;
    -        points[1].x = loader->pp2.x;
    -        points[1].y = loader->pp2.y;
    -
    -        points[2].x = loader->pp3.x;
    -        points[2].y = loader->pp3.y;
    -        points[3].x = loader->pp4.x;
    -        points[3].y = loader->pp4.y;
    +        points[0] = loader->pp1;
    +        points[1] = loader->pp2;
    +        points[2] = loader->pp3;
    +        points[3] = loader->pp4;
     
    -        outline.n_points   = 4;
    -        outline.n_contours = 4;
    +        outline.n_points   = 0;
    +        outline.n_contours = 0;
             outline.points     = points;
    -        outline.tags       = tags;
    -        outline.contours   = contours;
    +        outline.tags       = NULL;
    +        outline.contours   = NULL;
     
             /* this must be done before scaling */
    -        error = TT_Vary_Apply_Glyph_Deltas( loader->face,
    -                                            glyph_index,
    +        error = TT_Vary_Apply_Glyph_Deltas( loader,
                                                 &outline,
    -                                            unrounded,
    -                                            (FT_UInt)outline.n_points );
    +                                            unrounded );
             if ( error )
               goto Exit;
    -
    -        loader->pp1.x = points[0].x;
    -        loader->pp1.y = points[0].y;
    -        loader->pp2.x = points[1].x;
    -        loader->pp2.y = points[1].y;
    -
    -        loader->pp3.x = points[2].x;
    -        loader->pp3.y = points[2].y;
    -        loader->pp4.x = points[3].x;
    -        loader->pp4.y = points[3].y;
    -
    -        /* recalculate linear horizontal and vertical advances */
    -        /* if we don't have HVAR and VVAR, respectively        */
    -        if ( !( loader->face->variation_support & TT_FACE_FLAG_VAR_HADVANCE ) )
    -          loader->linear = FT_PIX_ROUND( unrounded[1].x -
    -                                         unrounded[0].x ) / 64;
    -        if ( !( loader->face->variation_support & TT_FACE_FLAG_VAR_VADVANCE ) )
    -          loader->vadvance = FT_PIX_ROUND( unrounded[3].x -
    -                                           unrounded[2].x ) / 64;
           }
     
     #endif /* TT_CONFIG_OPTION_GX_VAR_SUPPORT */
    @@ -1959,17 +1900,16 @@
     
             /* construct an outline structure for              */
             /* communication with `TT_Vary_Apply_Glyph_Deltas' */
    -        outline.n_points   = (short)( gloader->current.num_subglyphs + 4 );
    -        outline.n_contours = outline.n_points;
    +        outline.n_contours = outline.n_points = limit;
     
             outline.points   = NULL;
             outline.tags     = NULL;
             outline.contours = NULL;
     
    -        if ( FT_NEW_ARRAY( points, outline.n_points )    ||
    -             FT_NEW_ARRAY( tags, outline.n_points )      ||
    -             FT_NEW_ARRAY( contours, outline.n_points )  ||
    -             FT_NEW_ARRAY( unrounded, outline.n_points ) )
    +        if ( FT_NEW_ARRAY( points, limit + 4 )    ||
    +             FT_NEW_ARRAY( tags, limit + 4 )      ||
    +             FT_NEW_ARRAY( contours, limit + 4 )  ||
    +             FT_NEW_ARRAY( unrounded, limit + 4 ) )
               goto Exit1;
     
             subglyph = gloader->current.subglyphs;
    @@ -1985,28 +1925,10 @@
               contours[i] = i;
             }
     
    -        points[i].x = loader->pp1.x;
    -        points[i].y = loader->pp1.y;
    -        tags[i]     = 1;
    -        contours[i] = i;
    -
    -        i++;
    -        points[i].x = loader->pp2.x;
    -        points[i].y = loader->pp2.y;
    -        tags[i]     = 1;
    -        contours[i] = i;
    -
    -        i++;
    -        points[i].x = loader->pp3.x;
    -        points[i].y = loader->pp3.y;
    -        tags[i]     = 1;
    -        contours[i] = i;
    -
    -        i++;
    -        points[i].x = loader->pp4.x;
    -        points[i].y = loader->pp4.y;
    -        tags[i]     = 1;
    -        contours[i] = i;
    +        points[i++] = loader->pp1;
    +        points[i++] = loader->pp2;
    +        points[i++] = loader->pp3;
    +        points[i  ] = loader->pp4;
     
             outline.points   = points;
             outline.tags     = tags;
    @@ -2014,12 +1936,9 @@
     
             /* this call provides additional offsets */
             /* for each component's translation      */
    -        if ( FT_SET_ERROR( TT_Vary_Apply_Glyph_Deltas(
    -                             face,
    -                             glyph_index,
    -                             &outline,
    -                             unrounded,
    -                             (FT_UInt)outline.n_points ) ) )
    +        if ( FT_SET_ERROR( TT_Vary_Apply_Glyph_Deltas( loader,
    +                                                       &outline,
    +                                                       unrounded ) ) )
               goto Exit1;
     
             subglyph = gloader->current.subglyphs;
    @@ -2033,27 +1952,6 @@
               }
             }
     
    -        loader->pp1.x = points[i + 0].x;
    -        loader->pp1.y = points[i + 0].y;
    -        loader->pp2.x = points[i + 1].x;
    -        loader->pp2.y = points[i + 1].y;
    -
    -        loader->pp3.x = points[i + 2].x;
    -        loader->pp3.y = points[i + 2].y;
    -        loader->pp4.x = points[i + 3].x;
    -        loader->pp4.y = points[i + 3].y;
    -
    -        /* recalculate linear horizontal and vertical advances */
    -        /* if we don't have HVAR and VVAR, respectively        */
    -        if ( !( face->variation_support & TT_FACE_FLAG_VAR_HADVANCE ) )
    -          loader->linear =
    -            FT_PIX_ROUND( unrounded[outline.n_points - 3].x -
    -                          unrounded[outline.n_points - 4].x ) / 64;
    -        if ( !( face->variation_support & TT_FACE_FLAG_VAR_VADVANCE ) )
    -          loader->vadvance =
    -            FT_PIX_ROUND( unrounded[outline.n_points - 1].x -
    -                          unrounded[outline.n_points - 2].x ) / 64;
    -
           Exit1:
             FT_FREE( outline.points );
             FT_FREE( outline.tags );
    @@ -2229,12 +2127,11 @@
       compute_glyph_metrics( TT_Loader  loader,
                              FT_UInt    glyph_index )
       {
    -    TT_Face    face   = loader->face;
    -
    +    TT_Face       face  = loader->face;
    +    TT_Size       size  = loader->size;
    +    TT_GlyphSlot  glyph = loader->glyph;
         FT_BBox       bbox;
         FT_Fixed      y_scale;
    -    TT_GlyphSlot  glyph = loader->glyph;
    -    TT_Size       size  = loader->size;
     
     
         y_scale = 0x10000L;
    @@ -2372,17 +2269,13 @@
                        FT_UInt       glyph_index,
                        FT_Int32      load_flags )
       {
    -    TT_Face             face;
    -    SFNT_Service        sfnt;
    -    FT_Stream           stream;
    +    TT_Face             face   = (TT_Face)glyph->face;
    +    SFNT_Service        sfnt   = (SFNT_Service)face->sfnt;
    +    FT_Stream           stream = face->root.stream;
         FT_Error            error;
         TT_SBit_MetricsRec  sbit_metrics;
     
     
    -    face   = (TT_Face)glyph->face;
    -    sfnt   = (SFNT_Service)face->sfnt;
    -    stream = face->root.stream;
    -
         error = sfnt->load_sbit_image( face,
                                        size->strike_index,
                                        glyph_index,
    @@ -2433,22 +2326,19 @@
                       FT_Int32      load_flags,
                       FT_Bool       glyf_table_only )
       {
    -    TT_Face    face;
    -    FT_Stream  stream;
    +    TT_Face    face   = (TT_Face)glyph->face;
    +    FT_Stream  stream = face->root.stream;
     
     #ifdef TT_USE_BYTECODE_INTERPRETER
         FT_Error   error;
         FT_Bool    pedantic = FT_BOOL( load_flags & FT_LOAD_PEDANTIC );
     #if defined TT_SUPPORT_SUBPIXEL_HINTING_INFINALITY || \
         defined TT_SUPPORT_SUBPIXEL_HINTING_MINIMAL
    -    TT_Driver  driver = (TT_Driver)FT_FACE_DRIVER( (TT_Face)glyph->face );
    +    TT_Driver  driver   = (TT_Driver)FT_FACE_DRIVER( glyph->face );
     #endif
     #endif
     
     
    -    face   = (TT_Face)glyph->face;
    -    stream = face->root.stream;
    -
         FT_ZERO( loader );
     
     #ifdef TT_USE_BYTECODE_INTERPRETER
    @@ -2816,6 +2706,7 @@
                      FT_UInt       glyph_index,
                      FT_Int32      load_flags )
       {
    +    TT_Face       face = (TT_Face)glyph->face;
         FT_Error      error;
         TT_LoaderRec  loader;
     
    @@ -2840,8 +2731,6 @@
             /* if we have a bitmap-only font, return an empty glyph            */
             if ( !FT_IS_SCALABLE( glyph->face ) )
             {
    -          TT_Face  face = (TT_Face)glyph->face;
    -
               FT_Short  left_bearing = 0;
               FT_Short  top_bearing  = 0;
     
    @@ -2900,9 +2789,6 @@
             if ( FT_IS_SCALABLE( glyph->face ) ||
                  FT_HAS_SBIX( glyph->face )    )
             {
    -          TT_Face  face = (TT_Face)glyph->face;
    -
    -
               /* for the bbox we need the header only */
               (void)tt_loader_init( &loader, size, glyph, load_flags, TRUE );
               (void)load_truetype_glyph( &loader, glyph_index, 0, TRUE );
    @@ -2971,23 +2857,23 @@
     #ifdef FT_CONFIG_OPTION_SVG
     
         /* check for OT-SVG */
    -    if ( ( load_flags & FT_LOAD_COLOR ) && ( (TT_Face)glyph->face )->svg )
    +    if ( ( load_flags & FT_LOAD_COLOR ) && face->svg )
         {
    -      SFNT_Service  sfnt;
    -
    -      FT_Short   leftBearing;
    -      FT_Short   topBearing;
    -      FT_UShort  advanceX;
    -      FT_UShort  advanceY;
    +      SFNT_Service  sfnt = (SFNT_Service)face->sfnt;
     
     
           FT_TRACE3(( "Trying to load SVG glyph\n" ));
    -      sfnt = (SFNT_Service)( (TT_Face)glyph->face )->sfnt;
     
           error = sfnt->load_svg_doc( glyph, glyph_index );
           if ( !error )
           {
    -        TT_Face  face = (TT_Face)glyph->face;
    +        FT_Fixed  x_scale = size->root.metrics.x_scale;
    +        FT_Fixed  y_scale = size->root.metrics.y_scale;
    +
    +        FT_Short   leftBearing;
    +        FT_Short   topBearing;
    +        FT_UShort  advanceX;
    +        FT_UShort  advanceY;
     
     
             FT_TRACE3(( "Successfully loaded SVG glyph\n" ));
    @@ -3005,15 +2891,11 @@
                                &topBearing,
                                &advanceY );
     
    -        advanceX = (FT_UShort)FT_MulDiv( advanceX,
    -                                         glyph->face->size->metrics.x_ppem,
    -                                         glyph->face->units_per_EM );
    -        advanceY = (FT_UShort)FT_MulDiv( advanceY,
    -                                         glyph->face->size->metrics.y_ppem,
    -                                         glyph->face->units_per_EM );
    +        glyph->linearHoriAdvance = advanceX;
    +        glyph->linearVertAdvance = advanceY;
     
    -        glyph->metrics.horiAdvance = advanceX << 6;
    -        glyph->metrics.vertAdvance = advanceY << 6;
    +        glyph->metrics.horiAdvance = FT_MulFix( advanceX, x_scale );
    +        glyph->metrics.vertAdvance = FT_MulFix( advanceY, y_scale );
     
             return error;
           }
    diff --git a/src/java.desktop/share/native/libfreetype/src/truetype/ttgload.h b/src/java.desktop/share/native/libfreetype/src/truetype/ttgload.h
    index 3195351f78d42..f18637dce330b 100644
    --- a/src/java.desktop/share/native/libfreetype/src/truetype/ttgload.h
    +++ b/src/java.desktop/share/native/libfreetype/src/truetype/ttgload.h
    @@ -4,7 +4,7 @@
      *
      *   TrueType Glyph Loader (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/truetype/ttgxvar.c b/src/java.desktop/share/native/libfreetype/src/truetype/ttgxvar.c
    index 6a0edef29bd33..fc957320c843b 100644
    --- a/src/java.desktop/share/native/libfreetype/src/truetype/ttgxvar.c
    +++ b/src/java.desktop/share/native/libfreetype/src/truetype/ttgxvar.c
    @@ -4,7 +4,7 @@
      *
      *   TrueType GX Font Variation loader
      *
    - * Copyright (C) 2004-2022 by
    + * Copyright (C) 2004-2023 by
      * David Turner, Robert Wilhelm, Werner Lemberg, and George Williams.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -42,6 +42,7 @@
     #include 
     #include 
     #include FT_CONFIG_CONFIG_H
    +#include 
     #include 
     #include 
     #include 
    @@ -353,15 +354,24 @@
       static void
       ft_var_load_avar( TT_Face  face )
       {
    -    FT_Stream       stream = FT_FACE_STREAM( face );
    -    FT_Memory       memory = stream->memory;
    +    FT_Error   error;
    +    FT_Stream  stream = FT_FACE_STREAM( face );
    +    FT_Memory  memory = stream->memory;
    +    FT_Int     i, j;
    +
         GX_Blend        blend  = face->blend;
         GX_AVarSegment  segment;
    -    FT_Error        error;
    -    FT_Long         version;
    -    FT_Long         axisCount;
    -    FT_Int          i, j;
    -    FT_ULong        table_len;
    +    GX_AVarTable    table;
    +
    +    FT_Long   version;
    +    FT_Long   axisCount;
    +    FT_ULong  table_len;
    +
    +#ifndef TT_CONFIG_OPTION_NO_BORING_EXPANSION
    +    FT_ULong  table_offset;
    +    FT_ULong  store_offset;
    +    FT_ULong  axisMap_offset;
    +#endif
     
     
         FT_TRACE2(( "AVAR " ));
    @@ -374,13 +384,21 @@
           return;
         }
     
    +#ifndef TT_CONFIG_OPTION_NO_BORING_EXPANSION
    +    table_offset = FT_STREAM_POS();
    +#endif
    +
         if ( FT_FRAME_ENTER( table_len ) )
           return;
     
         version   = FT_GET_LONG();
         axisCount = FT_GET_LONG();
     
    -    if ( version != 0x00010000L )
    +    if ( version != 0x00010000L
    +#ifndef TT_CONFIG_OPTION_NO_BORING_EXPANSION
    +         && version != 0x00020000L
    +#endif
    +       )
         {
           FT_TRACE2(( "bad table version\n" ));
           goto Exit;
    @@ -396,10 +414,14 @@
           goto Exit;
         }
     
    -    if ( FT_QNEW_ARRAY( blend->avar_segment, axisCount ) )
    +    if ( FT_NEW( blend->avar_table ) )
           goto Exit;
    +    table = blend->avar_table;
     
    -    segment = &blend->avar_segment[0];
    +    if ( FT_QNEW_ARRAY( table->avar_segment, axisCount ) )
    +      goto Exit;
    +
    +    segment = &table->avar_segment[0];
         for ( i = 0; i < axisCount; i++, segment++ )
         {
           FT_TRACE5(( "  axis %d:\n", i ));
    @@ -412,9 +434,9 @@
             /* it right now since loading the `avar' table is optional.   */
     
             for ( j = i - 1; j >= 0; j-- )
    -          FT_FREE( blend->avar_segment[j].correspondence );
    +          FT_FREE( table->avar_segment[j].correspondence );
     
    -        FT_FREE( blend->avar_segment );
    +        FT_FREE( table->avar_segment );
             goto Exit;
           }
     
    @@ -426,20 +448,51 @@
               FT_fdot14ToFixed( FT_GET_SHORT() );
     
             FT_TRACE5(( "    mapping %.5f to %.5f\n",
    -                    segment->correspondence[j].fromCoord / 65536.0,
    -                    segment->correspondence[j].toCoord / 65536.0 ));
    +                    (double)segment->correspondence[j].fromCoord / 65536,
    +                    (double)segment->correspondence[j].toCoord / 65536 ));
           }
     
           FT_TRACE5(( "\n" ));
         }
     
    +#ifndef TT_CONFIG_OPTION_NO_BORING_EXPANSION
    +    if ( version < 0x00020000L )
    +      goto Exit;
    +
    +    axisMap_offset = FT_GET_ULONG();
    +    store_offset   = FT_GET_ULONG();
    +
    +    if ( store_offset )
    +    {
    +      error = tt_var_load_item_variation_store(
    +                face,
    +                table_offset + store_offset,
    +                &table->itemStore );
    +      if ( error )
    +        goto Exit;
    +    }
    +
    +    if ( axisMap_offset )
    +    {
    +      error = tt_var_load_delta_set_index_mapping(
    +                face,
    +                table_offset + axisMap_offset,
    +                &table->axisMap,
    +                &table->itemStore,
    +                table_len );
    +      if ( error )
    +        goto Exit;
    +    }
    +#endif
    +
    +
       Exit:
         FT_FRAME_EXIT();
       }
     
     
    -  static FT_Error
    -  ft_var_load_item_variation_store( TT_Face          face,
    +  FT_LOCAL_DEF( FT_Error )
    +  tt_var_load_item_variation_store( TT_Face          face,
                                         FT_ULong         offset,
                                         GX_ItemVarStore  itemStore )
       {
    @@ -449,13 +502,15 @@
         FT_Error   error;
         FT_UShort  format;
         FT_ULong   region_offset;
    -    FT_UInt    i, j, k;
    -    FT_UInt    wordDeltaCount;
    -    FT_Bool    long_words;
     
    -    GX_Blend        blend = face->blend;
    -    GX_ItemVarData  varData;
    +    FT_UInt    data_count;
    +    FT_UShort  axis_count;
    +    FT_UInt    region_count;
    +
    +    FT_UInt  i, j, k;
    +    FT_Bool  long_words;
     
    +    GX_Blend   blend           = face->blend;
         FT_ULong*  dataOffsetArray = NULL;
     
     
    @@ -465,31 +520,31 @@
     
         if ( format != 1 )
         {
    -      FT_TRACE2(( "ft_var_load_item_variation_store: bad store format %d\n",
    +      FT_TRACE2(( "tt_var_load_item_variation_store: bad store format %d\n",
                       format ));
           error = FT_THROW( Invalid_Table );
           goto Exit;
         }
     
         /* read top level fields */
    -    if ( FT_READ_ULONG( region_offset )         ||
    -         FT_READ_USHORT( itemStore->dataCount ) )
    +    if ( FT_READ_ULONG( region_offset ) ||
    +         FT_READ_USHORT( data_count )   )
           goto Exit;
     
         /* we need at least one entry in `itemStore->varData' */
    -    if ( !itemStore->dataCount )
    +    if ( !data_count )
         {
    -      FT_TRACE2(( "ft_var_load_item_variation_store: missing varData\n" ));
    +      FT_TRACE2(( "tt_var_load_item_variation_store: missing varData\n" ));
           error = FT_THROW( Invalid_Table );
           goto Exit;
         }
     
         /* make temporary copy of item variation data offsets; */
         /* we will parse region list first, then come back     */
    -    if ( FT_QNEW_ARRAY( dataOffsetArray, itemStore->dataCount ) )
    +    if ( FT_QNEW_ARRAY( dataOffsetArray, data_count ) )
           goto Exit;
     
    -    for ( i = 0; i < itemStore->dataCount; i++ )
    +    for ( i = 0; i < data_count; i++ )
         {
           if ( FT_READ_ULONG( dataOffsetArray[i] ) )
             goto Exit;
    @@ -499,39 +554,40 @@
         if ( FT_STREAM_SEEK( offset + region_offset ) )
           goto Exit;
     
    -    if ( FT_READ_USHORT( itemStore->axisCount )   ||
    -         FT_READ_USHORT( itemStore->regionCount ) )
    +    if ( FT_READ_USHORT( axis_count )   ||
    +         FT_READ_USHORT( region_count ) )
           goto Exit;
     
    -    if ( itemStore->axisCount != (FT_Long)blend->mmvar->num_axis )
    +    if ( axis_count != (FT_Long)blend->mmvar->num_axis )
         {
    -      FT_TRACE2(( "ft_var_load_item_variation_store:"
    +      FT_TRACE2(( "tt_var_load_item_variation_store:"
                       " number of axes in item variation store\n" ));
           FT_TRACE2(( "                                 "
                       " and `fvar' table are different\n" ));
           error = FT_THROW( Invalid_Table );
           goto Exit;
         }
    +    itemStore->axisCount = axis_count;
     
         /* new constraint in OpenType 1.8.4 */
    -    if ( itemStore->regionCount >= 32768U )
    +    if ( region_count >= 32768U )
         {
    -      FT_TRACE2(( "ft_var_load_item_variation_store:"
    +      FT_TRACE2(( "tt_var_load_item_variation_store:"
                       " too many variation region tables\n" ));
           error = FT_THROW( Invalid_Table );
           goto Exit;
         }
     
    -    if ( FT_NEW_ARRAY( itemStore->varRegionList, itemStore->regionCount ) )
    +    if ( FT_NEW_ARRAY( itemStore->varRegionList, region_count ) )
           goto Exit;
    +    itemStore->regionCount = region_count;
     
         for ( i = 0; i < itemStore->regionCount; i++ )
         {
           GX_AxisCoords  axisCoords;
     
     
    -      if ( FT_NEW_ARRAY( itemStore->varRegionList[i].axisList,
    -                         itemStore->axisCount ) )
    +      if ( FT_NEW_ARRAY( itemStore->varRegionList[i].axisList, axis_count ) )
             goto Exit;
     
           axisCoords = itemStore->varRegionList[i].axisList;
    @@ -555,47 +611,53 @@
         /* end of region list parse */
     
         /* use dataOffsetArray now to parse varData items */
    -    if ( FT_NEW_ARRAY( itemStore->varData, itemStore->dataCount ) )
    +    if ( FT_NEW_ARRAY( itemStore->varData, data_count ) )
           goto Exit;
    +    itemStore->dataCount = data_count;
     
    -    for ( i = 0; i < itemStore->dataCount; i++ )
    +    for ( i = 0; i < data_count; i++ )
         {
    -      varData = &itemStore->varData[i];
    +      GX_ItemVarData  varData = &itemStore->varData[i];
    +
    +      FT_UInt  item_count;
    +      FT_UInt  word_delta_count;
    +      FT_UInt  region_idx_count;
    +
     
           if ( FT_STREAM_SEEK( offset + dataOffsetArray[i] ) )
             goto Exit;
     
    -      if ( FT_READ_USHORT( varData->itemCount )      ||
    -           FT_READ_USHORT( wordDeltaCount )          ||
    -           FT_READ_USHORT( varData->regionIdxCount ) )
    +      if ( FT_READ_USHORT( item_count )       ||
    +           FT_READ_USHORT( word_delta_count ) ||
    +           FT_READ_USHORT( region_idx_count ) )
             goto Exit;
     
    -      long_words      = !!( wordDeltaCount & 0x8000 );
    -      wordDeltaCount &= 0x7FFF;
    +      long_words        = !!( word_delta_count & 0x8000 );
    +      word_delta_count &= 0x7FFF;
     
           /* check some data consistency */
    -      if ( wordDeltaCount > varData->regionIdxCount )
    +      if ( word_delta_count > region_idx_count )
           {
             FT_TRACE2(( "bad short count %d or region count %d\n",
    -                    wordDeltaCount,
    -                    varData->regionIdxCount ));
    +                    word_delta_count,
    +                    region_idx_count ));
             error = FT_THROW( Invalid_Table );
             goto Exit;
           }
     
    -      if ( varData->regionIdxCount > itemStore->regionCount )
    +      if ( region_idx_count > itemStore->regionCount )
           {
             FT_TRACE2(( "inconsistent regionCount %d in varData[%d]\n",
    -                    varData->regionIdxCount,
    +                    region_idx_count,
                         i ));
             error = FT_THROW( Invalid_Table );
             goto Exit;
           }
     
           /* parse region indices */
    -      if ( FT_NEW_ARRAY( varData->regionIndices,
    -                         varData->regionIdxCount ) )
    +      if ( FT_NEW_ARRAY( varData->regionIndices, region_idx_count ) )
             goto Exit;
    +      varData->regionIdxCount = region_idx_count;
     
           for ( j = 0; j < varData->regionIdxCount; j++ )
           {
    @@ -611,54 +673,35 @@
             }
           }
     
    -      /* Parse delta set.                                                */
    -      /*                                                                 */
    -      /* On input, deltas are (wordDeltaCount + regionIdxCount) bytes    */
    -      /* each if `long_words` isn't set, and twice as much otherwise.    */
    -      /*                                                                 */
    -      /* On output, deltas are expanded to `regionIdxCount` shorts each. */
    -      if ( FT_NEW_ARRAY( varData->deltaSet,
    -                         varData->regionIdxCount * varData->itemCount ) )
    +      /* Parse delta set.                                                  */
    +      /*                                                                   */
    +      /* On input, deltas are (word_delta_count + region_idx_count) bytes  */
    +      /* each if `long_words` isn't set, and twice as much otherwise.      */
    +      /*                                                                   */
    +      /* On output, deltas are expanded to `region_idx_count` shorts each. */
    +      if ( FT_NEW_ARRAY( varData->deltaSet, item_count * region_idx_count ) )
             goto Exit;
    +      varData->itemCount = item_count;
     
    -      /* the delta set is stored as a 2-dimensional array of shorts */
    -      if ( long_words )
    +      for ( j = 0; j < item_count * region_idx_count; )
           {
    -        /* new in OpenType 1.9, currently for 'COLR' table only;          */
    -        /* the deltas are interpreted as 16.16 fixed-point scaling values */
    -
    -        /* not supported yet */
    -
    -        error = FT_THROW( Invalid_Table );
    -        goto Exit;
    -      }
    -      else
    -      {
    -        for ( j = 0; j < varData->itemCount * varData->regionIdxCount; )
    +        if ( long_words )
             {
    -          for ( k = 0; k < wordDeltaCount; k++, j++ )
    -          {
    -            /* read the short deltas */
    -            FT_Short  delta;
    -
    -
    -            if ( FT_READ_SHORT( delta ) )
    +          for ( k = 0; k < word_delta_count; k++, j++ )
    +            if ( FT_READ_LONG( varData->deltaSet[j] ) )
                   goto Exit;
    -
    -            varData->deltaSet[j] = delta;
    -          }
    -
    -          for ( ; k < varData->regionIdxCount; k++, j++ )
    -          {
    -            /* read the (signed) byte deltas */
    -            FT_Char  delta;
    -
    -
    -            if ( FT_READ_CHAR( delta ) )
    +          for ( ; k < region_idx_count; k++, j++ )
    +            if ( FT_READ_SHORT( varData->deltaSet[j] ) )
    +              goto Exit;
    +        }
    +        else
    +        {
    +          for ( k = 0; k < word_delta_count; k++, j++ )
    +            if ( FT_READ_SHORT( varData->deltaSet[j] ) )
    +              goto Exit;
    +          for ( ; k < region_idx_count; k++, j++ )
    +            if ( FT_READ_CHAR( varData->deltaSet[j] ) )
                   goto Exit;
    -
    -            varData->deltaSet[j] = delta;
    -          }
             }
           }
         }
    @@ -670,8 +713,8 @@
       }
     
     
    -  static FT_Error
    -  ft_var_load_delta_set_index_mapping( TT_Face            face,
    +  FT_LOCAL_DEF( FT_Error )
    +  tt_var_load_delta_set_index_mapping( TT_Face            face,
                                            FT_ULong           offset,
                                            GX_DeltaSetIdxMap  map,
                                            GX_ItemVarStore    itemStore,
    @@ -728,7 +771,7 @@
         /* rough sanity check */
         if ( map->mapCount * entrySize > table_len )
         {
    -      FT_TRACE1(( "ft_var_load_delta_set_index_mapping:"
    +      FT_TRACE1(( "tt_var_load_delta_set_index_mapping:"
                       " invalid number of delta-set index mappings\n" ));
           error = FT_THROW( Invalid_Table );
           goto Exit;
    @@ -758,6 +801,16 @@
             mapData = ( mapData << 8 ) | data;
           }
     
    +      /* new in OpenType 1.8.4 */
    +      if ( mapData == 0xFFFFFFFFUL )
    +      {
    +        /* no variation data for this item */
    +        map->outerIndex[i] = 0xFFFFU;
    +        map->innerIndex[i] = 0xFFFFU;
    +
    +        continue;
    +      }
    +
           outerIndex = mapData >> innerBitCount;
     
           if ( outerIndex >= itemStore->dataCount )
    @@ -887,7 +940,7 @@
           table = blend->hvar_table;
         }
     
    -    error = ft_var_load_item_variation_store(
    +    error = tt_var_load_item_variation_store(
                   face,
                   table_offset + store_offset,
                   &table->itemStore );
    @@ -896,7 +949,7 @@
     
         if ( widthMap_offset )
         {
    -      error = ft_var_load_delta_set_index_mapping(
    +      error = tt_var_load_delta_set_index_mapping(
                     face,
                     table_offset + widthMap_offset,
                     &table->widthMap,
    @@ -938,26 +991,47 @@
       }
     
     
    -  static FT_Int
    -  ft_var_get_item_delta( TT_Face          face,
    +  FT_LOCAL_DEF( FT_ItemVarDelta )
    +  tt_var_get_item_delta( TT_Face          face,
                              GX_ItemVarStore  itemStore,
                              FT_UInt          outerIndex,
                              FT_UInt          innerIndex )
       {
    -    GX_ItemVarData  varData;
    -    FT_Short*       deltaSet;
    +    FT_Stream  stream = FT_FACE_STREAM( face );
    +    FT_Memory  memory = stream->memory;
    +    FT_Error   error  = FT_Err_Ok;
    +
    +    GX_ItemVarData    varData;
    +    FT_ItemVarDelta*  deltaSet;
     
    -    FT_UInt   master, j;
    -    FT_Fixed  netAdjustment = 0;     /* accumulated adjustment */
    -    FT_Fixed  scaledDelta;
    -    FT_Fixed  delta;
    +    FT_UInt          master, j;
    +    FT_Fixed*        scalars = NULL;
    +    FT_ItemVarDelta  returnValue;
     
     
    +    if ( !face->blend || !face->blend->normalizedcoords )
    +      return 0;
    +
    +    /* OpenType 1.8.4+: No variation data for this item */
    +    /* as indices have special value 0xFFFF.            */
    +    if ( outerIndex == 0xFFFF && innerIndex == 0xFFFF )
    +      return 0;
    +
         /* See pseudo code from `Font Variations Overview' */
         /* in the OpenType specification.                  */
     
    +    if ( outerIndex >= itemStore->dataCount )
    +      return 0; /* Out of range. */
    +
         varData  = &itemStore->varData[outerIndex];
    -    deltaSet = &varData->deltaSet[varData->regionIdxCount * innerIndex];
    +    deltaSet = FT_OFFSET( varData->deltaSet,
    +                          varData->regionIdxCount * innerIndex );
    +
    +    if ( innerIndex >= varData->itemCount )
    +      return 0; /* Out of range. */
    +
    +    if ( FT_QNEW_ARRAY( scalars, varData->regionIdxCount ) )
    +      return 0;
     
         /* outer loop steps through master designs to be blended */
         for ( master = 0; master < varData->regionIdxCount; master++ )
    @@ -1008,18 +1082,33 @@
                 FT_MulDiv( scalar,
                            axis->endCoord - face->blend->normalizedcoords[j],
                            axis->endCoord - axis->peakCoord );
    -      } /* per-axis loop */
     
    -      /* get the scaled delta for this region */
    -      delta       = FT_intToFixed( deltaSet[master] );
    -      scaledDelta = FT_MulFix( scalar, delta );
    +      } /* per-axis loop */
     
    -      /* accumulate the adjustments from each region */
    -      netAdjustment = netAdjustment + scaledDelta;
    +      scalars[master] = scalar;
     
         } /* per-region loop */
     
    -    return FT_fixedToInt( netAdjustment );
    +
    +    /* Compute the scaled delta for this region.
    +     *
    +     * From: https://docs.microsoft.com/en-us/typography/opentype/spec/otvarcommonformats#item-variation-store-header-and-item-variation-data-subtables:
    +     *
    +     *   `Fixed` is a 32-bit (16.16) type and, in the general case, requires
    +     *   32-bit deltas.  As described above, the `DeltaSet` record can
    +     *   accommodate deltas that are, logically, either 16-bit or 32-bit.
    +     *   When scaled deltas are applied to `Fixed` values, the `Fixed` value
    +     *   is treated like a 32-bit integer.
    +     *
    +     * `FT_MulAddFix` internally uses 64-bit precision; it thus can handle
    +     * deltas ranging from small 8-bit to large 32-bit values that are
    +     * applied to 16.16 `FT_Fixed` / OpenType `Fixed` values.
    +     */
    +    returnValue = FT_MulAddFix( scalars, deltaSet, varData->regionIdxCount );
    +
    +    FT_FREE( scalars );
    +
    +    return returnValue;
       }
     
     
    @@ -1112,35 +1201,27 @@
         }
         else
         {
    -      GX_ItemVarData  varData;
    -
    -
           /* no widthMap data */
           outerIndex = 0;
           innerIndex = gindex;
    -
    -      varData = &table->itemStore.varData[outerIndex];
    -      if ( gindex >= varData->itemCount )
    -      {
    -        FT_TRACE2(( "gindex %d out of range\n", gindex ));
    -        error = FT_THROW( Invalid_Argument );
    -        goto Exit;
    -      }
         }
     
    -    delta = ft_var_get_item_delta( face,
    +    delta = tt_var_get_item_delta( face,
                                        &table->itemStore,
                                        outerIndex,
                                        innerIndex );
     
    -    FT_TRACE5(( "%s value %d adjusted by %d unit%s (%s)\n",
    -                vertical ? "vertical height" : "horizontal width",
    -                *avalue,
    -                delta,
    -                delta == 1 ? "" : "s",
    -                vertical ? "VVAR" : "HVAR" ));
    -
    -    *avalue += delta;
    +    if ( delta )
    +    {
    +      FT_TRACE5(( "%s value %d adjusted by %d unit%s (%s)\n",
    +                  vertical ? "vertical height" : "horizontal width",
    +                  *avalue,
    +                  delta,
    +                  delta == 1 ? "" : "s",
    +                  vertical ? "VVAR" : "HVAR" ));
    +
    +      *avalue = ADD_INT( *avalue, delta );
    +    }
     
       Exit:
         return error;
    @@ -1307,7 +1388,7 @@
     
         records_offset = FT_STREAM_POS();
     
    -    error = ft_var_load_item_variation_store(
    +    error = tt_var_load_item_variation_store(
                   face,
                   table_offset + store_offset,
                   &blend->mvar_table->itemStore );
    @@ -1323,7 +1404,7 @@
           return;
     
         value     = blend->mvar_table->values;
    -    limit     = value + blend->mvar_table->valueCount;
    +    limit     = FT_OFFSET( value, blend->mvar_table->valueCount );
         itemStore = &blend->mvar_table->itemStore;
     
         for ( ; value < limit; value++ )
    @@ -1332,6 +1413,13 @@
           value->outerIndex = FT_GET_USHORT();
           value->innerIndex = FT_GET_USHORT();
     
    +      /* new in OpenType 1.8.4 */
    +      if ( value->outerIndex == 0xFFFFU && value->innerIndex == 0xFFFFU )
    +      {
    +        /* no variation data for this item */
    +        continue;
    +      }
    +
           if ( value->outerIndex >= itemStore->dataCount                  ||
                value->innerIndex >= itemStore->varData[value->outerIndex]
                                                       .itemCount          )
    @@ -1349,7 +1437,7 @@
         FT_TRACE2(( "loaded\n" ));
     
         value = blend->mvar_table->values;
    -    limit = value + blend->mvar_table->valueCount;
    +    limit = FT_OFFSET( value, blend->mvar_table->valueCount );
     
         /* save original values of the data MVAR is going to modify */
         for ( ; value < limit; value++ )
    @@ -1414,7 +1502,7 @@
           return;
     
         value = blend->mvar_table->values;
    -    limit = value + blend->mvar_table->valueCount;
    +    limit = FT_OFFSET( value, blend->mvar_table->valueCount );
     
         for ( ; value < limit; value++ )
         {
    @@ -1422,12 +1510,12 @@
           FT_Int     delta;
     
     
    -      delta = ft_var_get_item_delta( face,
    +      delta = tt_var_get_item_delta( face,
                                          &blend->mvar_table->itemStore,
                                          value->outerIndex,
                                          value->innerIndex );
     
    -      if ( p )
    +      if ( p && delta )
           {
             FT_TRACE5(( "value %c%c%c%c (%d unit%s) adjusted by %d unit%s (MVAR)\n",
                         (FT_Char)( value->tag >> 24 ),
    @@ -1725,7 +1813,7 @@
               blend->tuplecoords[i * gvar_head.axisCount + j] =
                 FT_fdot14ToFixed( FT_GET_SHORT() );
               FT_TRACE5(( "%.5f ",
    -            blend->tuplecoords[i * gvar_head.axisCount + j] / 65536.0 ));
    +            (double)blend->tuplecoords[i * gvar_head.axisCount + j] / 65536 ));
             }
             FT_TRACE5(( "]\n" ));
           }
    @@ -1796,7 +1884,7 @@
         for ( i = 0; i < blend->num_axis; i++ )
         {
           FT_TRACE6(( "    axis %d coordinate %.5f:\n",
    -                  i, blend->normalizedcoords[i] / 65536.0 ));
    +                  i, (double)blend->normalizedcoords[i] / 65536 ));
     
           /* It's not clear why (for intermediate tuples) we don't need     */
           /* to check against start/end -- the documentation says we don't. */
    @@ -1819,7 +1907,7 @@
           if ( blend->normalizedcoords[i] == tuple_coords[i] )
           {
             FT_TRACE6(( "      tuple coordinate %.5f fits perfectly\n",
    -                    tuple_coords[i] / 65536.0 ));
    +                    (double)tuple_coords[i] / 65536 ));
             /* `apply' does not change */
             continue;
           }
    @@ -1832,13 +1920,13 @@
                  blend->normalizedcoords[i] > FT_MAX( 0, tuple_coords[i] ) )
             {
               FT_TRACE6(( "      tuple coordinate %.5f is exceeded, stop\n",
    -                      tuple_coords[i] / 65536.0 ));
    +                      (double)tuple_coords[i] / 65536 ));
               apply = 0;
               break;
             }
     
             FT_TRACE6(( "      tuple coordinate %.5f fits\n",
    -                    tuple_coords[i] / 65536.0 ));
    +                    (double)tuple_coords[i] / 65536 ));
             apply = FT_MulDiv( apply,
                                blend->normalizedcoords[i],
                                tuple_coords[i] );
    @@ -1852,15 +1940,15 @@
             {
               FT_TRACE6(( "      intermediate tuple range ]%.5f;%.5f[ is exceeded,"
                           " stop\n",
    -                      im_start_coords[i] / 65536.0,
    -                      im_end_coords[i] / 65536.0 ));
    +                      (double)im_start_coords[i] / 65536,
    +                      (double)im_end_coords[i] / 65536 ));
               apply = 0;
               break;
             }
     
             FT_TRACE6(( "      intermediate tuple range ]%.5f;%.5f[ fits\n",
    -                    im_start_coords[i] / 65536.0,
    -                    im_end_coords[i] / 65536.0 ));
    +                    (double)im_start_coords[i] / 65536,
    +                    (double)im_end_coords[i] / 65536 ));
             if ( blend->normalizedcoords[i] < tuple_coords[i] )
               apply = FT_MulDiv( apply,
                                  blend->normalizedcoords[i] - im_start_coords[i],
    @@ -1872,7 +1960,7 @@
           }
         }
     
    -    FT_TRACE6(( "    apply factor is %.5f\n", apply / 65536.0 ));
    +    FT_TRACE6(( "    apply factor is %.5f\n", (double)apply / 65536 ));
     
         return apply;
       }
    @@ -1886,12 +1974,18 @@
                             FT_Fixed*  coords,
                             FT_Fixed*  normalized )
       {
    +    FT_Error   error  = FT_Err_Ok;
    +    FT_Memory  memory = face->root.memory;
    +    FT_UInt    i, j;
    +
         GX_Blend        blend;
         FT_MM_Var*      mmvar;
    -    FT_UInt         i, j;
         FT_Var_Axis*    a;
         GX_AVarSegment  av;
     
    +    FT_Fixed*  new_normalized = NULL;
    +    FT_Fixed*  old_normalized;
    +
     
         blend = face->blend;
         mmvar = blend->mmvar;
    @@ -1914,15 +2008,15 @@
           FT_Fixed  coord = coords[i];
     
     
    -      FT_TRACE5(( "    %d: %.5f\n", i, coord / 65536.0 ));
    +      FT_TRACE5(( "    %d: %.5f\n", i, (double)coord / 65536 ));
           if ( coord > a->maximum || coord < a->minimum )
           {
             FT_TRACE1(( "ft_var_to_normalized: design coordinate %.5f\n",
    -                    coord / 65536.0 ));
    +                    (double)coord / 65536 ));
             FT_TRACE1(( "                      is out of range [%.5f;%.5f];"
                         " clamping\n",
    -                    a->minimum / 65536.0,
    -                    a->maximum / 65536.0 ));
    +                    (double)a->minimum / 65536,
    +                    (double)a->maximum / 65536 ));
           }
     
           if ( coord > a->def )
    @@ -1942,30 +2036,91 @@
         for ( ; i < mmvar->num_axis; i++ )
           normalized[i] = 0;
     
    -    if ( blend->avar_segment )
    +    if ( blend->avar_table )
         {
    +      GX_AVarTable  table = blend->avar_table;
    +
    +
           FT_TRACE5(( "normalized design coordinates"
                       " before applying `avar' data:\n" ));
     
    -      av = blend->avar_segment;
    -      for ( i = 0; i < mmvar->num_axis; i++, av++ )
    +      if ( table->avar_segment )
           {
    -        for ( j = 1; j < (FT_UInt)av->pairCount; j++ )
    +        av = table->avar_segment;
    +
    +        for ( i = 0; i < mmvar->num_axis; i++, av++ )
             {
    -          if ( normalized[i] < av->correspondence[j].fromCoord )
    +          for ( j = 1; j < (FT_UInt)av->pairCount; j++ )
               {
    -            FT_TRACE5(( "  %.5f\n", normalized[i] / 65536.0 ));
    +            if ( normalized[i] < av->correspondence[j].fromCoord )
    +            {
    +              FT_TRACE5(( "  %.5f\n", (double)normalized[i] / 65536 ));
    +
    +              normalized[i] =
    +                FT_MulDiv( normalized[i] - av->correspondence[j - 1].fromCoord,
    +                           av->correspondence[j].toCoord -
    +                             av->correspondence[j - 1].toCoord,
    +                           av->correspondence[j].fromCoord -
    +                             av->correspondence[j - 1].fromCoord ) +
    +                av->correspondence[j - 1].toCoord;
    +              break;
    +            }
    +          }
    +        }
    +      }
     
    -            normalized[i] =
    -              FT_MulDiv( normalized[i] - av->correspondence[j - 1].fromCoord,
    -                         av->correspondence[j].toCoord -
    -                           av->correspondence[j - 1].toCoord,
    -                         av->correspondence[j].fromCoord -
    -                           av->correspondence[j - 1].fromCoord ) +
    -              av->correspondence[j - 1].toCoord;
    -            break;
    +      if ( table->itemStore.varData )
    +      {
    +        if ( FT_QNEW_ARRAY( new_normalized, mmvar->num_axis ) )
    +          return;
    +
    +        /* Install our half-normalized coordinates for the next */
    +        /* Item Variation Store to work with.                   */
    +        old_normalized                = face->blend->normalizedcoords;
    +        face->blend->normalizedcoords = normalized;
    +
    +        for ( i = 0; i < mmvar->num_axis; i++ )
    +        {
    +          FT_Fixed  v          = normalized[i];
    +          FT_UInt   innerIndex = i;
    +          FT_UInt   outerIndex = 0;
    +          FT_Int    delta;
    +
    +
    +          if ( table->axisMap.innerIndex )
    +          {
    +            FT_UInt  idx = i;
    +
    +
    +            if ( idx >= table->axisMap.mapCount )
    +              idx = table->axisMap.mapCount - 1;
    +
    +            outerIndex = table->axisMap.outerIndex[idx];
    +            innerIndex = table->axisMap.innerIndex[idx];
               }
    +
    +          delta = tt_var_get_item_delta( face,
    +                                         &table->itemStore,
    +                                         outerIndex,
    +                                         innerIndex );
    +
    +      v += delta << 2;
    +
    +      /* Clamp value range. */
    +      v = v >=  0x10000L ?  0x10000 : v;
    +      v = v <= -0x10000L ? -0x10000 : v;
    +
    +          new_normalized[i] = v;
    +        }
    +
    +        for ( i = 0; i < mmvar->num_axis; i++ )
    +        {
    +          normalized[i] = new_normalized[i];
             }
    +
    +        face->blend->normalizedcoords = old_normalized;
    +
    +        FT_FREE( new_normalized );
           }
         }
       }
    @@ -2003,9 +2158,9 @@
         for ( ; i < num_coords; i++ )
           design[i] = 0;
     
    -    if ( blend->avar_segment )
    +    if ( blend->avar_table && blend->avar_table->avar_segment )
         {
    -      GX_AVarSegment  av = blend->avar_segment;
    +      GX_AVarSegment  av = blend->avar_table->avar_segment;
     
     
           FT_TRACE5(( "design coordinates"
    @@ -2025,7 +2180,7 @@
                                av->correspondence[j - 1].toCoord ) +
                   av->correspondence[j - 1].fromCoord;
     
    -            FT_TRACE5(( "  %.5f\n", design[i] / 65536.0 ));
    +            FT_TRACE5(( "  %.5f\n", (double)design[i] / 65536 ));
                 break;
               }
             }
    @@ -2170,6 +2325,11 @@
           FT_FRAME_END
         };
     
    +    /* `num_instances` holds the number of all named instances including  */
    +    /* the default instance, which might be missing in the table of named */
    +    /* instances (in 'fvar').  This value is validated in `sfobjs.c` and  */
    +    /* may be reset to 0 if consistency checks fail.                      */
    +    num_instances = (FT_UInt)face->root.style_flags >> 16;
     
         /* read the font data and set up the internal representation */
         /* if not already done                                       */
    @@ -2180,20 +2340,6 @@
         {
           FT_TRACE2(( "FVAR " ));
     
    -      /* both `fvar' and `gvar' must be present */
    -      if ( FT_SET_ERROR( face->goto_table( face, TTAG_gvar,
    -                                           stream, &table_len ) ) )
    -      {
    -        /* CFF2 is an alternate to gvar here */
    -        if ( FT_SET_ERROR( face->goto_table( face, TTAG_CFF2,
    -                                             stream, &table_len ) ) )
    -        {
    -          FT_TRACE1(( "\n" ));
    -          FT_TRACE1(( "TT_Get_MM_Var: `gvar' or `CFF2' table is missing\n" ));
    -          goto Exit;
    -        }
    -      }
    -
           if ( FT_SET_ERROR( face->goto_table( face, TTAG_fvar,
                                                stream, &table_len ) ) )
           {
    @@ -2208,6 +2354,17 @@
           if ( FT_STREAM_READ_FIELDS( fvar_fields, &fvar_head ) )
             goto Exit;
     
    +      /* If `num_instances` is larger, synthetization of the default  */
    +      /* instance is required.  If `num_instances` is smaller,        */
    +      /* however, the value has been reset to 0 in `sfnt_init_face`   */
    +      /* (in `sfobjs.c`); in this case we have underallocated `mmvar` */
    +      /* structs.                                                     */
    +      if ( num_instances < fvar_head.instanceCount )
    +      {
    +        error = FT_THROW( Invalid_Table );
    +        goto Exit;
    +      }
    +
           usePsName = FT_BOOL( fvar_head.instanceSize ==
                                6 + 4 * fvar_head.axisCount );
     
    @@ -2226,11 +2383,6 @@
         else
           num_axes = face->blend->num_axis;
     
    -    /* `num_instances' holds the number of all named instances, */
    -    /* including the default instance which might be missing    */
    -    /* in fvar's table of named instances                       */
    -    num_instances = (FT_UInt)face->root.style_flags >> 16;
    -
         /* prepare storage area for MM data; this cannot overflow   */
         /* 32-bit arithmetic because of the size limits used in the */
         /* `fvar' table validity check in `sfnt_init_face'          */
    @@ -2358,9 +2510,9 @@
                         "  %10.5f  %10.5f  %10.5f  0x%04X%s\n",
                         i,
                         a->name,
    -                    a->minimum / 65536.0,
    -                    a->def / 65536.0,
    -                    a->maximum / 65536.0,
    +                    (double)a->minimum / 65536,
    +                    (double)a->def / 65536,
    +                    (double)a->maximum / 65536,
                         *axis_flags,
                         invalid ? " (invalid, disabled)" : "" ));
     #endif
    @@ -2561,6 +2713,8 @@
               a->name = (char*)"OpticalSize";
             else if ( a->tag == TTAG_slnt )
               a->name = (char*)"Slant";
    +        else if ( a->tag == TTAG_ital )
    +          a->name = (char*)"Italic";
     
             next_name += 5;
             a++;
    @@ -2622,11 +2776,11 @@
     
         for ( i = 0; i < num_coords; i++ )
         {
    -      FT_TRACE5(( "    %.5f\n", coords[i] / 65536.0 ));
    +      FT_TRACE5(( "    %.5f\n", (double)coords[i] / 65536 ));
           if ( coords[i] < -0x00010000L || coords[i] > 0x00010000L )
           {
             FT_TRACE1(( "TT_Set_MM_Blend: normalized design coordinate %.5f\n",
    -                    coords[i] / 65536.0 ));
    +                    (double)coords[i] / 65536 ));
             FT_TRACE1(( "                 is out of range [-1;1]\n" ));
             error = FT_THROW( Invalid_Argument );
             goto Exit;
    @@ -2636,8 +2790,16 @@
         FT_TRACE5(( "\n" ));
     
         if ( !face->is_cff2 && !blend->glyphoffsets )
    -      if ( FT_SET_ERROR( ft_var_load_gvar( face ) ) )
    +    {
    +      /* While a missing 'gvar' table is acceptable, for example for */
    +      /* fonts that only vary metrics information or 'COLR' v1       */
    +      /* `PaintVar*` tables, an incorrect SFNT table offset or size  */
    +      /* for 'gvar', or an inconsistent 'gvar' table is not.         */
    +      error = ft_var_load_gvar( face );
    +      if ( error != FT_Err_Table_Missing && error != FT_Err_Ok )
             goto Exit;
    +      error = FT_Err_Ok;
    +    }
     
         if ( !blend->coords )
         {
    @@ -3503,10 +3665,10 @@
               {
                 FT_TRACE7(( "      %d: %f -> %f\n",
                             j,
    -                        ( FT_fdot6ToFixed( face->cvt[j] ) +
    -                          old_cvt_delta ) / 65536.0,
    -                        ( FT_fdot6ToFixed( face->cvt[j] ) +
    -                          cvt_deltas[j] ) / 65536.0 ));
    +                        (double)( FT_fdot6ToFixed( face->cvt[j] ) +
    +                                    old_cvt_delta ) / 65536,
    +                        (double)( FT_fdot6ToFixed( face->cvt[j] ) +
    +                                    cvt_deltas[j] ) / 65536 ));
                 count++;
               }
     #endif
    @@ -3545,10 +3707,10 @@
               {
                 FT_TRACE7(( "      %d: %f -> %f\n",
                             pindex,
    -                        ( FT_fdot6ToFixed( face->cvt[pindex] ) +
    -                          old_cvt_delta ) / 65536.0,
    -                        ( FT_fdot6ToFixed( face->cvt[pindex] ) +
    -                          cvt_deltas[pindex] ) / 65536.0 ));
    +                        (double)( FT_fdot6ToFixed( face->cvt[pindex] ) +
    +                                    old_cvt_delta ) / 65536,
    +                        (double)( FT_fdot6ToFixed( face->cvt[pindex] ) +
    +                                    cvt_deltas[pindex] ) / 65536 ));
                 count++;
               }
     #endif
    @@ -3813,20 +3975,12 @@
        * @Description:
        *   Apply the appropriate deltas to the current glyph.
        *
    -   * @Input:
    -   *   face ::
    -   *     A handle to the target face object.
    -   *
    -   *   glyph_index ::
    -   *     The index of the glyph being modified.
    -   *
    -   *   n_points ::
    -   *     The number of the points in the glyph, including
    -   *     phantom points.
    -   *
        * @InOut:
    +   *   loader ::
    +   *     A handle to the loader object.
    +   *
        *   outline ::
    -   *     The outline to change.
    +   *     The outline to change, with appended phantom points.
        *
        * @Output:
        *   unrounded ::
    @@ -3837,15 +3991,16 @@
        *   FreeType error code.  0 means success.
        */
       FT_LOCAL_DEF( FT_Error )
    -  TT_Vary_Apply_Glyph_Deltas( TT_Face      face,
    -                              FT_UInt      glyph_index,
    +  TT_Vary_Apply_Glyph_Deltas( TT_Loader    loader,
                                   FT_Outline*  outline,
    -                              FT_Vector*   unrounded,
    -                              FT_UInt      n_points )
    +                              FT_Vector*   unrounded )
       {
         FT_Error   error;
    -    FT_Stream  stream = face->root.stream;
    -    FT_Memory  memory = stream->memory;
    +    TT_Face    face        = loader->face;
    +    FT_Stream  stream      = face->root.stream;
    +    FT_Memory  memory      = stream->memory;
    +    FT_UInt    glyph_index = loader->glyph_index;
    +    FT_UInt    n_points    = (FT_UInt)outline->n_points + 4;
     
         FT_Vector*  points_org = NULL;  /* coordinates in 16.16 format */
         FT_Vector*  points_out = NULL;  /* coordinates in 16.16 format */
    @@ -4063,50 +4218,22 @@
               FT_Fixed  point_delta_y = FT_MulFix( deltas_y[j], apply );
     
     
    -          if ( j < n_points - 4 )
    -          {
    -            point_deltas_x[j] = old_point_delta_x + point_delta_x;
    -            point_deltas_y[j] = old_point_delta_y + point_delta_y;
    -          }
    -          else
    -          {
    -            /* To avoid double adjustment of advance width or height, */
    -            /* adjust phantom points only if there is no HVAR or VVAR */
    -            /* support, respectively.                                 */
    -            if ( j == ( n_points - 4 )        &&
    -                 !( face->variation_support &
    -                    TT_FACE_FLAG_VAR_LSB    ) )
    -              point_deltas_x[j] = old_point_delta_x + point_delta_x;
    -
    -            else if ( j == ( n_points - 3 )          &&
    -                      !( face->variation_support   &
    -                         TT_FACE_FLAG_VAR_HADVANCE ) )
    -              point_deltas_x[j] = old_point_delta_x + point_delta_x;
    -
    -            else if ( j == ( n_points - 2 )        &&
    -                      !( face->variation_support &
    -                         TT_FACE_FLAG_VAR_TSB    ) )
    -              point_deltas_y[j] = old_point_delta_y + point_delta_y;
    -
    -            else if ( j == ( n_points - 1 )          &&
    -                      !( face->variation_support   &
    -                         TT_FACE_FLAG_VAR_VADVANCE ) )
    -              point_deltas_y[j] = old_point_delta_y + point_delta_y;
    -          }
    +          point_deltas_x[j] = old_point_delta_x + point_delta_x;
    +          point_deltas_y[j] = old_point_delta_y + point_delta_y;
     
     #ifdef FT_DEBUG_LEVEL_TRACE
               if ( point_delta_x || point_delta_y )
               {
                 FT_TRACE7(( "      %d: (%f, %f) -> (%f, %f)\n",
                             j,
    -                        ( FT_intToFixed( outline->points[j].x ) +
    -                          old_point_delta_x ) / 65536.0,
    -                        ( FT_intToFixed( outline->points[j].y ) +
    -                          old_point_delta_y ) / 65536.0,
    -                        ( FT_intToFixed( outline->points[j].x ) +
    -                          point_deltas_x[j] ) / 65536.0,
    -                        ( FT_intToFixed( outline->points[j].y ) +
    -                          point_deltas_y[j] ) / 65536.0 ));
    +                        (double)( FT_intToFixed( outline->points[j].x ) +
    +                                    old_point_delta_x ) / 65536,
    +                        (double)( FT_intToFixed( outline->points[j].y ) +
    +                                    old_point_delta_y ) / 65536,
    +                        (double)( FT_intToFixed( outline->points[j].x ) +
    +                                    point_deltas_x[j] ) / 65536,
    +                        (double)( FT_intToFixed( outline->points[j].y ) +
    +                                    point_deltas_y[j] ) / 65536 ));
                 count++;
               }
     #endif
    @@ -4165,50 +4292,22 @@
               FT_Pos  point_delta_y = points_out[j].y - points_org[j].y;
     
     
    -          if ( j < n_points - 4 )
    -          {
    -            point_deltas_x[j] = old_point_delta_x + point_delta_x;
    -            point_deltas_y[j] = old_point_delta_y + point_delta_y;
    -          }
    -          else
    -          {
    -            /* To avoid double adjustment of advance width or height, */
    -            /* adjust phantom points only if there is no HVAR or VVAR */
    -            /* support, respectively.                                 */
    -            if ( j == ( n_points - 4 )        &&
    -                 !( face->variation_support &
    -                    TT_FACE_FLAG_VAR_LSB    ) )
    -              point_deltas_x[j] = old_point_delta_x + point_delta_x;
    -
    -            else if ( j == ( n_points - 3 )          &&
    -                      !( face->variation_support   &
    -                         TT_FACE_FLAG_VAR_HADVANCE ) )
    -              point_deltas_x[j] = old_point_delta_x + point_delta_x;
    -
    -            else if ( j == ( n_points - 2 )        &&
    -                      !( face->variation_support &
    -                         TT_FACE_FLAG_VAR_TSB    ) )
    -              point_deltas_y[j] = old_point_delta_y + point_delta_y;
    -
    -            else if ( j == ( n_points - 1 )          &&
    -                      !( face->variation_support   &
    -                         TT_FACE_FLAG_VAR_VADVANCE ) )
    -              point_deltas_y[j] = old_point_delta_y + point_delta_y;
    -          }
    +          point_deltas_x[j] = old_point_delta_x + point_delta_x;
    +          point_deltas_y[j] = old_point_delta_y + point_delta_y;
     
     #ifdef FT_DEBUG_LEVEL_TRACE
               if ( point_delta_x || point_delta_y )
               {
                 FT_TRACE7(( "      %d: (%f, %f) -> (%f, %f)\n",
                             j,
    -                        ( FT_intToFixed( outline->points[j].x ) +
    -                          old_point_delta_x ) / 65536.0,
    -                        ( FT_intToFixed( outline->points[j].y ) +
    -                          old_point_delta_y ) / 65536.0,
    -                        ( FT_intToFixed( outline->points[j].x ) +
    -                          point_deltas_x[j] ) / 65536.0,
    -                        ( FT_intToFixed( outline->points[j].y ) +
    -                          point_deltas_y[j] ) / 65536.0 ));
    +                        (double)( FT_intToFixed( outline->points[j].x ) +
    +                                    old_point_delta_x ) / 65536,
    +                        (double)( FT_intToFixed( outline->points[j].y ) +
    +                                    old_point_delta_y ) / 65536,
    +                        (double)( FT_intToFixed( outline->points[j].x ) +
    +                                    point_deltas_x[j] ) / 65536,
    +                        (double)( FT_intToFixed( outline->points[j].y ) +
    +                                    point_deltas_y[j] ) / 65536 ));
                 count++;
               }
     #endif
    @@ -4232,6 +4331,24 @@
     
         FT_TRACE5(( "\n" ));
     
    +    /* To avoid double adjustment of advance width or height, */
    +    /* do not move phantom points if there is HVAR or VVAR    */
    +    /* support, respectively.                                 */
    +    if ( face->variation_support & TT_FACE_FLAG_VAR_HADVANCE )
    +    {
    +      point_deltas_x[n_points - 4] = 0;
    +      point_deltas_y[n_points - 4] = 0;
    +      point_deltas_x[n_points - 3] = 0;
    +      point_deltas_y[n_points - 3] = 0;
    +    }
    +    if ( face->variation_support & TT_FACE_FLAG_VAR_VADVANCE )
    +    {
    +      point_deltas_x[n_points - 2] = 0;
    +      point_deltas_y[n_points - 2] = 0;
    +      point_deltas_x[n_points - 1] = 0;
    +      point_deltas_y[n_points - 1] = 0;
    +    }
    +
         for ( i = 0; i < n_points; i++ )
         {
           unrounded[i].x += FT_fixedToFdot6( point_deltas_x[i] );
    @@ -4241,6 +4358,24 @@
           outline->points[i].y += FT_fixedToInt( point_deltas_y[i] );
         }
     
    +    /* To avoid double adjustment of advance width or height, */
    +    /* adjust phantom points only if there is no HVAR or VVAR */
    +    /* support, respectively.                                 */
    +    if ( !( face->variation_support & TT_FACE_FLAG_VAR_HADVANCE ) )
    +    {
    +      loader->pp1      = outline->points[n_points - 4];
    +      loader->pp2      = outline->points[n_points - 3];
    +      loader->linear   = FT_PIX_ROUND( unrounded[n_points - 3].x -
    +                                       unrounded[n_points - 4].x ) / 64;
    +    }
    +    if ( !( face->variation_support & TT_FACE_FLAG_VAR_VADVANCE ) )
    +    {
    +      loader->pp3      = outline->points[n_points - 2];
    +      loader->pp4      = outline->points[n_points - 1];
    +      loader->vadvance = FT_PIX_ROUND( unrounded[n_points - 1].y -
    +                                       unrounded[n_points - 2].y ) / 64;
    +    }
    +
       Fail3:
         FT_FREE( point_deltas_x );
         FT_FREE( point_deltas_y );
    @@ -4305,8 +4440,8 @@
       }
     
     
    -  static void
    -  ft_var_done_item_variation_store( TT_Face          face,
    +  FT_LOCAL_DEF( void )
    +  tt_var_done_item_variation_store( TT_Face          face,
                                         GX_ItemVarStore  itemStore )
       {
         FT_Memory  memory = FT_FACE_MEMORY( face );
    @@ -4334,6 +4469,18 @@
       }
     
     
    +  FT_LOCAL_DEF( void )
    +  tt_var_done_delta_set_index_map( TT_Face            face,
    +                                   GX_DeltaSetIdxMap  deltaSetIdxMap )
    +  {
    +    FT_Memory  memory = FT_FACE_MEMORY( face );
    +
    +
    +    FT_FREE( deltaSetIdxMap->innerIndex );
    +    FT_FREE( deltaSetIdxMap->outerIndex );
    +  }
    +
    +
       /**************************************************************************
        *
        * @Function:
    @@ -4362,36 +4509,47 @@
           FT_FREE( blend->normalized_stylecoords );
           FT_FREE( blend->mmvar );
     
    -      if ( blend->avar_segment )
    +      if ( blend->avar_table )
           {
    -        for ( i = 0; i < num_axes; i++ )
    -          FT_FREE( blend->avar_segment[i].correspondence );
    -        FT_FREE( blend->avar_segment );
    +        if ( blend->avar_table->avar_segment )
    +        {
    +          for ( i = 0; i < num_axes; i++ )
    +            FT_FREE( blend->avar_table->avar_segment[i].correspondence );
    +          FT_FREE( blend->avar_table->avar_segment );
    +        }
    +
    +        tt_var_done_item_variation_store( face,
    +                                          &blend->avar_table->itemStore );
    +
    +        tt_var_done_delta_set_index_map( face,
    +                                         &blend->avar_table->axisMap );
    +
    +        FT_FREE( blend->avar_table );
           }
     
           if ( blend->hvar_table )
           {
    -        ft_var_done_item_variation_store( face,
    +        tt_var_done_item_variation_store( face,
                                               &blend->hvar_table->itemStore );
     
    -        FT_FREE( blend->hvar_table->widthMap.innerIndex );
    -        FT_FREE( blend->hvar_table->widthMap.outerIndex );
    +        tt_var_done_delta_set_index_map( face,
    +                                         &blend->hvar_table->widthMap );
             FT_FREE( blend->hvar_table );
           }
     
           if ( blend->vvar_table )
           {
    -        ft_var_done_item_variation_store( face,
    +        tt_var_done_item_variation_store( face,
                                               &blend->vvar_table->itemStore );
     
    -        FT_FREE( blend->vvar_table->widthMap.innerIndex );
    -        FT_FREE( blend->vvar_table->widthMap.outerIndex );
    +        tt_var_done_delta_set_index_map( face,
    +                                         &blend->vvar_table->widthMap );
             FT_FREE( blend->vvar_table );
           }
     
           if ( blend->mvar_table )
           {
    -        ft_var_done_item_variation_store( face,
    +        tt_var_done_item_variation_store( face,
                                               &blend->mvar_table->itemStore );
     
             FT_FREE( blend->mvar_table->values );
    diff --git a/src/java.desktop/share/native/libfreetype/src/truetype/ttgxvar.h b/src/java.desktop/share/native/libfreetype/src/truetype/ttgxvar.h
    index 17915f00d3e41..4fec980dcc066 100644
    --- a/src/java.desktop/share/native/libfreetype/src/truetype/ttgxvar.h
    +++ b/src/java.desktop/share/native/libfreetype/src/truetype/ttgxvar.h
    @@ -4,7 +4,7 @@
      *
      *   TrueType GX Font Variation loader (specification)
      *
    - * Copyright (C) 2004-2022 by
    + * Copyright (C) 2004-2023 by
      * David Turner, Robert Wilhelm, Werner Lemberg and George Williams.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -20,6 +20,7 @@
     #define TTGXVAR_H_
     
     
    +#include 
     #include "ttobjs.h"
     
     
    @@ -62,55 +63,21 @@ FT_BEGIN_HEADER
       } GX_AVarSegmentRec, *GX_AVarSegment;
     
     
    -  typedef struct  GX_ItemVarDataRec_
    -  {
    -    FT_UInt    itemCount;      /* number of delta sets per item         */
    -    FT_UInt    regionIdxCount; /* number of region indices in this data */
    -    FT_UInt*   regionIndices;  /* array of `regionCount' indices;       */
    -                               /* these index `varRegionList'           */
    -    FT_Short*  deltaSet;       /* array of `itemCount' deltas           */
    -                               /* use `innerIndex' for this array       */
    -
    -  } GX_ItemVarDataRec, *GX_ItemVarData;
    -
    -
    -  /* contribution of one axis to a region */
    -  typedef struct  GX_AxisCoordsRec_
    -  {
    -    FT_Fixed  startCoord;
    -    FT_Fixed  peakCoord;      /* zero means no effect (factor = 1) */
    -    FT_Fixed  endCoord;
    -
    -  } GX_AxisCoordsRec, *GX_AxisCoords;
    -
    -
    -  typedef struct  GX_VarRegionRec_
    -  {
    -    GX_AxisCoords  axisList;               /* array of axisCount records */
    -
    -  } GX_VarRegionRec, *GX_VarRegion;
    -
    -
    -  /* item variation store */
    -  typedef struct  GX_ItemVarStoreRec_
    -  {
    -    FT_UInt         dataCount;
    -    GX_ItemVarData  varData;            /* array of dataCount records;     */
    -                                        /* use `outerIndex' for this array */
    -    FT_UShort     axisCount;
    -    FT_UInt       regionCount;          /* total number of regions defined */
    -    GX_VarRegion  varRegionList;
    -
    -  } GX_ItemVarStoreRec, *GX_ItemVarStore;
    -
    -
    -  typedef struct  GX_DeltaSetIdxMapRec_
    +  /**************************************************************************
    +   *
    +   * @Struct:
    +   *   GX_AVarTableRec
    +   *
    +   * @Description:
    +   *   Data from the `avar' table.
    +   */
    +  typedef struct  GX_AVarTableRec_
       {
    -    FT_ULong  mapCount;
    -    FT_UInt*  outerIndex;               /* indices to item var data */
    -    FT_UInt*  innerIndex;               /* indices to delta set     */
    +    GX_AVarSegment        avar_segment;   /* avar_segment[num_axis] */
    +    GX_ItemVarStoreRec    itemStore;      /* Item Variation Store   */
    +    GX_DeltaSetIdxMapRec  axisMap;        /* Axis Mapping           */
     
    -  } GX_DeltaSetIdxMapRec, *GX_DeltaSetIdxMap;
    +  } GX_AVarTableRec, *GX_AVarTable;
     
     
       /**************************************************************************
    @@ -245,7 +212,7 @@ FT_BEGIN_HEADER
        *     A Boolean; if set, FreeType tried to load (and parse) the `avar'
        *     table.
        *
    -   *   avar_segment ::
    +   *   avar_table ::
        *     Data from the `avar' table.
        *
        *   hvar_loaded ::
    @@ -310,7 +277,7 @@ FT_BEGIN_HEADER
                           /* normalized_stylecoords[num_namedstyles][num_axis] */
     
         FT_Bool         avar_loaded;
    -    GX_AVarSegment  avar_segment;                /* avar_segment[num_axis] */
    +    GX_AVarTable    avar_table;
     
         FT_Bool         hvar_loaded;
         FT_Bool         hvar_checked;
    @@ -376,6 +343,7 @@ FT_BEGIN_HEADER
     #define TTAG_wdth  FT_MAKE_TAG( 'w', 'd', 't', 'h' )
     #define TTAG_opsz  FT_MAKE_TAG( 'o', 'p', 's', 'z' )
     #define TTAG_slnt  FT_MAKE_TAG( 's', 'l', 'n', 't' )
    +#define TTAG_ital  FT_MAKE_TAG( 'i', 't', 'a', 'l' )
     
     
       FT_LOCAL( FT_Error )
    @@ -412,11 +380,9 @@ FT_BEGIN_HEADER
     
     
       FT_LOCAL( FT_Error )
    -  TT_Vary_Apply_Glyph_Deltas( TT_Face      face,
    -                              FT_UInt      glyph_index,
    +  TT_Vary_Apply_Glyph_Deltas( TT_Loader    loader,
                                   FT_Outline*  outline,
    -                              FT_Vector*   unrounded,
    -                              FT_UInt      n_points );
    +                              FT_Vector*   unrounded );
     
       FT_LOCAL( FT_Error )
       tt_hadvance_adjust( TT_Face  face,
    @@ -431,6 +397,34 @@ FT_BEGIN_HEADER
       FT_LOCAL( void )
       tt_apply_mvar( TT_Face  face );
     
    +
    +  FT_LOCAL( FT_Error )
    +  tt_var_load_item_variation_store( TT_Face          face,
    +                                    FT_ULong         offset,
    +                                    GX_ItemVarStore  itemStore );
    +
    +  FT_LOCAL( FT_Error )
    +  tt_var_load_delta_set_index_mapping( TT_Face            face,
    +                                       FT_ULong           offset,
    +                                       GX_DeltaSetIdxMap  map,
    +                                       GX_ItemVarStore    itemStore,
    +                                       FT_ULong           table_len );
    +
    +  FT_LOCAL( FT_ItemVarDelta )
    +  tt_var_get_item_delta( TT_Face          face,
    +                         GX_ItemVarStore  itemStore,
    +                         FT_UInt          outerIndex,
    +                         FT_UInt          innerIndex );
    +
    +  FT_LOCAL( void )
    +  tt_var_done_item_variation_store( TT_Face          face,
    +                                    GX_ItemVarStore  itemStore );
    +
    +  FT_LOCAL( void )
    +  tt_var_done_delta_set_index_map( TT_Face            face,
    +                                   GX_DeltaSetIdxMap  deltaSetIdxMap );
    +
    +
       FT_LOCAL( FT_Error )
       tt_get_var_blend( TT_Face      face,
                         FT_UInt     *num_coords,
    diff --git a/src/java.desktop/share/native/libfreetype/src/truetype/ttinterp.c b/src/java.desktop/share/native/libfreetype/src/truetype/ttinterp.c
    index e16565c3a57f2..4fcfaa3e4300c 100644
    --- a/src/java.desktop/share/native/libfreetype/src/truetype/ttinterp.c
    +++ b/src/java.desktop/share/native/libfreetype/src/truetype/ttinterp.c
    @@ -4,7 +4,7 @@
      *
      *   TrueType bytecode interpreter (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -1527,9 +1527,8 @@
       static void
       Modify_CVT_Check( TT_ExecContext  exc )
       {
    -    /* TT_RunIns sets origCvt and restores cvt to origCvt when done. */
         if ( exc->iniRange == tt_coderange_glyph &&
    -         exc->cvt == exc->origCvt            )
    +         exc->cvt != exc->glyfCvt            )
         {
           exc->error = Update_Max( exc->memory,
                                    &exc->glyfCvtSize,
    @@ -3115,10 +3114,8 @@
         }
         else
         {
    -      /* TT_RunIns sets origStorage and restores storage to origStorage */
    -      /* when done.                                                     */
           if ( exc->iniRange == tt_coderange_glyph &&
    -           exc->storage == exc->origStorage    )
    +           exc->storage != exc->glyfStorage    )
           {
             FT_ULong  tmp = (FT_ULong)exc->glyfStoreSize;
     
    @@ -6874,7 +6871,7 @@
     
     
       static void
    -  _iup_worker_shift( IUP_Worker  worker,
    +  iup_worker_shift_( IUP_Worker  worker,
                          FT_UInt     p1,
                          FT_UInt     p2,
                          FT_UInt     p )
    @@ -6896,7 +6893,7 @@
     
     
       static void
    -  _iup_worker_interpolate( IUP_Worker  worker,
    +  iup_worker_interpolate_( IUP_Worker  worker,
                                FT_UInt     p1,
                                FT_UInt     p2,
                                FT_UInt     ref1,
    @@ -7090,7 +7087,7 @@
             {
               if ( ( exc->pts.tags[point] & mask ) != 0 )
               {
    -            _iup_worker_interpolate( &V,
    +            iup_worker_interpolate_( &V,
                                          cur_touched + 1,
                                          point - 1,
                                          cur_touched,
    @@ -7102,17 +7099,17 @@
             }
     
             if ( cur_touched == first_touched )
    -          _iup_worker_shift( &V, first_point, end_point, cur_touched );
    +          iup_worker_shift_( &V, first_point, end_point, cur_touched );
             else
             {
    -          _iup_worker_interpolate( &V,
    +          iup_worker_interpolate_( &V,
                                        (FT_UShort)( cur_touched + 1 ),
                                        end_point,
                                        cur_touched,
                                        first_touched );
     
               if ( first_touched > 0 )
    -            _iup_worker_interpolate( &V,
    +            iup_worker_interpolate_( &V,
                                          first_point,
                                          first_touched - 1,
                                          cur_touched,
    @@ -7832,8 +7829,6 @@
           exc->func_move_cvt  = Move_CVT;
         }
     
    -    exc->origCvt     = exc->cvt;
    -    exc->origStorage = exc->storage;
         exc->iniRange    = exc->curRange;
     
         Compute_Funcs( exc );
    @@ -8570,7 +8565,8 @@
     
           /* increment instruction counter and check if we didn't */
           /* run this program for too long (e.g. infinite loops). */
    -      if ( ++ins_counter > TT_CONFIG_OPTION_MAX_RUNNABLE_OPCODES ) {
    +      if ( ++ins_counter > TT_CONFIG_OPTION_MAX_RUNNABLE_OPCODES )
    +      {
             exc->error = FT_THROW( Execution_Too_Long );
             goto LErrorLabel_;
           }
    @@ -8593,9 +8589,6 @@
                     ins_counter,
                     ins_counter == 1 ? "" : "s" ));
     
    -    exc->cvt     = exc->origCvt;
    -    exc->storage = exc->origStorage;
    -
         return FT_Err_Ok;
     
       LErrorCodeOverflow_:
    @@ -8605,9 +8598,6 @@
         if ( exc->error && !exc->instruction_trap )
           FT_TRACE1(( "  The interpreter returned error 0x%x\n", exc->error ));
     
    -    exc->cvt     = exc->origCvt;
    -    exc->storage = exc->origStorage;
    -
         return exc->error;
       }
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/truetype/ttinterp.h b/src/java.desktop/share/native/libfreetype/src/truetype/ttinterp.h
    index 48f618dc9d481..c54c053b29e03 100644
    --- a/src/java.desktop/share/native/libfreetype/src/truetype/ttinterp.h
    +++ b/src/java.desktop/share/native/libfreetype/src/truetype/ttinterp.h
    @@ -4,7 +4,7 @@
      *
      *   TrueType bytecode interpreter (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -193,7 +193,6 @@ FT_BEGIN_HEADER
         FT_Long*           cvt;       /* ! */
         FT_ULong           glyfCvtSize;
         FT_Long*           glyfCvt;   /* cvt working copy for glyph */
    -    FT_Long*           origCvt;
     
         FT_UInt            glyphSize; /* ! glyph instructions buffer size */
         FT_Byte*           glyphIns;  /* ! glyph instructions buffer      */
    @@ -224,7 +223,6 @@ FT_BEGIN_HEADER
         FT_Long*           storage;      /* ! storage area            */
         FT_UShort          glyfStoreSize;
         FT_Long*           glyfStorage;  /* storage working copy for glyph */
    -    FT_Long*           origStorage;
     
         FT_F26Dot6         period;     /* values used for the */
         FT_F26Dot6         phase;      /* `SuperRounding'     */
    diff --git a/src/java.desktop/share/native/libfreetype/src/truetype/ttobjs.c b/src/java.desktop/share/native/libfreetype/src/truetype/ttobjs.c
    index f4f3c69336a3c..4a8873fd8c845 100644
    --- a/src/java.desktop/share/native/libfreetype/src/truetype/ttobjs.c
    +++ b/src/java.desktop/share/native/libfreetype/src/truetype/ttobjs.c
    @@ -4,7 +4,7 @@
      *
      *   Objects manager (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -1004,7 +1004,7 @@
         {
           size->cvt[i] = FT_MulFix( face->cvt[i], scale );
           FT_TRACE6(( "  %3d: %f (%f)\n",
    -                  i, face->cvt[i] / 64.0, size->cvt[i] / 64.0 ));
    +                  i, (double)face->cvt[i] / 64, (double)size->cvt[i] / 64 ));
         }
         FT_TRACE6(( "\n" ));
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/truetype/ttobjs.h b/src/java.desktop/share/native/libfreetype/src/truetype/ttobjs.h
    index 5fa239d43aa3e..bc6fbe7f19600 100644
    --- a/src/java.desktop/share/native/libfreetype/src/truetype/ttobjs.h
    +++ b/src/java.desktop/share/native/libfreetype/src/truetype/ttobjs.h
    @@ -4,7 +4,7 @@
      *
      *   Objects manager (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/truetype/ttpload.c b/src/java.desktop/share/native/libfreetype/src/truetype/ttpload.c
    index 6982c717aba5e..e08bf309e3ca9 100644
    --- a/src/java.desktop/share/native/libfreetype/src/truetype/ttpload.c
    +++ b/src/java.desktop/share/native/libfreetype/src/truetype/ttpload.c
    @@ -4,7 +4,7 @@
      *
      *   TrueType-specific tables loader (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/truetype/ttpload.h b/src/java.desktop/share/native/libfreetype/src/truetype/ttpload.h
    index fa5d96ed35c42..939e02fe4f158 100644
    --- a/src/java.desktop/share/native/libfreetype/src/truetype/ttpload.h
    +++ b/src/java.desktop/share/native/libfreetype/src/truetype/ttpload.h
    @@ -4,7 +4,7 @@
      *
      *   TrueType-specific tables loader (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/truetype/ttsubpix.c b/src/java.desktop/share/native/libfreetype/src/truetype/ttsubpix.c
    index 2438d3a2a29b1..d811beef0df21 100644
    --- a/src/java.desktop/share/native/libfreetype/src/truetype/ttsubpix.c
    +++ b/src/java.desktop/share/native/libfreetype/src/truetype/ttsubpix.c
    @@ -4,7 +4,7 @@
      *
      *   TrueType Subpixel Hinting.
      *
    - * Copyright (C) 2010-2022 by
    + * Copyright (C) 2010-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/truetype/ttsubpix.h b/src/java.desktop/share/native/libfreetype/src/truetype/ttsubpix.h
    index 181f83810ce6d..62af4c272d119 100644
    --- a/src/java.desktop/share/native/libfreetype/src/truetype/ttsubpix.h
    +++ b/src/java.desktop/share/native/libfreetype/src/truetype/ttsubpix.h
    @@ -4,7 +4,7 @@
      *
      *   TrueType Subpixel Hinting.
      *
    - * Copyright (C) 2010-2022 by
    + * Copyright (C) 2010-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/type1/t1afm.c b/src/java.desktop/share/native/libfreetype/src/type1/t1afm.c
    index 6009e9ee2ee38..608582c9a5739 100644
    --- a/src/java.desktop/share/native/libfreetype/src/type1/t1afm.c
    +++ b/src/java.desktop/share/native/libfreetype/src/type1/t1afm.c
    @@ -4,7 +4,7 @@
      *
      *   AFM support for Type 1 fonts (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -178,7 +178,6 @@
         /* temporarily.  If we find no PostScript charmap, then just use    */
         /* the default and hope it is the right one.                        */
         oldcharmap = t1_face->charmap;
    -    charmap    = NULL;
     
         for ( n = 0; n < t1_face->num_charmaps; n++ )
         {
    @@ -186,9 +185,7 @@
           /* check against PostScript pseudo platform */
           if ( charmap->platform_id == 7 )
           {
    -        error = FT_Set_Charmap( t1_face, charmap );
    -        if ( error )
    -          goto Exit;
    +        t1_face->charmap = charmap;
             break;
           }
         }
    @@ -209,10 +206,7 @@
           kp++;
         }
     
    -    if ( oldcharmap )
    -      error = FT_Set_Charmap( t1_face, oldcharmap );
    -    if ( error )
    -      goto Exit;
    +    t1_face->charmap = oldcharmap;
     
         /* now, sort the kern pairs according to their glyph indices */
         ft_qsort( fi->KernPairs, fi->NumKernPair, sizeof ( AFM_KernPairRec ),
    @@ -302,9 +296,14 @@
           t1_face->bbox.xMax = ( fi->FontBBox.xMax + 0xFFFF ) >> 16;
           t1_face->bbox.yMax = ( fi->FontBBox.yMax + 0xFFFF ) >> 16;
     
    -      /* no `U' suffix here to 0x8000! */
    -      t1_face->ascender  = (FT_Short)( ( fi->Ascender  + 0x8000 ) >> 16 );
    -      t1_face->descender = (FT_Short)( ( fi->Descender + 0x8000 ) >> 16 );
    +      /* ascender and descender are optional and could both be zero */
    +      /* check if values are meaningful before overriding defaults  */
    +      if ( fi->Ascender > fi->Descender )
    +      {
    +        /* no `U' suffix here to 0x8000! */
    +        t1_face->ascender  = (FT_Short)( ( fi->Ascender  + 0x8000 ) >> 16 );
    +        t1_face->descender = (FT_Short)( ( fi->Descender + 0x8000 ) >> 16 );
    +      }
     
           if ( fi->NumKernPair )
           {
    diff --git a/src/java.desktop/share/native/libfreetype/src/type1/t1afm.h b/src/java.desktop/share/native/libfreetype/src/type1/t1afm.h
    index 040ed68298134..e0d5aa5a88277 100644
    --- a/src/java.desktop/share/native/libfreetype/src/type1/t1afm.h
    +++ b/src/java.desktop/share/native/libfreetype/src/type1/t1afm.h
    @@ -4,7 +4,7 @@
      *
      *   AFM support for Type 1 fonts (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/type1/t1driver.c b/src/java.desktop/share/native/libfreetype/src/type1/t1driver.c
    index dd31545cf627b..ded3b264e8517 100644
    --- a/src/java.desktop/share/native/libfreetype/src/type1/t1driver.c
    +++ b/src/java.desktop/share/native/libfreetype/src/type1/t1driver.c
    @@ -4,7 +4,7 @@
      *
      *   Type 1 driver interface (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -121,19 +121,30 @@
     #ifndef T1_CONFIG_OPTION_NO_MM_SUPPORT
       static const FT_Service_MultiMastersRec  t1_service_multi_masters =
       {
    -    (FT_Get_MM_Func)             T1_Get_Multi_Master,    /* get_mm              */
    -    (FT_Set_MM_Design_Func)      T1_Set_MM_Design,       /* set_mm_design       */
    -    (FT_Set_MM_Blend_Func)       T1_Set_MM_Blend,        /* set_mm_blend        */
    -    (FT_Get_MM_Blend_Func)       T1_Get_MM_Blend,        /* get_mm_blend        */
    -    (FT_Get_MM_Var_Func)         T1_Get_MM_Var,          /* get_mm_var          */
    -    (FT_Set_Var_Design_Func)     T1_Set_Var_Design,      /* set_var_design      */
    -    (FT_Get_Var_Design_Func)     T1_Get_Var_Design,      /* get_var_design      */
    -    (FT_Set_Instance_Func)       T1_Reset_MM_Blend,      /* set_instance        */
    -    (FT_Set_MM_WeightVector_Func)T1_Set_MM_WeightVector, /* set_mm_weightvector */
    -    (FT_Get_MM_WeightVector_Func)T1_Get_MM_WeightVector, /* get_mm_weightvector */
    -
    -    (FT_Get_Var_Blend_Func)      NULL,                   /* get_var_blend       */
    -    (FT_Done_Blend_Func)         T1_Done_Blend           /* done_blend          */
    +    (FT_Get_MM_Func)        T1_Get_Multi_Master,    /* get_mm                    */
    +    (FT_Set_MM_Design_Func) T1_Set_MM_Design,       /* set_mm_design             */
    +    (FT_Set_MM_Blend_Func)  T1_Set_MM_Blend,        /* set_mm_blend              */
    +    (FT_Get_MM_Blend_Func)  T1_Get_MM_Blend,        /* get_mm_blend              */
    +    (FT_Get_MM_Var_Func)    T1_Get_MM_Var,          /* get_mm_var                */
    +    (FT_Set_Var_Design_Func)T1_Set_Var_Design,      /* set_var_design            */
    +    (FT_Get_Var_Design_Func)T1_Get_Var_Design,      /* get_var_design            */
    +    (FT_Set_Instance_Func)  T1_Reset_MM_Blend,      /* set_instance              */
    +    (FT_Set_MM_WeightVector_Func)
    +                            T1_Set_MM_WeightVector, /* set_mm_weightvector       */
    +    (FT_Get_MM_WeightVector_Func)
    +                            T1_Get_MM_WeightVector, /* get_mm_weightvector       */
    +    (FT_Var_Load_Delta_Set_Idx_Map_Func)
    +                            NULL,                   /* load_delta_set_idx_map    */
    +    (FT_Var_Load_Item_Var_Store_Func)
    +                            NULL,                   /* load_item_variation_store */
    +    (FT_Var_Get_Item_Delta_Func)
    +                            NULL,                   /* get_item_delta            */
    +    (FT_Var_Done_Item_Var_Store_Func)
    +                            NULL,                   /* done_item_variation_store */
    +    (FT_Var_Done_Delta_Set_Idx_Map_Func)
    +                            NULL,                   /* done_delta_set_index_map  */
    +    (FT_Get_Var_Blend_Func) NULL,                   /* get_var_blend             */
    +    (FT_Done_Blend_Func)    T1_Done_Blend           /* done_blend                */
       };
     #endif
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/type1/t1driver.h b/src/java.desktop/share/native/libfreetype/src/type1/t1driver.h
    index 9fe19403343b9..ee7fcf43e01db 100644
    --- a/src/java.desktop/share/native/libfreetype/src/type1/t1driver.h
    +++ b/src/java.desktop/share/native/libfreetype/src/type1/t1driver.h
    @@ -4,7 +4,7 @@
      *
      *   High-level Type 1 driver interface (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/type1/t1errors.h b/src/java.desktop/share/native/libfreetype/src/type1/t1errors.h
    index 1b87c42f18b6b..2fbd1e513f3ec 100644
    --- a/src/java.desktop/share/native/libfreetype/src/type1/t1errors.h
    +++ b/src/java.desktop/share/native/libfreetype/src/type1/t1errors.h
    @@ -4,7 +4,7 @@
      *
      *   Type 1 error codes (specification only).
      *
    - * Copyright (C) 2001-2022 by
    + * Copyright (C) 2001-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/type1/t1gload.c b/src/java.desktop/share/native/libfreetype/src/type1/t1gload.c
    index 540231561c110..a32a4649d6d53 100644
    --- a/src/java.desktop/share/native/libfreetype/src/type1/t1gload.c
    +++ b/src/java.desktop/share/native/libfreetype/src/type1/t1gload.c
    @@ -4,7 +4,7 @@
      *
      *   Type 1 Glyph Loader (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -264,7 +264,7 @@
         }
     
         FT_TRACE6(( "T1_Compute_Max_Advance: max advance: %f\n",
    -                *max_advance / 65536.0 ));
    +                (double)*max_advance / 65536 ));
     
         psaux->t1_decoder_funcs->done( &decoder );
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/type1/t1gload.h b/src/java.desktop/share/native/libfreetype/src/type1/t1gload.h
    index fdb985264f9d9..c06484758a597 100644
    --- a/src/java.desktop/share/native/libfreetype/src/type1/t1gload.h
    +++ b/src/java.desktop/share/native/libfreetype/src/type1/t1gload.h
    @@ -4,7 +4,7 @@
      *
      *   Type 1 Glyph Loader (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/type1/t1load.c b/src/java.desktop/share/native/libfreetype/src/type1/t1load.c
    index 66bebd560f366..5a1afd8d9f5a5 100644
    --- a/src/java.desktop/share/native/libfreetype/src/type1/t1load.c
    +++ b/src/java.desktop/share/native/libfreetype/src/type1/t1load.c
    @@ -4,7 +4,7 @@
      *
      *   Type 1 font loader (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -355,6 +355,10 @@
             mmvar->axis[i].tag = FT_MAKE_TAG( 'w', 'd', 't', 'h' );
           else if ( ft_strcmp( mmvar->axis[i].name, "OpticalSize" ) == 0 )
             mmvar->axis[i].tag = FT_MAKE_TAG( 'o', 'p', 's', 'z' );
    +      else if ( ft_strcmp( mmvar->axis[i].name, "Slant" ) == 0 )
    +        mmvar->axis[i].tag = FT_MAKE_TAG( 's', 'l', 'n', 't' );
    +      else if ( ft_strcmp( mmvar->axis[i].name, "Italic" ) == 0 )
    +        mmvar->axis[i].tag = FT_MAKE_TAG( 'i', 't', 'a', 'l' );
         }
     
         mm_weights_unmap( blend->default_weight_vector,
    diff --git a/src/java.desktop/share/native/libfreetype/src/type1/t1load.h b/src/java.desktop/share/native/libfreetype/src/type1/t1load.h
    index a6d46eb1e4a70..f8511cccf6074 100644
    --- a/src/java.desktop/share/native/libfreetype/src/type1/t1load.h
    +++ b/src/java.desktop/share/native/libfreetype/src/type1/t1load.h
    @@ -4,7 +4,7 @@
      *
      *   Type 1 font loader (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/type1/t1objs.c b/src/java.desktop/share/native/libfreetype/src/type1/t1objs.c
    index 847ae0e64bfdf..1bb2f15f3a8d9 100644
    --- a/src/java.desktop/share/native/libfreetype/src/type1/t1objs.c
    +++ b/src/java.desktop/share/native/libfreetype/src/type1/t1objs.c
    @@ -4,7 +4,7 @@
      *
      *   Type 1 objects manager (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -146,7 +146,9 @@
       FT_LOCAL_DEF( void )
       T1_GlyphSlot_Done( FT_GlyphSlot  slot )
       {
    -    slot->internal->glyph_hints = NULL;
    +    /* `slot->internal` might be NULL in out-of-memory situations. */
    +    if ( slot->internal )
    +      slot->internal->glyph_hints = NULL;
       }
     
     
    diff --git a/src/java.desktop/share/native/libfreetype/src/type1/t1objs.h b/src/java.desktop/share/native/libfreetype/src/type1/t1objs.h
    index e632fb58bdc3c..03847b27e96ac 100644
    --- a/src/java.desktop/share/native/libfreetype/src/type1/t1objs.h
    +++ b/src/java.desktop/share/native/libfreetype/src/type1/t1objs.h
    @@ -4,7 +4,7 @@
      *
      *   Type 1 objects manager (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/type1/t1parse.c b/src/java.desktop/share/native/libfreetype/src/type1/t1parse.c
    index 95dc97d79acaa..6dec6c16c3e70 100644
    --- a/src/java.desktop/share/native/libfreetype/src/type1/t1parse.c
    +++ b/src/java.desktop/share/native/libfreetype/src/type1/t1parse.c
    @@ -4,7 +4,7 @@
      *
      *   Type 1 parser (body).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    @@ -330,50 +330,25 @@
           /* the private dict.  Otherwise, simply overwrite into the base  */
           /* dictionary block in the heap.                                 */
     
    -      /* first of all, look at the `eexec' keyword */
    +      /* First look for the `eexec' keyword. Ensure `eexec' is real -- */
    +      /* it could be in a comment or string (as e.g. in u003043t.gsf   */
    +      /* from ghostscript).                                            */
           FT_Byte*    cur   = parser->base_dict;
           FT_Byte*    limit = cur + parser->base_len;
           FT_Pointer  pos_lf;
           FT_Bool     test_cr;
     
     
    -    Again:
    -      for (;;)
    -      {
    -        if ( cur[0] == 'e'   &&
    -             cur + 9 < limit )      /* 9 = 5 letters for `eexec' + */
    -                                    /* whitespace + 4 chars        */
    -        {
    -          if ( cur[1] == 'e' &&
    -               cur[2] == 'x' &&
    -               cur[3] == 'e' &&
    -               cur[4] == 'c' )
    -            break;
    -        }
    -        cur++;
    -        if ( cur >= limit )
    -        {
    -          FT_ERROR(( "T1_Get_Private_Dict:"
    -                     " could not find `eexec' keyword\n" ));
    -          error = FT_THROW( Invalid_File_Format );
    -          goto Exit;
    -        }
    -      }
    -
    -      /* check whether `eexec' was real -- it could be in a comment */
    -      /* or string (as e.g. in u003043t.gsf from ghostscript)       */
    -
           parser->root.cursor = parser->base_dict;
    -      /* set limit to `eexec' + whitespace + 4 characters */
    -      parser->root.limit  = cur + 10;
    +      parser->root.limit  = parser->base_dict + parser->base_len;
     
           cur   = parser->root.cursor;
           limit = parser->root.limit;
     
           while ( cur < limit )
           {
    -        if ( cur[0] == 'e'   &&
    -             cur + 5 < limit )
    +        /* 9 = 5 letters for `eexec' + whitespace + 4 chars */
    +        if ( cur[0] == 'e' && cur + 9 < limit )
             {
               if ( cur[1] == 'e' &&
                    cur[2] == 'x' &&
    @@ -389,21 +364,9 @@
             cur = parser->root.cursor;
           }
     
    -      /* we haven't found the correct `eexec'; go back and continue */
    -      /* searching                                                  */
    -
    -      cur   = limit;
    -      limit = parser->base_dict + parser->base_len;
    -
    -      if ( cur >= limit )
    -      {
    -        FT_ERROR(( "T1_Get_Private_Dict:"
    -                   " premature end in private dictionary\n" ));
    -        error = FT_THROW( Invalid_File_Format );
    -        goto Exit;
    -      }
    -
    -      goto Again;
    +      FT_ERROR(( "T1_Get_Private_Dict: could not find `eexec' keyword\n" ));
    +      error = FT_THROW( Invalid_File_Format );
    +      goto Exit;
     
           /* now determine where to write the _encrypted_ binary private  */
           /* dictionary.  We overwrite the base dictionary for disk-based */
    diff --git a/src/java.desktop/share/native/libfreetype/src/type1/t1parse.h b/src/java.desktop/share/native/libfreetype/src/type1/t1parse.h
    index d9c7e3b56adfe..0d9a2865df08e 100644
    --- a/src/java.desktop/share/native/libfreetype/src/type1/t1parse.h
    +++ b/src/java.desktop/share/native/libfreetype/src/type1/t1parse.h
    @@ -4,7 +4,7 @@
      *
      *   Type 1 parser (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libfreetype/src/type1/t1tokens.h b/src/java.desktop/share/native/libfreetype/src/type1/t1tokens.h
    index 79080d9e4db0b..40f3609262203 100644
    --- a/src/java.desktop/share/native/libfreetype/src/type1/t1tokens.h
    +++ b/src/java.desktop/share/native/libfreetype/src/type1/t1tokens.h
    @@ -4,7 +4,7 @@
      *
      *   Type 1 tokenizer (specification).
      *
    - * Copyright (C) 1996-2022 by
    + * Copyright (C) 1996-2023 by
      * David Turner, Robert Wilhelm, and Werner Lemberg.
      *
      * This file is part of the FreeType project, and may only be used,
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-color-cbdt-table.hh b/src/java.desktop/share/native/libharfbuzz/OT/Color/CBDT/CBDT.hh
    similarity index 93%
    rename from src/java.desktop/share/native/libharfbuzz/hb-ot-color-cbdt-table.hh
    rename to src/java.desktop/share/native/libharfbuzz/OT/Color/CBDT/CBDT.hh
    index 70d78c74d3b82..11aeda5297f4e 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-color-cbdt-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Color/CBDT/CBDT.hh
    @@ -24,10 +24,11 @@
      * Google Author(s): Seigo Nonaka, Calder Kitagawa
      */
     
    -#ifndef HB_OT_COLOR_CBDT_TABLE_HH
    -#define HB_OT_COLOR_CBDT_TABLE_HH
    +#ifndef OT_COLOR_CBDT_CBDT_HH
    +#define OT_COLOR_CBDT_CBDT_HH
     
    -#include "hb-open-type.hh"
    +#include "../../../hb-open-type.hh"
    +#include "../../../hb-paint.hh"
     
     /*
      * CBLC -- Color Bitmap Location
    @@ -67,7 +68,7 @@ _copy_data_to_cbdt (hb_vector_t *cbdt_prime,
     {
       unsigned int new_len = cbdt_prime->length + length;
       if (unlikely (!cbdt_prime->alloc (new_len))) return false;
    -  memcpy (cbdt_prime->arrayZ + cbdt_prime->length, data, length);
    +  hb_memcpy (cbdt_prime->arrayZ + cbdt_prime->length, data, length);
       cbdt_prime->length = new_len;
       return true;
     }
    @@ -80,12 +81,15 @@ struct SmallGlyphMetrics
         return_trace (c->check_struct (this));
       }
     
    -  void get_extents (hb_font_t *font, hb_glyph_extents_t *extents) const
    +  void get_extents (hb_font_t *font, hb_glyph_extents_t *extents, bool scale) const
       {
    -    extents->x_bearing = font->em_scale_x (bearingX);
    -    extents->y_bearing = font->em_scale_y (bearingY);
    -    extents->width = font->em_scale_x (width);
    -    extents->height = font->em_scale_y (-static_cast(height));
    +    extents->x_bearing = bearingX;
    +    extents->y_bearing = bearingY;
    +    extents->width = width;
    +    extents->height = -static_cast (height);
    +
    +    if (scale)
    +      font->scale_glyph_extents (extents);
       }
     
       HBUINT8       height;
    @@ -307,7 +311,7 @@ struct IndexSubtable
         }
       }
     
    -  bool get_extents (hb_glyph_extents_t *extents HB_UNUSED) const
    +  bool get_extents (hb_glyph_extents_t *extents HB_UNUSED, bool scale HB_UNUSED) const
       {
         switch (u.header.indexFormat)
         {
    @@ -468,13 +472,13 @@ struct IndexSubtableRecord
         if (unlikely (!c->serializer->check_success (records->resize (records->length + 1))))
           return_trace (false);
     
    -    (*records)[records->length - 1].firstGlyphIndex = 1;
    -    (*records)[records->length - 1].lastGlyphIndex = 0;
    +    records->tail ().firstGlyphIndex = 1;
    +    records->tail ().lastGlyphIndex = 0;
         bitmap_size_context->size += IndexSubtableRecord::min_size;
     
         c->serializer->push ();
     
    -    if (unlikely (!add_new_subtable (c, bitmap_size_context, &((*records)[records->length - 1]), lookup, base, start)))
    +    if (unlikely (!add_new_subtable (c, bitmap_size_context, &(records->tail ()), lookup, base, start)))
         {
           c->serializer->pop_discard ();
           c->serializer->revert (snap);
    @@ -504,8 +508,8 @@ struct IndexSubtableRecord
         return num_missing;
       }
     
    -  bool get_extents (hb_glyph_extents_t *extents, const void *base) const
    -  { return (base+offsetToSubtable).get_extents (extents); }
    +  bool get_extents (hb_glyph_extents_t *extents, const void *base, bool scale) const
    +  { return (base+offsetToSubtable).get_extents (extents, scale); }
     
       bool get_image_data (unsigned int  gid,
                            const void   *base,
    @@ -833,7 +837,7 @@ struct CBDT
         }
     
         bool
    -    get_extents (hb_font_t *font, hb_codepoint_t glyph, hb_glyph_extents_t *extents) const
    +    get_extents (hb_font_t *font, hb_codepoint_t glyph, hb_glyph_extents_t *extents, bool scale = true) const
         {
           const void *base;
           const BitmapSizeTable &strike = this->cblc->choose_strike (font);
    @@ -841,7 +845,7 @@ struct CBDT
           if (!subtable_record || !strike.ppemX || !strike.ppemY)
             return false;
     
    -      if (subtable_record->get_extents (extents, base))
    +      if (subtable_record->get_extents (extents, base, scale))
             return true;
     
           unsigned int image_offset = 0, image_length = 0, image_format = 0;
    @@ -858,26 +862,29 @@ struct CBDT
             if (unlikely (image_length < GlyphBitmapDataFormat17::min_size))
               return false;
             auto &glyphFormat17 = StructAtOffset (this->cbdt, image_offset);
    -        glyphFormat17.glyphMetrics.get_extents (font, extents);
    +        glyphFormat17.glyphMetrics.get_extents (font, extents, scale);
             break;
           }
           case 18: {
             if (unlikely (image_length < GlyphBitmapDataFormat18::min_size))
               return false;
             auto &glyphFormat18 = StructAtOffset (this->cbdt, image_offset);
    -        glyphFormat18.glyphMetrics.get_extents (font, extents);
    +        glyphFormat18.glyphMetrics.get_extents (font, extents, scale);
             break;
           }
           default: return false; /* TODO: Support other image formats. */
           }
     
           /* Convert to font units. */
    -      float x_scale = upem / (float) strike.ppemX;
    -      float y_scale = upem / (float) strike.ppemY;
    -      extents->x_bearing = roundf (extents->x_bearing * x_scale);
    -      extents->y_bearing = roundf (extents->y_bearing * y_scale);
    -      extents->width = roundf (extents->width * x_scale);
    -      extents->height = roundf (extents->height * y_scale);
    +      if (scale)
    +      {
    +        float x_scale = upem / (float) strike.ppemX;
    +        float y_scale = upem / (float) strike.ppemY;
    +        extents->x_bearing = roundf (extents->x_bearing * x_scale);
    +        extents->y_bearing = roundf (extents->y_bearing * y_scale);
    +        extents->width = roundf (extents->width * x_scale);
    +        extents->height = roundf (extents->height * y_scale);
    +      }
     
           return true;
         }
    @@ -934,6 +941,32 @@ struct CBDT
     
         bool has_data () const { return cbdt.get_length (); }
     
    +    bool paint_glyph (hb_font_t *font, hb_codepoint_t glyph, hb_paint_funcs_t *funcs, void *data) const
    +    {
    +      hb_glyph_extents_t extents;
    +      hb_glyph_extents_t pixel_extents;
    +      hb_blob_t *blob = reference_png (font, glyph);
    +
    +      if (unlikely (blob == hb_blob_get_empty ()))
    +        return false;
    +
    +      if (unlikely (!hb_font_get_glyph_extents (font, glyph, &extents)))
    +        return false;
    +
    +      if (unlikely (!get_extents (font, glyph, &pixel_extents, false)))
    +        return false;
    +
    +      bool ret = funcs->image (data,
    +                               blob,
    +                               pixel_extents.width, -pixel_extents.height,
    +                               HB_PAINT_IMAGE_FORMAT_PNG,
    +                               font->slant_xy,
    +                               &extents);
    +
    +      hb_blob_destroy (blob);
    +      return ret;
    +    }
    +
         private:
         hb_blob_ptr_t cblc;
         hb_blob_ptr_t cbdt;
    @@ -994,4 +1027,4 @@ struct CBDT_accelerator_t : CBDT::accelerator_t {
     
     } /* namespace OT */
     
    -#endif /* HB_OT_COLOR_CBDT_TABLE_HH */
    +#endif /* OT_COLOR_CBDT_CBDT_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-color-colr-table.hh b/src/java.desktop/share/native/libharfbuzz/OT/Color/COLR/COLR.hh
    similarity index 56%
    rename from src/java.desktop/share/native/libharfbuzz/hb-ot-color-colr-table.hh
    rename to src/java.desktop/share/native/libharfbuzz/OT/Color/COLR/COLR.hh
    index 6b9e734615b61..e7c34a83fd6f6 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-color-colr-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Color/COLR/COLR.hh
    @@ -25,12 +25,14 @@
      * Google Author(s): Calder Kitagawa
      */
     
    -#ifndef HB_OT_COLOR_COLR_TABLE_HH
    -#define HB_OT_COLOR_COLR_TABLE_HH
    +#ifndef OT_COLOR_COLR_COLR_HH
    +#define OT_COLOR_COLR_COLR_HH
     
    -#include "hb-open-type.hh"
    -#include "hb-ot-layout-common.hh"
    -#include "hb-ot-var-common.hh"
    +#include "../../../hb.hh"
    +#include "../../../hb-open-type.hh"
    +#include "../../../hb-ot-var-common.hh"
    +#include "../../../hb-paint.hh"
    +#include "../../../hb-paint-extents.hh"
     
     /*
      * COLR -- Color
    @@ -38,17 +40,81 @@
      */
     #define HB_OT_TAG_COLR HB_TAG('C','O','L','R')
     
    -#ifndef HB_COLRV1_MAX_NESTING_LEVEL
    -#define HB_COLRV1_MAX_NESTING_LEVEL     100
    -#endif
    -
    -#ifndef COLRV1_ENABLE_SUBSETTING
    -#define COLRV1_ENABLE_SUBSETTING 1
    -#endif
    +namespace OT {
    +struct hb_paint_context_t;
    +}
     
     namespace OT {
     
     struct COLR;
    +
    +struct Paint;
    +
    +struct hb_paint_context_t :
    +       hb_dispatch_context_t
    +{
    +  template 
    +  return_t dispatch (const T &obj) { obj.paint_glyph (this); return hb_empty_t (); }
    +  static return_t default_return_value () { return hb_empty_t (); }
    +
    +  const COLR* get_colr_table () const
    +  { return reinterpret_cast (base); }
    +
    +public:
    +  const void *base;
    +  hb_paint_funcs_t *funcs;
    +  void *data;
    +  hb_font_t *font;
    +  unsigned int palette_index;
    +  hb_color_t foreground;
    +  VarStoreInstancer &instancer;
    +  int depth_left = HB_MAX_NESTING_LEVEL;
    +  int edge_count = HB_COLRV1_MAX_EDGE_COUNT;
    +
    +  hb_paint_context_t (const void *base_,
    +                      hb_paint_funcs_t *funcs_,
    +                      void *data_,
    +                      hb_font_t *font_,
    +                      unsigned int palette_,
    +                      hb_color_t foreground_,
    +                      VarStoreInstancer &instancer_) :
    +    base (base_),
    +    funcs (funcs_),
    +    data (data_),
    +    font (font_),
    +    palette_index (palette_),
    +    foreground (foreground_),
    +    instancer (instancer_)
    +  { }
    +
    +  hb_color_t get_color (unsigned int color_index, float alpha, hb_bool_t *is_foreground)
    +  {
    +    hb_color_t color = foreground;
    +
    +    *is_foreground = true;
    +
    +    if (color_index != 0xffff)
    +    {
    +      if (!funcs->custom_palette_color (data, color_index, &color))
    +      {
    +        unsigned int clen = 1;
    +        hb_face_t *face = hb_font_get_face (font);
    +
    +        hb_ot_color_palette_get_colors (face, palette_index, color_index, &clen, &color);
    +      }
    +
    +      *is_foreground = false;
    +    }
    +
    +    return HB_COLOR (hb_color_get_blue (color),
    +                     hb_color_get_green (color),
    +                     hb_color_get_red (color),
    +                     hb_color_get_alpha (color) * alpha);
    +  }
    +
    +  inline void recurse (const Paint &paint);
    +};
    +
     struct hb_colrv1_closure_context_t :
            hb_dispatch_context_t
     {
    @@ -102,7 +168,7 @@ struct hb_colrv1_closure_context_t :
                                    hb_set_t *glyphs_,
                                    hb_set_t *layer_indices_,
                                    hb_set_t *palette_indices_,
    -                               unsigned nesting_level_left_ = HB_COLRV1_MAX_NESTING_LEVEL) :
    +                               unsigned nesting_level_left_ = HB_MAX_NESTING_LEVEL) :
                               base (base_),
                               glyphs (glyphs_),
                               layer_indices (layer_indices_),
    @@ -145,7 +211,7 @@ struct BaseGlyphRecord
       bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
    -    return_trace (likely (c->check_struct (this)));
    +    return_trace (c->check_struct (this));
       }
     
       public:
    @@ -164,6 +230,8 @@ struct BaseGlyphRecord
     template 
     struct Variable
     {
    +  static constexpr bool is_variable = true;
    +
       Variable* copy (hb_serialize_context_t *c) const
       {
         TRACE_SERIALIZE (this);
    @@ -173,10 +241,15 @@ struct Variable
       void closurev1 (hb_colrv1_closure_context_t* c) const
       { value.closurev1 (c); }
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer) const
       {
         TRACE_SUBSET (this);
    -    if (!value.subset (c)) return_trace (false);
    +    if (!value.subset (c, instancer, varIdxBase)) return_trace (false);
    +    if (c->plan->all_axes_pinned)
    +      return_trace (true);
    +
    +    //TODO: update varIdxBase for partial-instancing
         return_trace (c->serializer->embed (varIdxBase));
       }
     
    @@ -186,8 +259,26 @@ struct Variable
         return_trace (c->check_struct (this) && value.sanitize (c));
       }
     
    +  void paint_glyph (hb_paint_context_t *c) const
    +  {
    +    value.paint_glyph (c, varIdxBase);
    +  }
    +
    +  void get_color_stop (hb_paint_context_t *c,
    +                       hb_color_stop_t *stop,
    +                       const VarStoreInstancer &instancer) const
    +  {
    +    value.get_color_stop (c, stop, varIdxBase, instancer);
    +  }
    +
    +  hb_paint_extend_t get_extend () const
    +  {
    +    return value.get_extend ();
    +  }
    +
       protected:
       T      value;
    +  public:
       VarIdx varIdxBase;
       public:
       DEFINE_SIZE_STATIC (4 + T::static_size);
    @@ -196,6 +287,10 @@ struct Variable
     template 
     struct NoVariable
     {
    +  static constexpr bool is_variable = false;
    +
    +  static constexpr uint32_t varIdxBase = VarIdx::NO_VARIATION;
    +
       NoVariable* copy (hb_serialize_context_t *c) const
       {
         TRACE_SERIALIZE (this);
    @@ -205,10 +300,11 @@ struct NoVariable
       void closurev1 (hb_colrv1_closure_context_t* c) const
       { value.closurev1 (c); }
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer) const
       {
         TRACE_SUBSET (this);
    -    return_trace (value.subset (c));
    +    return_trace (value.subset (c, instancer, varIdxBase));
       }
     
       bool sanitize (hb_sanitize_context_t *c) const
    @@ -217,6 +313,23 @@ struct NoVariable
         return_trace (c->check_struct (this) && value.sanitize (c));
       }
     
    +  void paint_glyph (hb_paint_context_t *c) const
    +  {
    +    value.paint_glyph (c, varIdxBase);
    +  }
    +
    +  void get_color_stop (hb_paint_context_t *c,
    +                       hb_color_stop_t *stop,
    +                       const VarStoreInstancer &instancer) const
    +  {
    +    value.get_color_stop (c, stop, VarIdx::NO_VARIATION, instancer);
    +  }
    +
    +  hb_paint_extend_t get_extend () const
    +  {
    +    return value.get_extend ();
    +  }
    +
       T      value;
       public:
       DEFINE_SIZE_STATIC (T::static_size);
    @@ -229,12 +342,21 @@ struct ColorStop
       void closurev1 (hb_colrv1_closure_context_t* c) const
       { c->add_palette_index (paletteIndex); }
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer,
    +               uint32_t varIdxBase) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (*this);
         if (unlikely (!out)) return_trace (false);
    -    return_trace (c->serializer->check_assign (out->paletteIndex, c->plan->colr_palettes->get (paletteIndex),
    +
    +    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
    +    {
    +      out->stopOffset.set_float (stopOffset.to_float(instancer (varIdxBase, 0)));
    +      out->alpha.set_float (alpha.to_float (instancer (varIdxBase, 1)));
    +    }
    +
    +    return_trace (c->serializer->check_assign (out->paletteIndex, c->plan->colr_palettes.get (paletteIndex),
                                                    HB_SERIALIZE_ERROR_INT_OVERFLOW));
       }
     
    @@ -244,6 +366,17 @@ struct ColorStop
         return_trace (c->check_struct (this));
       }
     
    +  void get_color_stop (hb_paint_context_t *c,
    +                       hb_color_stop_t *out,
    +                       uint32_t varIdx,
    +                       const VarStoreInstancer &instancer) const
    +  {
    +    out->offset = stopOffset.to_float(instancer (varIdx, 0));
    +    out->color = c->get_color (paletteIndex,
    +                               alpha.to_float (instancer (varIdx, 1)),
    +                               &out->is_foreground);
    +  }
    +
       F2DOT14       stopOffset;
       HBUINT16      paletteIndex;
       F2DOT14       alpha;
    @@ -271,7 +404,8 @@ struct ColorLine
           stop.closurev1 (c);
       }
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->start_embed (this);
    @@ -283,7 +417,7 @@ struct ColorLine
     
         for (const auto& stop : stops.iter ())
         {
    -      if (!stop.subset (c)) return_trace (false);
    +      if (!stop.subset (c, instancer)) return_trace (false);
         }
         return_trace (true);
       }
    @@ -295,6 +429,52 @@ struct ColorLine
                       stops.sanitize (c));
       }
     
    +  /* get up to count stops from start */
    +  unsigned int
    +  get_color_stops (hb_paint_context_t *c,
    +                   unsigned int start,
    +                   unsigned int *count,
    +                   hb_color_stop_t *color_stops,
    +                   const VarStoreInstancer &instancer) const
    +  {
    +    unsigned int len = stops.len;
    +
    +    if (count && color_stops)
    +    {
    +      unsigned int i;
    +      for (i = 0; i < *count && start + i < len; i++)
    +        stops[start + i].get_color_stop (c, &color_stops[i], instancer);
    +      *count = i;
    +    }
    +
    +    return len;
    +  }
    +
    +  HB_INTERNAL static unsigned int static_get_color_stops (hb_color_line_t *color_line,
    +                                                          void *color_line_data,
    +                                                          unsigned int start,
    +                                                          unsigned int *count,
    +                                                          hb_color_stop_t *color_stops,
    +                                                          void *user_data)
    +  {
    +    const ColorLine *thiz = (const ColorLine *) color_line_data;
    +    hb_paint_context_t *c = (hb_paint_context_t *) user_data;
    +    return thiz->get_color_stops (c, start, count, color_stops, c->instancer);
    +  }
    +
    +  hb_paint_extend_t get_extend () const
    +  {
    +    return (hb_paint_extend_t) (unsigned int) extend;
    +  }
    +
    +  HB_INTERNAL static hb_paint_extend_t static_get_extend (hb_color_line_t *color_line,
    +                                                          void *color_line_data,
    +                                                          void *user_data)
    +  {
    +    const ColorLine *thiz = (const ColorLine *) color_line_data;
    +    return thiz->get_extend ();
    +  }
    +
       Extend        extend;
       Array16Of>     stops;
       public:
    @@ -358,26 +538,57 @@ struct Affine2x3
         return_trace (c->check_struct (this));
       }
     
    -  HBFixed xx;
    -  HBFixed yx;
    -  HBFixed xy;
    -  HBFixed yy;
    -  HBFixed dx;
    -  HBFixed dy;
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer,
    +               uint32_t varIdxBase) const
    +  {
    +    TRACE_SUBSET (this);
    +    auto *out = c->serializer->embed (*this);
    +    if (unlikely (!out)) return_trace (false);
    +    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
    +    {
    +      out->xx.set_float (xx.to_float(instancer (varIdxBase, 0)));
    +      out->yx.set_float (yx.to_float(instancer (varIdxBase, 1)));
    +      out->xy.set_float (xy.to_float(instancer (varIdxBase, 2)));
    +      out->yy.set_float (yy.to_float(instancer (varIdxBase, 3)));
    +      out->dx.set_float (dx.to_float(instancer (varIdxBase, 4)));
    +      out->dy.set_float (dy.to_float(instancer (varIdxBase, 5)));
    +    }
    +    return_trace (true);
    +  }
    +
    +  void paint_glyph (hb_paint_context_t *c, uint32_t varIdxBase) const
    +  {
    +    c->funcs->push_transform (c->data,
    +                              xx.to_float (c->instancer (varIdxBase, 0)),
    +                              yx.to_float (c->instancer (varIdxBase, 1)),
    +                              xy.to_float (c->instancer (varIdxBase, 2)),
    +                              yy.to_float (c->instancer (varIdxBase, 3)),
    +                              dx.to_float (c->instancer (varIdxBase, 4)),
    +                              dy.to_float (c->instancer (varIdxBase, 5)));
    +  }
    +
    +  F16DOT16 xx;
    +  F16DOT16 yx;
    +  F16DOT16 xy;
    +  F16DOT16 yy;
    +  F16DOT16 dx;
    +  F16DOT16 dy;
       public:
    -  DEFINE_SIZE_STATIC (6 * HBFixed::static_size);
    +  DEFINE_SIZE_STATIC (6 * F16DOT16::static_size);
     };
     
     struct PaintColrLayers
     {
       void closurev1 (hb_colrv1_closure_context_t* c) const;
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer HB_UNUSED) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (this);
         if (unlikely (!out)) return_trace (false);
    -    return_trace (c->serializer->check_assign (out->firstLayerIndex, c->plan->colrv1_layers->get (firstLayerIndex),
    +    return_trace (c->serializer->check_assign (out->firstLayerIndex, c->plan->colrv1_layers.get (firstLayerIndex),
                                                    HB_SERIALIZE_ERROR_INT_OVERFLOW));
     
         return_trace (true);
    @@ -389,6 +600,8 @@ struct PaintColrLayers
         return_trace (c->check_struct (this));
       }
     
    +  inline void paint_glyph (hb_paint_context_t *c) const;
    +
       HBUINT8       format; /* format = 1 */
       HBUINT8       numLayers;
       HBUINT32      firstLayerIndex;  /* index into COLRv1::layerList */
    @@ -401,12 +614,21 @@ struct PaintSolid
       void closurev1 (hb_colrv1_closure_context_t* c) const
       { c->add_palette_index (paletteIndex); }
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer,
    +               uint32_t varIdxBase) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (*this);
         if (unlikely (!out)) return_trace (false);
    -    return_trace (c->serializer->check_assign (out->paletteIndex, c->plan->colr_palettes->get (paletteIndex),
    +
    +    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
    +      out->alpha.set_float (alpha.to_float (instancer (varIdxBase, 0)));
    +
    +    if (format == 3 && c->plan->all_axes_pinned)
    +        out->format = 2;
    +
    +    return_trace (c->serializer->check_assign (out->paletteIndex, c->plan->colr_palettes.get (paletteIndex),
                                                    HB_SERIALIZE_ERROR_INT_OVERFLOW));
       }
     
    @@ -416,6 +638,17 @@ struct PaintSolid
         return_trace (c->check_struct (this));
       }
     
    +  void paint_glyph (hb_paint_context_t *c, uint32_t varIdxBase) const
    +  {
    +    hb_bool_t is_foreground;
    +    hb_color_t color;
    +
    +    color = c->get_color (paletteIndex,
    +                          alpha.to_float (c->instancer (varIdxBase, 0)),
    +                          &is_foreground);
    +    c->funcs->color (c->data, is_foreground, color);
    +  }
    +
       HBUINT8       format; /* format = 2(noVar) or 3(Var)*/
       HBUINT16      paletteIndex;
       F2DOT14       alpha;
    @@ -429,13 +662,28 @@ struct PaintLinearGradient
       void closurev1 (hb_colrv1_closure_context_t* c) const
       { (this+colorLine).closurev1 (c); }
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer,
    +               uint32_t varIdxBase) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (this);
         if (unlikely (!out)) return_trace (false);
     
    -    return_trace (out->colorLine.serialize_subset (c, colorLine, this));
    +    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
    +    {
    +      out->x0 = x0 + (int) roundf (instancer (varIdxBase, 0));
    +      out->y0 = y0 + (int) roundf (instancer (varIdxBase, 1));
    +      out->x1 = x1 + (int) roundf (instancer (varIdxBase, 2));
    +      out->y1 = y1 + (int) roundf (instancer (varIdxBase, 3));
    +      out->x2 = x2 + (int) roundf (instancer (varIdxBase, 4));
    +      out->y2 = y2 + (int) roundf (instancer (varIdxBase, 5));
    +    }
    +
    +    if (format == 5 && c->plan->all_axes_pinned)
    +        out->format = 4;
    +
    +    return_trace (out->colorLine.serialize_subset (c, colorLine, this, instancer));
       }
     
       bool sanitize (hb_sanitize_context_t *c) const
    @@ -444,6 +692,23 @@ struct PaintLinearGradient
         return_trace (c->check_struct (this) && colorLine.sanitize (c, this));
       }
     
    +  void paint_glyph (hb_paint_context_t *c, uint32_t varIdxBase) const
    +  {
    +    hb_color_line_t cl = {
    +      (void *) &(this+colorLine),
    +      (this+colorLine).static_get_color_stops, c,
    +      (this+colorLine).static_get_extend, nullptr
    +    };
    +
    +    c->funcs->linear_gradient (c->data, &cl,
    +                               x0 + c->instancer (varIdxBase, 0),
    +                               y0 + c->instancer (varIdxBase, 1),
    +                               x1 + c->instancer (varIdxBase, 2),
    +                               y1 + c->instancer (varIdxBase, 3),
    +                               x2 + c->instancer (varIdxBase, 4),
    +                               y2 + c->instancer (varIdxBase, 5));
    +  }
    +
       HBUINT8                       format; /* format = 4(noVar) or 5 (Var) */
       Offset24To>    colorLine; /* Offset (from beginning of PaintLinearGradient
                                                 * table) to ColorLine subtable. */
    @@ -463,13 +728,28 @@ struct PaintRadialGradient
       void closurev1 (hb_colrv1_closure_context_t* c) const
       { (this+colorLine).closurev1 (c); }
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer,
    +               uint32_t varIdxBase) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (this);
         if (unlikely (!out)) return_trace (false);
     
    -    return_trace (out->colorLine.serialize_subset (c, colorLine, this));
    +    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
    +    {
    +      out->x0 = x0 + (int) roundf (instancer (varIdxBase, 0));
    +      out->y0 = y0 + (int) roundf (instancer (varIdxBase, 1));
    +      out->radius0 = radius0 + (unsigned) roundf (instancer (varIdxBase, 2));
    +      out->x1 = x1 + (int) roundf (instancer (varIdxBase, 3));
    +      out->y1 = y1 + (int) roundf (instancer (varIdxBase, 4));
    +      out->radius1 = radius1 + (unsigned) roundf (instancer (varIdxBase, 5));
    +    }
    +
    +    if (format == 7 && c->plan->all_axes_pinned)
    +        out->format = 6;
    +
    +    return_trace (out->colorLine.serialize_subset (c, colorLine, this, instancer));
       }
     
       bool sanitize (hb_sanitize_context_t *c) const
    @@ -478,6 +758,23 @@ struct PaintRadialGradient
         return_trace (c->check_struct (this) && colorLine.sanitize (c, this));
       }
     
    +  void paint_glyph (hb_paint_context_t *c, uint32_t varIdxBase) const
    +  {
    +    hb_color_line_t cl = {
    +      (void *) &(this+colorLine),
    +      (this+colorLine).static_get_color_stops, c,
    +      (this+colorLine).static_get_extend, nullptr
    +    };
    +
    +    c->funcs->radial_gradient (c->data, &cl,
    +                               x0 + c->instancer (varIdxBase, 0),
    +                               y0 + c->instancer (varIdxBase, 1),
    +                               radius0 + c->instancer (varIdxBase, 2),
    +                               x1 + c->instancer (varIdxBase, 3),
    +                               y1 + c->instancer (varIdxBase, 4),
    +                               radius1 + c->instancer (varIdxBase, 5));
    +  }
    +
       HBUINT8                       format; /* format = 6(noVar) or 7 (Var) */
       Offset24To>    colorLine; /* Offset (from beginning of PaintRadialGradient
                                                 * table) to ColorLine subtable. */
    @@ -497,13 +794,26 @@ struct PaintSweepGradient
       void closurev1 (hb_colrv1_closure_context_t* c) const
       { (this+colorLine).closurev1 (c); }
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer,
    +               uint32_t varIdxBase) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (this);
         if (unlikely (!out)) return_trace (false);
     
    -    return_trace (out->colorLine.serialize_subset (c, colorLine, this));
    +    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
    +    {
    +      out->centerX = centerX + (int) roundf (instancer (varIdxBase, 0));
    +      out->centerY = centerY + (int) roundf (instancer (varIdxBase, 1));
    +      out->startAngle.set_float (startAngle.to_float (instancer (varIdxBase, 2)));
    +      out->endAngle.set_float (endAngle.to_float (instancer (varIdxBase, 3)));
    +    }
    +
    +    if (format == 9 && c->plan->all_axes_pinned)
    +        out->format = 8;
    +
    +    return_trace (out->colorLine.serialize_subset (c, colorLine, this, instancer));
       }
     
       bool sanitize (hb_sanitize_context_t *c) const
    @@ -512,6 +822,21 @@ struct PaintSweepGradient
         return_trace (c->check_struct (this) && colorLine.sanitize (c, this));
       }
     
    +  void paint_glyph (hb_paint_context_t *c, uint32_t varIdxBase) const
    +  {
    +    hb_color_line_t cl = {
    +      (void *) &(this+colorLine),
    +      (this+colorLine).static_get_color_stops, c,
    +      (this+colorLine).static_get_extend, nullptr
    +    };
    +
    +    c->funcs->sweep_gradient (c->data, &cl,
    +                              centerX + c->instancer (varIdxBase, 0),
    +                              centerY + c->instancer (varIdxBase, 1),
    +                              (startAngle.to_float (c->instancer (varIdxBase, 2)) + 1) * HB_PI,
    +                              (endAngle.to_float   (c->instancer (varIdxBase, 3)) + 1) * HB_PI);
    +  }
    +
       HBUINT8                       format; /* format = 8(noVar) or 9 (Var) */
       Offset24To>    colorLine; /* Offset (from beginning of PaintSweepGradient
                                                 * table) to ColorLine subtable. */
    @@ -523,13 +848,13 @@ struct PaintSweepGradient
       DEFINE_SIZE_STATIC (4 + 2 * FWORD::static_size + 2 * F2DOT14::static_size);
     };
     
    -struct Paint;
     // Paint a non-COLR glyph, filled as indicated by paint.
     struct PaintGlyph
     {
       void closurev1 (hb_colrv1_closure_context_t* c) const;
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (this);
    @@ -539,7 +864,7 @@ struct PaintGlyph
                                            HB_SERIALIZE_ERROR_INT_OVERFLOW))
           return_trace (false);
     
    -    return_trace (out->paint.serialize_subset (c, paint, this));
    +    return_trace (out->paint.serialize_subset (c, paint, this, instancer));
       }
     
       bool sanitize (hb_sanitize_context_t *c) const
    @@ -548,6 +873,17 @@ struct PaintGlyph
         return_trace (c->check_struct (this) && paint.sanitize (c, this));
       }
     
    +  void paint_glyph (hb_paint_context_t *c) const
    +  {
    +    c->funcs->push_inverse_root_transform (c->data, c->font);
    +    c->funcs->push_clip_glyph (c->data, gid, c->font);
    +    c->funcs->push_root_transform (c->data, c->font);
    +    c->recurse (this+paint);
    +    c->funcs->pop_transform (c->data);
    +    c->funcs->pop_clip (c->data);
    +    c->funcs->pop_transform (c->data);
    +  }
    +
       HBUINT8               format; /* format = 10 */
       Offset24To     paint;  /* Offset (from beginning of PaintGlyph table) to Paint subtable. */
       HBUINT16              gid;
    @@ -559,7 +895,8 @@ struct PaintColrGlyph
     {
       void closurev1 (hb_colrv1_closure_context_t* c) const;
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer HB_UNUSED) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (this);
    @@ -575,6 +912,8 @@ struct PaintColrGlyph
         return_trace (c->check_struct (this));
       }
     
    +  inline void paint_glyph (hb_paint_context_t *c) const;
    +
       HBUINT8       format; /* format = 11 */
       HBUINT16      gid;
       public:
    @@ -586,13 +925,16 @@ struct PaintTransform
     {
       HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (this);
         if (unlikely (!out)) return_trace (false);
    -    if (!out->transform.serialize_copy (c->serializer, transform, this)) return_trace (false);
    -    return_trace (out->src.serialize_subset (c, src, this));
    +    if (!out->transform.serialize_subset (c, transform, this, instancer)) return_trace (false);
    +    if (format == 13 && c->plan->all_axes_pinned)
    +      out->format = 12;
    +    return_trace (out->src.serialize_subset (c, src, this, instancer));
       }
     
       bool sanitize (hb_sanitize_context_t *c) const
    @@ -603,6 +945,13 @@ struct PaintTransform
                       transform.sanitize (c, this));
       }
     
    +  void paint_glyph (hb_paint_context_t *c) const
    +  {
    +    (this+transform).paint_glyph (c);
    +    c->recurse (this+src);
    +    c->funcs->pop_transform (c->data);
    +  }
    +
       HBUINT8                       format; /* format = 12(noVar) or 13 (Var) */
       Offset24To             src; /* Offset (from beginning of PaintTransform table) to Paint subtable. */
       Offset24To>    transform;
    @@ -614,13 +963,24 @@ struct PaintTranslate
     {
       HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer,
    +               uint32_t varIdxBase) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (this);
         if (unlikely (!out)) return_trace (false);
     
    -    return_trace (out->src.serialize_subset (c, src, this));
    +    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
    +    {
    +      out->dx = dx + (int) roundf (instancer (varIdxBase, 0));
    +      out->dy = dy + (int) roundf (instancer (varIdxBase, 1));
    +    }
    +
    +    if (format == 15 && c->plan->all_axes_pinned)
    +        out->format = 14;
    +
    +    return_trace (out->src.serialize_subset (c, src, this, instancer));
       }
     
       bool sanitize (hb_sanitize_context_t *c) const
    @@ -629,6 +989,16 @@ struct PaintTranslate
         return_trace (c->check_struct (this) && src.sanitize (c, this));
       }
     
    +  void paint_glyph (hb_paint_context_t *c, uint32_t varIdxBase) const
    +  {
    +    float ddx = dx + c->instancer (varIdxBase, 0);
    +    float ddy = dy + c->instancer (varIdxBase, 1);
    +
    +    bool p1 = c->funcs->push_translate (c->data, ddx, ddy);
    +    c->recurse (this+src);
    +    if (p1) c->funcs->pop_transform (c->data);
    +  }
    +
       HBUINT8               format; /* format = 14(noVar) or 15 (Var) */
       Offset24To     src; /* Offset (from beginning of PaintTranslate table) to Paint subtable. */
       FWORD         dx;
    @@ -641,13 +1011,24 @@ struct PaintScale
     {
       HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer,
    +               uint32_t varIdxBase) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (this);
         if (unlikely (!out)) return_trace (false);
     
    -    return_trace (out->src.serialize_subset (c, src, this));
    +    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
    +    {
    +      out->scaleX.set_float (scaleX.to_float (instancer (varIdxBase, 0)));
    +      out->scaleY.set_float (scaleY.to_float (instancer (varIdxBase, 1)));
    +    }
    +
    +    if (format == 17 && c->plan->all_axes_pinned)
    +        out->format = 16;
    +
    +    return_trace (out->src.serialize_subset (c, src, this, instancer));
       }
     
       bool sanitize (hb_sanitize_context_t *c) const
    @@ -656,6 +1037,16 @@ struct PaintScale
         return_trace (c->check_struct (this) && src.sanitize (c, this));
       }
     
    +  void paint_glyph (hb_paint_context_t *c, uint32_t varIdxBase) const
    +  {
    +    float sx = scaleX.to_float (c->instancer (varIdxBase, 0));
    +    float sy = scaleY.to_float (c->instancer (varIdxBase, 1));
    +
    +    bool p1 = c->funcs->push_scale (c->data, sx, sy);
    +    c->recurse (this+src);
    +    if (p1) c->funcs->pop_transform (c->data);
    +  }
    +
       HBUINT8               format; /* format = 16 (noVar) or 17(Var) */
       Offset24To     src; /* Offset (from beginning of PaintScale table) to Paint subtable. */
       F2DOT14               scaleX;
    @@ -668,13 +1059,26 @@ struct PaintScaleAroundCenter
     {
       HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer,
    +               uint32_t varIdxBase) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (this);
         if (unlikely (!out)) return_trace (false);
     
    -    return_trace (out->src.serialize_subset (c, src, this));
    +    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
    +    {
    +      out->scaleX.set_float (scaleX.to_float (instancer (varIdxBase, 0)));
    +      out->scaleY.set_float (scaleY.to_float (instancer (varIdxBase, 1)));
    +      out->centerX = centerX + (int) roundf (instancer (varIdxBase, 2));
    +      out->centerY = centerY + (int) roundf (instancer (varIdxBase, 3));
    +    }
    +
    +    if (format == 19 && c->plan->all_axes_pinned)
    +        out->format = 18;
    +
    +    return_trace (out->src.serialize_subset (c, src, this, instancer));
       }
     
       bool sanitize (hb_sanitize_context_t *c) const
    @@ -683,6 +1087,22 @@ struct PaintScaleAroundCenter
         return_trace (c->check_struct (this) && src.sanitize (c, this));
       }
     
    +  void paint_glyph (hb_paint_context_t *c, uint32_t varIdxBase) const
    +  {
    +    float sx = scaleX.to_float (c->instancer (varIdxBase, 0));
    +    float sy = scaleY.to_float (c->instancer (varIdxBase, 1));
    +    float tCenterX = centerX + c->instancer (varIdxBase, 2);
    +    float tCenterY = centerY + c->instancer (varIdxBase, 3);
    +
    +    bool p1 = c->funcs->push_translate (c->data, +tCenterX, +tCenterY);
    +    bool p2 = c->funcs->push_scale (c->data, sx, sy);
    +    bool p3 = c->funcs->push_translate (c->data, -tCenterX, -tCenterY);
    +    c->recurse (this+src);
    +    if (p3) c->funcs->pop_transform (c->data);
    +    if (p2) c->funcs->pop_transform (c->data);
    +    if (p1) c->funcs->pop_transform (c->data);
    +  }
    +
       HBUINT8               format; /* format = 18 (noVar) or 19(Var) */
       Offset24To     src; /* Offset (from beginning of PaintScaleAroundCenter table) to Paint subtable. */
       F2DOT14       scaleX;
    @@ -697,13 +1117,21 @@ struct PaintScaleUniform
     {
       HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer,
    +               uint32_t varIdxBase) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (this);
         if (unlikely (!out)) return_trace (false);
     
    -    return_trace (out->src.serialize_subset (c, src, this));
    +    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
    +      out->scale.set_float (scale.to_float (instancer (varIdxBase, 0)));
    +
    +    if (format == 21 && c->plan->all_axes_pinned)
    +        out->format = 20;
    +
    +    return_trace (out->src.serialize_subset (c, src, this, instancer));
       }
     
       bool sanitize (hb_sanitize_context_t *c) const
    @@ -712,6 +1140,15 @@ struct PaintScaleUniform
         return_trace (c->check_struct (this) && src.sanitize (c, this));
       }
     
    +  void paint_glyph (hb_paint_context_t *c, uint32_t varIdxBase) const
    +  {
    +    float s = scale.to_float (c->instancer (varIdxBase, 0));
    +
    +    bool p1 = c->funcs->push_scale (c->data, s, s);
    +    c->recurse (this+src);
    +    if (p1) c->funcs->pop_transform (c->data);
    +  }
    +
       HBUINT8               format; /* format = 20 (noVar) or 21(Var) */
       Offset24To     src; /* Offset (from beginning of PaintScaleUniform table) to Paint subtable. */
       F2DOT14               scale;
    @@ -723,13 +1160,25 @@ struct PaintScaleUniformAroundCenter
     {
       HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer,
    +               uint32_t varIdxBase) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (this);
         if (unlikely (!out)) return_trace (false);
     
    -    return_trace (out->src.serialize_subset (c, src, this));
    +    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
    +    {
    +      out->scale.set_float (scale.to_float (instancer (varIdxBase, 0)));
    +      out->centerX = centerX + (int) roundf (instancer (varIdxBase, 1));
    +      out->centerY = centerY + (int) roundf (instancer (varIdxBase, 2));
    +    }
    +
    +    if (format == 23 && c->plan->all_axes_pinned)
    +        out->format = 22;
    +
    +    return_trace (out->src.serialize_subset (c, src, this, instancer));
       }
     
       bool sanitize (hb_sanitize_context_t *c) const
    @@ -738,6 +1187,21 @@ struct PaintScaleUniformAroundCenter
         return_trace (c->check_struct (this) && src.sanitize (c, this));
       }
     
    +  void paint_glyph (hb_paint_context_t *c, uint32_t varIdxBase) const
    +  {
    +    float s = scale.to_float (c->instancer (varIdxBase, 0));
    +    float tCenterX = centerX + c->instancer (varIdxBase, 1);
    +    float tCenterY = centerY + c->instancer (varIdxBase, 2);
    +
    +    bool p1 = c->funcs->push_translate (c->data, +tCenterX, +tCenterY);
    +    bool p2 = c->funcs->push_scale (c->data, s, s);
    +    bool p3 = c->funcs->push_translate (c->data, -tCenterX, -tCenterY);
    +    c->recurse (this+src);
    +    if (p3) c->funcs->pop_transform (c->data);
    +    if (p2) c->funcs->pop_transform (c->data);
    +    if (p1) c->funcs->pop_transform (c->data);
    +  }
    +
       HBUINT8               format; /* format = 22 (noVar) or 23(Var) */
       Offset24To     src; /* Offset (from beginning of PaintScaleUniformAroundCenter table) to Paint subtable. */
       F2DOT14       scale;
    @@ -751,13 +1215,21 @@ struct PaintRotate
     {
       HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer,
    +               uint32_t varIdxBase) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (this);
         if (unlikely (!out)) return_trace (false);
     
    -    return_trace (out->src.serialize_subset (c, src, this));
    +    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
    +      out->angle.set_float (angle.to_float (instancer (varIdxBase, 0)));
    +
    +    if (format == 25 && c->plan->all_axes_pinned)
    +      out->format = 24;
    +
    +    return_trace (out->src.serialize_subset (c, src, this, instancer));
       }
     
       bool sanitize (hb_sanitize_context_t *c) const
    @@ -766,6 +1238,15 @@ struct PaintRotate
         return_trace (c->check_struct (this) && src.sanitize (c, this));
       }
     
    +  void paint_glyph (hb_paint_context_t *c, uint32_t varIdxBase) const
    +  {
    +    float a = angle.to_float (c->instancer (varIdxBase, 0));
    +
    +    bool p1 = c->funcs->push_rotate (c->data, a);
    +    c->recurse (this+src);
    +    if (p1) c->funcs->pop_transform (c->data);
    +  }
    +
       HBUINT8               format; /* format = 24 (noVar) or 25(Var) */
       Offset24To     src; /* Offset (from beginning of PaintRotate table) to Paint subtable. */
       F2DOT14               angle;
    @@ -777,13 +1258,25 @@ struct PaintRotateAroundCenter
     {
       HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer,
    +               uint32_t varIdxBase) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (this);
         if (unlikely (!out)) return_trace (false);
     
    -    return_trace (out->src.serialize_subset (c, src, this));
    +    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
    +    {
    +      out->angle.set_float (angle.to_float (instancer (varIdxBase, 0)));
    +      out->centerX = centerX + (int) roundf (instancer (varIdxBase, 1));
    +      out->centerY = centerY + (int) roundf (instancer (varIdxBase, 2));
    +    }
    +
    +    if (format ==27 && c->plan->all_axes_pinned)
    +        out->format = 26;
    +
    +    return_trace (out->src.serialize_subset (c, src, this, instancer));
       }
     
       bool sanitize (hb_sanitize_context_t *c) const
    @@ -792,6 +1285,21 @@ struct PaintRotateAroundCenter
         return_trace (c->check_struct (this) && src.sanitize (c, this));
       }
     
    +  void paint_glyph (hb_paint_context_t *c, uint32_t varIdxBase) const
    +  {
    +    float a = angle.to_float (c->instancer (varIdxBase, 0));
    +    float tCenterX = centerX + c->instancer (varIdxBase, 1);
    +    float tCenterY = centerY + c->instancer (varIdxBase, 2);
    +
    +    bool p1 = c->funcs->push_translate (c->data, +tCenterX, +tCenterY);
    +    bool p2 = c->funcs->push_rotate (c->data, a);
    +    bool p3 = c->funcs->push_translate (c->data, -tCenterX, -tCenterY);
    +    c->recurse (this+src);
    +    if (p3) c->funcs->pop_transform (c->data);
    +    if (p2) c->funcs->pop_transform (c->data);
    +    if (p1) c->funcs->pop_transform (c->data);
    +  }
    +
       HBUINT8               format; /* format = 26 (noVar) or 27(Var) */
       Offset24To     src; /* Offset (from beginning of PaintRotateAroundCenter table) to Paint subtable. */
       F2DOT14       angle;
    @@ -805,13 +1313,24 @@ struct PaintSkew
     {
       HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer,
    +               uint32_t varIdxBase) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (this);
         if (unlikely (!out)) return_trace (false);
     
    -    return_trace (out->src.serialize_subset (c, src, this));
    +    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
    +    {
    +      out->xSkewAngle.set_float (xSkewAngle.to_float (instancer (varIdxBase, 0)));
    +      out->ySkewAngle.set_float (ySkewAngle.to_float (instancer (varIdxBase, 1)));
    +    }
    +
    +    if (format == 29 && c->plan->all_axes_pinned)
    +        out->format = 28;
    +
    +    return_trace (out->src.serialize_subset (c, src, this, instancer));
       }
     
       bool sanitize (hb_sanitize_context_t *c) const
    @@ -820,6 +1339,16 @@ struct PaintSkew
         return_trace (c->check_struct (this) && src.sanitize (c, this));
       }
     
    +  void paint_glyph (hb_paint_context_t *c, uint32_t varIdxBase) const
    +  {
    +    float sx = xSkewAngle.to_float(c->instancer (varIdxBase, 0));
    +    float sy = ySkewAngle.to_float(c->instancer (varIdxBase, 1));
    +
    +    bool p1 = c->funcs->push_skew (c->data, sx, sy);
    +    c->recurse (this+src);
    +    if (p1) c->funcs->pop_transform (c->data);
    +  }
    +
       HBUINT8               format; /* format = 28(noVar) or 29 (Var) */
       Offset24To     src; /* Offset (from beginning of PaintSkew table) to Paint subtable. */
       F2DOT14               xSkewAngle;
    @@ -832,13 +1361,26 @@ struct PaintSkewAroundCenter
     {
       HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer,
    +               uint32_t varIdxBase) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (this);
         if (unlikely (!out)) return_trace (false);
     
    -    return_trace (out->src.serialize_subset (c, src, this));
    +    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
    +    {
    +      out->xSkewAngle.set_float (xSkewAngle.to_float (instancer (varIdxBase, 0)));
    +      out->ySkewAngle.set_float (ySkewAngle.to_float (instancer (varIdxBase, 1)));
    +      out->centerX = centerX + (int) roundf (instancer (varIdxBase, 2));
    +      out->centerY = centerY + (int) roundf (instancer (varIdxBase, 3));
    +    }
    +
    +    if (format == 31 && c->plan->all_axes_pinned)
    +        out->format = 30;
    +
    +    return_trace (out->src.serialize_subset (c, src, this, instancer));
       }
     
       bool sanitize (hb_sanitize_context_t *c) const
    @@ -847,6 +1389,22 @@ struct PaintSkewAroundCenter
         return_trace (c->check_struct (this) && src.sanitize (c, this));
       }
     
    +  void paint_glyph (hb_paint_context_t *c, uint32_t varIdxBase) const
    +  {
    +    float sx = xSkewAngle.to_float(c->instancer (varIdxBase, 0));
    +    float sy = ySkewAngle.to_float(c->instancer (varIdxBase, 1));
    +    float tCenterX = centerX + c->instancer (varIdxBase, 2);
    +    float tCenterY = centerY + c->instancer (varIdxBase, 3);
    +
    +    bool p1 = c->funcs->push_translate (c->data, +tCenterX, +tCenterY);
    +    bool p2 = c->funcs->push_skew (c->data, sx, sy);
    +    bool p3 = c->funcs->push_translate (c->data, -tCenterX, -tCenterY);
    +    c->recurse (this+src);
    +    if (p3) c->funcs->pop_transform (c->data);
    +    if (p2) c->funcs->pop_transform (c->data);
    +    if (p1) c->funcs->pop_transform (c->data);
    +  }
    +
       HBUINT8               format; /* format = 30(noVar) or 31 (Var) */
       Offset24To     src; /* Offset (from beginning of PaintSkewAroundCenter table) to Paint subtable. */
       F2DOT14       xSkewAngle;
    @@ -861,14 +1419,15 @@ struct PaintComposite
     {
       void closurev1 (hb_colrv1_closure_context_t* c) const;
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->embed (this);
         if (unlikely (!out)) return_trace (false);
     
    -    if (!out->src.serialize_subset (c, src, this)) return_trace (false);
    -    return_trace (out->backdrop.serialize_subset (c, backdrop, this));
    +    if (!out->src.serialize_subset (c, src, this, instancer)) return_trace (false);
    +    return_trace (out->backdrop.serialize_subset (c, backdrop, this, instancer));
       }
     
       bool sanitize (hb_sanitize_context_t *c) const
    @@ -879,6 +1438,14 @@ struct PaintComposite
                       backdrop.sanitize (c, this));
       }
     
    +  void paint_glyph (hb_paint_context_t *c) const
    +  {
    +    c->recurse (this+backdrop);
    +    c->funcs->push_group (c->data);
    +    c->recurse (this+src);
    +    c->funcs->pop_group (c->data, (hb_paint_composite_mode_t) (int) mode);
    +  }
    +
       HBUINT8               format; /* format = 32 */
       Offset24To     src; /* Offset (from beginning of PaintComposite table) to source Paint subtable. */
       CompositeMode         mode;   /* If mode is unrecognized use COMPOSITE_CLEAR */
    @@ -887,6 +1454,11 @@ struct PaintComposite
       DEFINE_SIZE_STATIC (8);
     };
     
    +struct ClipBoxData
    +{
    +  int xMin, yMin, xMax, yMax;
    +};
    +
     struct ClipBoxFormat1
     {
       bool sanitize (hb_sanitize_context_t *c) const
    @@ -895,6 +1467,36 @@ struct ClipBoxFormat1
         return_trace (c->check_struct (this));
       }
     
    +  void get_clip_box (ClipBoxData &clip_box, const VarStoreInstancer &instancer HB_UNUSED) const
    +  {
    +    clip_box.xMin = xMin;
    +    clip_box.yMin = yMin;
    +    clip_box.xMax = xMax;
    +    clip_box.yMax = yMax;
    +  }
    +
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer,
    +               uint32_t varIdxBase) const
    +  {
    +    TRACE_SUBSET (this);
    +    auto *out = c->serializer->embed (*this);
    +    if (unlikely (!out)) return_trace (false);
    +
    +    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
    +    {
    +      out->xMin = xMin + (int) roundf (instancer (varIdxBase, 0));
    +      out->yMin = yMin + (int) roundf (instancer (varIdxBase, 1));
    +      out->xMax = xMax + (int) roundf (instancer (varIdxBase, 2));
    +      out->yMax = yMax + (int) roundf (instancer (varIdxBase, 3));
    +    }
    +
    +    if (format == 2 && c->plan->all_axes_pinned)
    +        out->format = 1;
    +
    +    return_trace (true);
    +  }
    +
       public:
       HBUINT8       format; /* format = 1(noVar) or 2(Var)*/
       FWORD         xMin;
    @@ -905,25 +1507,39 @@ struct ClipBoxFormat1
       DEFINE_SIZE_STATIC (1 + 4 * FWORD::static_size);
     };
     
    -struct ClipBoxFormat2 : Variable {};
    +struct ClipBoxFormat2 : Variable
    +{
    +  void get_clip_box (ClipBoxData &clip_box, const VarStoreInstancer &instancer) const
    +  {
    +    value.get_clip_box(clip_box, instancer);
    +    if (instancer)
    +    {
    +      clip_box.xMin += _hb_roundf (instancer (varIdxBase, 0));
    +      clip_box.yMin += _hb_roundf (instancer (varIdxBase, 1));
    +      clip_box.xMax += _hb_roundf (instancer (varIdxBase, 2));
    +      clip_box.yMax += _hb_roundf (instancer (varIdxBase, 3));
    +    }
    +  }
    +};
     
     struct ClipBox
     {
    -  ClipBox* copy (hb_serialize_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer) const
       {
    -    TRACE_SERIALIZE (this);
    +    TRACE_SUBSET (this);
         switch (u.format) {
    -    case 1: return_trace (reinterpret_cast (c->embed (u.format1)));
    -    case 2: return_trace (reinterpret_cast (c->embed (u.format2)));
    -    default:return_trace (nullptr);
    +    case 1: return_trace (u.format1.subset (c, instancer, VarIdx::NO_VARIATION));
    +    case 2: return_trace (u.format2.subset (c, instancer));
    +    default:return_trace (c->default_return_value ());
         }
       }
     
       template 
       typename context_t::return_t dispatch (context_t *c, Ts&&... ds) const
       {
    +    if (unlikely (!c->may_dispatch (this, &u.format))) return c->no_dispatch_return_value ();
         TRACE_DISPATCH (this, u.format);
    -    if (unlikely (!c->may_dispatch (this, &u.format))) return_trace (c->no_dispatch_return_value ());
         switch (u.format) {
         case 1: return_trace (c->dispatch (u.format1, std::forward (ds)...));
         case 2: return_trace (c->dispatch (u.format2, std::forward (ds)...));
    @@ -931,6 +1547,28 @@ struct ClipBox
         }
       }
     
    +  bool get_extents (hb_glyph_extents_t *extents,
    +                    const VarStoreInstancer &instancer) const
    +  {
    +    ClipBoxData clip_box;
    +    switch (u.format) {
    +    case 1:
    +      u.format1.get_clip_box (clip_box, instancer);
    +      break;
    +    case 2:
    +      u.format2.get_clip_box (clip_box, instancer);
    +      break;
    +    default:
    +      return false;
    +    }
    +
    +    extents->x_bearing = clip_box.xMin;
    +    extents->y_bearing = clip_box.yMax;
    +    extents->width = clip_box.xMax - clip_box.xMin;
    +    extents->height = clip_box.yMin - clip_box.yMax;
    +    return true;
    +  }
    +
       protected:
       union {
       HBUINT8               format;         /* Format identifier */
    @@ -941,13 +1579,18 @@ struct ClipBox
     
     struct ClipRecord
     {
    -  ClipRecord* copy (hb_serialize_context_t *c, const void *base) const
    +  int cmp (hb_codepoint_t g) const
    +  { return g < startGlyphID ? -1 : g <= endGlyphID ? 0 : +1; }
    +
    +  bool subset (hb_subset_context_t *c,
    +               const void *base,
    +               const VarStoreInstancer &instancer) const
       {
    -    TRACE_SERIALIZE (this);
    -    auto *out = c->embed (this);
    -    if (unlikely (!out)) return_trace (nullptr);
    -    if (!out->clipBox.serialize_copy (c, clipBox, base)) return_trace (nullptr);
    -    return_trace (out);
    +    TRACE_SUBSET (this);
    +    auto *out = c->serializer->embed (*this);
    +    if (unlikely (!out)) return_trace (false);
    +
    +    return_trace (out->clipBox.serialize_subset (c, clipBox, base, instancer));
       }
     
       bool sanitize (hb_sanitize_context_t *c, const void *base) const
    @@ -956,6 +1599,13 @@ struct ClipRecord
         return_trace (c->check_struct (this) && clipBox.sanitize (c, base));
       }
     
    +  bool get_extents (hb_glyph_extents_t *extents,
    +                    const void *base,
    +                    const VarStoreInstancer &instancer) const
    +  {
    +    return (base+clipBox).get_extents (extents, instancer);
    +  }
    +
       public:
       HBUINT16              startGlyphID;  // first gid clip applies to
       HBUINT16              endGlyphID;    // last gid clip applies to, inclusive
    @@ -963,10 +1613,12 @@ struct ClipRecord
       public:
       DEFINE_SIZE_STATIC (7);
     };
    +DECLARE_NULL_NAMESPACE_BYTES (OT, ClipRecord);
     
     struct ClipList
     {
    -  unsigned serialize_clip_records (hb_serialize_context_t *c,
    +  unsigned serialize_clip_records (hb_subset_context_t *c,
    +                                   const VarStoreInstancer &instancer,
                                        const hb_set_t& gids,
                                        const hb_map_t& gid_offset_map) const
       {
    @@ -998,7 +1650,7 @@ struct ClipList
           record.endGlyphID = prev_gid;
           record.clipBox = prev_offset;
     
    -      if (!c->copy (record, this)) return_trace (0);
    +      if (!record.subset (c, this, instancer)) return_trace (0);
           count++;
     
           start_gid = _;
    @@ -1012,20 +1664,21 @@ struct ClipList
           record.startGlyphID = start_gid;
           record.endGlyphID = prev_gid;
           record.clipBox = prev_offset;
    -      if (!c->copy (record, this)) return_trace (0);
    +      if (!record.subset (c, this, instancer)) return_trace (0);
           count++;
         }
         return_trace (count);
       }
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->start_embed (*this);
         if (unlikely (!c->serializer->extend_min (out))) return_trace (false);
         if (!c->serializer->check_assign (out->format, format, HB_SERIALIZE_ERROR_INT_OVERFLOW)) return_trace (false);
     
    -    const hb_set_t& glyphset = *c->plan->_glyphset_colred;
    +    const hb_set_t& glyphset = c->plan->_glyphset_colred;
         const hb_map_t &glyph_map = *c->plan->glyph_map;
     
         hb_map_t new_gid_offset_map;
    @@ -1043,7 +1696,7 @@ struct ClipList
           }
         }
     
    -    unsigned count = serialize_clip_records (c->serializer, new_gids, new_gid_offset_map);
    +    unsigned count = serialize_clip_records (c, instancer, new_gids, new_gid_offset_map);
         if (!count) return_trace (false);
         return_trace (c->serializer->check_assign (out->clips.len, count, HB_SERIALIZE_ERROR_INT_OVERFLOW));
       }
    @@ -1051,11 +1704,26 @@ struct ClipList
       bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
    +    // TODO Make a formatted struct!
         return_trace (c->check_struct (this) && clips.sanitize (c, this));
       }
     
    +  bool
    +  get_extents (hb_codepoint_t gid,
    +               hb_glyph_extents_t *extents,
    +               const VarStoreInstancer &instancer) const
    +  {
    +    auto *rec = clips.as_array ().bsearch (gid);
    +    if (rec)
    +    {
    +      rec->get_extents (extents, this, instancer);
    +      return true;
    +    }
    +    return false;
    +  }
    +
       HBUINT8                       format;  // Set to 1.
    -  Array32Of         clips;  // Clip records, sorted by startGlyphID
    +  SortedArray32Of   clips;  // Clip records, sorted by startGlyphID
       public:
       DEFINE_SIZE_ARRAY_SIZED (5, clips);
     };
    @@ -1068,7 +1736,7 @@ struct Paint
       {
         TRACE_SANITIZE (this);
     
    -    if (unlikely (!c->check_start_recursion (HB_COLRV1_MAX_NESTING_LEVEL)))
    +    if (unlikely (!c->check_start_recursion (HB_MAX_NESTING_LEVEL)))
           return_trace (c->no_dispatch_return_value ());
     
         return_trace (c->end_recursion (this->dispatch (c, std::forward (ds)...)));
    @@ -1077,8 +1745,8 @@ struct Paint
       template 
       typename context_t::return_t dispatch (context_t *c, Ts&&... ds) const
       {
    +    if (unlikely (!c->may_dispatch (this, &u.format))) return c->no_dispatch_return_value ();
         TRACE_DISPATCH (this, u.format);
    -    if (unlikely (!c->may_dispatch (this, &u.format))) return_trace (c->no_dispatch_return_value ());
         switch (u.format) {
         case 1: return_trace (c->dispatch (u.paintformat1, std::forward (ds)...));
         case 2: return_trace (c->dispatch (u.paintformat2, std::forward (ds)...));
    @@ -1120,38 +1788,40 @@ struct Paint
       union {
       HBUINT8                                       format;
       PaintColrLayers                               paintformat1;
    -  PaintSolid                                    paintformat2;
    +  NoVariable                        paintformat2;
       Variable                          paintformat3;
    -  PaintLinearGradient               paintformat4;
    +  NoVariable>   paintformat4;
       Variable>       paintformat5;
    -  PaintRadialGradient               paintformat6;
    +  NoVariable>   paintformat6;
       Variable>       paintformat7;
    -  PaintSweepGradient                paintformat8;
    +  NoVariable>    paintformat8;
       Variable>        paintformat9;
       PaintGlyph                                    paintformat10;
       PaintColrGlyph                                paintformat11;
       PaintTransform                    paintformat12;
       PaintTransform                      paintformat13;
    -  PaintTranslate                                paintformat14;
    +  NoVariable                    paintformat14;
       Variable                      paintformat15;
    -  PaintScale                                    paintformat16;
    +  NoVariable                        paintformat16;
       Variable                          paintformat17;
    -  PaintScaleAroundCenter                        paintformat18;
    +  NoVariable            paintformat18;
       Variable              paintformat19;
    -  PaintScaleUniform                             paintformat20;
    +  NoVariable                 paintformat20;
       Variable                   paintformat21;
    -  PaintScaleUniformAroundCenter                 paintformat22;
    +  NoVariable     paintformat22;
       Variable       paintformat23;
    -  PaintRotate                                   paintformat24;
    +  NoVariable                       paintformat24;
       Variable                         paintformat25;
    -  PaintRotateAroundCenter                       paintformat26;
    +  NoVariable           paintformat26;
       Variable             paintformat27;
    -  PaintSkew                                     paintformat28;
    +  NoVariable                         paintformat28;
       Variable                           paintformat29;
    -  PaintSkewAroundCenter                         paintformat30;
    +  NoVariable             paintformat30;
       Variable               paintformat31;
       PaintComposite                                paintformat32;
       } u;
    +  public:
    +  DEFINE_SIZE_MIN (2);
     };
     
     struct BaseGlyphPaintRecord
    @@ -1160,7 +1830,8 @@ struct BaseGlyphPaintRecord
       { return g < glyphId ? -1 : g > glyphId ? 1 : 0; }
     
       bool serialize (hb_serialize_context_t *s, const hb_map_t* glyph_map,
    -                  const void* src_base, hb_subset_context_t *c) const
    +                  const void* src_base, hb_subset_context_t *c,
    +                  const VarStoreInstancer &instancer) const
       {
         TRACE_SERIALIZE (this);
         auto *out = s->embed (this);
    @@ -1169,7 +1840,7 @@ struct BaseGlyphPaintRecord
                               HB_SERIALIZE_ERROR_INT_OVERFLOW))
           return_trace (false);
     
    -    return_trace (out->paint.serialize_subset (c, paint, src_base));
    +    return_trace (out->paint.serialize_subset (c, paint, src_base, instancer));
       }
     
       bool sanitize (hb_sanitize_context_t *c, const void *base) const
    @@ -1188,19 +1859,20 @@ struct BaseGlyphPaintRecord
     
     struct BaseGlyphList : SortedArray32Of
     {
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->start_embed (this);
         if (unlikely (!c->serializer->extend_min (out)))  return_trace (false);
    -    const hb_set_t* glyphset = c->plan->_glyphset_colred;
    +    const hb_set_t* glyphset = &c->plan->_glyphset_colred;
     
         for (const auto& _ : as_array ())
         {
           unsigned gid = _.glyphId;
           if (!glyphset->has (gid)) continue;
     
    -      if (_.serialize (c->serializer, c->plan->glyph_map, this, c)) out->len++;
    +      if (_.serialize (c->serializer, c->plan->glyph_map, this, c, instancer)) out->len++;
           else return_trace (false);
         }
     
    @@ -1219,7 +1891,8 @@ struct LayerList : Array32OfOffset32To
       const Paint& get_paint (unsigned i) const
       { return this+(*this)[i]; }
     
    -  bool subset (hb_subset_context_t *c) const
    +  bool subset (hb_subset_context_t *c,
    +               const VarStoreInstancer &instancer) const
       {
         TRACE_SUBSET (this);
         auto *out = c->serializer->start_embed (this);
    @@ -1230,7 +1903,7 @@ struct LayerList : Array32OfOffset32To
     
         {
           auto *o = out->serialize_append (c->serializer);
    -      if (unlikely (!o) || !o->serialize_subset (c, _.second, this))
    +      if (unlikely (!o) || !o->serialize_subset (c, _.second, this, instancer))
             return_trace (false);
         }
         return_trace (true);
    @@ -1247,7 +1920,14 @@ struct COLR
     {
       static constexpr hb_tag_t tableTag = HB_OT_TAG_COLR;
     
    -  bool has_data () const { return numBaseGlyphs; }
    +  bool has_v0_data () const { return numBaseGlyphs; }
    +  bool has_v1_data () const
    +  {
    +    if (version == 1)
    +      return (this+baseGlyphList).len > 0;
    +
    +    return false;
    +  }
     
       unsigned int get_glyph_layers (hb_codepoint_t       glyph,
                                      unsigned int         start_offset,
    @@ -1356,7 +2036,7 @@ struct COLR
                       (this+baseGlyphsZ).sanitize (c, numBaseGlyphs) &&
                       (this+layersZ).sanitize (c, numLayers) &&
                       (version == 0 ||
    -                   (COLRV1_ENABLE_SUBSETTING && version == 1 &&
    +                   (version == 1 &&
                         baseGlyphList.sanitize (c, this) &&
                         layerList.sanitize (c, this) &&
                         clipList.sanitize (c, this) &&
    @@ -1425,9 +2105,8 @@ struct COLR
       bool subset (hb_subset_context_t *c) const
       {
         TRACE_SUBSET (this);
    -
         const hb_map_t &reverse_glyph_map = *c->plan->reverse_glyph_map;
    -    const hb_set_t& glyphset = *c->plan->_glyphset_colred;
    +    const hb_set_t& glyphset = c->plan->_glyphset_colred;
     
         auto base_it =
         + hb_range (c->plan->num_output_glyphs ())
    @@ -1476,7 +2155,7 @@ struct COLR
                                       if (unlikely (!c->plan->new_gid_for_old_gid (out_layers[i].glyphId, &new_gid)))
                                         return hb_pair_t> (false, out_layers);
                                       out_layers[i].glyphId = new_gid;
    -                                  out_layers[i].colorIdx = c->plan->colr_palettes->get (layers[i].colorIdx);
    +                                  out_layers[i].colorIdx = c->plan->colr_palettes.get (layers[i].colorIdx);
                                     }
     
                                     return hb_pair_t> (true, out_layers);
    @@ -1496,7 +2175,12 @@ struct COLR
     
         auto snap = c->serializer->snapshot ();
         if (!c->serializer->allocate_size (5 * HBUINT32::static_size)) return_trace (false);
    -    if (!colr_prime->baseGlyphList.serialize_subset (c, baseGlyphList, this))
    +
    +    VarStoreInstancer instancer (varStore ? &(this+varStore) : nullptr,
    +                                 varIdxMap ? &(this+varIdxMap) : nullptr,
    +                                 c->plan->normalized_coords.as_array ());
    +
    +    if (!colr_prime->baseGlyphList.serialize_subset (c, baseGlyphList, this, instancer))
         {
           if (c->serializer->in_error ()) return_trace (false);
           //no more COLRv1 glyphs: downgrade to version 0
    @@ -1506,13 +2190,181 @@ struct COLR
     
         if (!colr_prime->serialize_V0 (c->serializer, version, base_it, layer_it)) return_trace (false);
     
    -    colr_prime->layerList.serialize_subset (c, layerList, this);
    -    colr_prime->clipList.serialize_subset (c, clipList, this);
    +    colr_prime->layerList.serialize_subset (c, layerList, this, instancer);
    +    colr_prime->clipList.serialize_subset (c, clipList, this, instancer);
    +    if (!varStore || c->plan->all_axes_pinned)
    +      return_trace (true);
    +
         colr_prime->varIdxMap.serialize_copy (c->serializer, varIdxMap, this);
    -    //TODO: subset varStore once it's implemented in fonttools
    +    colr_prime->varStore.serialize_copy (c->serializer, varStore, this);
         return_trace (true);
       }
     
    +  const Paint *get_base_glyph_paint (hb_codepoint_t glyph) const
    +  {
    +    const BaseGlyphList &baseglyph_paintrecords = this+baseGlyphList;
    +    const BaseGlyphPaintRecord* record = get_base_glyph_paintrecord (glyph);
    +    if (record)
    +    {
    +      const Paint &paint = &baseglyph_paintrecords+record->paint;
    +      return &paint;
    +    }
    +    else
    +      return nullptr;
    +  }
    +
    +#ifndef HB_NO_PAINT
    +  bool
    +  get_extents (hb_font_t *font, hb_codepoint_t glyph, hb_glyph_extents_t *extents) const
    +  {
    +    if (version != 1)
    +      return false;
    +
    +    VarStoreInstancer instancer (&(this+varStore),
    +                                 &(this+varIdxMap),
    +                                 hb_array (font->coords, font->num_coords));
    +
    +    if (get_clip (glyph, extents, instancer))
    +    {
    +      font->scale_glyph_extents (extents);
    +      return true;
    +    }
    +
    +    auto *extents_funcs = hb_paint_extents_get_funcs ();
    +    hb_paint_extents_context_t extents_data;
    +    bool ret = paint_glyph (font, glyph, extents_funcs, &extents_data, 0, HB_COLOR(0,0,0,0));
    +
    +    hb_extents_t e = extents_data.get_extents ();
    +    if (e.is_void ())
    +    {
    +      extents->x_bearing = 0;
    +      extents->y_bearing = 0;
    +      extents->width = 0;
    +      extents->height = 0;
    +    }
    +    else
    +    {
    +      extents->x_bearing = e.xmin;
    +      extents->y_bearing = e.ymax;
    +      extents->width = e.xmax - e.xmin;
    +      extents->height = e.ymin - e.ymax;
    +    }
    +
    +    return ret;
    +  }
    +#endif
    +
    +  bool
    +  has_paint_for_glyph (hb_codepoint_t glyph) const
    +  {
    +    if (version == 1)
    +    {
    +      const Paint *paint = get_base_glyph_paint (glyph);
    +
    +      return paint != nullptr;
    +    }
    +
    +    return false;
    +  }
    +
    +  bool get_clip (hb_codepoint_t glyph,
    +                 hb_glyph_extents_t *extents,
    +                 const VarStoreInstancer instancer) const
    +  {
    +    return (this+clipList).get_extents (glyph,
    +                                        extents,
    +                                        instancer);
    +  }
    +
    +#ifndef HB_NO_PAINT
    +  bool
    +  paint_glyph (hb_font_t *font, hb_codepoint_t glyph, hb_paint_funcs_t *funcs, void *data, unsigned int palette_index, hb_color_t foreground, bool clip = true) const
    +  {
    +    VarStoreInstancer instancer (&(this+varStore),
    +                                 &(this+varIdxMap),
    +                                 hb_array (font->coords, font->num_coords));
    +    hb_paint_context_t c (this, funcs, data, font, palette_index, foreground, instancer);
    +
    +    if (version == 1)
    +    {
    +      const Paint *paint = get_base_glyph_paint (glyph);
    +      if (paint)
    +      {
    +        // COLRv1 glyph
    +
    +        VarStoreInstancer instancer (&(this+varStore),
    +                                     &(this+varIdxMap),
    +                                     hb_array (font->coords, font->num_coords));
    +
    +        bool is_bounded = true;
    +        if (clip)
    +        {
    +          hb_glyph_extents_t extents;
    +          if (get_clip (glyph, &extents, instancer))
    +          {
    +            font->scale_glyph_extents (&extents);
    +            c.funcs->push_clip_rectangle (c.data,
    +                                          extents.x_bearing,
    +                                          extents.y_bearing + extents.height,
    +                                          extents.x_bearing + extents.width,
    +                                          extents.y_bearing);
    +          }
    +          else
    +          {
    +            auto *extents_funcs = hb_paint_extents_get_funcs ();
    +            hb_paint_extents_context_t extents_data;
    +
    +            paint_glyph (font, glyph,
    +                         extents_funcs, &extents_data,
    +                         palette_index, foreground,
    +                         false);
    +
    +            hb_extents_t extents = extents_data.get_extents ();
    +            is_bounded = extents_data.is_bounded ();
    +
    +            c.funcs->push_clip_rectangle (c.data,
    +                                          extents.xmin,
    +                                          extents.ymin,
    +                                          extents.xmax,
    +                                          extents.ymax);
    +          }
    +        }
    +
    +        c.funcs->push_root_transform (c.data, font);
    +
    +        if (is_bounded)
    +          c.recurse (*paint);
    +
    +        c.funcs->pop_transform (c.data);
    +
    +        if (clip)
    +          c.funcs->pop_clip (c.data);
    +
    +        return true;
    +      }
    +    }
    +
    +    const BaseGlyphRecord *record = get_base_glyph_record (glyph);
    +    if (record && ((hb_codepoint_t) record->glyphId == glyph))
    +    {
    +      // COLRv0 glyph
    +      for (const auto &r : (this+layersZ).as_array (numLayers)
    +                           .sub_array (record->firstLayerIdx, record->numLayers))
    +      {
    +        hb_bool_t is_foreground;
    +        hb_color_t color = c.get_color (r.colorIdx, 1., &is_foreground);
    +        c.funcs->push_clip_glyph (c.data, r.glyphId, c.font);
    +        c.funcs->color (c.data, is_foreground, color);
    +        c.funcs->pop_clip (c.data);
    +      }
    +
    +      return true;
    +    }
    +
    +    return false;
    +  }
    +#endif
    +
       protected:
       HBUINT16      version;        /* Table version number (starts at 0). */
       HBUINT16      numBaseGlyphs;  /* Number of Base Glyph Records. */
    @@ -1535,7 +2387,50 @@ struct COLR_accelerator_t : COLR::accelerator_t {
       COLR_accelerator_t (hb_face_t *face) : COLR::accelerator_t (face) {}
     };
     
    -} /* namespace OT */
    +void
    +hb_paint_context_t::recurse (const Paint &paint)
    +{
    +  if (unlikely (depth_left <= 0 || edge_count <= 0)) return;
    +  depth_left--;
    +  edge_count--;
    +  paint.dispatch (this);
    +  depth_left++;
    +}
    +
    +void PaintColrLayers::paint_glyph (hb_paint_context_t *c) const
    +{
    +  const LayerList &paint_offset_lists = c->get_colr_table ()->get_layerList ();
    +  for (unsigned i = firstLayerIndex; i < firstLayerIndex + numLayers; i++)
    +  {
    +    const Paint &paint = paint_offset_lists.get_paint (i);
    +    c->funcs->push_group (c->data);
    +    c->recurse (paint);
    +    c->funcs->pop_group (c->data, HB_PAINT_COMPOSITE_MODE_SRC_OVER);
    +  }
    +}
    +
    +void PaintColrGlyph::paint_glyph (hb_paint_context_t *c) const
    +{
    +  const COLR *colr_table = c->get_colr_table ();
    +  const Paint *paint = colr_table->get_base_glyph_paint (gid);
     
    +  hb_glyph_extents_t extents = {0};
    +  bool has_clip_box = colr_table->get_clip (gid, &extents, c->instancer);
    +
    +  if (has_clip_box)
    +    c->funcs->push_clip_rectangle (c->data,
    +                                   extents.x_bearing,
    +                                   extents.y_bearing + extents.height,
    +                                   extents.x_bearing + extents.width,
    +                                   extents.y_bearing);
    +
    +  if (paint)
    +    c->recurse (*paint);
    +
    +  if (has_clip_box)
    +    c->funcs->pop_clip (c->data);
    +}
    +
    +} /* namespace OT */
     
    -#endif /* HB_OT_COLOR_COLR_TABLE_HH */
    +#endif /* OT_COLOR_COLR_COLR_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-color-colrv1-closure.hh b/src/java.desktop/share/native/libharfbuzz/OT/Color/COLR/colrv1-closure.hh
    similarity index 94%
    rename from src/java.desktop/share/native/libharfbuzz/hb-ot-color-colrv1-closure.hh
    rename to src/java.desktop/share/native/libharfbuzz/OT/Color/COLR/colrv1-closure.hh
    index fbaf2ec26b6b4..705863d4ade5c 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-color-colrv1-closure.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Color/COLR/colrv1-closure.hh
    @@ -24,12 +24,11 @@
      *
      */
     
    -#ifndef HB_OT_COLR_COLRV1_CLOSURE_HH
    -#define HB_OT_COLR_COLRV1_CLOSURE_HH
    +#ifndef OT_COLOR_COLR_COLRV1_CLOSURE_HH
    +#define OT_COLOR_COLR_COLRV1_CLOSURE_HH
     
    -#include "hb-open-type.hh"
    -#include "hb-ot-layout-common.hh"
    -#include "hb-ot-color-colr-table.hh"
    +#include "../../../hb-open-type.hh"
    +#include "COLR.hh"
     
     /*
      * COLR -- Color
    @@ -105,4 +104,4 @@ HB_INTERNAL void PaintComposite::closurev1 (hb_colrv1_closure_context_t* c) cons
     } /* namespace OT */
     
     
    -#endif /* HB_OT_COLR_COLRV1_CLOSURE_HH */
    +#endif /* OT_COLOR_COLR_COLRV1_CLOSURE_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-color-cpal-table.hh b/src/java.desktop/share/native/libharfbuzz/OT/Color/CPAL/CPAL.hh
    similarity index 89%
    rename from src/java.desktop/share/native/libharfbuzz/hb-ot-color-cpal-table.hh
    rename to src/java.desktop/share/native/libharfbuzz/OT/Color/CPAL/CPAL.hh
    index 5558d518190d8..ed8f5957e9615 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-color-cpal-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Color/CPAL/CPAL.hh
    @@ -25,12 +25,12 @@
      * Google Author(s): Sascha Brawer
      */
     
    -#ifndef HB_OT_COLOR_CPAL_TABLE_HH
    -#define HB_OT_COLOR_CPAL_TABLE_HH
    +#ifndef OT_COLOR_CPAL_CPAL_HH
    +#define OT_COLOR_CPAL_CPAL_HH
     
    -#include "hb-open-type.hh"
    -#include "hb-ot-color.h"
    -#include "hb-ot-name.h"
    +#include "../../../hb-open-type.hh"
    +#include "../../../hb-ot-color.h"
    +#include "../../../hb-ot-name.h"
     
     
     /*
    @@ -73,6 +73,30 @@ struct CPALV1Tail
       }
     
       public:
    +  void collect_name_ids (const void *base,
    +                         unsigned palette_count,
    +                         unsigned color_count,
    +                         const hb_map_t *color_index_map,
    +                         hb_set_t *nameids_to_retain /* OUT */) const
    +  {
    +    if (paletteLabelsZ)
    +    {
    +      + (base+paletteLabelsZ).as_array (palette_count)
    +      | hb_sink (nameids_to_retain)
    +      ;
    +    }
    +
    +    if (colorLabelsZ)
    +    {
    +      const hb_array_t colorLabels = (base+colorLabelsZ).as_array (color_count);
    +      for (unsigned i = 0; i < color_count; i++)
    +      {
    +        if (!color_index_map->has (i)) continue;
    +        nameids_to_retain->add (colorLabels[i]);
    +      }
    +    }
    +  }
    +
       bool serialize (hb_serialize_context_t *c,
                       unsigned palette_count,
                       unsigned color_count,
    @@ -95,13 +119,10 @@ struct CPALV1Tail
         if (colorLabelsZ)
         {
           c->push ();
    -      for (const auto _ : colorLabels)
    +      for (unsigned i = 0; i < color_count; i++)
           {
    -        const hb_codepoint_t *v;
    -        if (!color_index_map->has (_, &v)) continue;
    -        NameID new_color_idx;
    -        new_color_idx = *v;
    -        if (!c->copy (new_color_idx))
    +        if (!color_index_map->has (i)) continue;
    +        if (!c->copy (colorLabels[i]))
             {
               c->pop_discard ();
               return_trace (false);
    @@ -189,6 +210,13 @@ struct CPAL
         return numColors;
       }
     
    +  void collect_name_ids (const hb_map_t *color_index_map,
    +                         hb_set_t *nameids_to_retain /* OUT */) const
    +  {
    +    if (version == 1)
    +      v1 ().collect_name_ids (this, numPalettes, numColors, color_index_map, nameids_to_retain);
    +  }
    +
       private:
       const CPALV1Tail& v1 () const
       {
    @@ -239,7 +267,7 @@ struct CPAL
         TRACE_SUBSET (this);
         if (!numPalettes) return_trace (false);
     
    -    const hb_map_t *color_index_map = c->plan->colr_palettes;
    +    const hb_map_t *color_index_map = &c->plan->colr_palettes;
         if (color_index_map->is_empty ()) return_trace (false);
     
         hb_set_t retained_color_indices;
    @@ -319,4 +347,4 @@ struct CPAL
     } /* namespace OT */
     
     
    -#endif /* HB_OT_COLOR_CPAL_TABLE_HH */
    +#endif /* OT_COLOR_CPAL_CPAL_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-color-sbix-table.hh b/src/java.desktop/share/native/libharfbuzz/OT/Color/sbix/sbix.hh
    similarity index 88%
    rename from src/java.desktop/share/native/libharfbuzz/hb-ot-color-sbix-table.hh
    rename to src/java.desktop/share/native/libharfbuzz/OT/Color/sbix/sbix.hh
    index a46b226477020..55d770e928360 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-color-sbix-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Color/sbix/sbix.hh
    @@ -25,11 +25,11 @@
      * Google Author(s): Calder Kitagawa
      */
     
    -#ifndef HB_OT_COLOR_SBIX_TABLE_HH
    -#define HB_OT_COLOR_SBIX_TABLE_HH
    +#ifndef OT_COLOR_SBIX_SBIX_HH
    +#define OT_COLOR_SBIX_SBIX_HH
     
    -#include "hb-open-type.hh"
    -#include "hb-ot-layout-common.hh"
    +#include "../../../hb-open-type.hh"
    +#include "../../../hb-paint.hh"
     
     /*
      * sbix -- Standard Bitmap Graphics
    @@ -213,10 +213,11 @@ struct sbix
     
         bool get_extents (hb_font_t          *font,
                           hb_codepoint_t      glyph,
    -                      hb_glyph_extents_t *extents) const
    +                      hb_glyph_extents_t *extents,
    +                      bool                scale = true) const
         {
           /* We only support PNG right now, and following function checks type. */
    -      return get_png_extents (font, glyph, extents);
    +      return get_png_extents (font, glyph, extents, scale);
         }
     
         hb_blob_t *reference_png (hb_font_t      *font,
    @@ -231,6 +232,37 @@ struct sbix
                                                       num_glyphs, available_ppem);
         }
     
    +    bool paint_glyph (hb_font_t *font, hb_codepoint_t glyph, hb_paint_funcs_t *funcs, void *data) const
    +    {
    +      if (!has_data ())
    +        return false;
    +
    +      int x_offset = 0, y_offset = 0;
    +      unsigned int strike_ppem = 0;
    +      hb_blob_t *blob = reference_png (font, glyph, &x_offset, &y_offset, &strike_ppem);
    +      hb_glyph_extents_t extents;
    +      hb_glyph_extents_t pixel_extents;
    +
    +      if (blob == hb_blob_get_empty ())
    +        return false;
    +
    +      if (!hb_font_get_glyph_extents (font, glyph, &extents))
    +        return false;
    +
    +      if (unlikely (!get_extents (font, glyph, &pixel_extents, false)))
    +        return false;
    +
    +      bool ret = funcs->image (data,
    +                               blob,
    +                               pixel_extents.width, -pixel_extents.height,
    +                               HB_PAINT_IMAGE_FORMAT_PNG,
    +                               font->slant_xy,
    +                               &extents);
    +
    +      hb_blob_destroy (blob);
    +      return ret;
    +    }
    +
         private:
     
         const SBIXStrike &choose_strike (hb_font_t *font) const
    @@ -285,7 +317,8 @@ struct sbix
     
         bool get_png_extents (hb_font_t          *font,
                               hb_codepoint_t      glyph,
    -                          hb_glyph_extents_t *extents) const
    +                          hb_glyph_extents_t *extents,
    +                          bool                scale = true) const
         {
           /* Following code is safe to call even without data.
            * But faster to short-circuit. */
    @@ -310,22 +343,18 @@ struct sbix
           extents->height    = -1 * png.IHDR.height;
     
           /* Convert to font units. */
    -      if (strike_ppem)
    +      if (strike_ppem && scale)
           {
             float scale = font->face->get_upem () / (float) strike_ppem;
    -        extents->x_bearing = font->em_scalef_x (extents->x_bearing * scale);
    -        extents->y_bearing = font->em_scalef_y (extents->y_bearing * scale);
    -        extents->width = font->em_scalef_x (extents->width * scale);
    -        extents->height = font->em_scalef_y (extents->height * scale);
    -      }
    -      else
    -      {
    -        extents->x_bearing = font->em_scale_x (extents->x_bearing);
    -        extents->y_bearing = font->em_scale_y (extents->y_bearing);
    -        extents->width = font->em_scale_x (extents->width);
    -        extents->height = font->em_scale_y (extents->height);
    +        extents->x_bearing = roundf (extents->x_bearing * scale);
    +        extents->y_bearing = roundf (extents->y_bearing * scale);
    +        extents->width = roundf (extents->width * scale);
    +        extents->height = roundf (extents->height * scale);
           }
     
    +      if (scale)
    +        font->scale_glyph_extents (extents);
    +
           hb_blob_destroy (blob);
     
           return strike_ppem;
    @@ -420,4 +449,4 @@ struct sbix_accelerator_t : sbix::accelerator_t {
     
     } /* namespace OT */
     
    -#endif /* HB_OT_COLOR_SBIX_TABLE_HH */
    +#endif /* OT_COLOR_SBIX_SBIX_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-color-svg-table.hh b/src/java.desktop/share/native/libharfbuzz/OT/Color/svg/svg.hh
    similarity index 83%
    rename from src/java.desktop/share/native/libharfbuzz/hb-ot-color-svg-table.hh
    rename to src/java.desktop/share/native/libharfbuzz/OT/Color/svg/svg.hh
    index 8d4dee92878d4..c8ff6ab743e8f 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-color-svg-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Color/svg/svg.hh
    @@ -22,10 +22,12 @@
      * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
      */
     
    -#ifndef HB_OT_COLOR_SVG_TABLE_HH
    -#define HB_OT_COLOR_SVG_TABLE_HH
    +#ifndef OT_COLOR_SVG_SVG_HH
    +#define OT_COLOR_SVG_SVG_HH
     
    -#include "hb-open-type.hh"
    +#include "../../../hb-open-type.hh"
    +#include "../../../hb-blob.hh"
    +#include "../../../hb-paint.hh"
     
     /*
      * SVG -- SVG (Scalable Vector Graphics)
    @@ -91,8 +93,31 @@ struct SVG
     
         bool has_data () const { return table->has_data (); }
     
    +    bool paint_glyph (hb_font_t *font HB_UNUSED, hb_codepoint_t glyph, hb_paint_funcs_t *funcs, void *data) const
    +    {
    +      if (!has_data ())
    +        return false;
    +
    +      hb_blob_t *blob = reference_blob_for_glyph (glyph);
    +
    +      if (blob == hb_blob_get_empty ())
    +        return false;
    +
    +      funcs->image (data,
    +                    blob,
    +                    0, 0,
    +                    HB_PAINT_IMAGE_FORMAT_SVG,
    +                    font->slant_xy,
    +                    nullptr);
    +
    +      hb_blob_destroy (blob);
    +      return true;
    +    }
    +
         private:
         hb_blob_ptr_t table;
    +    public:
    +    DEFINE_SIZE_STATIC (sizeof (hb_blob_ptr_t));
       };
     
       const SVGDocumentIndexEntry &get_glyph_entry (hb_codepoint_t glyph_id) const
    @@ -123,4 +148,4 @@ struct SVG_accelerator_t : SVG::accelerator_t {
     } /* namespace OT */
     
     
    -#endif /* HB_OT_COLOR_SVG_TABLE_HH */
    +#endif /* OT_COLOR_SVG_SVG_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/Common/Coverage.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/Common/Coverage.hh
    new file mode 100644
    index 0000000000000..016a9402e3cd2
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/Common/Coverage.hh
    @@ -0,0 +1,337 @@
    +/*
    + * Copyright © 2007,2008,2009  Red Hat, Inc.
    + * Copyright © 2010,2012  Google, Inc.
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + *
    + * Red Hat Author(s): Behdad Esfahbod
    + * Google Author(s): Behdad Esfahbod, Garret Rieger
    + */
    +
    +#ifndef OT_LAYOUT_COMMON_COVERAGE_HH
    +#define OT_LAYOUT_COMMON_COVERAGE_HH
    +
    +#include "../types.hh"
    +#include "CoverageFormat1.hh"
    +#include "CoverageFormat2.hh"
    +
    +namespace OT {
    +namespace Layout {
    +namespace Common {
    +
    +template
    +static inline void Coverage_serialize (hb_serialize_context_t *c,
    +                                       Iterator it);
    +
    +struct Coverage
    +{
    +
    +  protected:
    +  union {
    +  HBUINT16                      format;         /* Format identifier */
    +  CoverageFormat1_3 format1;
    +  CoverageFormat2_4 format2;
    +#ifndef HB_NO_BEYOND_64K
    +  CoverageFormat1_3format3;
    +  CoverageFormat2_4format4;
    +#endif
    +  } u;
    +  public:
    +  DEFINE_SIZE_UNION (2, format);
    +
    +  bool sanitize (hb_sanitize_context_t *c) const
    +  {
    +    TRACE_SANITIZE (this);
    +    if (!u.format.sanitize (c)) return_trace (false);
    +    switch (u.format)
    +    {
    +    case 1: return_trace (u.format1.sanitize (c));
    +    case 2: return_trace (u.format2.sanitize (c));
    +#ifndef HB_NO_BEYOND_64K
    +    case 3: return_trace (u.format3.sanitize (c));
    +    case 4: return_trace (u.format4.sanitize (c));
    +#endif
    +    default:return_trace (true);
    +    }
    +  }
    +
    +  /* Has interface. */
    +  unsigned operator [] (hb_codepoint_t k) const { return get (k); }
    +  bool has (hb_codepoint_t k) const { return (*this)[k] != NOT_COVERED; }
    +  /* Predicate. */
    +  bool operator () (hb_codepoint_t k) const { return has (k); }
    +
    +  unsigned int get (hb_codepoint_t k) const { return get_coverage (k); }
    +  unsigned int get_coverage (hb_codepoint_t glyph_id) const
    +  {
    +    switch (u.format) {
    +    case 1: return u.format1.get_coverage (glyph_id);
    +    case 2: return u.format2.get_coverage (glyph_id);
    +#ifndef HB_NO_BEYOND_64K
    +    case 3: return u.format3.get_coverage (glyph_id);
    +    case 4: return u.format4.get_coverage (glyph_id);
    +#endif
    +    default:return NOT_COVERED;
    +    }
    +  }
    +
    +  unsigned get_population () const
    +  {
    +    switch (u.format) {
    +    case 1: return u.format1.get_population ();
    +    case 2: return u.format2.get_population ();
    +#ifndef HB_NO_BEYOND_64K
    +    case 3: return u.format3.get_population ();
    +    case 4: return u.format4.get_population ();
    +#endif
    +    default:return NOT_COVERED;
    +    }
    +  }
    +
    +  template 
    +  bool serialize (hb_serialize_context_t *c, Iterator glyphs)
    +  {
    +    TRACE_SERIALIZE (this);
    +    if (unlikely (!c->extend_min (this))) return_trace (false);
    +
    +    unsigned count = 0;
    +    unsigned num_ranges = 0;
    +    hb_codepoint_t last = (hb_codepoint_t) -2;
    +    for (auto g: glyphs)
    +    {
    +      if (last + 1 != g)
    +        num_ranges++;
    +      last = g;
    +      count++;
    +    }
    +    u.format = count <= num_ranges * 3 ? 1 : 2;
    +
    +#ifndef HB_NO_BEYOND_64K
    +    if (count && last > 0xFFFFu)
    +      u.format += 2;
    +#endif
    +
    +    switch (u.format)
    +    {
    +    case 1: return_trace (u.format1.serialize (c, glyphs));
    +    case 2: return_trace (u.format2.serialize (c, glyphs));
    +#ifndef HB_NO_BEYOND_64K
    +    case 3: return_trace (u.format3.serialize (c, glyphs));
    +    case 4: return_trace (u.format4.serialize (c, glyphs));
    +#endif
    +    default:return_trace (false);
    +    }
    +  }
    +
    +  bool subset (hb_subset_context_t *c) const
    +  {
    +    TRACE_SUBSET (this);
    +    auto it =
    +    + iter ()
    +    | hb_take (c->plan->source->get_num_glyphs ())
    +    | hb_filter (c->plan->glyph_map_gsub)
    +    | hb_map_retains_sorting (c->plan->glyph_map_gsub)
    +    ;
    +
    +    // Cache the iterator result as it will be iterated multiple times
    +    // by the serialize code below.
    +    hb_sorted_vector_t glyphs (it);
    +    Coverage_serialize (c->serializer, glyphs.iter ());
    +    return_trace (bool (glyphs));
    +  }
    +
    +  bool intersects (const hb_set_t *glyphs) const
    +  {
    +    switch (u.format)
    +    {
    +    case 1: return u.format1.intersects (glyphs);
    +    case 2: return u.format2.intersects (glyphs);
    +#ifndef HB_NO_BEYOND_64K
    +    case 3: return u.format3.intersects (glyphs);
    +    case 4: return u.format4.intersects (glyphs);
    +#endif
    +    default:return false;
    +    }
    +  }
    +  bool intersects_coverage (const hb_set_t *glyphs, unsigned int index) const
    +  {
    +    switch (u.format)
    +    {
    +    case 1: return u.format1.intersects_coverage (glyphs, index);
    +    case 2: return u.format2.intersects_coverage (glyphs, index);
    +#ifndef HB_NO_BEYOND_64K
    +    case 3: return u.format3.intersects_coverage (glyphs, index);
    +    case 4: return u.format4.intersects_coverage (glyphs, index);
    +#endif
    +    default:return false;
    +    }
    +  }
    +
    +  /* Might return false if array looks unsorted.
    +   * Used for faster rejection of corrupt data. */
    +  template 
    +  bool collect_coverage (set_t *glyphs) const
    +  {
    +    switch (u.format)
    +    {
    +    case 1: return u.format1.collect_coverage (glyphs);
    +    case 2: return u.format2.collect_coverage (glyphs);
    +#ifndef HB_NO_BEYOND_64K
    +    case 3: return u.format3.collect_coverage (glyphs);
    +    case 4: return u.format4.collect_coverage (glyphs);
    +#endif
    +    default:return false;
    +    }
    +  }
    +
    +  template 
    +  void intersect_set (const hb_set_t &glyphs, IterableOut&& intersect_glyphs) const
    +  {
    +    switch (u.format)
    +    {
    +    case 1: return u.format1.intersect_set (glyphs, intersect_glyphs);
    +    case 2: return u.format2.intersect_set (glyphs, intersect_glyphs);
    +#ifndef HB_NO_BEYOND_64K
    +    case 3: return u.format3.intersect_set (glyphs, intersect_glyphs);
    +    case 4: return u.format4.intersect_set (glyphs, intersect_glyphs);
    +#endif
    +    default:return ;
    +    }
    +  }
    +
    +  struct iter_t : hb_iter_with_fallback_t
    +  {
    +    static constexpr bool is_sorted_iterator = true;
    +    iter_t (const Coverage &c_ = Null (Coverage))
    +    {
    +      hb_memset (this, 0, sizeof (*this));
    +      format = c_.u.format;
    +      switch (format)
    +      {
    +      case 1: u.format1.init (c_.u.format1); return;
    +      case 2: u.format2.init (c_.u.format2); return;
    +#ifndef HB_NO_BEYOND_64K
    +      case 3: u.format3.init (c_.u.format3); return;
    +      case 4: u.format4.init (c_.u.format4); return;
    +#endif
    +      default:                               return;
    +      }
    +    }
    +    bool __more__ () const
    +    {
    +      switch (format)
    +      {
    +      case 1: return u.format1.__more__ ();
    +      case 2: return u.format2.__more__ ();
    +#ifndef HB_NO_BEYOND_64K
    +      case 3: return u.format3.__more__ ();
    +      case 4: return u.format4.__more__ ();
    +#endif
    +      default:return false;
    +      }
    +    }
    +    void __next__ ()
    +    {
    +      switch (format)
    +      {
    +      case 1: u.format1.__next__ (); break;
    +      case 2: u.format2.__next__ (); break;
    +#ifndef HB_NO_BEYOND_64K
    +      case 3: u.format3.__next__ (); break;
    +      case 4: u.format4.__next__ (); break;
    +#endif
    +      default:                   break;
    +      }
    +    }
    +    typedef hb_codepoint_t __item_t__;
    +    __item_t__ __item__ () const { return get_glyph (); }
    +
    +    hb_codepoint_t get_glyph () const
    +    {
    +      switch (format)
    +      {
    +      case 1: return u.format1.get_glyph ();
    +      case 2: return u.format2.get_glyph ();
    +#ifndef HB_NO_BEYOND_64K
    +      case 3: return u.format3.get_glyph ();
    +      case 4: return u.format4.get_glyph ();
    +#endif
    +      default:return 0;
    +      }
    +    }
    +    bool operator != (const iter_t& o) const
    +    {
    +      if (unlikely (format != o.format)) return true;
    +      switch (format)
    +      {
    +      case 1: return u.format1 != o.u.format1;
    +      case 2: return u.format2 != o.u.format2;
    +#ifndef HB_NO_BEYOND_64K
    +      case 3: return u.format3 != o.u.format3;
    +      case 4: return u.format4 != o.u.format4;
    +#endif
    +      default:return false;
    +      }
    +    }
    +    iter_t __end__ () const
    +    {
    +      iter_t it = {};
    +      it.format = format;
    +      switch (format)
    +      {
    +      case 1: it.u.format1 = u.format1.__end__ (); break;
    +      case 2: it.u.format2 = u.format2.__end__ (); break;
    +#ifndef HB_NO_BEYOND_64K
    +      case 3: it.u.format3 = u.format3.__end__ (); break;
    +      case 4: it.u.format4 = u.format4.__end__ (); break;
    +#endif
    +      default: break;
    +      }
    +      return it;
    +    }
    +
    +    private:
    +    unsigned int format;
    +    union {
    +#ifndef HB_NO_BEYOND_64K
    +    CoverageFormat2_4::iter_t      format4; /* Put this one first since it's larger; helps shut up compiler. */
    +    CoverageFormat1_3::iter_t      format3;
    +#endif
    +    CoverageFormat2_4::iter_t       format2; /* Put this one first since it's larger; helps shut up compiler. */
    +    CoverageFormat1_3::iter_t       format1;
    +    } u;
    +  };
    +  iter_t iter () const { return iter_t (*this); }
    +};
    +
    +template
    +static inline void
    +Coverage_serialize (hb_serialize_context_t *c,
    +                    Iterator it)
    +{ c->start_embed ()->serialize (c, it); }
    +
    +}
    +}
    +}
    +
    +#endif  // #ifndef OT_LAYOUT_COMMON_COVERAGE_HH
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/Common/CoverageFormat1.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/Common/CoverageFormat1.hh
    new file mode 100644
    index 0000000000000..1fa92df3620a8
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/Common/CoverageFormat1.hh
    @@ -0,0 +1,133 @@
    +/*
    + * Copyright © 2007,2008,2009  Red Hat, Inc.
    + * Copyright © 2010,2012  Google, Inc.
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + *
    + * Red Hat Author(s): Behdad Esfahbod
    + * Google Author(s): Behdad Esfahbod, Garret Rieger
    + */
    +
    +
    +#ifndef OT_LAYOUT_COMMON_COVERAGEFORMAT1_HH
    +#define OT_LAYOUT_COMMON_COVERAGEFORMAT1_HH
    +
    +namespace OT {
    +namespace Layout {
    +namespace Common {
    +
    +#define NOT_COVERED             ((unsigned int) -1)
    +
    +template 
    +struct CoverageFormat1_3
    +{
    +  friend struct Coverage;
    +
    +  protected:
    +  HBUINT16      coverageFormat; /* Format identifier--format = 1 */
    +  SortedArray16Of
    +                glyphArray;     /* Array of GlyphIDs--in numerical order */
    +  public:
    +  DEFINE_SIZE_ARRAY (4, glyphArray);
    +
    +  private:
    +  bool sanitize (hb_sanitize_context_t *c) const
    +  {
    +    TRACE_SANITIZE (this);
    +    return_trace (glyphArray.sanitize (c));
    +  }
    +
    +  unsigned int get_coverage (hb_codepoint_t glyph_id) const
    +  {
    +    unsigned int i;
    +    glyphArray.bfind (glyph_id, &i, HB_NOT_FOUND_STORE, NOT_COVERED);
    +    return i;
    +  }
    +
    +  unsigned get_population () const
    +  {
    +    return glyphArray.len;
    +  }
    +
    +  template 
    +  bool serialize (hb_serialize_context_t *c, Iterator glyphs)
    +  {
    +    TRACE_SERIALIZE (this);
    +    return_trace (glyphArray.serialize (c, glyphs));
    +  }
    +
    +  bool intersects (const hb_set_t *glyphs) const
    +  {
    +    if (glyphArray.len > glyphs->get_population () * hb_bit_storage ((unsigned) glyphArray.len) / 2)
    +    {
    +      for (hb_codepoint_t g = HB_SET_VALUE_INVALID; glyphs->next (&g);)
    +        if (get_coverage (g) != NOT_COVERED)
    +          return true;
    +      return false;
    +    }
    +
    +    for (const auto& g : glyphArray.as_array ())
    +      if (glyphs->has (g))
    +        return true;
    +    return false;
    +  }
    +  bool intersects_coverage (const hb_set_t *glyphs, unsigned int index) const
    +  { return glyphs->has (glyphArray[index]); }
    +
    +  template 
    +  void intersect_set (const hb_set_t &glyphs, IterableOut&& intersect_glyphs) const
    +  {
    +    unsigned count = glyphArray.len;
    +    for (unsigned i = 0; i < count; i++)
    +      if (glyphs.has (glyphArray[i]))
    +        intersect_glyphs << glyphArray[i];
    +  }
    +
    +  template 
    +  bool collect_coverage (set_t *glyphs) const
    +  { return glyphs->add_sorted_array (glyphArray.as_array ()); }
    +
    +  public:
    +  /* Older compilers need this to be public. */
    +  struct iter_t
    +  {
    +    void init (const struct CoverageFormat1_3 &c_) { c = &c_; i = 0; }
    +    bool __more__ () const { return i < c->glyphArray.len; }
    +    void __next__ () { i++; }
    +    hb_codepoint_t get_glyph () const { return c->glyphArray[i]; }
    +    bool operator != (const iter_t& o) const
    +    { return i != o.i; }
    +    iter_t __end__ () const { iter_t it; it.init (*c); it.i = c->glyphArray.len; return it; }
    +
    +    private:
    +    const struct CoverageFormat1_3 *c;
    +    unsigned int i;
    +  };
    +  private:
    +};
    +
    +}
    +}
    +}
    +
    +#endif  // #ifndef OT_LAYOUT_COMMON_COVERAGEFORMAT1_HH
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/Common/CoverageFormat2.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/Common/CoverageFormat2.hh
    new file mode 100644
    index 0000000000000..2650d198f4c97
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/Common/CoverageFormat2.hh
    @@ -0,0 +1,232 @@
    +/*
    + * Copyright © 2007,2008,2009  Red Hat, Inc.
    + * Copyright © 2010,2012  Google, Inc.
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + *
    + * Red Hat Author(s): Behdad Esfahbod
    + * Google Author(s): Behdad Esfahbod, Garret Rieger
    + */
    +
    +#ifndef OT_LAYOUT_COMMON_COVERAGEFORMAT2_HH
    +#define OT_LAYOUT_COMMON_COVERAGEFORMAT2_HH
    +
    +#include "RangeRecord.hh"
    +
    +namespace OT {
    +namespace Layout {
    +namespace Common {
    +
    +template 
    +struct CoverageFormat2_4
    +{
    +  friend struct Coverage;
    +
    +  protected:
    +  HBUINT16      coverageFormat; /* Format identifier--format = 2 */
    +  SortedArray16Of>
    +                rangeRecord;    /* Array of glyph ranges--ordered by
    +                                 * Start GlyphID. rangeCount entries
    +                                 * long */
    +  public:
    +  DEFINE_SIZE_ARRAY (4, rangeRecord);
    +
    +  private:
    +
    +  bool sanitize (hb_sanitize_context_t *c) const
    +  {
    +    TRACE_SANITIZE (this);
    +    return_trace (rangeRecord.sanitize (c));
    +  }
    +
    +  unsigned int get_coverage (hb_codepoint_t glyph_id) const
    +  {
    +    const RangeRecord &range = rangeRecord.bsearch (glyph_id);
    +    return likely (range.first <= range.last)
    +         ? (unsigned int) range.value + (glyph_id - range.first)
    +         : NOT_COVERED;
    +  }
    +
    +  unsigned get_population () const
    +  {
    +    typename Types::large_int ret = 0;
    +    for (const auto &r : rangeRecord)
    +      ret += r.get_population ();
    +    return ret > UINT_MAX ? UINT_MAX : (unsigned) ret;
    +  }
    +
    +  template 
    +  bool serialize (hb_serialize_context_t *c, Iterator glyphs)
    +  {
    +    TRACE_SERIALIZE (this);
    +    if (unlikely (!c->extend_min (this))) return_trace (false);
    +
    +    unsigned num_ranges = 0;
    +    hb_codepoint_t last = (hb_codepoint_t) -2;
    +    for (auto g: glyphs)
    +    {
    +      if (last + 1 != g)
    +        num_ranges++;
    +      last = g;
    +    }
    +
    +    if (unlikely (!rangeRecord.serialize (c, num_ranges))) return_trace (false);
    +    if (!num_ranges) return_trace (true);
    +
    +    unsigned count = 0;
    +    unsigned range = (unsigned) -1;
    +    last = (hb_codepoint_t) -2;
    +    for (auto g: glyphs)
    +    {
    +      if (last + 1 != g)
    +      {
    +        range++;
    +        rangeRecord[range].first = g;
    +        rangeRecord[range].value = count;
    +      }
    +      rangeRecord[range].last = g;
    +      last = g;
    +      count++;
    +    }
    +
    +    return_trace (true);
    +  }
    +
    +  bool intersects (const hb_set_t *glyphs) const
    +  {
    +    if (rangeRecord.len > glyphs->get_population () * hb_bit_storage ((unsigned) rangeRecord.len) / 2)
    +    {
    +      for (hb_codepoint_t g = HB_SET_VALUE_INVALID; glyphs->next (&g);)
    +        if (get_coverage (g) != NOT_COVERED)
    +          return true;
    +      return false;
    +    }
    +
    +    return hb_any (+ hb_iter (rangeRecord)
    +                   | hb_map ([glyphs] (const RangeRecord &range) { return range.intersects (*glyphs); }));
    +  }
    +  bool intersects_coverage (const hb_set_t *glyphs, unsigned int index) const
    +  {
    +    auto *range = rangeRecord.as_array ().bsearch (index);
    +    if (range)
    +      return range->intersects (*glyphs);
    +    return false;
    +  }
    +
    +  template 
    +  void intersect_set (const hb_set_t &glyphs, IterableOut&& intersect_glyphs) const
    +  {
    +    /* Break out of loop for overlapping, broken, tables,
    +     * to avoid fuzzer timouts. */
    +    hb_codepoint_t last = 0;
    +    for (const auto& range : rangeRecord)
    +    {
    +      if (unlikely (range.first < last))
    +        break;
    +      last = range.last;
    +      for (hb_codepoint_t g = range.first - 1;
    +           glyphs.next (&g) && g <= last;)
    +        intersect_glyphs << g;
    +    }
    +  }
    +
    +  template 
    +  bool collect_coverage (set_t *glyphs) const
    +  {
    +    for (const auto& range: rangeRecord)
    +      if (unlikely (!range.collect_coverage (glyphs)))
    +        return false;
    +    return true;
    +  }
    +
    +  public:
    +  /* Older compilers need this to be public. */
    +  struct iter_t
    +  {
    +    void init (const CoverageFormat2_4 &c_)
    +    {
    +      c = &c_;
    +      coverage = 0;
    +      i = 0;
    +      j = c->rangeRecord.len ? c->rangeRecord[0].first : 0;
    +      if (unlikely (c->rangeRecord[0].first > c->rangeRecord[0].last))
    +      {
    +        /* Broken table. Skip. */
    +        i = c->rangeRecord.len;
    +        j = 0;
    +      }
    +    }
    +    bool __more__ () const { return i < c->rangeRecord.len; }
    +    void __next__ ()
    +    {
    +      if (j >= c->rangeRecord[i].last)
    +      {
    +        i++;
    +        if (__more__ ())
    +        {
    +          unsigned int old = coverage;
    +          j = c->rangeRecord[i].first;
    +          coverage = c->rangeRecord[i].value;
    +          if (unlikely (coverage != old + 1))
    +          {
    +            /* Broken table. Skip. Important to avoid DoS.
    +             * Also, our callers depend on coverage being
    +             * consecutive and monotonically increasing,
    +             * ie. iota(). */
    +           i = c->rangeRecord.len;
    +           j = 0;
    +           return;
    +          }
    +        }
    +        else
    +          j = 0;
    +        return;
    +      }
    +      coverage++;
    +      j++;
    +    }
    +    hb_codepoint_t get_glyph () const { return j; }
    +    bool operator != (const iter_t& o) const
    +    { return i != o.i || j != o.j; }
    +    iter_t __end__ () const
    +    {
    +      iter_t it;
    +      it.init (*c);
    +      it.i = c->rangeRecord.len;
    +      it.j = 0;
    +      return it;
    +    }
    +
    +    private:
    +    const struct CoverageFormat2_4 *c;
    +    unsigned int i, coverage;
    +    hb_codepoint_t j;
    +  };
    +  private:
    +};
    +
    +}
    +}
    +}
    +
    +#endif  // #ifndef OT_LAYOUT_COMMON_COVERAGEFORMAT2_HH
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/Common/RangeRecord.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/Common/RangeRecord.hh
    new file mode 100644
    index 0000000000000..a62629fad3439
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/Common/RangeRecord.hh
    @@ -0,0 +1,85 @@
    +/*
    + * Copyright © 2007,2008,2009  Red Hat, Inc.
    + * Copyright © 2010,2012  Google, Inc.
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + *
    + * Red Hat Author(s): Behdad Esfahbod
    + * Google Author(s): Behdad Esfahbod, Garret Rieger
    + */
    +
    +#ifndef OT_LAYOUT_COMMON_RANGERECORD_HH
    +#define OT_LAYOUT_COMMON_RANGERECORD_HH
    +
    +namespace OT {
    +namespace Layout {
    +namespace Common {
    +
    +template 
    +struct RangeRecord
    +{
    +  typename Types::HBGlyphID     first;          /* First GlyphID in the range */
    +  typename Types::HBGlyphID     last;           /* Last GlyphID in the range */
    +  HBUINT16                      value;          /* Value */
    +
    +  DEFINE_SIZE_STATIC (2 + 2 * Types::size);
    +
    +  bool sanitize (hb_sanitize_context_t *c) const
    +  {
    +    TRACE_SANITIZE (this);
    +    return_trace (c->check_struct (this));
    +  }
    +
    +  int cmp (hb_codepoint_t g) const
    +  { return g < first ? -1 : g <= last ? 0 : +1; }
    +
    +  unsigned get_population () const
    +  {
    +    if (unlikely (last < first)) return 0;
    +    return (last - first + 1);
    +  }
    +
    +  bool intersects (const hb_set_t &glyphs) const
    +  { return glyphs.intersects (first, last); }
    +
    +  template 
    +  bool collect_coverage (set_t *glyphs) const
    +  { return glyphs->add_range (first, last); }
    +};
    +
    +}
    +}
    +}
    +
    +// TODO(garretrieger): This was previously implemented using
    +//    DECLARE_NULL_NAMESPACE_BYTES_TEMPLATE1 (OT, RangeRecord, 9);
    +//    but that only works when there is only a single namespace level.
    +//    The macro should probably be fixed so it can work in this situation.
    +extern HB_INTERNAL const unsigned char _hb_Null_OT_RangeRecord[9];
    +template 
    +struct Null> {
    +  static OT::Layout::Common::RangeRecord const & get_null () {
    +    return *reinterpret_cast *> (_hb_Null_OT_RangeRecord);
    +  }
    +};
    +
    +
    +#endif  // #ifndef OT_LAYOUT_COMMON_RANGERECORD_HH
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GDEF/GDEF.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GDEF/GDEF.hh
    new file mode 100644
    index 0000000000000..0cf5753b0e5fd
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GDEF/GDEF.hh
    @@ -0,0 +1,918 @@
    +/*
    + * Copyright © 2007,2008,2009  Red Hat, Inc.
    + * Copyright © 2010,2011,2012  Google, Inc.
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + *
    + * Red Hat Author(s): Behdad Esfahbod
    + * Google Author(s): Behdad Esfahbod
    + */
    +
    +#ifndef OT_LAYOUT_GDEF_GDEF_HH
    +#define OT_LAYOUT_GDEF_GDEF_HH
    +
    +#include "../../../hb-ot-layout-common.hh"
    +
    +#include "../../../hb-font.hh"
    +
    +
    +namespace OT {
    +
    +
    +/*
    + * Attachment List Table
    + */
    +
    +/* Array of contour point indices--in increasing numerical order */
    +struct AttachPoint : Array16Of
    +{
    +  bool subset (hb_subset_context_t *c) const
    +  {
    +    TRACE_SUBSET (this);
    +    auto *out = c->serializer->start_embed (*this);
    +    if (unlikely (!out)) return_trace (false);
    +
    +    return_trace (out->serialize (c->serializer, + iter ()));
    +  }
    +};
    +
    +struct AttachList
    +{
    +  unsigned int get_attach_points (hb_codepoint_t glyph_id,
    +                                  unsigned int start_offset,
    +                                  unsigned int *point_count /* IN/OUT */,
    +                                  unsigned int *point_array /* OUT */) const
    +  {
    +    unsigned int index = (this+coverage).get_coverage (glyph_id);
    +    if (index == NOT_COVERED)
    +    {
    +      if (point_count)
    +        *point_count = 0;
    +      return 0;
    +    }
    +
    +    const AttachPoint &points = this+attachPoint[index];
    +
    +    if (point_count)
    +    {
    +      + points.as_array ().sub_array (start_offset, point_count)
    +      | hb_sink (hb_array (point_array, *point_count))
    +      ;
    +    }
    +
    +    return points.len;
    +  }
    +
    +  bool subset (hb_subset_context_t *c) const
    +  {
    +    TRACE_SUBSET (this);
    +    const hb_set_t &glyphset = *c->plan->glyphset_gsub ();
    +    const hb_map_t &glyph_map = *c->plan->glyph_map;
    +
    +    auto *out = c->serializer->start_embed (*this);
    +    if (unlikely (!c->serializer->extend_min (out))) return_trace (false);
    +
    +    hb_sorted_vector_t new_coverage;
    +    + hb_zip (this+coverage, attachPoint)
    +    | hb_filter (glyphset, hb_first)
    +    | hb_filter (subset_offset_array (c, out->attachPoint, this), hb_second)
    +    | hb_map (hb_first)
    +    | hb_map (glyph_map)
    +    | hb_sink (new_coverage)
    +    ;
    +    out->coverage.serialize_serialize (c->serializer, new_coverage.iter ());
    +    return_trace (bool (new_coverage));
    +  }
    +
    +  bool sanitize (hb_sanitize_context_t *c) const
    +  {
    +    TRACE_SANITIZE (this);
    +    return_trace (coverage.sanitize (c, this) && attachPoint.sanitize (c, this));
    +  }
    +
    +  protected:
    +  Offset16To
    +                coverage;               /* Offset to Coverage table -- from
    +                                         * beginning of AttachList table */
    +  Array16OfOffset16To
    +                attachPoint;            /* Array of AttachPoint tables
    +                                         * in Coverage Index order */
    +  public:
    +  DEFINE_SIZE_ARRAY (4, attachPoint);
    +};
    +
    +/*
    + * Ligature Caret Table
    + */
    +
    +struct CaretValueFormat1
    +{
    +  friend struct CaretValue;
    +  bool subset (hb_subset_context_t *c) const
    +  {
    +    TRACE_SUBSET (this);
    +    auto *out = c->serializer->embed (this);
    +    if (unlikely (!out)) return_trace (false);
    +    return_trace (true);
    +  }
    +
    +  private:
    +  hb_position_t get_caret_value (hb_font_t *font, hb_direction_t direction) const
    +  {
    +    return HB_DIRECTION_IS_HORIZONTAL (direction) ? font->em_scale_x (coordinate) : font->em_scale_y (coordinate);
    +  }
    +
    +  bool sanitize (hb_sanitize_context_t *c) const
    +  {
    +    TRACE_SANITIZE (this);
    +    return_trace (c->check_struct (this));
    +  }
    +
    +  protected:
    +  HBUINT16      caretValueFormat;       /* Format identifier--format = 1 */
    +  FWORD         coordinate;             /* X or Y value, in design units */
    +  public:
    +  DEFINE_SIZE_STATIC (4);
    +};
    +
    +struct CaretValueFormat2
    +{
    +  friend struct CaretValue;
    +  bool subset (hb_subset_context_t *c) const
    +  {
    +    TRACE_SUBSET (this);
    +    auto *out = c->serializer->embed (this);
    +    if (unlikely (!out)) return_trace (false);
    +    return_trace (true);
    +  }
    +
    +  private:
    +  hb_position_t get_caret_value (hb_font_t *font, hb_direction_t direction, hb_codepoint_t glyph_id) const
    +  {
    +    hb_position_t x, y;
    +    font->get_glyph_contour_point_for_origin (glyph_id, caretValuePoint, direction, &x, &y);
    +    return HB_DIRECTION_IS_HORIZONTAL (direction) ? x : y;
    +  }
    +
    +  bool sanitize (hb_sanitize_context_t *c) const
    +  {
    +    TRACE_SANITIZE (this);
    +    return_trace (c->check_struct (this));
    +  }
    +
    +  protected:
    +  HBUINT16      caretValueFormat;       /* Format identifier--format = 2 */
    +  HBUINT16      caretValuePoint;        /* Contour point index on glyph */
    +  public:
    +  DEFINE_SIZE_STATIC (4);
    +};
    +
    +struct CaretValueFormat3
    +{
    +  friend struct CaretValue;
    +
    +  hb_position_t get_caret_value (hb_font_t *font, hb_direction_t direction,
    +                                 const VariationStore &var_store) const
    +  {
    +    return HB_DIRECTION_IS_HORIZONTAL (direction) ?
    +           font->em_scale_x (coordinate) + (this+deviceTable).get_x_delta (font, var_store) :
    +           font->em_scale_y (coordinate) + (this+deviceTable).get_y_delta (font, var_store);
    +  }
    +
    +  bool subset (hb_subset_context_t *c) const
    +  {
    +    TRACE_SUBSET (this);
    +    auto *out = c->serializer->start_embed (*this);
    +    if (unlikely (!out)) return_trace (false);
    +    if (!c->serializer->embed (caretValueFormat)) return_trace (false);
    +    if (!c->serializer->embed (coordinate)) return_trace (false);
    +
    +    unsigned varidx = (this+deviceTable).get_variation_index ();
    +    if (c->plan->layout_variation_idx_delta_map.has (varidx))
    +    {
    +      int delta = hb_second (c->plan->layout_variation_idx_delta_map.get (varidx));
    +      if (delta != 0)
    +      {
    +        if (!c->serializer->check_assign (out->coordinate, coordinate + delta, HB_SERIALIZE_ERROR_INT_OVERFLOW))
    +          return_trace (false);
    +      }
    +    }
    +
    +    if (c->plan->all_axes_pinned)
    +      return_trace (c->serializer->check_assign (out->caretValueFormat, 1, HB_SERIALIZE_ERROR_INT_OVERFLOW));
    +
    +    if (!c->serializer->embed (deviceTable))
    +      return_trace (false);
    +
    +    return_trace (out->deviceTable.serialize_copy (c->serializer, deviceTable, this, c->serializer->to_bias (out),
    +                                                   hb_serialize_context_t::Head, &c->plan->layout_variation_idx_delta_map));
    +  }
    +
    +  void collect_variation_indices (hb_collect_variation_indices_context_t *c) const
    +  { (this+deviceTable).collect_variation_indices (c); }
    +
    +  bool sanitize (hb_sanitize_context_t *c) const
    +  {
    +    TRACE_SANITIZE (this);
    +    return_trace (c->check_struct (this) && deviceTable.sanitize (c, this));
    +  }
    +
    +  protected:
    +  HBUINT16      caretValueFormat;       /* Format identifier--format = 3 */
    +  FWORD         coordinate;             /* X or Y value, in design units */
    +  Offset16To
    +                deviceTable;            /* Offset to Device table for X or Y
    +                                         * value--from beginning of CaretValue
    +                                         * table */
    +  public:
    +  DEFINE_SIZE_STATIC (6);
    +};
    +
    +struct CaretValue
    +{
    +  hb_position_t get_caret_value (hb_font_t *font,
    +                                 hb_direction_t direction,
    +                                 hb_codepoint_t glyph_id,
    +                                 const VariationStore &var_store) const
    +  {
    +    switch (u.format) {
    +    case 1: return u.format1.get_caret_value (font, direction);
    +    case 2: return u.format2.get_caret_value (font, direction, glyph_id);
    +    case 3: return u.format3.get_caret_value (font, direction, var_store);
    +    default:return 0;
    +    }
    +  }
    +
    +  template 
    +  typename context_t::return_t dispatch (context_t *c, Ts&&... ds) const
    +  {
    +    if (unlikely (!c->may_dispatch (this, &u.format))) return c->no_dispatch_return_value ();
    +    TRACE_DISPATCH (this, u.format);
    +    switch (u.format) {
    +    case 1: return_trace (c->dispatch (u.format1, std::forward (ds)...));
    +    case 2: return_trace (c->dispatch (u.format2, std::forward (ds)...));
    +    case 3: return_trace (c->dispatch (u.format3, std::forward (ds)...));
    +    default:return_trace (c->default_return_value ());
    +    }
    +  }
    +
    +  void collect_variation_indices (hb_collect_variation_indices_context_t *c) const
    +  {
    +    switch (u.format) {
    +    case 1:
    +    case 2:
    +      return;
    +    case 3:
    +      u.format3.collect_variation_indices (c);
    +      return;
    +    default: return;
    +    }
    +  }
    +
    +  bool sanitize (hb_sanitize_context_t *c) const
    +  {
    +    TRACE_SANITIZE (this);
    +    if (!u.format.sanitize (c)) return_trace (false);
    +    switch (u.format) {
    +    case 1: return_trace (u.format1.sanitize (c));
    +    case 2: return_trace (u.format2.sanitize (c));
    +    case 3: return_trace (u.format3.sanitize (c));
    +    default:return_trace (true);
    +    }
    +  }
    +
    +  protected:
    +  union {
    +  HBUINT16              format;         /* Format identifier */
    +  CaretValueFormat1     format1;
    +  CaretValueFormat2     format2;
    +  CaretValueFormat3     format3;
    +  } u;
    +  public:
    +  DEFINE_SIZE_UNION (2, format);
    +};
    +
    +struct LigGlyph
    +{
    +  unsigned get_lig_carets (hb_font_t            *font,
    +                           hb_direction_t        direction,
    +                           hb_codepoint_t        glyph_id,
    +                           const VariationStore &var_store,
    +                           unsigned              start_offset,
    +                           unsigned             *caret_count /* IN/OUT */,
    +                           hb_position_t        *caret_array /* OUT */) const
    +  {
    +    if (caret_count)
    +    {
    +      + carets.as_array ().sub_array (start_offset, caret_count)
    +      | hb_map (hb_add (this))
    +      | hb_map ([&] (const CaretValue &value) { return value.get_caret_value (font, direction, glyph_id, var_store); })
    +      | hb_sink (hb_array (caret_array, *caret_count))
    +      ;
    +    }
    +
    +    return carets.len;
    +  }
    +
    +  bool subset (hb_subset_context_t *c) const
    +  {
    +    TRACE_SUBSET (this);
    +    auto *out = c->serializer->start_embed (*this);
    +    if (unlikely (!c->serializer->extend_min (out))) return_trace (false);
    +
    +    + hb_iter (carets)
    +    | hb_apply (subset_offset_array (c, out->carets, this))
    +    ;
    +
    +    return_trace (bool (out->carets));
    +  }
    +
    +  void collect_variation_indices (hb_collect_variation_indices_context_t *c) const
    +  {
    +    for (const Offset16To& offset : carets.iter ())
    +      (this+offset).collect_variation_indices (c);
    +  }
    +
    +  bool sanitize (hb_sanitize_context_t *c) const
    +  {
    +    TRACE_SANITIZE (this);
    +    return_trace (carets.sanitize (c, this));
    +  }
    +
    +  protected:
    +  Array16OfOffset16To
    +                carets;                 /* Offset array of CaretValue tables
    +                                         * --from beginning of LigGlyph table
    +                                         * --in increasing coordinate order */
    +  public:
    +  DEFINE_SIZE_ARRAY (2, carets);
    +};
    +
    +struct LigCaretList
    +{
    +  unsigned int get_lig_carets (hb_font_t *font,
    +                               hb_direction_t direction,
    +                               hb_codepoint_t glyph_id,
    +                               const VariationStore &var_store,
    +                               unsigned int start_offset,
    +                               unsigned int *caret_count /* IN/OUT */,
    +                               hb_position_t *caret_array /* OUT */) const
    +  {
    +    unsigned int index = (this+coverage).get_coverage (glyph_id);
    +    if (index == NOT_COVERED)
    +    {
    +      if (caret_count)
    +        *caret_count = 0;
    +      return 0;
    +    }
    +    const LigGlyph &lig_glyph = this+ligGlyph[index];
    +    return lig_glyph.get_lig_carets (font, direction, glyph_id, var_store, start_offset, caret_count, caret_array);
    +  }
    +
    +  bool subset (hb_subset_context_t *c) const
    +  {
    +    TRACE_SUBSET (this);
    +    const hb_set_t &glyphset = *c->plan->glyphset_gsub ();
    +    const hb_map_t &glyph_map = *c->plan->glyph_map;
    +
    +    auto *out = c->serializer->start_embed (*this);
    +    if (unlikely (!c->serializer->extend_min (out))) return_trace (false);
    +
    +    hb_sorted_vector_t new_coverage;
    +    + hb_zip (this+coverage, ligGlyph)
    +    | hb_filter (glyphset, hb_first)
    +    | hb_filter (subset_offset_array (c, out->ligGlyph, this), hb_second)
    +    | hb_map (hb_first)
    +    | hb_map (glyph_map)
    +    | hb_sink (new_coverage)
    +    ;
    +    out->coverage.serialize_serialize (c->serializer, new_coverage.iter ());
    +    return_trace (bool (new_coverage));
    +  }
    +
    +  void collect_variation_indices (hb_collect_variation_indices_context_t *c) const
    +  {
    +    + hb_zip (this+coverage, ligGlyph)
    +    | hb_filter (c->glyph_set, hb_first)
    +    | hb_map (hb_second)
    +    | hb_map (hb_add (this))
    +    | hb_apply ([c] (const LigGlyph& _) { _.collect_variation_indices (c); })
    +    ;
    +  }
    +
    +  bool sanitize (hb_sanitize_context_t *c) const
    +  {
    +    TRACE_SANITIZE (this);
    +    return_trace (coverage.sanitize (c, this) && ligGlyph.sanitize (c, this));
    +  }
    +
    +  protected:
    +  Offset16To
    +                coverage;               /* Offset to Coverage table--from
    +                                         * beginning of LigCaretList table */
    +  Array16OfOffset16To
    +                ligGlyph;               /* Array of LigGlyph tables
    +                                         * in Coverage Index order */
    +  public:
    +  DEFINE_SIZE_ARRAY (4, ligGlyph);
    +};
    +
    +
    +struct MarkGlyphSetsFormat1
    +{
    +  bool covers (unsigned int set_index, hb_codepoint_t glyph_id) const
    +  { return (this+coverage[set_index]).get_coverage (glyph_id) != NOT_COVERED; }
    +
    +  bool subset (hb_subset_context_t *c) const
    +  {
    +    TRACE_SUBSET (this);
    +    auto *out = c->serializer->start_embed (*this);
    +    if (unlikely (!c->serializer->extend_min (out))) return_trace (false);
    +    out->format = format;
    +
    +    bool ret = true;
    +    for (const Offset32To& offset : coverage.iter ())
    +    {
    +      auto *o = out->coverage.serialize_append (c->serializer);
    +      if (unlikely (!o))
    +      {
    +        ret = false;
    +        break;
    +      }
    +
    +      //not using o->serialize_subset (c, offset, this, out) here because
    +      //OTS doesn't allow null offset.
    +      //See issue: https://github.com/khaledhosny/ots/issues/172
    +      c->serializer->push ();
    +      c->dispatch (this+offset);
    +      c->serializer->add_link (*o, c->serializer->pop_pack ());
    +    }
    +
    +    return_trace (ret && out->coverage.len);
    +  }
    +
    +  bool sanitize (hb_sanitize_context_t *c) const
    +  {
    +    TRACE_SANITIZE (this);
    +    return_trace (coverage.sanitize (c, this));
    +  }
    +
    +  protected:
    +  HBUINT16      format;                 /* Format identifier--format = 1 */
    +  Array16Of>
    +                coverage;               /* Array of long offsets to mark set
    +                                         * coverage tables */
    +  public:
    +  DEFINE_SIZE_ARRAY (4, coverage);
    +};
    +
    +struct MarkGlyphSets
    +{
    +  bool covers (unsigned int set_index, hb_codepoint_t glyph_id) const
    +  {
    +    switch (u.format) {
    +    case 1: return u.format1.covers (set_index, glyph_id);
    +    default:return false;
    +    }
    +  }
    +
    +  bool subset (hb_subset_context_t *c) const
    +  {
    +    TRACE_SUBSET (this);
    +    switch (u.format) {
    +    case 1: return_trace (u.format1.subset (c));
    +    default:return_trace (false);
    +    }
    +  }
    +
    +  bool sanitize (hb_sanitize_context_t *c) const
    +  {
    +    TRACE_SANITIZE (this);
    +    if (!u.format.sanitize (c)) return_trace (false);
    +    switch (u.format) {
    +    case 1: return_trace (u.format1.sanitize (c));
    +    default:return_trace (true);
    +    }
    +  }
    +
    +  protected:
    +  union {
    +  HBUINT16              format;         /* Format identifier */
    +  MarkGlyphSetsFormat1  format1;
    +  } u;
    +  public:
    +  DEFINE_SIZE_UNION (2, format);
    +};
    +
    +
    +/*
    + * GDEF -- Glyph Definition
    + * https://docs.microsoft.com/en-us/typography/opentype/spec/gdef
    + */
    +
    +
    +template 
    +struct GDEFVersion1_2
    +{
    +  friend struct GDEF;
    +
    +  protected:
    +  FixedVersion<>version;                /* Version of the GDEF table--currently
    +                                         * 0x00010003u */
    +  typename Types::template OffsetTo
    +                glyphClassDef;          /* Offset to class definition table
    +                                         * for glyph type--from beginning of
    +                                         * GDEF header (may be Null) */
    +  typename Types::template OffsetTo
    +                attachList;             /* Offset to list of glyphs with
    +                                         * attachment points--from beginning
    +                                         * of GDEF header (may be Null) */
    +  typename Types::template OffsetTo
    +                ligCaretList;           /* Offset to list of positioning points
    +                                         * for ligature carets--from beginning
    +                                         * of GDEF header (may be Null) */
    +  typename Types::template OffsetTo
    +                markAttachClassDef;     /* Offset to class definition table for
    +                                         * mark attachment type--from beginning
    +                                         * of GDEF header (may be Null) */
    +  typename Types::template OffsetTo
    +                markGlyphSetsDef;       /* Offset to the table of mark set
    +                                         * definitions--from beginning of GDEF
    +                                         * header (may be NULL).  Introduced
    +                                         * in version 0x00010002. */
    +  Offset32To
    +                varStore;               /* Offset to the table of Item Variation
    +                                         * Store--from beginning of GDEF
    +                                         * header (may be NULL).  Introduced
    +                                         * in version 0x00010003. */
    +  public:
    +  DEFINE_SIZE_MIN (4 + 4 * Types::size);
    +
    +  unsigned int get_size () const
    +  {
    +    return min_size +
    +           (version.to_int () >= 0x00010002u ? markGlyphSetsDef.static_size : 0) +
    +           (version.to_int () >= 0x00010003u ? varStore.static_size : 0);
    +  }
    +
    +  bool sanitize (hb_sanitize_context_t *c) const
    +  {
    +    TRACE_SANITIZE (this);
    +    return_trace (version.sanitize (c) &&
    +                  glyphClassDef.sanitize (c, this) &&
    +                  attachList.sanitize (c, this) &&
    +                  ligCaretList.sanitize (c, this) &&
    +                  markAttachClassDef.sanitize (c, this) &&
    +                  (version.to_int () < 0x00010002u || markGlyphSetsDef.sanitize (c, this)) &&
    +                  (version.to_int () < 0x00010003u || varStore.sanitize (c, this)));
    +  }
    +
    +  bool subset (hb_subset_context_t *c) const
    +  {
    +    TRACE_SUBSET (this);
    +    auto *out = c->serializer->embed (*this);
    +    if (unlikely (!out)) return_trace (false);
    +
    +    bool subset_glyphclassdef = out->glyphClassDef.serialize_subset (c, glyphClassDef, this, nullptr, false, true);
    +    bool subset_attachlist = out->attachList.serialize_subset (c, attachList, this);
    +    bool subset_ligcaretlist = out->ligCaretList.serialize_subset (c, ligCaretList, this);
    +    bool subset_markattachclassdef = out->markAttachClassDef.serialize_subset (c, markAttachClassDef, this, nullptr, false, true);
    +
    +    bool subset_markglyphsetsdef = false;
    +    if (version.to_int () >= 0x00010002u)
    +    {
    +      subset_markglyphsetsdef = out->markGlyphSetsDef.serialize_subset (c, markGlyphSetsDef, this);
    +    }
    +
    +    bool subset_varstore = false;
    +    if (version.to_int () >= 0x00010003u)
    +    {
    +      if (c->plan->all_axes_pinned)
    +        out->varStore = 0;
    +      else
    +        subset_varstore = out->varStore.serialize_subset (c, varStore, this, c->plan->gdef_varstore_inner_maps.as_array ());
    +    }
    +
    +    if (subset_varstore)
    +    {
    +      out->version.minor = 3;
    +    } else if (subset_markglyphsetsdef) {
    +      out->version.minor = 2;
    +    } else  {
    +      out->version.minor = 0;
    +    }
    +
    +    return_trace (subset_glyphclassdef || subset_attachlist ||
    +                  subset_ligcaretlist || subset_markattachclassdef ||
    +                  (out->version.to_int () >= 0x00010002u && subset_markglyphsetsdef) ||
    +                  (out->version.to_int () >= 0x00010003u && subset_varstore));
    +  }
    +};
    +
    +struct GDEF
    +{
    +  static constexpr hb_tag_t tableTag = HB_OT_TAG_GDEF;
    +
    +  enum GlyphClasses {
    +    UnclassifiedGlyph   = 0,
    +    BaseGlyph           = 1,
    +    LigatureGlyph       = 2,
    +    MarkGlyph           = 3,
    +    ComponentGlyph      = 4
    +  };
    +
    +  unsigned int get_size () const
    +  {
    +    switch (u.version.major) {
    +    case 1: return u.version1.get_size ();
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return u.version2.get_size ();
    +#endif
    +    default: return u.version.static_size;
    +    }
    +  }
    +
    +  bool sanitize (hb_sanitize_context_t *c) const
    +  {
    +    TRACE_SANITIZE (this);
    +    if (unlikely (!u.version.sanitize (c))) return_trace (false);
    +    switch (u.version.major) {
    +    case 1: return_trace (u.version1.sanitize (c));
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return_trace (u.version2.sanitize (c));
    +#endif
    +    default: return_trace (true);
    +    }
    +  }
    +
    +  bool subset (hb_subset_context_t *c) const
    +  {
    +    switch (u.version.major) {
    +    case 1: return u.version1.subset (c);
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return u.version2.subset (c);
    +#endif
    +    default: return false;
    +    }
    +  }
    +
    +  bool has_glyph_classes () const
    +  {
    +    switch (u.version.major) {
    +    case 1: return u.version1.glyphClassDef != 0;
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return u.version2.glyphClassDef != 0;
    +#endif
    +    default: return false;
    +    }
    +  }
    +  const ClassDef &get_glyph_class_def () const
    +  {
    +    switch (u.version.major) {
    +    case 1: return this+u.version1.glyphClassDef;
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return this+u.version2.glyphClassDef;
    +#endif
    +    default: return Null(ClassDef);
    +    }
    +  }
    +  bool has_attach_list () const
    +  {
    +    switch (u.version.major) {
    +    case 1: return u.version1.attachList != 0;
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return u.version2.attachList != 0;
    +#endif
    +    default: return false;
    +    }
    +  }
    +  const AttachList &get_attach_list () const
    +  {
    +    switch (u.version.major) {
    +    case 1: return this+u.version1.attachList;
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return this+u.version2.attachList;
    +#endif
    +    default: return Null(AttachList);
    +    }
    +  }
    +  bool has_lig_carets () const
    +  {
    +    switch (u.version.major) {
    +    case 1: return u.version1.ligCaretList != 0;
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return u.version2.ligCaretList != 0;
    +#endif
    +    default: return false;
    +    }
    +  }
    +  const LigCaretList &get_lig_caret_list () const
    +  {
    +    switch (u.version.major) {
    +    case 1: return this+u.version1.ligCaretList;
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return this+u.version2.ligCaretList;
    +#endif
    +    default: return Null(LigCaretList);
    +    }
    +  }
    +  bool has_mark_attachment_types () const
    +  {
    +    switch (u.version.major) {
    +    case 1: return u.version1.markAttachClassDef != 0;
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return u.version2.markAttachClassDef != 0;
    +#endif
    +    default: return false;
    +    }
    +  }
    +  const ClassDef &get_mark_attach_class_def () const
    +  {
    +    switch (u.version.major) {
    +    case 1: return this+u.version1.markAttachClassDef;
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return this+u.version2.markAttachClassDef;
    +#endif
    +    default: return Null(ClassDef);
    +    }
    +  }
    +  bool has_mark_glyph_sets () const
    +  {
    +    switch (u.version.major) {
    +    case 1: return u.version.to_int () >= 0x00010002u && u.version1.markGlyphSetsDef != 0;
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return u.version2.markGlyphSetsDef != 0;
    +#endif
    +    default: return false;
    +    }
    +  }
    +  const MarkGlyphSets &get_mark_glyph_sets () const
    +  {
    +    switch (u.version.major) {
    +    case 1: return u.version.to_int () >= 0x00010002u ? this+u.version1.markGlyphSetsDef : Null(MarkGlyphSets);
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return this+u.version2.markGlyphSetsDef;
    +#endif
    +    default: return Null(MarkGlyphSets);
    +    }
    +  }
    +  bool has_var_store () const
    +  {
    +    switch (u.version.major) {
    +    case 1: return u.version.to_int () >= 0x00010003u && u.version1.varStore != 0;
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return u.version2.varStore != 0;
    +#endif
    +    default: return false;
    +    }
    +  }
    +  const VariationStore &get_var_store () const
    +  {
    +    switch (u.version.major) {
    +    case 1: return u.version.to_int () >= 0x00010003u ? this+u.version1.varStore : Null(VariationStore);
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return this+u.version2.varStore;
    +#endif
    +    default: return Null(VariationStore);
    +    }
    +  }
    +
    +
    +  bool has_data () const { return u.version.to_int (); }
    +  unsigned int get_glyph_class (hb_codepoint_t glyph) const
    +  { return get_glyph_class_def ().get_class (glyph); }
    +  void get_glyphs_in_class (unsigned int klass, hb_set_t *glyphs) const
    +  { get_glyph_class_def ().collect_class (glyphs, klass); }
    +
    +  unsigned int get_mark_attachment_type (hb_codepoint_t glyph) const
    +  { return get_mark_attach_class_def ().get_class (glyph); }
    +
    +  unsigned int get_attach_points (hb_codepoint_t glyph_id,
    +                                  unsigned int start_offset,
    +                                  unsigned int *point_count /* IN/OUT */,
    +                                  unsigned int *point_array /* OUT */) const
    +  { return get_attach_list ().get_attach_points (glyph_id, start_offset, point_count, point_array); }
    +
    +  unsigned int get_lig_carets (hb_font_t *font,
    +                               hb_direction_t direction,
    +                               hb_codepoint_t glyph_id,
    +                               unsigned int start_offset,
    +                               unsigned int *caret_count /* IN/OUT */,
    +                               hb_position_t *caret_array /* OUT */) const
    +  { return get_lig_caret_list ().get_lig_carets (font,
    +                                                 direction, glyph_id, get_var_store(),
    +                                                 start_offset, caret_count, caret_array); }
    +
    +  bool mark_set_covers (unsigned int set_index, hb_codepoint_t glyph_id) const
    +  { return get_mark_glyph_sets ().covers (set_index, glyph_id); }
    +
    +  /* glyph_props is a 16-bit integer where the lower 8-bit have bits representing
    +   * glyph class and other bits, and high 8-bit the mark attachment type (if any).
    +   * Not to be confused with lookup_props which is very similar. */
    +  unsigned int get_glyph_props (hb_codepoint_t glyph) const
    +  {
    +    unsigned int klass = get_glyph_class (glyph);
    +
    +    static_assert (((unsigned int) HB_OT_LAYOUT_GLYPH_PROPS_BASE_GLYPH == (unsigned int) LookupFlag::IgnoreBaseGlyphs), "");
    +    static_assert (((unsigned int) HB_OT_LAYOUT_GLYPH_PROPS_LIGATURE == (unsigned int) LookupFlag::IgnoreLigatures), "");
    +    static_assert (((unsigned int) HB_OT_LAYOUT_GLYPH_PROPS_MARK == (unsigned int) LookupFlag::IgnoreMarks), "");
    +
    +    switch (klass) {
    +    default:                    return HB_OT_LAYOUT_GLYPH_CLASS_UNCLASSIFIED;
    +    case BaseGlyph:             return HB_OT_LAYOUT_GLYPH_PROPS_BASE_GLYPH;
    +    case LigatureGlyph:         return HB_OT_LAYOUT_GLYPH_PROPS_LIGATURE;
    +    case MarkGlyph:
    +          klass = get_mark_attachment_type (glyph);
    +          return HB_OT_LAYOUT_GLYPH_PROPS_MARK | (klass << 8);
    +    }
    +  }
    +
    +  HB_INTERNAL bool is_blocklisted (hb_blob_t *blob,
    +                                   hb_face_t *face) const;
    +
    +  struct accelerator_t
    +  {
    +    accelerator_t (hb_face_t *face)
    +    {
    +      table = hb_sanitize_context_t ().reference_table (face);
    +      if (unlikely (table->is_blocklisted (table.get_blob (), face)))
    +      {
    +        hb_blob_destroy (table.get_blob ());
    +        table = hb_blob_get_empty ();
    +      }
    +    }
    +    ~accelerator_t () { table.destroy (); }
    +
    +    hb_blob_ptr_t table;
    +  };
    +
    +  void collect_variation_indices (hb_collect_variation_indices_context_t *c) const
    +  { get_lig_caret_list ().collect_variation_indices (c); }
    +
    +  void remap_layout_variation_indices (const hb_set_t *layout_variation_indices,
    +                                       hb_hashmap_t> *layout_variation_idx_delta_map /* OUT */) const
    +  {
    +    if (!has_var_store ()) return;
    +    if (layout_variation_indices->is_empty ()) return;
    +
    +    unsigned new_major = 0, new_minor = 0;
    +    unsigned last_major = (layout_variation_indices->get_min ()) >> 16;
    +    for (unsigned idx : layout_variation_indices->iter ())
    +    {
    +      uint16_t major = idx >> 16;
    +      if (major >= get_var_store ().get_sub_table_count ()) break;
    +      if (major != last_major)
    +      {
    +        new_minor = 0;
    +        ++new_major;
    +      }
    +
    +      unsigned new_idx = (new_major << 16) + new_minor;
    +      if (!layout_variation_idx_delta_map->has (idx))
    +        continue;
    +      int delta = hb_second (layout_variation_idx_delta_map->get (idx));
    +
    +      layout_variation_idx_delta_map->set (idx, hb_pair_t (new_idx, delta));
    +      ++new_minor;
    +      last_major = major;
    +    }
    +  }
    +
    +  protected:
    +  union {
    +  FixedVersion<>                version;        /* Version identifier */
    +  GDEFVersion1_2    version1;
    +#ifndef HB_NO_BEYOND_64K
    +  GDEFVersion1_2   version2;
    +#endif
    +  } u;
    +  public:
    +  DEFINE_SIZE_MIN (4);
    +};
    +
    +struct GDEF_accelerator_t : GDEF::accelerator_t {
    +  GDEF_accelerator_t (hb_face_t *face) : GDEF::accelerator_t (face) {}
    +};
    +
    +} /* namespace OT */
    +
    +
    +#endif /* OT_LAYOUT_GDEF_GDEF_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/Anchor.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/Anchor.hh
    index bfe6b36afd34b..49e76e77509e6 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/Anchor.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/Anchor.hh
    @@ -58,8 +58,7 @@ struct Anchor
             return_trace (bool (reinterpret_cast (u.format1.copy (c->serializer))));
           }
           return_trace (bool (reinterpret_cast (u.format2.copy (c->serializer))));
    -    case 3: return_trace (bool (reinterpret_cast (u.format3.copy (c->serializer,
    -                                                                            c->plan->layout_variation_idx_map))));
    +    case 3: return_trace (u.format3.subset (c));
         default:return_trace (false);
         }
       }
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/AnchorFormat3.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/AnchorFormat3.hh
    index d77b4699be951..e7e3c5c6d1ea4 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/AnchorFormat3.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/AnchorFormat3.hh
    @@ -41,24 +41,54 @@ struct AnchorFormat3
           *y += (this+yDeviceTable).get_y_delta (font, c->var_store, c->var_store_cache);
       }
     
    -  AnchorFormat3* copy (hb_serialize_context_t *c,
    -                       const hb_map_t *layout_variation_idx_map) const
    +  bool subset (hb_subset_context_t *c) const
       {
    -    TRACE_SERIALIZE (this);
    -    if (!layout_variation_idx_map) return_trace (nullptr);
    +    TRACE_SUBSET (this);
    +    auto *out = c->serializer->start_embed (*this);
    +    if (unlikely (!out)) return_trace (false);
    +    if (unlikely (!c->serializer->embed (format))) return_trace (false);
    +    if (unlikely (!c->serializer->embed (xCoordinate))) return_trace (false);
    +    if (unlikely (!c->serializer->embed (yCoordinate))) return_trace (false);
     
    -    auto *out = c->embed (this);
    -    if (unlikely (!out)) return_trace (nullptr);
    +    unsigned x_varidx = xDeviceTable ? (this+xDeviceTable).get_variation_index () : HB_OT_LAYOUT_NO_VARIATIONS_INDEX;
    +    if (c->plan->layout_variation_idx_delta_map.has (x_varidx))
    +    {
    +      int delta = hb_second (c->plan->layout_variation_idx_delta_map.get (x_varidx));
    +      if (delta != 0)
    +      {
    +        if (!c->serializer->check_assign (out->xCoordinate, xCoordinate + delta,
    +                                          HB_SERIALIZE_ERROR_INT_OVERFLOW))
    +          return_trace (false);
    +      }
    +    }
     
    -    out->xDeviceTable.serialize_copy (c, xDeviceTable, this, 0, hb_serialize_context_t::Head, layout_variation_idx_map);
    -    out->yDeviceTable.serialize_copy (c, yDeviceTable, this, 0, hb_serialize_context_t::Head, layout_variation_idx_map);
    +    unsigned y_varidx = yDeviceTable ? (this+yDeviceTable).get_variation_index () : HB_OT_LAYOUT_NO_VARIATIONS_INDEX;
    +    if (c->plan->layout_variation_idx_delta_map.has (y_varidx))
    +    {
    +      int delta = hb_second (c->plan->layout_variation_idx_delta_map.get (y_varidx));
    +      if (delta != 0)
    +      {
    +        if (!c->serializer->check_assign (out->yCoordinate, yCoordinate + delta,
    +                                          HB_SERIALIZE_ERROR_INT_OVERFLOW))
    +          return_trace (false);
    +      }
    +    }
    +
    +    if (c->plan->all_axes_pinned)
    +      return_trace (c->serializer->check_assign (out->format, 1, HB_SERIALIZE_ERROR_INT_OVERFLOW));
    +
    +    if (!c->serializer->embed (xDeviceTable)) return_trace (false);
    +    if (!c->serializer->embed (yDeviceTable)) return_trace (false);
    +
    +    out->xDeviceTable.serialize_copy (c->serializer, xDeviceTable, this, 0, hb_serialize_context_t::Head, &c->plan->layout_variation_idx_delta_map);
    +    out->yDeviceTable.serialize_copy (c->serializer, yDeviceTable, this, 0, hb_serialize_context_t::Head, &c->plan->layout_variation_idx_delta_map);
         return_trace (out);
       }
     
       void collect_variation_indices (hb_collect_variation_indices_context_t *c) const
       {
    -    (this+xDeviceTable).collect_variation_indices (c->layout_variation_indices);
    -    (this+yDeviceTable).collect_variation_indices (c->layout_variation_indices);
    +    (this+xDeviceTable).collect_variation_indices (c);
    +    (this+yDeviceTable).collect_variation_indices (c);
       }
     };
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/Common.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/Common.hh
    index e16c06729d229..408197454f1c7 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/Common.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/Common.hh
    @@ -22,7 +22,8 @@ template
     static void SinglePos_serialize (hb_serialize_context_t *c,
                                      const SrcLookup *src,
                                      Iterator it,
    -                                 const hb_map_t *layout_variation_idx_map);
    +                                 const hb_hashmap_t> *layout_variation_idx_delta_map,
    +                                 bool all_axes_pinned);
     
     
     }
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/CursivePos.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/CursivePos.hh
    index c105cfb091650..0105a9b854275 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/CursivePos.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/CursivePos.hh
    @@ -19,8 +19,8 @@ struct CursivePos
       template 
       typename context_t::return_t dispatch (context_t *c, Ts&&... ds) const
       {
    +    if (unlikely (!c->may_dispatch (this, &u.format))) return c->no_dispatch_return_value ();
         TRACE_DISPATCH (this, u.format);
    -    if (unlikely (!c->may_dispatch (this, &u.format))) return_trace (c->no_dispatch_return_value ());
         switch (u.format) {
         case 1: return_trace (c->dispatch (u.format1, std::forward (ds)...));
         default:return_trace (c->default_return_value ());
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/CursivePosFormat1.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/CursivePosFormat1.hh
    index e212fab976b1f..13a435d00bfae 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/CursivePosFormat1.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/CursivePosFormat1.hh
    @@ -140,7 +140,14 @@ struct CursivePosFormat1
         unsigned int i = skippy_iter.idx;
         unsigned int j = buffer->idx;
     
    -    buffer->unsafe_to_break (i, j);
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->message (c->font,
    +                          "cursive attaching glyph at %u to glyph at %u",
    +                          i, j);
    +    }
    +
    +    buffer->unsafe_to_break (i, j + 1);
         float entry_x, entry_y, exit_x, exit_y;
         (this+prev_record.exitAnchor).get_anchor (c, buffer->info[i].codepoint, &exit_x, &exit_y);
         (this+this_record.entryAnchor).get_anchor (c, buffer->info[j].codepoint, &entry_x, &entry_y);
    @@ -223,7 +230,20 @@ struct CursivePosFormat1
          * https://github.com/harfbuzz/harfbuzz/issues/2469
          */
         if (unlikely (pos[parent].attach_chain() == -pos[child].attach_chain()))
    +    {
           pos[parent].attach_chain() = 0;
    +      if (likely (HB_DIRECTION_IS_HORIZONTAL (c->direction)))
    +        pos[parent].y_offset = 0;
    +      else
    +        pos[parent].x_offset = 0;
    +    }
    +
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->message (c->font,
    +                          "cursive attached glyph at %u to glyph at %u",
    +                          i, j);
    +    }
     
         buffer->idx++;
         return_trace (true);
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/GPOS.hh
    similarity index 84%
    rename from src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS.hh
    rename to src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/GPOS.hh
    index 224d6b746bdaf..9493ec987e864 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/GPOS.hh
    @@ -1,12 +1,15 @@
    -#ifndef OT_LAYOUT_GPOS_HH
    -#define OT_LAYOUT_GPOS_HH
    +#ifndef OT_LAYOUT_GPOS_GPOS_HH
    +#define OT_LAYOUT_GPOS_GPOS_HH
     
    -#include "../../hb-ot-layout-common.hh"
    -#include "../../hb-ot-layout-gsubgpos.hh"
    -#include "GPOS/Common.hh"
    -#include "GPOS/PosLookup.hh"
    +#include "../../../hb-ot-layout-common.hh"
    +#include "../../../hb-ot-layout-gsubgpos.hh"
    +#include "Common.hh"
    +#include "PosLookup.hh"
     
     namespace OT {
    +
    +using Layout::GPOS_impl::PosLookup;
    +
     namespace Layout {
     
     static void
    @@ -25,10 +28,10 @@ struct GPOS : GSUBGPOS
     {
       static constexpr hb_tag_t tableTag = HB_OT_TAG_GPOS;
     
    -  using Lookup = GPOS_impl::PosLookup;
    +  using Lookup = PosLookup;
     
    -  const GPOS_impl::PosLookup& get_lookup (unsigned int i) const
    -  { return static_cast (GSUBGPOS::get_lookup (i)); }
    +  const PosLookup& get_lookup (unsigned int i) const
    +  { return static_cast (GSUBGPOS::get_lookup (i)); }
     
       static inline void position_start (hb_font_t *font, hb_buffer_t *buffer);
       static inline void position_finish_advances (hb_font_t *font, hb_buffer_t *buffer);
    @@ -36,12 +39,15 @@ struct GPOS : GSUBGPOS
     
       bool subset (hb_subset_context_t *c) const
       {
    -    hb_subset_layout_context_t l (c, tableTag, c->plan->gpos_lookups, c->plan->gpos_langsys, c->plan->gpos_features);
    -    return GSUBGPOS::subset (&l);
    +    hb_subset_layout_context_t l (c, tableTag);
    +    return GSUBGPOS::subset (&l);
       }
     
       bool sanitize (hb_sanitize_context_t *c) const
    -  { return GSUBGPOS::sanitize (c); }
    +  {
    +    TRACE_SANITIZE (this);
    +    return_trace (GSUBGPOS::sanitize (c));
    +  }
     
       HB_INTERNAL bool is_blocklisted (hb_blob_t *blob,
                                        hb_face_t *face) const;
    @@ -51,7 +57,7 @@ struct GPOS : GSUBGPOS
         for (unsigned i = 0; i < GSUBGPOS::get_lookup_count (); i++)
         {
           if (!c->gpos_lookups->has (i)) continue;
    -      const GPOS_impl::PosLookup &l = get_lookup (i);
    +      const PosLookup &l = get_lookup (i);
           l.dispatch (c);
         }
       }
    @@ -59,7 +65,7 @@ struct GPOS : GSUBGPOS
       void closure_lookups (hb_face_t      *face,
                             const hb_set_t *glyphs,
                             hb_set_t       *lookup_indexes /* IN/OUT */) const
    -  { GSUBGPOS::closure_lookups (face, glyphs, lookup_indexes); }
    +  { GSUBGPOS::closure_lookups (face, glyphs, lookup_indexes); }
     
       typedef GSUBGPOS::accelerator_t accelerator_t;
     };
    @@ -162,4 +168,4 @@ struct GPOS_accelerator_t : Layout::GPOS::accelerator_t {
     
     }
     
    -#endif  /* OT_LAYOUT_GPOS_HH */
    +#endif  /* OT_LAYOUT_GPOS_GPOS_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/LigatureArray.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/LigatureArray.hh
    new file mode 100644
    index 0000000000000..a2d807cc3206c
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/LigatureArray.hh
    @@ -0,0 +1,56 @@
    +#ifndef OT_LAYOUT_GPOS_LIGATUREARRAY_HH
    +#define OT_LAYOUT_GPOS_LIGATUREARRAY_HH
    +
    +namespace OT {
    +namespace Layout {
    +namespace GPOS_impl {
    +
    +
    +typedef AnchorMatrix LigatureAttach;    /* component-major--
    +                                         * in order of writing direction--,
    +                                         * mark-minor--
    +                                         * ordered by class--zero-based. */
    +
    +/* Array of LigatureAttach tables ordered by LigatureCoverage Index */
    +struct LigatureArray : List16OfOffset16To
    +{
    +  template 
    +  bool subset (hb_subset_context_t *c,
    +               Iterator             coverage,
    +               unsigned             class_count,
    +               const hb_map_t      *klass_mapping) const
    +  {
    +    TRACE_SUBSET (this);
    +    const hb_set_t &glyphset = *c->plan->glyphset_gsub ();
    +
    +    auto *out = c->serializer->start_embed (this);
    +    if (unlikely (!c->serializer->extend_min (out)))  return_trace (false);
    +
    +    for (const auto _ : + hb_zip (coverage, *this)
    +                  | hb_filter (glyphset, hb_first))
    +    {
    +      auto *matrix = out->serialize_append (c->serializer);
    +      if (unlikely (!matrix)) return_trace (false);
    +
    +      const LigatureAttach& src = (this + _.second);
    +      auto indexes =
    +          + hb_range (src.rows * class_count)
    +          | hb_filter ([=] (unsigned index) { return klass_mapping->has (index % class_count); })
    +          ;
    +      matrix->serialize_subset (c,
    +                                _.second,
    +                                this,
    +                                src.rows,
    +                                indexes);
    +    }
    +    return_trace (this->len);
    +  }
    +};
    +
    +
    +}
    +}
    +}
    +
    +#endif /* OT_LAYOUT_GPOS_LIGATUREARRAY_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkArray.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkArray.hh
    index f8cddd199186d..e80c05cd2708a 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkArray.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkArray.hh
    @@ -39,6 +39,13 @@ struct MarkArray : Array16Of        /* Array of MarkRecords--in Cove
         mark_anchor.get_anchor (c, buffer->cur().codepoint, &mark_x, &mark_y);
         glyph_anchor.get_anchor (c, buffer->info[glyph_pos].codepoint, &base_x, &base_y);
     
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->message (c->font,
    +                          "attaching mark glyph at %u to glyph at %u",
    +                          c->buffer->idx, glyph_pos);
    +    }
    +
         hb_glyph_position_t &o = buffer->cur_pos();
         o.x_offset = roundf (base_x - mark_x);
         o.y_offset = roundf (base_y - mark_y);
    @@ -46,6 +53,13 @@ struct MarkArray : Array16Of        /* Array of MarkRecords--in Cove
         o.attach_chain() = (int) glyph_pos - (int) buffer->idx;
         buffer->scratch_flags |= HB_BUFFER_SCRATCH_FLAG_HAS_GPOS_ATTACHMENT;
     
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->message (c->font,
    +                          "attached mark glyph at %u to glyph at %u",
    +                          c->buffer->idx, glyph_pos);
    +    }
    +
         buffer->idx++;
         return_trace (true);
       }
    @@ -83,10 +97,11 @@ struct MarkArray : Array16Of        /* Array of MarkRecords--in Cove
       }
     };
     
    -static void Markclass_closure_and_remap_indexes (const Coverage  &mark_coverage,
    -                                                 const MarkArray &mark_array,
    -                                                 const hb_set_t  &glyphset,
    -                                                 hb_map_t*        klass_mapping /* INOUT */)
    +HB_INTERNAL inline
    +void Markclass_closure_and_remap_indexes (const Coverage  &mark_coverage,
    +                                          const MarkArray &mark_array,
    +                                          const hb_set_t  &glyphset,
    +                                          hb_map_t*        klass_mapping /* INOUT */)
     {
       hb_set_t orig_classes;
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkBasePos.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkBasePos.hh
    index e99e13ff84f3d..b1d1118a86b3c 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkBasePos.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkBasePos.hh
    @@ -11,18 +11,24 @@ struct MarkBasePos
     {
       protected:
       union {
    -  HBUINT16              format;         /* Format identifier */
    -  MarkBasePosFormat1    format1;
    +  HBUINT16                              format;         /* Format identifier */
    +  MarkBasePosFormat1_2      format1;
    +#ifndef HB_NO_BEYOND_64K
    +  MarkBasePosFormat1_2     format2;
    +#endif
       } u;
     
       public:
       template 
       typename context_t::return_t dispatch (context_t *c, Ts&&... ds) const
       {
    +    if (unlikely (!c->may_dispatch (this, &u.format))) return c->no_dispatch_return_value ();
         TRACE_DISPATCH (this, u.format);
    -    if (unlikely (!c->may_dispatch (this, &u.format))) return_trace (c->no_dispatch_return_value ());
         switch (u.format) {
         case 1: return_trace (c->dispatch (u.format1, std::forward (ds)...));
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return_trace (c->dispatch (u.format2, std::forward (ds)...));
    +#endif
         default:return_trace (c->default_return_value ());
         }
       }
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkBasePosFormat1.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkBasePosFormat1.hh
    index a10b806fe5f7b..bf129a4a0e976 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkBasePosFormat1.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkBasePosFormat1.hh
    @@ -12,26 +12,27 @@ typedef AnchorMatrix BaseArray;         /* base-major--
                                              * mark-minor--
                                              * ordered by class--zero-based. */
     
    -struct MarkBasePosFormat1
    +template 
    +struct MarkBasePosFormat1_2
     {
       protected:
       HBUINT16      format;                 /* Format identifier--format = 1 */
    -  Offset16To
    +  typename Types::template OffsetTo
                     markCoverage;           /* Offset to MarkCoverage table--from
                                              * beginning of MarkBasePos subtable */
    -  Offset16To
    +  typename Types::template OffsetTo
                     baseCoverage;           /* Offset to BaseCoverage table--from
                                              * beginning of MarkBasePos subtable */
       HBUINT16      classCount;             /* Number of classes defined for marks */
    -  Offset16To
    +  typename Types::template OffsetTo
                     markArray;              /* Offset to MarkArray table--from
                                              * beginning of MarkBasePos subtable */
    -  Offset16To
    +  typename Types::template OffsetTo
                     baseArray;              /* Offset to BaseArray table--from
                                              * beginning of MarkBasePos subtable */
     
       public:
    -  DEFINE_SIZE_STATIC (12);
    +  DEFINE_SIZE_STATIC (4 + 4 * Types::size);
     
         bool sanitize (hb_sanitize_context_t *c) const
       {
    @@ -89,6 +90,25 @@ struct MarkBasePosFormat1
     
       const Coverage &get_coverage () const { return this+markCoverage; }
     
    +  static inline bool accept (hb_buffer_t *buffer, unsigned idx)
    +  {
    +    /* We only want to attach to the first of a MultipleSubst sequence.
    +     * https://github.com/harfbuzz/harfbuzz/issues/740
    +     * Reject others...
    +     * ...but stop if we find a mark in the MultipleSubst sequence:
    +     * https://github.com/harfbuzz/harfbuzz/issues/1020 */
    +    return !_hb_glyph_info_multiplied (&buffer->info[idx]) ||
    +           0 == _hb_glyph_info_get_lig_comp (&buffer->info[idx]) ||
    +           (idx == 0 ||
    +            _hb_glyph_info_is_mark (&buffer->info[idx - 1]) ||
    +            !_hb_glyph_info_multiplied (&buffer->info[idx - 1]) ||
    +            _hb_glyph_info_get_lig_id (&buffer->info[idx]) !=
    +            _hb_glyph_info_get_lig_id (&buffer->info[idx - 1]) ||
    +            _hb_glyph_info_get_lig_comp (&buffer->info[idx]) !=
    +            _hb_glyph_info_get_lig_comp (&buffer->info[idx - 1]) + 1
    +            );
    +  }
    +
       bool apply (hb_ot_apply_context_t *c) const
       {
         TRACE_APPLY (this);
    @@ -96,47 +116,54 @@ struct MarkBasePosFormat1
         unsigned int mark_index = (this+markCoverage).get_coverage  (buffer->cur().codepoint);
         if (likely (mark_index == NOT_COVERED)) return_trace (false);
     
    -    /* Now we search backwards for a non-mark glyph */
    +    /* Now we search backwards for a non-mark glyph.
    +     * We don't use skippy_iter.prev() to avoid O(n^2) behavior. */
    +
         hb_ot_apply_context_t::skipping_iterator_t &skippy_iter = c->iter_input;
    -    skippy_iter.reset (buffer->idx, 1);
         skippy_iter.set_lookup_props (LookupFlag::IgnoreMarks);
    -    do {
    -      unsigned unsafe_from;
    -      if (!skippy_iter.prev (&unsafe_from))
    +
    +    if (c->last_base_until > buffer->idx)
    +    {
    +      c->last_base_until = 0;
    +      c->last_base = -1;
    +    }
    +    unsigned j;
    +    for (j = buffer->idx; j > c->last_base_until; j--)
    +    {
    +      auto match = skippy_iter.match (buffer->info[j - 1]);
    +      if (match == skippy_iter.MATCH)
           {
    -        buffer->unsafe_to_concat_from_outbuffer (unsafe_from, buffer->idx + 1);
    -        return_trace (false);
    +        // https://github.com/harfbuzz/harfbuzz/issues/4124
    +        if (!accept (buffer, j - 1) &&
    +            NOT_COVERED == (this+baseCoverage).get_coverage  (buffer->info[j - 1].codepoint))
    +          match = skippy_iter.SKIP;
           }
    -
    -      /* We only want to attach to the first of a MultipleSubst sequence.
    -       * https://github.com/harfbuzz/harfbuzz/issues/740
    -       * Reject others...
    -       * ...but stop if we find a mark in the MultipleSubst sequence:
    -       * https://github.com/harfbuzz/harfbuzz/issues/1020 */
    -      if (!_hb_glyph_info_multiplied (&buffer->info[skippy_iter.idx]) ||
    -          0 == _hb_glyph_info_get_lig_comp (&buffer->info[skippy_iter.idx]) ||
    -          (skippy_iter.idx == 0 ||
    -           _hb_glyph_info_is_mark (&buffer->info[skippy_iter.idx - 1]) ||
    -           _hb_glyph_info_get_lig_id (&buffer->info[skippy_iter.idx]) !=
    -           _hb_glyph_info_get_lig_id (&buffer->info[skippy_iter.idx - 1]) ||
    -           _hb_glyph_info_get_lig_comp (&buffer->info[skippy_iter.idx]) !=
    -           _hb_glyph_info_get_lig_comp (&buffer->info[skippy_iter.idx - 1]) + 1
    -           ))
    +      if (match == skippy_iter.MATCH)
    +      {
    +        c->last_base = (signed) j - 1;
             break;
    -      skippy_iter.reject ();
    -    } while (true);
    +      }
    +    }
    +    c->last_base_until = buffer->idx;
    +    if (c->last_base == -1)
    +    {
    +      buffer->unsafe_to_concat_from_outbuffer (0, buffer->idx + 1);
    +      return_trace (false);
    +    }
    +
    +    unsigned idx = (unsigned) c->last_base;
     
         /* Checking that matched glyph is actually a base glyph by GDEF is too strong; disabled */
    -    //if (!_hb_glyph_info_is_base_glyph (&buffer->info[skippy_iter.idx])) { return_trace (false); }
    +    //if (!_hb_glyph_info_is_base_glyph (&buffer->info[idx])) { return_trace (false); }
     
    -    unsigned int base_index = (this+baseCoverage).get_coverage  (buffer->info[skippy_iter.idx].codepoint);
    +    unsigned int base_index = (this+baseCoverage).get_coverage  (buffer->info[idx].codepoint);
         if (base_index == NOT_COVERED)
         {
    -      buffer->unsafe_to_concat_from_outbuffer (skippy_iter.idx, buffer->idx + 1);
    +      buffer->unsafe_to_concat_from_outbuffer (idx, buffer->idx + 1);
           return_trace (false);
         }
     
    -    return_trace ((this+markArray).apply (c, mark_index, base_index, this+baseArray, classCount, skippy_iter.idx));
    +    return_trace ((this+markArray).apply (c, mark_index, base_index, this+baseArray, classCount, idx));
       }
     
       bool subset (hb_subset_context_t *c) const
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkLigPos.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkLigPos.hh
    index 7e74aa73e0a39..b10102880c0c3 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkLigPos.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkLigPos.hh
    @@ -11,18 +11,24 @@ struct MarkLigPos
     {
       protected:
       union {
    -  HBUINT16              format;         /* Format identifier */
    -  MarkLigPosFormat1     format1;
    +  HBUINT16                              format;         /* Format identifier */
    +  MarkLigPosFormat1_2       format1;
    +#ifndef HB_NO_BEYOND_64K
    +  MarkLigPosFormat1_2      format2;
    +#endif
       } u;
     
       public:
       template 
       typename context_t::return_t dispatch (context_t *c, Ts&&... ds) const
       {
    +    if (unlikely (!c->may_dispatch (this, &u.format))) return c->no_dispatch_return_value ();
         TRACE_DISPATCH (this, u.format);
    -    if (unlikely (!c->may_dispatch (this, &u.format))) return_trace (c->no_dispatch_return_value ());
         switch (u.format) {
         case 1: return_trace (c->dispatch (u.format1, std::forward (ds)...));
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return_trace (c->dispatch (u.format2, std::forward (ds)...));
    +#endif
         default:return_trace (c->default_return_value ());
         }
       }
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkLigPosFormat1.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkLigPosFormat1.hh
    index 4382aa6c6cb3d..30bba4b0d8c33 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkLigPosFormat1.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkLigPosFormat1.hh
    @@ -1,72 +1,34 @@
     #ifndef OT_LAYOUT_GPOS_MARKLIGPOSFORMAT1_HH
     #define OT_LAYOUT_GPOS_MARKLIGPOSFORMAT1_HH
     
    +#include "LigatureArray.hh"
    +
     namespace OT {
     namespace Layout {
     namespace GPOS_impl {
     
    -typedef AnchorMatrix LigatureAttach;    /* component-major--
    -                                         * in order of writing direction--,
    -                                         * mark-minor--
    -                                         * ordered by class--zero-based. */
     
    -/* Array of LigatureAttach tables ordered by LigatureCoverage Index */
    -struct LigatureArray : List16OfOffset16To
    -{
    -  template 
    -  bool subset (hb_subset_context_t *c,
    -               Iterator             coverage,
    -               unsigned             class_count,
    -               const hb_map_t      *klass_mapping) const
    -  {
    -    TRACE_SUBSET (this);
    -    const hb_set_t &glyphset = *c->plan->glyphset_gsub ();
    -
    -    auto *out = c->serializer->start_embed (this);
    -    if (unlikely (!c->serializer->extend_min (out)))  return_trace (false);
    -
    -    for (const auto _ : + hb_zip (coverage, *this)
    -                  | hb_filter (glyphset, hb_first))
    -    {
    -      auto *matrix = out->serialize_append (c->serializer);
    -      if (unlikely (!matrix)) return_trace (false);
    -
    -      const LigatureAttach& src = (this + _.second);
    -      auto indexes =
    -          + hb_range (src.rows * class_count)
    -          | hb_filter ([=] (unsigned index) { return klass_mapping->has (index % class_count); })
    -          ;
    -      matrix->serialize_subset (c,
    -                                _.second,
    -                                this,
    -                                src.rows,
    -                                indexes);
    -    }
    -    return_trace (this->len);
    -  }
    -};
    -
    -struct MarkLigPosFormat1
    +template 
    +struct MarkLigPosFormat1_2
     {
       protected:
       HBUINT16      format;                 /* Format identifier--format = 1 */
    -  Offset16To
    +  typename Types::template OffsetTo
                     markCoverage;           /* Offset to Mark Coverage table--from
                                              * beginning of MarkLigPos subtable */
    -  Offset16To
    +  typename Types::template OffsetTo
                     ligatureCoverage;       /* Offset to Ligature Coverage
                                              * table--from beginning of MarkLigPos
                                              * subtable */
       HBUINT16      classCount;             /* Number of defined mark classes */
    -  Offset16To
    +  typename Types::template OffsetTo
                     markArray;              /* Offset to MarkArray table--from
                                              * beginning of MarkLigPos subtable */
    -  Offset16To
    +  typename Types::template OffsetTo
                     ligatureArray;          /* Offset to LigatureArray table--from
                                              * beginning of MarkLigPos subtable */
       public:
    -  DEFINE_SIZE_STATIC (12);
    +  DEFINE_SIZE_STATIC (4 + 4 * Types::size);
     
       bool sanitize (hb_sanitize_context_t *c) const
       {
    @@ -138,24 +100,41 @@ struct MarkLigPosFormat1
         if (likely (mark_index == NOT_COVERED)) return_trace (false);
     
         /* Now we search backwards for a non-mark glyph */
    +
         hb_ot_apply_context_t::skipping_iterator_t &skippy_iter = c->iter_input;
    -    skippy_iter.reset (buffer->idx, 1);
         skippy_iter.set_lookup_props (LookupFlag::IgnoreMarks);
    -    unsigned unsafe_from;
    -    if (!skippy_iter.prev (&unsafe_from))
    +
    +    if (c->last_base_until > buffer->idx)
    +    {
    +      c->last_base_until = 0;
    +      c->last_base = -1;
    +    }
    +    unsigned j;
    +    for (j = buffer->idx; j > c->last_base_until; j--)
         {
    -      buffer->unsafe_to_concat_from_outbuffer (unsafe_from, buffer->idx + 1);
    +      auto match = skippy_iter.match (buffer->info[j - 1]);
    +      if (match == skippy_iter.MATCH)
    +      {
    +        c->last_base = (signed) j - 1;
    +        break;
    +      }
    +    }
    +    c->last_base_until = buffer->idx;
    +    if (c->last_base == -1)
    +    {
    +      buffer->unsafe_to_concat_from_outbuffer (0, buffer->idx + 1);
           return_trace (false);
         }
     
    +    unsigned idx = (unsigned) c->last_base;
    +
         /* Checking that matched glyph is actually a ligature by GDEF is too strong; disabled */
    -    //if (!_hb_glyph_info_is_ligature (&buffer->info[skippy_iter.idx])) { return_trace (false); }
    +    //if (!_hb_glyph_info_is_ligature (&buffer->info[idx])) { return_trace (false); }
     
    -    unsigned int j = skippy_iter.idx;
    -    unsigned int lig_index = (this+ligatureCoverage).get_coverage  (buffer->info[j].codepoint);
    +    unsigned int lig_index = (this+ligatureCoverage).get_coverage  (buffer->info[idx].codepoint);
         if (lig_index == NOT_COVERED)
         {
    -      buffer->unsafe_to_concat_from_outbuffer (skippy_iter.idx, buffer->idx + 1);
    +      buffer->unsafe_to_concat_from_outbuffer (idx, buffer->idx + 1);
           return_trace (false);
         }
     
    @@ -166,7 +145,7 @@ struct MarkLigPosFormat1
         unsigned int comp_count = lig_attach.rows;
         if (unlikely (!comp_count))
         {
    -      buffer->unsafe_to_concat_from_outbuffer (skippy_iter.idx, buffer->idx + 1);
    +      buffer->unsafe_to_concat_from_outbuffer (idx, buffer->idx + 1);
           return_trace (false);
         }
     
    @@ -175,7 +154,7 @@ struct MarkLigPosFormat1
          * can directly use the component index.  If not, we attach the mark
          * glyph to the last component of the ligature. */
         unsigned int comp_index;
    -    unsigned int lig_id = _hb_glyph_info_get_lig_id (&buffer->info[j]);
    +    unsigned int lig_id = _hb_glyph_info_get_lig_id (&buffer->info[idx]);
         unsigned int mark_id = _hb_glyph_info_get_lig_id (&buffer->cur());
         unsigned int mark_comp = _hb_glyph_info_get_lig_comp (&buffer->cur());
         if (lig_id && lig_id == mark_id && mark_comp > 0)
    @@ -183,7 +162,7 @@ struct MarkLigPosFormat1
         else
           comp_index = comp_count - 1;
     
    -    return_trace ((this+markArray).apply (c, mark_index, comp_index, lig_attach, classCount, j));
    +    return_trace ((this+markArray).apply (c, mark_index, comp_index, lig_attach, classCount, idx));
       }
     
       bool subset (hb_subset_context_t *c) const
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkMarkPos.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkMarkPos.hh
    index c0eee6d54cf89..e0d9eca02806b 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkMarkPos.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkMarkPos.hh
    @@ -11,18 +11,24 @@ struct MarkMarkPos
     {
       protected:
       union {
    -  HBUINT16              format;         /* Format identifier */
    -  MarkMarkPosFormat1    format1;
    +  HBUINT16                              format;         /* Format identifier */
    +  MarkMarkPosFormat1_2      format1;
    +#ifndef HB_NO_BEYOND_64K
    +  MarkMarkPosFormat1_2     format2;
    +#endif
       } u;
     
       public:
       template 
       typename context_t::return_t dispatch (context_t *c, Ts&&... ds) const
       {
    +    if (unlikely (!c->may_dispatch (this, &u.format))) return c->no_dispatch_return_value ();
         TRACE_DISPATCH (this, u.format);
    -    if (unlikely (!c->may_dispatch (this, &u.format))) return_trace (c->no_dispatch_return_value ());
         switch (u.format) {
         case 1: return_trace (c->dispatch (u.format1, std::forward (ds)...));
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return_trace (c->dispatch (u.format2, std::forward (ds)...));
    +#endif
         default:return_trace (c->default_return_value ());
         }
       }
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkMarkPosFormat1.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkMarkPosFormat1.hh
    index c48a74f77389b..fbcebb8044169 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkMarkPosFormat1.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkMarkPosFormat1.hh
    @@ -12,27 +12,28 @@ typedef AnchorMatrix Mark2Array;        /* mark2-major--
                                              * mark1-minor--
                                              * ordered by class--zero-based. */
     
    -struct MarkMarkPosFormat1
    +template 
    +struct MarkMarkPosFormat1_2
     {
       protected:
       HBUINT16      format;                 /* Format identifier--format = 1 */
    -  Offset16To
    +  typename Types::template OffsetTo
                     mark1Coverage;          /* Offset to Combining Mark1 Coverage
                                              * table--from beginning of MarkMarkPos
                                              * subtable */
    -  Offset16To
    +  typename Types::template OffsetTo
                     mark2Coverage;          /* Offset to Combining Mark2 Coverage
                                              * table--from beginning of MarkMarkPos
                                              * subtable */
       HBUINT16      classCount;             /* Number of defined mark classes */
    -  Offset16To
    +  typename Types::template OffsetTo
                     mark1Array;             /* Offset to Mark1Array table--from
                                              * beginning of MarkMarkPos subtable */
    -  Offset16To
    +  typename Types::template OffsetTo
                     mark2Array;             /* Offset to Mark2Array table--from
                                              * beginning of MarkMarkPos subtable */
       public:
    -  DEFINE_SIZE_STATIC (12);
    +  DEFINE_SIZE_STATIC (4 + 4 * Types::size);
     
       bool sanitize (hb_sanitize_context_t *c) const
       {
    @@ -100,7 +101,7 @@ struct MarkMarkPosFormat1
         /* now we search backwards for a suitable mark glyph until a non-mark glyph */
         hb_ot_apply_context_t::skipping_iterator_t &skippy_iter = c->iter_input;
         skippy_iter.reset (buffer->idx, 1);
    -    skippy_iter.set_lookup_props (c->lookup_props & ~LookupFlag::IgnoreFlags);
    +    skippy_iter.set_lookup_props (c->lookup_props & ~(uint32_t)LookupFlag::IgnoreFlags);
         unsigned unsafe_from;
         if (!skippy_iter.prev (&unsafe_from))
         {
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkRecord.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkRecord.hh
    index 7a514453aeeed..a7d489d2a51f0 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkRecord.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/MarkRecord.hh
    @@ -9,7 +9,7 @@ struct MarkRecord
     {
       friend struct MarkArray;
     
    -  protected:
    +  public:
       HBUINT16      klass;                  /* Class defined for this mark */
       Offset16To
                     markAnchor;             /* Offset to Anchor table--from
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/PairPos.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/PairPos.hh
    index 8479178d384a5..e3794ea9ed59a 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/PairPos.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/PairPos.hh
    @@ -12,20 +12,28 @@ struct PairPos
     {
       protected:
       union {
    -  HBUINT16              format;         /* Format identifier */
    -  PairPosFormat1        format1;
    -  PairPosFormat2        format2;
    +  HBUINT16                      format;         /* Format identifier */
    +  PairPosFormat1_3  format1;
    +  PairPosFormat2_4  format2;
    +#ifndef HB_NO_BEYOND_64K
    +  PairPosFormat1_3 format3;
    +  PairPosFormat2_4 format4;
    +#endif
       } u;
     
       public:
       template 
       typename context_t::return_t dispatch (context_t *c, Ts&&... ds) const
       {
    +    if (unlikely (!c->may_dispatch (this, &u.format))) return c->no_dispatch_return_value ();
         TRACE_DISPATCH (this, u.format);
    -    if (unlikely (!c->may_dispatch (this, &u.format))) return_trace (c->no_dispatch_return_value ());
         switch (u.format) {
         case 1: return_trace (c->dispatch (u.format1, std::forward (ds)...));
         case 2: return_trace (c->dispatch (u.format2, std::forward (ds)...));
    +#ifndef HB_NO_BEYOND_64K
    +    case 3: return_trace (c->dispatch (u.format3, std::forward (ds)...));
    +    case 4: return_trace (c->dispatch (u.format4, std::forward (ds)...));
    +#endif
         default:return_trace (c->default_return_value ());
         }
       }
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/PairPosFormat1.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/PairPosFormat1.hh
    index 35a2db2d45eb5..468eccfd50169 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/PairPosFormat1.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/PairPosFormat1.hh
    @@ -1,248 +1,22 @@
     #ifndef OT_LAYOUT_GPOS_PAIRPOSFORMAT1_HH
     #define OT_LAYOUT_GPOS_PAIRPOSFORMAT1_HH
     
    +#include "PairSet.hh"
    +
     namespace OT {
     namespace Layout {
     namespace GPOS_impl {
     
    -struct PairValueRecord
    -{
    -  friend struct PairSet;
    -
    -  int cmp (hb_codepoint_t k) const
    -  { return secondGlyph.cmp (k); }
    -
    -  struct context_t
    -  {
    -    const void          *base;
    -    const ValueFormat   *valueFormats;
    -    const ValueFormat   *newFormats;
    -    unsigned            len1; /* valueFormats[0].get_len() */
    -    const hb_map_t      *glyph_map;
    -    const hb_map_t      *layout_variation_idx_map;
    -  };
    -
    -  bool subset (hb_subset_context_t *c,
    -               context_t *closure) const
    -  {
    -    TRACE_SERIALIZE (this);
    -    auto *s = c->serializer;
    -    auto *out = s->start_embed (*this);
    -    if (unlikely (!s->extend_min (out))) return_trace (false);
    -
    -    out->secondGlyph = (*closure->glyph_map)[secondGlyph];
    -
    -    closure->valueFormats[0].copy_values (s,
    -                                          closure->newFormats[0],
    -                                          closure->base, &values[0],
    -                                          closure->layout_variation_idx_map);
    -    closure->valueFormats[1].copy_values (s,
    -                                          closure->newFormats[1],
    -                                          closure->base,
    -                                          &values[closure->len1],
    -                                          closure->layout_variation_idx_map);
    -
    -    return_trace (true);
    -  }
    -
    -  void collect_variation_indices (hb_collect_variation_indices_context_t *c,
    -                                  const ValueFormat *valueFormats,
    -                                  const void *base) const
    -  {
    -    unsigned record1_len = valueFormats[0].get_len ();
    -    unsigned record2_len = valueFormats[1].get_len ();
    -    const hb_array_t values_array = values.as_array (record1_len + record2_len);
    -
    -    if (valueFormats[0].has_device ())
    -      valueFormats[0].collect_variation_indices (c, base, values_array.sub_array (0, record1_len));
    -
    -    if (valueFormats[1].has_device ())
    -      valueFormats[1].collect_variation_indices (c, base, values_array.sub_array (record1_len, record2_len));
    -  }
    -
    -  bool intersects (const hb_set_t& glyphset) const
    -  {
    -    return glyphset.has(secondGlyph);
    -  }
    -
    -  const Value* get_values_1 () const
    -  {
    -    return &values[0];
    -  }
    -
    -  const Value* get_values_2 (ValueFormat format1) const
    -  {
    -    return &values[format1.get_len ()];
    -  }
    -
    -  protected:
    -  HBGlyphID16   secondGlyph;            /* GlyphID of second glyph in the
    -                                         * pair--first glyph is listed in the
    -                                         * Coverage table */
    -  ValueRecord   values;                 /* Positioning data for the first glyph
    -                                         * followed by for second glyph */
    -  public:
    -  DEFINE_SIZE_ARRAY (2, values);
    -};
     
    -struct PairSet
    +template 
    +struct PairPosFormat1_3
     {
    -  friend struct PairPosFormat1;
    -
    -  bool intersects (const hb_set_t *glyphs,
    -                   const ValueFormat *valueFormats) const
    -  {
    -    unsigned int len1 = valueFormats[0].get_len ();
    -    unsigned int len2 = valueFormats[1].get_len ();
    -    unsigned int record_size = HBUINT16::static_size * (1 + len1 + len2);
    -
    -    const PairValueRecord *record = &firstPairValueRecord;
    -    unsigned int count = len;
    -    for (unsigned int i = 0; i < count; i++)
    -    {
    -      if (glyphs->has (record->secondGlyph))
    -        return true;
    -      record = &StructAtOffset (record, record_size);
    -    }
    -    return false;
    -  }
    -
    -  void collect_glyphs (hb_collect_glyphs_context_t *c,
    -                       const ValueFormat *valueFormats) const
    -  {
    -    unsigned int len1 = valueFormats[0].get_len ();
    -    unsigned int len2 = valueFormats[1].get_len ();
    -    unsigned int record_size = HBUINT16::static_size * (1 + len1 + len2);
    -
    -    const PairValueRecord *record = &firstPairValueRecord;
    -    c->input->add_array (&record->secondGlyph, len, record_size);
    -  }
    -
    -  void collect_variation_indices (hb_collect_variation_indices_context_t *c,
    -                                  const ValueFormat *valueFormats) const
    -  {
    -    unsigned len1 = valueFormats[0].get_len ();
    -    unsigned len2 = valueFormats[1].get_len ();
    -    unsigned record_size = HBUINT16::static_size * (1 + len1 + len2);
    -
    -    const PairValueRecord *record = &firstPairValueRecord;
    -    unsigned count = len;
    -    for (unsigned i = 0; i < count; i++)
    -    {
    -      if (c->glyph_set->has (record->secondGlyph))
    -      { record->collect_variation_indices (c, valueFormats, this); }
    -
    -      record = &StructAtOffset (record, record_size);
    -    }
    -  }
    -
    -  bool apply (hb_ot_apply_context_t *c,
    -              const ValueFormat *valueFormats,
    -              unsigned int pos) const
    -  {
    -    TRACE_APPLY (this);
    -    hb_buffer_t *buffer = c->buffer;
    -    unsigned int len1 = valueFormats[0].get_len ();
    -    unsigned int len2 = valueFormats[1].get_len ();
    -    unsigned int record_size = HBUINT16::static_size * (1 + len1 + len2);
    -
    -    const PairValueRecord *record = hb_bsearch (buffer->info[pos].codepoint,
    -                                                &firstPairValueRecord,
    -                                                len,
    -                                                record_size);
    -    if (record)
    -    {
    -      bool applied_first = valueFormats[0].apply_value (c, this, &record->values[0], buffer->cur_pos());
    -      bool applied_second = valueFormats[1].apply_value (c, this, &record->values[len1], buffer->pos[pos]);
    -      if (applied_first || applied_second)
    -        buffer->unsafe_to_break (buffer->idx, pos + 1);
    -      if (len2)
    -        pos++;
    -      buffer->idx = pos;
    -      return_trace (true);
    -    }
    -    buffer->unsafe_to_concat (buffer->idx, pos + 1);
    -    return_trace (false);
    -  }
    -
    -  bool subset (hb_subset_context_t *c,
    -               const ValueFormat valueFormats[2],
    -               const ValueFormat newFormats[2]) const
    -  {
    -    TRACE_SUBSET (this);
    -    auto snap = c->serializer->snapshot ();
    -
    -    auto *out = c->serializer->start_embed (*this);
    -    if (unlikely (!c->serializer->extend_min (out))) return_trace (false);
    -    out->len = 0;
    -
    -    const hb_set_t &glyphset = *c->plan->glyphset_gsub ();
    -    const hb_map_t &glyph_map = *c->plan->glyph_map;
    -
    -    unsigned len1 = valueFormats[0].get_len ();
    -    unsigned len2 = valueFormats[1].get_len ();
    -    unsigned record_size = HBUINT16::static_size + Value::static_size * (len1 + len2);
    -
    -    PairValueRecord::context_t context =
    -    {
    -      this,
    -      valueFormats,
    -      newFormats,
    -      len1,
    -      &glyph_map,
    -      c->plan->layout_variation_idx_map
    -    };
    -
    -    const PairValueRecord *record = &firstPairValueRecord;
    -    unsigned count = len, num = 0;
    -    for (unsigned i = 0; i < count; i++)
    -    {
    -      if (glyphset.has (record->secondGlyph)
    -         && record->subset (c, &context)) num++;
    -      record = &StructAtOffset (record, record_size);
    -    }
    -
    -    out->len = num;
    -    if (!num) c->serializer->revert (snap);
    -    return_trace (num);
    -  }
    -
    -  struct sanitize_closure_t
    -  {
    -    const ValueFormat *valueFormats;
    -    unsigned int len1; /* valueFormats[0].get_len() */
    -    unsigned int stride; /* 1 + len1 + len2 */
    -  };
    -
    -  bool sanitize (hb_sanitize_context_t *c, const sanitize_closure_t *closure) const
    -  {
    -    TRACE_SANITIZE (this);
    -    if (!(c->check_struct (this)
    -       && c->check_range (&firstPairValueRecord,
    -                          len,
    -                          HBUINT16::static_size,
    -                          closure->stride))) return_trace (false);
    -
    -    unsigned int count = len;
    -    const PairValueRecord *record = &firstPairValueRecord;
    -    return_trace (closure->valueFormats[0].sanitize_values_stride_unsafe (c, this, &record->values[0], count, closure->stride) &&
    -                  closure->valueFormats[1].sanitize_values_stride_unsafe (c, this, &record->values[closure->len1], count, closure->stride));
    -  }
    +  using PairSet = GPOS_impl::PairSet;
    +  using PairValueRecord = GPOS_impl::PairValueRecord;
     
    -  protected:
    -  HBUINT16              len;    /* Number of PairValueRecords */
    -  PairValueRecord       firstPairValueRecord;
    -                                /* Array of PairValueRecords--ordered
    -                                 * by GlyphID of the second glyph */
    -  public:
    -  DEFINE_SIZE_MIN (2);
    -};
    -
    -struct PairPosFormat1
    -{
       protected:
       HBUINT16      format;                 /* Format identifier--format = 1 */
    -  Offset16To
    +  typename Types::template OffsetTo
                     coverage;               /* Offset to Coverage table--from
                                              * beginning of subtable */
       ValueFormat   valueFormat[2];         /* [0] Defines the types of data in
    @@ -251,11 +25,11 @@ struct PairPosFormat1
                                             /* [1] Defines the types of data in
                                              * ValueRecord2--for the second glyph
                                              * in the pair--may be zero (0) */
    -  Array16OfOffset16To
    +  Array16Of>
                     pairSet;                /* Array of PairSet tables
                                              * ordered by Coverage Index */
       public:
    -  DEFINE_SIZE_ARRAY (10, pairSet);
    +  DEFINE_SIZE_ARRAY (8 + Types::size, pairSet);
     
       bool sanitize (hb_sanitize_context_t *c) const
       {
    @@ -265,24 +39,36 @@ struct PairPosFormat1
     
         unsigned int len1 = valueFormat[0].get_len ();
         unsigned int len2 = valueFormat[1].get_len ();
    -    PairSet::sanitize_closure_t closure =
    +    typename PairSet::sanitize_closure_t closure =
         {
           valueFormat,
           len1,
    -      1 + len1 + len2
    +      PairSet::get_size (len1, len2)
         };
     
         return_trace (coverage.sanitize (c, this) && pairSet.sanitize (c, this, &closure));
       }
     
    -
       bool intersects (const hb_set_t *glyphs) const
       {
    +    auto &cov = this+coverage;
    +
    +    if (pairSet.len > glyphs->get_population () * hb_bit_storage ((unsigned) pairSet.len) / 4)
    +    {
    +      for (hb_codepoint_t g : glyphs->iter())
    +      {
    +        unsigned i = cov.get_coverage (g);
    +        if ((this+pairSet[i]).intersects (glyphs, valueFormat))
    +          return true;
    +      }
    +      return false;
    +    }
    +
         return
    -    + hb_zip (this+coverage, pairSet)
    +    + hb_zip (cov, pairSet)
         | hb_filter (*glyphs, hb_first)
         | hb_map (hb_second)
    -    | hb_map ([glyphs, this] (const Offset16To &_)
    +    | hb_map ([glyphs, this] (const typename Types::template OffsetTo &_)
                   { return (this+_).intersects (glyphs, valueFormat); })
         | hb_any
         ;
    @@ -354,11 +140,17 @@ struct PairPosFormat1
           out->valueFormat[1] = newFormats.second;
         }
     
    +    if (c->plan->all_axes_pinned)
    +    {
    +      out->valueFormat[0] = out->valueFormat[0].drop_device_table_flags ();
    +      out->valueFormat[1] = out->valueFormat[1].drop_device_table_flags ();
    +    }
    +
         hb_sorted_vector_t new_coverage;
     
         + hb_zip (this+coverage, pairSet)
         | hb_filter (glyphset, hb_first)
    -    | hb_filter ([this, c, out] (const Offset16To& _)
    +    | hb_filter ([this, c, out] (const typename Types::template OffsetTo& _)
                      {
                        auto snap = c->serializer->snapshot ();
                        auto *o = out->pairSet.serialize_append (c->serializer);
    @@ -385,19 +177,21 @@ struct PairPosFormat1
     
       hb_pair_t compute_effective_value_formats (const hb_set_t& glyphset) const
       {
    -    unsigned len1 = valueFormat[0].get_len ();
    -    unsigned len2 = valueFormat[1].get_len ();
    -    unsigned record_size = HBUINT16::static_size + Value::static_size * (len1 + len2);
    +    unsigned record_size = PairSet::get_size (valueFormat);
     
         unsigned format1 = 0;
         unsigned format2 = 0;
    -    for (const Offset16To& _ :
    -             + hb_zip (this+coverage, pairSet) | hb_filter (glyphset, hb_first) | hb_map (hb_second))
    +    for (const auto & _ :
    +          + hb_zip (this+coverage, pairSet)
    +          | hb_filter (glyphset, hb_first)
    +          | hb_map (hb_second)
    +        )
         {
           const PairSet& set = (this + _);
           const PairValueRecord *record = &set.firstPairValueRecord;
     
    -      for (unsigned i = 0; i < set.len; i++)
    +      unsigned count = set.len;
    +      for (unsigned i = 0; i < count; i++)
           {
             if (record->intersects (glyphset))
             {
    @@ -406,6 +200,9 @@ struct PairPosFormat1
             }
             record = &StructAtOffset (record, record_size);
           }
    +
    +      if (format1 == valueFormat[0] && format2 == valueFormat[1])
    +        break;
         }
     
         return hb_pair (format1, format2);
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/PairPosFormat2.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/PairPosFormat2.hh
    index 3f5f9959c4675..17486dddaf7de 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/PairPosFormat2.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/PairPosFormat2.hh
    @@ -7,11 +7,12 @@ namespace OT {
     namespace Layout {
     namespace GPOS_impl {
     
    -struct PairPosFormat2
    +template 
    +struct PairPosFormat2_4
     {
       protected:
       HBUINT16      format;                 /* Format identifier--format = 2 */
    -  Offset16To
    +  typename Types::template OffsetTo
                     coverage;               /* Offset to Coverage table--from
                                              * beginning of subtable */
       ValueFormat   valueFormat1;           /* ValueRecord definition--for the
    @@ -20,11 +21,11 @@ struct PairPosFormat2
       ValueFormat   valueFormat2;           /* ValueRecord definition--for the
                                              * second glyph of the pair--may be
                                              * zero (0) */
    -  Offset16To
    +  typename Types::template OffsetTo
                     classDef1;              /* Offset to ClassDef table--from
                                              * beginning of PairPos subtable--for
                                              * the first glyph of the pair */
    -  Offset16To
    +  typename Types::template OffsetTo
                     classDef2;              /* Offset to ClassDef table--from
                                              * beginning of PairPos subtable--for
                                              * the second glyph of the pair */
    @@ -36,7 +37,7 @@ struct PairPosFormat2
                                              * class1-major, class2-minor,
                                              * Each entry has value1 and value2 */
       public:
    -  DEFINE_SIZE_ARRAY (16, values);
    +  DEFINE_SIZE_ARRAY (10 + 3 * Types::size, values);
     
       bool sanitize (hb_sanitize_context_t *c) const
       {
    @@ -48,7 +49,7 @@ struct PairPosFormat2
     
         unsigned int len1 = valueFormat1.get_len ();
         unsigned int len2 = valueFormat2.get_len ();
    -    unsigned int stride = len1 + len2;
    +    unsigned int stride = HBUINT16::static_size * (len1 + len2);
         unsigned int record_size = valueFormat1.get_size () + valueFormat2.get_size ();
         unsigned int count = (unsigned int) class1Count * (unsigned int) class2Count;
         return_trace (c->check_range ((const void *) values,
    @@ -216,10 +217,31 @@ struct PairPosFormat2
         }
         bail:
     
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->message (c->font,
    +                          "try kerning glyphs at %u,%u",
    +                          c->buffer->idx, skippy_iter.idx);
    +    }
     
         applied_first = valueFormat1.apply_value (c, this, v, buffer->cur_pos());
         applied_second = valueFormat2.apply_value (c, this, v + len1, buffer->pos[skippy_iter.idx]);
     
    +    if (applied_first || applied_second)
    +      if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +      {
    +        c->buffer->message (c->font,
    +                            "kerned glyphs at %u,%u",
    +                            c->buffer->idx, skippy_iter.idx);
    +      }
    +
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->message (c->font,
    +                          "tried kerning glyphs at %u,%u",
    +                          c->buffer->idx, skippy_iter.idx);
    +    }
    +
         success:
         if (applied_first || applied_second)
           buffer->unsafe_to_break (buffer->idx, skippy_iter.idx + 1);
    @@ -227,10 +249,15 @@ struct PairPosFormat2
         boring:
           buffer->unsafe_to_concat (buffer->idx, skippy_iter.idx + 1);
     
    +    if (len2)
    +    {
    +      skippy_iter.idx++;
    +      // https://github.com/harfbuzz/harfbuzz/issues/3824
    +      // https://github.com/harfbuzz/harfbuzz/issues/3888#issuecomment-1326781116
    +      buffer->unsafe_to_break (buffer->idx, skippy_iter.idx + 1);
    +    }
     
         buffer->idx = skippy_iter.idx;
    -    if (len2)
    -      buffer->idx++;
     
         return_trace (true);
       }
    @@ -260,13 +287,19 @@ struct PairPosFormat2
         out->valueFormat1 = newFormats.first;
         out->valueFormat2 = newFormats.second;
     
    +    if (c->plan->all_axes_pinned)
    +    {
    +      out->valueFormat1 = out->valueFormat1.drop_device_table_flags ();
    +      out->valueFormat2 = out->valueFormat2.drop_device_table_flags ();
    +    }
    +
         for (unsigned class1_idx : + hb_range ((unsigned) class1Count) | hb_filter (klass1_map))
         {
           for (unsigned class2_idx : + hb_range ((unsigned) class2Count) | hb_filter (klass2_map))
           {
             unsigned idx = (class1_idx * (unsigned) class2Count + class2_idx) * (len1 + len2);
    -        valueFormat1.copy_values (c->serializer, newFormats.first, this, &values[idx], c->plan->layout_variation_idx_map);
    -        valueFormat2.copy_values (c->serializer, newFormats.second, this, &values[idx + len1], c->plan->layout_variation_idx_map);
    +        valueFormat1.copy_values (c->serializer, out->valueFormat1, this, &values[idx], &c->plan->layout_variation_idx_delta_map);
    +        valueFormat2.copy_values (c->serializer, out->valueFormat2, this, &values[idx + len1], &c->plan->layout_variation_idx_delta_map);
           }
         }
     
    @@ -289,6 +322,7 @@ struct PairPosFormat2
       {
         unsigned len1 = valueFormat1.get_len ();
         unsigned len2 = valueFormat2.get_len ();
    +    unsigned record_size = len1 + len2;
     
         unsigned format1 = 0;
         unsigned format2 = 0;
    @@ -297,10 +331,13 @@ struct PairPosFormat2
         {
           for (unsigned class2_idx : + hb_range ((unsigned) class2Count) | hb_filter (klass2_map))
           {
    -        unsigned idx = (class1_idx * (unsigned) class2Count + class2_idx) * (len1 + len2);
    +        unsigned idx = (class1_idx * (unsigned) class2Count + class2_idx) * record_size;
             format1 = format1 | valueFormat1.get_effective_format (&values[idx]);
             format2 = format2 | valueFormat2.get_effective_format (&values[idx + len1]);
           }
    +
    +      if (format1 == valueFormat1 && format2 == valueFormat2)
    +        break;
         }
     
         return hb_pair (format1, format2);
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/PairSet.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/PairSet.hh
    new file mode 100644
    index 0000000000000..adeb08e910b82
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/PairSet.hh
    @@ -0,0 +1,207 @@
    +#ifndef OT_LAYOUT_GPOS_PAIRSET_HH
    +#define OT_LAYOUT_GPOS_PAIRSET_HH
    +
    +#include "PairValueRecord.hh"
    +
    +namespace OT {
    +namespace Layout {
    +namespace GPOS_impl {
    +
    +
    +template 
    +struct PairSet
    +{
    +  template 
    +  friend struct PairPosFormat1_3;
    +
    +  using PairValueRecord = GPOS_impl::PairValueRecord;
    +
    +  protected:
    +  HBUINT16              len;    /* Number of PairValueRecords */
    +  PairValueRecord       firstPairValueRecord;
    +                                /* Array of PairValueRecords--ordered
    +                                 * by GlyphID of the second glyph */
    +  public:
    +  DEFINE_SIZE_MIN (2);
    +
    +  static unsigned get_size (unsigned len1, unsigned len2)
    +  {
    +    return Types::HBGlyphID::static_size + Value::static_size * (len1 + len2);
    +  }
    +  static unsigned get_size (const ValueFormat valueFormats[2])
    +  {
    +    unsigned len1 = valueFormats[0].get_len ();
    +    unsigned len2 = valueFormats[1].get_len ();
    +    return get_size (len1, len2);
    +  }
    +
    +  struct sanitize_closure_t
    +  {
    +    const ValueFormat *valueFormats;
    +    unsigned int len1; /* valueFormats[0].get_len() */
    +    unsigned int stride; /* bytes */
    +  };
    +
    +  bool sanitize (hb_sanitize_context_t *c, const sanitize_closure_t *closure) const
    +  {
    +    TRACE_SANITIZE (this);
    +    if (!(c->check_struct (this)
    +       && c->check_range (&firstPairValueRecord,
    +                          len,
    +                          closure->stride))) return_trace (false);
    +
    +    unsigned int count = len;
    +    const PairValueRecord *record = &firstPairValueRecord;
    +    return_trace (closure->valueFormats[0].sanitize_values_stride_unsafe (c, this, &record->values[0], count, closure->stride) &&
    +                  closure->valueFormats[1].sanitize_values_stride_unsafe (c, this, &record->values[closure->len1], count, closure->stride));
    +  }
    +
    +  bool intersects (const hb_set_t *glyphs,
    +                   const ValueFormat *valueFormats) const
    +  {
    +    unsigned record_size = get_size (valueFormats);
    +
    +    const PairValueRecord *record = &firstPairValueRecord;
    +    unsigned int count = len;
    +    for (unsigned int i = 0; i < count; i++)
    +    {
    +      if (glyphs->has (record->secondGlyph))
    +        return true;
    +      record = &StructAtOffset (record, record_size);
    +    }
    +    return false;
    +  }
    +
    +  void collect_glyphs (hb_collect_glyphs_context_t *c,
    +                       const ValueFormat *valueFormats) const
    +  {
    +    unsigned record_size = get_size (valueFormats);
    +
    +    const PairValueRecord *record = &firstPairValueRecord;
    +    c->input->add_array (&record->secondGlyph, len, record_size);
    +  }
    +
    +  void collect_variation_indices (hb_collect_variation_indices_context_t *c,
    +                                  const ValueFormat *valueFormats) const
    +  {
    +    unsigned record_size = get_size (valueFormats);
    +
    +    const PairValueRecord *record = &firstPairValueRecord;
    +    unsigned count = len;
    +    for (unsigned i = 0; i < count; i++)
    +    {
    +      if (c->glyph_set->has (record->secondGlyph))
    +      { record->collect_variation_indices (c, valueFormats, this); }
    +
    +      record = &StructAtOffset (record, record_size);
    +    }
    +  }
    +
    +  bool apply (hb_ot_apply_context_t *c,
    +              const ValueFormat *valueFormats,
    +              unsigned int pos) const
    +  {
    +    TRACE_APPLY (this);
    +    hb_buffer_t *buffer = c->buffer;
    +    unsigned int len1 = valueFormats[0].get_len ();
    +    unsigned int len2 = valueFormats[1].get_len ();
    +    unsigned record_size = get_size (len1, len2);
    +
    +    const PairValueRecord *record = hb_bsearch (buffer->info[pos].codepoint,
    +                                                &firstPairValueRecord,
    +                                                len,
    +                                                record_size);
    +    if (record)
    +    {
    +      if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +      {
    +        c->buffer->message (c->font,
    +                            "try kerning glyphs at %u,%u",
    +                            c->buffer->idx, pos);
    +      }
    +
    +      bool applied_first = valueFormats[0].apply_value (c, this, &record->values[0], buffer->cur_pos());
    +      bool applied_second = valueFormats[1].apply_value (c, this, &record->values[len1], buffer->pos[pos]);
    +
    +      if (applied_first || applied_second)
    +        if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +        {
    +          c->buffer->message (c->font,
    +                              "kerned glyphs at %u,%u",
    +                              c->buffer->idx, pos);
    +        }
    +
    +      if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +      {
    +        c->buffer->message (c->font,
    +                            "tried kerning glyphs at %u,%u",
    +                            c->buffer->idx, pos);
    +      }
    +
    +      if (applied_first || applied_second)
    +        buffer->unsafe_to_break (buffer->idx, pos + 1);
    +
    +      if (len2)
    +      {
    +        pos++;
    +      // https://github.com/harfbuzz/harfbuzz/issues/3824
    +      // https://github.com/harfbuzz/harfbuzz/issues/3888#issuecomment-1326781116
    +      buffer->unsafe_to_break (buffer->idx, pos + 1);
    +      }
    +
    +      buffer->idx = pos;
    +      return_trace (true);
    +    }
    +    buffer->unsafe_to_concat (buffer->idx, pos + 1);
    +    return_trace (false);
    +  }
    +
    +  bool subset (hb_subset_context_t *c,
    +               const ValueFormat valueFormats[2],
    +               const ValueFormat newFormats[2]) const
    +  {
    +    TRACE_SUBSET (this);
    +    auto snap = c->serializer->snapshot ();
    +
    +    auto *out = c->serializer->start_embed (*this);
    +    if (unlikely (!c->serializer->extend_min (out))) return_trace (false);
    +    out->len = 0;
    +
    +    const hb_set_t &glyphset = *c->plan->glyphset_gsub ();
    +    const hb_map_t &glyph_map = *c->plan->glyph_map;
    +
    +    unsigned len1 = valueFormats[0].get_len ();
    +    unsigned len2 = valueFormats[1].get_len ();
    +    unsigned record_size = get_size (len1, len2);
    +
    +    typename PairValueRecord::context_t context =
    +    {
    +      this,
    +      valueFormats,
    +      newFormats,
    +      len1,
    +      &glyph_map,
    +      &c->plan->layout_variation_idx_delta_map
    +    };
    +
    +    const PairValueRecord *record = &firstPairValueRecord;
    +    unsigned count = len, num = 0;
    +    for (unsigned i = 0; i < count; i++)
    +    {
    +      if (glyphset.has (record->secondGlyph)
    +         && record->subset (c, &context)) num++;
    +      record = &StructAtOffset (record, record_size);
    +    }
    +
    +    out->len = num;
    +    if (!num) c->serializer->revert (snap);
    +    return_trace (num);
    +  }
    +};
    +
    +
    +}
    +}
    +}
    +
    +#endif  // OT_LAYOUT_GPOS_PAIRSET_HH
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/PairValueRecord.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/PairValueRecord.hh
    new file mode 100644
    index 0000000000000..d4f549a480dcf
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/PairValueRecord.hh
    @@ -0,0 +1,99 @@
    +#ifndef OT_LAYOUT_GPOS_PAIRVALUERECORD_HH
    +#define OT_LAYOUT_GPOS_PAIRVALUERECORD_HH
    +
    +#include "ValueFormat.hh"
    +
    +namespace OT {
    +namespace Layout {
    +namespace GPOS_impl {
    +
    +
    +template 
    +struct PairValueRecord
    +{
    +  template 
    +  friend struct PairSet;
    +
    +  protected:
    +  typename Types::HBGlyphID
    +                secondGlyph;            /* GlyphID of second glyph in the
    +                                         * pair--first glyph is listed in the
    +                                         * Coverage table */
    +  ValueRecord   values;                 /* Positioning data for the first glyph
    +                                         * followed by for second glyph */
    +  public:
    +  DEFINE_SIZE_ARRAY (Types::size, values);
    +
    +  int cmp (hb_codepoint_t k) const
    +  { return secondGlyph.cmp (k); }
    +
    +  struct context_t
    +  {
    +    const void          *base;
    +    const ValueFormat   *valueFormats;
    +    const ValueFormat   *newFormats;
    +    unsigned            len1; /* valueFormats[0].get_len() */
    +    const hb_map_t      *glyph_map;
    +    const hb_hashmap_t> *layout_variation_idx_delta_map;
    +  };
    +
    +  bool subset (hb_subset_context_t *c,
    +               context_t *closure) const
    +  {
    +    TRACE_SERIALIZE (this);
    +    auto *s = c->serializer;
    +    auto *out = s->start_embed (*this);
    +    if (unlikely (!s->extend_min (out))) return_trace (false);
    +
    +    out->secondGlyph = (*closure->glyph_map)[secondGlyph];
    +
    +    closure->valueFormats[0].copy_values (s,
    +                                          closure->newFormats[0],
    +                                          closure->base, &values[0],
    +                                          closure->layout_variation_idx_delta_map);
    +    closure->valueFormats[1].copy_values (s,
    +                                          closure->newFormats[1],
    +                                          closure->base,
    +                                          &values[closure->len1],
    +                                          closure->layout_variation_idx_delta_map);
    +
    +    return_trace (true);
    +  }
    +
    +  void collect_variation_indices (hb_collect_variation_indices_context_t *c,
    +                                  const ValueFormat *valueFormats,
    +                                  const void *base) const
    +  {
    +    unsigned record1_len = valueFormats[0].get_len ();
    +    unsigned record2_len = valueFormats[1].get_len ();
    +    const hb_array_t values_array = values.as_array (record1_len + record2_len);
    +
    +    if (valueFormats[0].has_device ())
    +      valueFormats[0].collect_variation_indices (c, base, values_array.sub_array (0, record1_len));
    +
    +    if (valueFormats[1].has_device ())
    +      valueFormats[1].collect_variation_indices (c, base, values_array.sub_array (record1_len, record2_len));
    +  }
    +
    +  bool intersects (const hb_set_t& glyphset) const
    +  {
    +    return glyphset.has(secondGlyph);
    +  }
    +
    +  const Value* get_values_1 () const
    +  {
    +    return &values[0];
    +  }
    +
    +  const Value* get_values_2 (ValueFormat format1) const
    +  {
    +    return &values[format1.get_len ()];
    +  }
    +};
    +
    +
    +}
    +}
    +}
    +
    +#endif  // OT_LAYOUT_GPOS_PAIRVALUERECORD_HH
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/SinglePos.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/SinglePos.hh
    index 57e146befd62e..3af6c49965943 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/SinglePos.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/SinglePos.hh
    @@ -38,17 +38,18 @@ struct SinglePos
       void serialize (hb_serialize_context_t *c,
                       const SrcLookup* src,
                       Iterator glyph_val_iter_pairs,
    -                  const hb_map_t *layout_variation_idx_map)
    +                  const hb_hashmap_t> *layout_variation_idx_delta_map,
    +                  bool all_axes_pinned)
       {
         if (unlikely (!c->extend_min (u.format))) return;
         unsigned format = 2;
         ValueFormat new_format = src->get_value_format ();
     
    +    if (all_axes_pinned)
    +      new_format = new_format.drop_device_table_flags ();
    +
         if (glyph_val_iter_pairs)
    -    {
           format = get_format (glyph_val_iter_pairs);
    -      new_format = src->get_value_format ().get_effective_format (+ glyph_val_iter_pairs | hb_map (hb_second));
    -    }
     
         u.format = format;
         switch (u.format) {
    @@ -56,13 +57,13 @@ struct SinglePos
                                      src,
                                      glyph_val_iter_pairs,
                                      new_format,
    -                                 layout_variation_idx_map);
    +                                 layout_variation_idx_delta_map);
           return;
         case 2: u.format2.serialize (c,
                                      src,
                                      glyph_val_iter_pairs,
                                      new_format,
    -                                 layout_variation_idx_map);
    +                                 layout_variation_idx_delta_map);
           return;
         default:return;
         }
    @@ -71,8 +72,8 @@ struct SinglePos
       template 
       typename context_t::return_t dispatch (context_t *c, Ts&&... ds) const
       {
    +    if (unlikely (!c->may_dispatch (this, &u.format))) return c->no_dispatch_return_value ();
         TRACE_DISPATCH (this, u.format);
    -    if (unlikely (!c->may_dispatch (this, &u.format))) return_trace (c->no_dispatch_return_value ());
         switch (u.format) {
         case 1: return_trace (c->dispatch (u.format1, std::forward (ds)...));
         case 2: return_trace (c->dispatch (u.format2, std::forward (ds)...));
    @@ -87,8 +88,9 @@ static void
     SinglePos_serialize (hb_serialize_context_t *c,
                          const SrcLookup *src,
                          Iterator it,
    -                     const hb_map_t *layout_variation_idx_map)
    -{ c->start_embed ()->serialize (c, src, it, layout_variation_idx_map); }
    +                     const hb_hashmap_t> *layout_variation_idx_delta_map,
    +                     bool all_axes_pinned)
    +{ c->start_embed ()->serialize (c, src, it, layout_variation_idx_delta_map, all_axes_pinned); }
     
     
     }
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/SinglePosFormat1.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/SinglePosFormat1.hh
    index 8b7840ed0ebfd..47391c77028d5 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/SinglePosFormat1.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/SinglePosFormat1.hh
    @@ -28,7 +28,15 @@ struct SinglePosFormat1
         TRACE_SANITIZE (this);
         return_trace (c->check_struct (this) &&
                       coverage.sanitize (c, this) &&
    +                  /* The coverage  table may use a range to represent a set
    +                   * of glyphs, which means a small number of bytes can
    +                   * generate a large glyph set. Manually modify the
    +                   * sanitizer max ops to take this into account.
    +                   *
    +                   * Note: This check *must* be right after coverage sanitize. */
    +                  c->check_ops ((this + coverage).get_population () >> 1) &&
                       valueFormat.sanitize_value (c, this, values));
    +
       }
     
       bool intersects (const hb_set_t *glyphs) const
    @@ -39,12 +47,10 @@ struct SinglePosFormat1
       {
         if (!valueFormat.has_device ()) return;
     
    -    auto it =
    -    + hb_iter (this+coverage)
    -    | hb_filter (c->glyph_set)
    -    ;
    +    hb_set_t intersection;
    +    (this+coverage).intersect_set (*c->glyph_set, intersection);
    +    if (!intersection) return;
     
    -    if (!it) return;
         valueFormat.collect_variation_indices (c, this, values.as_array (valueFormat.get_len ()));
       }
     
    @@ -62,12 +68,44 @@ struct SinglePosFormat1
         unsigned int index = (this+coverage).get_coverage  (buffer->cur().codepoint);
         if (likely (index == NOT_COVERED)) return_trace (false);
     
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->message (c->font,
    +                          "positioning glyph at %u",
    +                          c->buffer->idx);
    +    }
    +
         valueFormat.apply_value (c, this, values, buffer->cur_pos());
     
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->message (c->font,
    +                          "positioned glyph at %u",
    +                          c->buffer->idx);
    +    }
    +
         buffer->idx++;
         return_trace (true);
       }
     
    +  bool
    +  position_single (hb_font_t           *font,
    +                   hb_direction_t       direction,
    +                   hb_codepoint_t       gid,
    +                   hb_glyph_position_t &pos) const
    +  {
    +    unsigned int index = (this+coverage).get_coverage  (gid);
    +    if (likely (index == NOT_COVERED)) return false;
    +
    +    /* This is ugly... */
    +    hb_buffer_t buffer;
    +    buffer.props.direction = direction;
    +    OT::hb_ot_apply_context_t c (1, font, &buffer);
    +
    +    valueFormat.apply_value (&c, this, values, pos);
    +    return true;
    +  }
    +
       template
    @@ -75,7 +113,7 @@ struct SinglePosFormat1
                       const SrcLookup *src,
                       Iterator it,
                       ValueFormat newFormat,
    -                  const hb_map_t *layout_variation_idx_map)
    +                  const hb_hashmap_t> *layout_variation_idx_delta_map)
       {
         if (unlikely (!c->extend_min (this))) return;
         if (unlikely (!c->check_assign (valueFormat,
    @@ -84,7 +122,7 @@ struct SinglePosFormat1
     
         for (const hb_array_t& _ : + it | hb_map (hb_second))
         {
    -      src->get_value_format ().copy_values (c, newFormat, src,  &_, layout_variation_idx_map);
    +      src->get_value_format ().copy_values (c, newFormat, src,  &_, layout_variation_idx_delta_map);
           // Only serialize the first entry in the iterator, the rest are assumed to
           // be the same.
           break;
    @@ -104,15 +142,17 @@ struct SinglePosFormat1
         const hb_set_t &glyphset = *c->plan->glyphset_gsub ();
         const hb_map_t &glyph_map = *c->plan->glyph_map;
     
    +    hb_set_t intersection;
    +    (this+coverage).intersect_set (glyphset, intersection);
    +
         auto it =
    -    + hb_iter (this+coverage)
    -    | hb_filter (glyphset)
    +    + hb_iter (intersection)
         | hb_map_retains_sorting (glyph_map)
         | hb_zip (hb_repeat (values.as_array (valueFormat.get_len ())))
         ;
     
         bool ret = bool (it);
    -    SinglePos_serialize (c->serializer, this, it, c->plan->layout_variation_idx_map);
    +    SinglePos_serialize (c->serializer, this, it, &c->plan->layout_variation_idx_delta_map, c->plan->all_axes_pinned);
         return_trace (ret);
       }
     };
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/SinglePosFormat2.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/SinglePosFormat2.hh
    index 0d038b44220e1..6546eb167037c 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/SinglePosFormat2.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/SinglePosFormat2.hh
    @@ -68,16 +68,52 @@ struct SinglePosFormat2
         unsigned int index = (this+coverage).get_coverage  (buffer->cur().codepoint);
         if (likely (index == NOT_COVERED)) return_trace (false);
     
    -    if (likely (index >= valueCount)) return_trace (false);
    +    if (unlikely (index >= valueCount)) return_trace (false);
    +
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->message (c->font,
    +                          "positioning glyph at %u",
    +                          c->buffer->idx);
    +    }
     
         valueFormat.apply_value (c, this,
                                  &values[index * valueFormat.get_len ()],
                                  buffer->cur_pos());
     
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->message (c->font,
    +                          "positioned glyph at %u",
    +                          c->buffer->idx);
    +    }
    +
         buffer->idx++;
         return_trace (true);
       }
     
    +  bool
    +  position_single (hb_font_t           *font,
    +                   hb_direction_t       direction,
    +                   hb_codepoint_t       gid,
    +                   hb_glyph_position_t &pos) const
    +  {
    +    unsigned int index = (this+coverage).get_coverage  (gid);
    +    if (likely (index == NOT_COVERED)) return false;
    +    if (unlikely (index >= valueCount)) return false;
    +
    +    /* This is ugly... */
    +    hb_buffer_t buffer;
    +    buffer.props.direction = direction;
    +    OT::hb_ot_apply_context_t c (1, font, &buffer);
    +
    +    valueFormat.apply_value (&c, this,
    +                             &values[index * valueFormat.get_len ()],
    +                             pos);
    +    return true;
    +  }
    +
    +
       template
    @@ -85,7 +121,7 @@ struct SinglePosFormat2
                       const SrcLookup *src,
                       Iterator it,
                       ValueFormat newFormat,
    -                  const hb_map_t *layout_variation_idx_map)
    +                  const hb_hashmap_t> *layout_variation_idx_delta_map)
       {
         auto out = c->extend_min (this);
         if (unlikely (!out)) return;
    @@ -95,7 +131,7 @@ struct SinglePosFormat2
         + it
         | hb_map (hb_second)
         | hb_apply ([&] (hb_array_t _)
    -    { src->get_value_format ().copy_values (c, newFormat, src, &_, layout_variation_idx_map); })
    +    { src->get_value_format ().copy_values (c, newFormat, src, &_, layout_variation_idx_delta_map); })
         ;
     
         auto glyphs =
    @@ -127,7 +163,7 @@ struct SinglePosFormat2
         ;
     
         bool ret = bool (it);
    -    SinglePos_serialize (c->serializer, this, it, c->plan->layout_variation_idx_map);
    +    SinglePos_serialize (c->serializer, this, it, &c->plan->layout_variation_idx_delta_map, c->plan->all_axes_pinned);
         return_trace (ret);
       }
     };
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/ValueFormat.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/ValueFormat.hh
    index b29f287bcee9a..1aa451abcc83b 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/ValueFormat.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GPOS/ValueFormat.hh
    @@ -59,6 +59,24 @@ struct ValueFormat : HBUINT16
       unsigned int get_len () const  { return hb_popcount ((unsigned int) *this); }
       unsigned int get_size () const { return get_len () * Value::static_size; }
     
    +  hb_vector_t get_device_table_indices () const {
    +    unsigned i = 0;
    +    hb_vector_t result;
    +    unsigned format = *this;
    +
    +    if (format & xPlacement) i++;
    +    if (format & yPlacement) i++;
    +    if (format & xAdvance)   i++;
    +    if (format & yAdvance)   i++;
    +
    +    if (format & xPlaDevice) result.push (i++);
    +    if (format & yPlaDevice) result.push (i++);
    +    if (format & xAdvDevice) result.push (i++);
    +    if (format & yAdvDevice) result.push (i++);
    +
    +    return result;
    +  }
    +
       bool apply_value (hb_ot_apply_context_t *c,
                         const void            *base,
                         const Value           *values,
    @@ -145,30 +163,50 @@ struct ValueFormat : HBUINT16
                         unsigned int new_format,
                         const void *base,
                         const Value *values,
    -                    const hb_map_t *layout_variation_idx_map) const
    +                    const hb_hashmap_t> *layout_variation_idx_delta_map) const
       {
         unsigned int format = *this;
         if (!format) return;
     
    -    if (format & xPlacement) copy_value (c, new_format, xPlacement, *values++);
    -    if (format & yPlacement) copy_value (c, new_format, yPlacement, *values++);
    -    if (format & xAdvance)   copy_value (c, new_format, xAdvance, *values++);
    -    if (format & yAdvance)   copy_value (c, new_format, yAdvance, *values++);
    +    HBINT16 *x_placement = nullptr, *y_placement = nullptr, *x_adv = nullptr, *y_adv = nullptr;
    +    if (format & xPlacement) x_placement = copy_value (c, new_format, xPlacement, *values++);
    +    if (format & yPlacement) y_placement = copy_value (c, new_format, yPlacement, *values++);
    +    if (format & xAdvance)   x_adv = copy_value (c, new_format, xAdvance, *values++);
    +    if (format & yAdvance)   y_adv = copy_value (c, new_format, yAdvance, *values++);
     
    -    if (format & xPlaDevice) copy_device (c, base, values++, layout_variation_idx_map);
    -    if (format & yPlaDevice) copy_device (c, base, values++, layout_variation_idx_map);
    -    if (format & xAdvDevice) copy_device (c, base, values++, layout_variation_idx_map);
    -    if (format & yAdvDevice) copy_device (c, base, values++, layout_variation_idx_map);
    +    if (format & xPlaDevice)
    +    {
    +      add_delta_to_value (x_placement, base, values, layout_variation_idx_delta_map);
    +      copy_device (c, base, values++, layout_variation_idx_delta_map, new_format, xPlaDevice);
    +    }
    +
    +    if (format & yPlaDevice)
    +    {
    +      add_delta_to_value (y_placement, base, values, layout_variation_idx_delta_map);
    +      copy_device (c, base, values++, layout_variation_idx_delta_map, new_format, yPlaDevice);
    +    }
    +
    +    if (format & xAdvDevice)
    +    {
    +      add_delta_to_value (x_adv, base, values, layout_variation_idx_delta_map);
    +      copy_device (c, base, values++, layout_variation_idx_delta_map, new_format, xAdvDevice);
    +    }
    +
    +    if (format & yAdvDevice)
    +    {
    +      add_delta_to_value (y_adv, base, values, layout_variation_idx_delta_map);
    +      copy_device (c, base, values++, layout_variation_idx_delta_map, new_format, yAdvDevice);
    +    }
       }
     
    -  void copy_value (hb_serialize_context_t *c,
    -                   unsigned int new_format,
    -                   Flags flag,
    -                   Value value) const
    +  HBINT16* copy_value (hb_serialize_context_t *c,
    +                       unsigned int new_format,
    +                       Flags flag,
    +                       Value value) const
       {
         // Filter by new format.
    -    if (!(new_format & flag)) return;
    -    c->copy (value);
    +    if (!(new_format & flag)) return nullptr;
    +    return reinterpret_cast (c->copy (value));
       }
     
       void collect_variation_indices (hb_collect_variation_indices_context_t *c,
    @@ -183,31 +221,40 @@ struct ValueFormat : HBUINT16
         if (format & yAdvance) i++;
         if (format & xPlaDevice)
         {
    -      (base + get_device (&(values[i]))).collect_variation_indices (c->layout_variation_indices);
    +      (base + get_device (&(values[i]))).collect_variation_indices (c);
           i++;
         }
     
         if (format & ValueFormat::yPlaDevice)
         {
    -      (base + get_device (&(values[i]))).collect_variation_indices (c->layout_variation_indices);
    +      (base + get_device (&(values[i]))).collect_variation_indices (c);
           i++;
         }
     
         if (format & ValueFormat::xAdvDevice)
         {
     
    -      (base + get_device (&(values[i]))).collect_variation_indices (c->layout_variation_indices);
    +      (base + get_device (&(values[i]))).collect_variation_indices (c);
           i++;
         }
     
         if (format & ValueFormat::yAdvDevice)
         {
     
    -      (base + get_device (&(values[i]))).collect_variation_indices (c->layout_variation_indices);
    +      (base + get_device (&(values[i]))).collect_variation_indices (c);
           i++;
         }
       }
     
    +  unsigned drop_device_table_flags () const
    +  {
    +    unsigned format = *this;
    +    for (unsigned flag = xPlaDevice; flag <= yAdvDevice; flag = flag << 1)
    +      format = format & ~flag;
    +
    +    return format;
    +  }
    +
       private:
       bool sanitize_value_devices (hb_sanitize_context_t *c, const void *base, const Value *values) const
       {
    @@ -236,9 +283,27 @@ struct ValueFormat : HBUINT16
         return *static_cast *> (value);
       }
     
    +  void add_delta_to_value (HBINT16 *value,
    +                           const void *base,
    +                           const Value *src_value,
    +                           const hb_hashmap_t> *layout_variation_idx_delta_map) const
    +  {
    +    if (!value) return;
    +    unsigned varidx = (base + get_device (src_value)).get_variation_index ();
    +    hb_pair_t *varidx_delta;
    +    if (!layout_variation_idx_delta_map->has (varidx, &varidx_delta)) return;
    +
    +    *value += hb_second (*varidx_delta);
    +  }
    +
       bool copy_device (hb_serialize_context_t *c, const void *base,
    -                    const Value *src_value, const hb_map_t *layout_variation_idx_map) const
    +                    const Value *src_value,
    +                    const hb_hashmap_t> *layout_variation_idx_delta_map,
    +                    unsigned int new_format, Flags flag) const
       {
    +    // Filter by new format.
    +    if (!(new_format & flag)) return true;
    +
         Value       *dst_value = c->copy (*src_value);
     
         if (!dst_value) return false;
    @@ -246,7 +311,7 @@ struct ValueFormat : HBUINT16
     
         *dst_value = 0;
         c->push ();
    -    if ((base + get_device (src_value)).copy (c, layout_variation_idx_map))
    +    if ((base + get_device (src_value)).copy (c, layout_variation_idx_delta_map))
         {
           c->add_link (*dst_value, c->pop_pack ());
           return true;
    @@ -306,7 +371,7 @@ struct ValueFormat : HBUINT16
         for (unsigned int i = 0; i < count; i++) {
           if (!sanitize_value_devices (c, base, values))
             return_trace (false);
    -      values += stride;
    +      values = &StructAtOffset (values, stride);
         }
     
         return_trace (true);
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/AlternateSet.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/AlternateSet.hh
    index 484f3474686cf..b5d506f36f9e7 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/AlternateSet.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/AlternateSet.hh
    @@ -5,12 +5,13 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
    +template 
     struct AlternateSet
     {
       protected:
    -  Array16Of
    +  Array16Of
                     alternates;             /* Array of alternate GlyphIDs--in
                                              * arbitrary order */
       public:
    @@ -56,8 +57,23 @@ struct AlternateSet
     
         if (unlikely (alt_index > count || alt_index == 0)) return_trace (false);
     
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->sync_so_far ();
    +      c->buffer->message (c->font,
    +                          "replacing glyph at %u (alternate substitution)",
    +                          c->buffer->idx);
    +    }
    +
         c->replace_glyph (alternates[alt_index - 1]);
     
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->message (c->font,
    +                          "replaced glyph at %u (alternate substitution)",
    +                          c->buffer->idx - 1u);
    +    }
    +
         return_trace (true);
       }
     
    @@ -68,7 +84,7 @@ struct AlternateSet
       {
         if (alternates.len && alternate_count)
         {
    -      + alternates.sub_array (start_offset, alternate_count)
    +      + alternates.as_array ().sub_array (start_offset, alternate_count)
           | hb_sink (hb_array (alternate_glyphs, *alternate_count))
           ;
         }
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/AlternateSubst.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/AlternateSubst.hh
    index e5d999261fc58..8951f5a7a1717 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/AlternateSubst.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/AlternateSubst.hh
    @@ -6,28 +6,36 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
     struct AlternateSubst
     {
       protected:
       union {
    -  HBUINT16              format;         /* Format identifier */
    -  AlternateSubstFormat1 format1;
    +  HBUINT16                              format;         /* Format identifier */
    +  AlternateSubstFormat1_2   format1;
    +#ifndef HB_NO_BEYOND_64K
    +  AlternateSubstFormat1_2  format2;
    +#endif
       } u;
       public:
     
       template 
       typename context_t::return_t dispatch (context_t *c, Ts&&... ds) const
       {
    +    if (unlikely (!c->may_dispatch (this, &u.format))) return c->no_dispatch_return_value ();
         TRACE_DISPATCH (this, u.format);
    -    if (unlikely (!c->may_dispatch (this, &u.format))) return_trace (c->no_dispatch_return_value ());
         switch (u.format) {
         case 1: return_trace (c->dispatch (u.format1, std::forward (ds)...));
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return_trace (c->dispatch (u.format2, std::forward (ds)...));
    +#endif
         default:return_trace (c->default_return_value ());
         }
       }
     
    +  /* TODO This function is unused and not updated to 24bit GIDs. Should be done by using
    +   * iterators. While at it perhaps using iterator of arrays of hb_codepoint_t instead. */
       bool serialize (hb_serialize_context_t *c,
                       hb_sorted_array_t glyphs,
                       hb_array_t alternate_len_list,
    @@ -42,6 +50,9 @@ struct AlternateSubst
         default:return_trace (false);
         }
       }
    +
    +  /* TODO subset() should choose format. */
    +
     };
     
     }
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/AlternateSubstFormat1.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/AlternateSubstFormat1.hh
    index af1cd7bedbad6..adec65d58646f 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/AlternateSubstFormat1.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/AlternateSubstFormat1.hh
    @@ -6,20 +6,21 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
    -struct AlternateSubstFormat1
    +template 
    +struct AlternateSubstFormat1_2
     {
       protected:
       HBUINT16      format;                 /* Format identifier--format = 1 */
    -  Offset16To
    +  typename Types::template OffsetTo
                     coverage;               /* Offset to Coverage table--from
                                              * beginning of Substitution table */
    -  Array16OfOffset16To
    +  Array16Of>>
                     alternateSet;           /* Array of AlternateSet tables
                                              * ordered by Coverage Index */
       public:
    -  DEFINE_SIZE_ARRAY (6, alternateSet);
    +  DEFINE_SIZE_ARRAY (2 + 2 * Types::size, alternateSet);
     
       bool sanitize (hb_sanitize_context_t *c) const
       {
    @@ -39,9 +40,8 @@ struct AlternateSubstFormat1
         | hb_filter (c->parent_active_glyphs (), hb_first)
         | hb_map (hb_second)
         | hb_map (hb_add (this))
    -    | hb_apply ([c] (const AlternateSet &_) { _.closure (c); })
    +    | hb_apply ([c] (const AlternateSet &_) { _.closure (c); })
         ;
    -
       }
     
       void closure_lookups (hb_closure_lookups_context_t *c) const {}
    @@ -52,7 +52,7 @@ struct AlternateSubstFormat1
         + hb_zip (this+coverage, alternateSet)
         | hb_map (hb_second)
         | hb_map (hb_add (this))
    -    | hb_apply ([c] (const AlternateSet &_) { _.collect_glyphs (c); })
    +    | hb_apply ([c] (const AlternateSet &_) { _.collect_glyphs (c); })
         ;
       }
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ChainContextSubst.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ChainContextSubst.hh
    index bbb88b222f838..08fd779f73096 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ChainContextSubst.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ChainContextSubst.hh
    @@ -7,7 +7,7 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
     struct ChainContextSubst : ChainContext {};
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/Common.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/Common.hh
    index f4c78a9f02058..968bba0481adf 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/Common.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/Common.hh
    @@ -6,7 +6,7 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
     typedef hb_pair_t hb_codepoint_pair_t;
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ContextSubst.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ContextSubst.hh
    index 2af54e8ff47ac..9f8cb46b5e209 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ContextSubst.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ContextSubst.hh
    @@ -7,7 +7,7 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
     struct ContextSubst : Context {};
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ExtensionSubst.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ExtensionSubst.hh
    index 40a3ff439f1b1..831a7dfa2d1b7 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ExtensionSubst.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ExtensionSubst.hh
    @@ -7,7 +7,7 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
     struct ExtensionSubst : Extension
     {
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/GSUB.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/GSUB.hh
    index 3f0c4b2ad9acb..900cf603e45a3 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/GSUB.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/GSUB.hh
    @@ -1,16 +1,15 @@
     #ifndef OT_LAYOUT_GSUB_GSUB_HH
     #define OT_LAYOUT_GSUB_GSUB_HH
     
    -// TODO(garretrieger): move to new layout.
     #include "../../../hb-ot-layout-gsubgpos.hh"
     #include "Common.hh"
     #include "SubstLookup.hh"
     
    -using OT::Layout::GSUB::SubstLookup;
    -
     namespace OT {
    +
    +using Layout::GSUB_impl::SubstLookup;
    +
     namespace Layout {
    -namespace GSUB {
     
     /*
      * GSUB -- Glyph Substitution
    @@ -28,12 +27,15 @@ struct GSUB : GSUBGPOS
     
       bool subset (hb_subset_context_t *c) const
       {
    -    hb_subset_layout_context_t l (c, tableTag, c->plan->gsub_lookups, c->plan->gsub_langsys, c->plan->gsub_features);
    +    hb_subset_layout_context_t l (c, tableTag);
         return GSUBGPOS::subset (&l);
       }
     
       bool sanitize (hb_sanitize_context_t *c) const
    -  { return GSUBGPOS::sanitize (c); }
    +  {
    +    TRACE_SANITIZE (this);
    +    return_trace (GSUBGPOS::sanitize (c));
    +  }
     
       HB_INTERNAL bool is_blocklisted (hb_blob_t *blob,
                                        hb_face_t *face) const;
    @@ -47,11 +49,10 @@ struct GSUB : GSUBGPOS
     };
     
     
    -}
     }
     
    -struct GSUB_accelerator_t : Layout::GSUB::GSUB::accelerator_t {
    -  GSUB_accelerator_t (hb_face_t *face) : Layout::GSUB::GSUB::accelerator_t (face) {}
    +struct GSUB_accelerator_t : Layout::GSUB::accelerator_t {
    +  GSUB_accelerator_t (hb_face_t *face) : Layout::GSUB::accelerator_t (face) {}
     };
     
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/Ligature.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/Ligature.hh
    index 0448d925d125a..308da587d1e72 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/Ligature.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/Ligature.hh
    @@ -5,18 +5,20 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
    +template 
     struct Ligature
     {
       protected:
    -  HBGlyphID16   ligGlyph;               /* GlyphID of ligature to substitute */
    -  HeadlessArrayOf
    +  typename Types::HBGlyphID
    +                ligGlyph;               /* GlyphID of ligature to substitute */
    +  HeadlessArrayOf
                     component;              /* Array of component GlyphIDs--start
                                              * with the second  component--ordered
                                              * in writing direction */
       public:
    -  DEFINE_SIZE_ARRAY (4, component);
    +  DEFINE_SIZE_ARRAY (Types::size + 2, component);
     
       bool sanitize (hb_sanitize_context_t *c) const
       {
    @@ -27,6 +29,9 @@ struct Ligature
       bool intersects (const hb_set_t *glyphs) const
       { return hb_all (component, glyphs); }
     
    +  bool intersects_lig_glyph (const hb_set_t *glyphs) const
    +  { return glyphs->has(ligGlyph); }
    +
       void closure (hb_closure_context_t *c) const
       {
         if (!intersects (c->glyphs)) return;
    @@ -62,7 +67,24 @@ struct Ligature
          * as a "ligated" substitution. */
         if (unlikely (count == 1))
         {
    +
    +      if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +      {
    +        c->buffer->sync_so_far ();
    +        c->buffer->message (c->font,
    +                            "replacing glyph at %u (ligature substitution)",
    +                            c->buffer->idx);
    +      }
    +
           c->replace_glyph (ligGlyph);
    +
    +      if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +      {
    +        c->buffer->message (c->font,
    +                            "replaced glyph at %u (ligature substitution)",
    +                            c->buffer->idx - 1u);
    +      }
    +
           return_trace (true);
         }
     
    @@ -83,6 +105,31 @@ struct Ligature
           return_trace (false);
         }
     
    +    unsigned pos = 0;
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      unsigned delta = c->buffer->sync_so_far ();
    +
    +      pos = c->buffer->idx;
    +
    +      char buf[HB_MAX_CONTEXT_LENGTH * 16] = {0};
    +      char *p = buf;
    +
    +      match_end += delta;
    +      for (unsigned i = 0; i < count; i++)
    +      {
    +        match_positions[i] += delta;
    +        if (i)
    +          *p++ = ',';
    +        snprintf (p, sizeof(buf) - (p - buf), "%u", match_positions[i]);
    +        p += strlen(p);
    +      }
    +
    +      c->buffer->message (c->font,
    +                          "ligating glyphs at %s",
    +                          buf);
    +    }
    +
         ligate_input (c,
                       count,
                       match_positions,
    @@ -90,6 +137,14 @@ struct Ligature
                       ligGlyph,
                       total_component_count);
     
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->sync_so_far ();
    +      c->buffer->message (c->font,
    +                          "ligated glyph at %u",
    +                          pos);
    +    }
    +
         return_trace (true);
       }
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/LigatureSet.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/LigatureSet.hh
    index 185b324b35bc6..2b23262280205 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/LigatureSet.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/LigatureSet.hh
    @@ -6,12 +6,13 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
    +template 
     struct LigatureSet
     {
       protected:
    -  Array16OfOffset16To
    +  Array16OfOffset16To>
                     ligature;               /* Array LigatureSet tables
                                              * ordered by preference */
       public:
    @@ -28,7 +29,19 @@ struct LigatureSet
         return
         + hb_iter (ligature)
         | hb_map (hb_add (this))
    -    | hb_map ([glyphs] (const Ligature &_) { return _.intersects (glyphs); })
    +    | hb_map ([glyphs] (const Ligature &_) { return _.intersects (glyphs); })
    +    | hb_any
    +    ;
    +  }
    +
    +  bool intersects_lig_glyph (const hb_set_t *glyphs) const
    +  {
    +    return
    +    + hb_iter (ligature)
    +    | hb_map (hb_add (this))
    +    | hb_map ([glyphs] (const Ligature &_) {
    +      return _.intersects_lig_glyph (glyphs) && _.intersects (glyphs);
    +    })
         | hb_any
         ;
       }
    @@ -37,7 +50,7 @@ struct LigatureSet
       {
         + hb_iter (ligature)
         | hb_map (hb_add (this))
    -    | hb_apply ([c] (const Ligature &_) { _.closure (c); })
    +    | hb_apply ([c] (const Ligature &_) { _.closure (c); })
         ;
       }
     
    @@ -45,7 +58,7 @@ struct LigatureSet
       {
         + hb_iter (ligature)
         | hb_map (hb_add (this))
    -    | hb_apply ([c] (const Ligature &_) { _.collect_glyphs (c); })
    +    | hb_apply ([c] (const Ligature &_) { _.collect_glyphs (c); })
         ;
       }
     
    @@ -54,7 +67,7 @@ struct LigatureSet
         return
         + hb_iter (ligature)
         | hb_map (hb_add (this))
    -    | hb_map ([c] (const Ligature &_) { return _.would_apply (c); })
    +    | hb_map ([c] (const Ligature &_) { return _.would_apply (c); })
         | hb_any
         ;
       }
    @@ -65,7 +78,7 @@ struct LigatureSet
         unsigned int num_ligs = ligature.len;
         for (unsigned int i = 0; i < num_ligs; i++)
         {
    -      const Ligature &lig = this+ligature[i];
    +      const auto &lig = this+ligature[i];
           if (lig.apply (c)) return_trace (true);
         }
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/LigatureSubst.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/LigatureSubst.hh
    index a029bf5e9f47f..cffa910295f8c 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/LigatureSubst.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/LigatureSubst.hh
    @@ -6,28 +6,37 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
     struct LigatureSubst
     {
       protected:
       union {
    -  HBUINT16              format;         /* Format identifier */
    -  LigatureSubstFormat1  format1;
    +  HBUINT16                              format;         /* Format identifier */
    +  LigatureSubstFormat1_2    format1;
    +#ifndef HB_NO_BEYOND_64K
    +  LigatureSubstFormat1_2   format2;
    +#endif
       } u;
     
       public:
       template 
       typename context_t::return_t dispatch (context_t *c, Ts&&... ds) const
       {
    +    if (unlikely (!c->may_dispatch (this, &u.format))) return c->no_dispatch_return_value ();
         TRACE_DISPATCH (this, u.format);
    -    if (unlikely (!c->may_dispatch (this, &u.format))) return_trace (c->no_dispatch_return_value ());
         switch (u.format) {
         case 1: return_trace (c->dispatch (u.format1, std::forward (ds)...));
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return_trace (c->dispatch (u.format2, std::forward (ds)...));
    +#endif
         default:return_trace (c->default_return_value ());
         }
       }
     
    +  /* TODO This function is only used by small GIDs, and not updated to 24bit GIDs. Should
    +   * be done by using iterators. While at it perhaps using iterator of arrays of hb_codepoint_t
    +   * instead. */
       bool serialize (hb_serialize_context_t *c,
                       hb_sorted_array_t first_glyphs,
                       hb_array_t ligature_per_first_glyph_count_list,
    @@ -49,6 +58,9 @@ struct LigatureSubst
         default:return_trace (false);
         }
       }
    +
    +  /* TODO subset() should choose format. */
    +
     };
     
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/LigatureSubstFormat1.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/LigatureSubstFormat1.hh
    index 19dfe98469477..5c7df97d13aea 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/LigatureSubstFormat1.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/LigatureSubstFormat1.hh
    @@ -6,20 +6,21 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
    -struct LigatureSubstFormat1
    +template 
    +struct LigatureSubstFormat1_2
     {
       protected:
       HBUINT16      format;                 /* Format identifier--format = 1 */
    -  Offset16To
    +  typename Types::template OffsetTo
                     coverage;               /* Offset to Coverage table--from
                                              * beginning of Substitution table */
    -  Array16OfOffset16To
    +  Array16Of>>
                     ligatureSet;            /* Array LigatureSet tables
                                              * ordered by Coverage Index */
       public:
    -  DEFINE_SIZE_ARRAY (6, ligatureSet);
    +  DEFINE_SIZE_ARRAY (4 + Types::size, ligatureSet);
     
       bool sanitize (hb_sanitize_context_t *c) const
       {
    @@ -33,7 +34,7 @@ struct LigatureSubstFormat1
         + hb_zip (this+coverage, ligatureSet)
         | hb_filter (*glyphs, hb_first)
         | hb_map (hb_second)
    -    | hb_map ([this, glyphs] (const Offset16To &_)
    +    | hb_map ([this, glyphs] (const typename Types::template OffsetTo> &_)
                   { return (this+_).intersects (glyphs); })
         | hb_any
         ;
    @@ -48,7 +49,7 @@ struct LigatureSubstFormat1
         | hb_filter (c->parent_active_glyphs (), hb_first)
         | hb_map (hb_second)
         | hb_map (hb_add (this))
    -    | hb_apply ([c] (const LigatureSet &_) { _.closure (c); })
    +    | hb_apply ([c] (const LigatureSet &_) { _.closure (c); })
         ;
     
       }
    @@ -62,7 +63,7 @@ struct LigatureSubstFormat1
         + hb_zip (this+coverage, ligatureSet)
         | hb_map (hb_second)
         | hb_map (hb_add (this))
    -    | hb_apply ([c] (const LigatureSet &_) { _.collect_glyphs (c); })
    +    | hb_apply ([c] (const LigatureSet &_) { _.collect_glyphs (c); })
         ;
       }
     
    @@ -73,7 +74,7 @@ struct LigatureSubstFormat1
         unsigned int index = (this+coverage).get_coverage (c->glyphs[0]);
         if (likely (index == NOT_COVERED)) return false;
     
    -    const LigatureSet &lig_set = this+ligatureSet[index];
    +    const auto &lig_set = this+ligatureSet[index];
         return lig_set.would_apply (c);
       }
     
    @@ -84,7 +85,7 @@ struct LigatureSubstFormat1
         unsigned int index = (this+coverage).get_coverage (c->buffer->cur ().codepoint);
         if (likely (index == NOT_COVERED)) return_trace (false);
     
    -    const LigatureSet &lig_set = this+ligatureSet[index];
    +    const auto &lig_set = this+ligatureSet[index];
         return_trace (lig_set.apply (c));
       }
     
    @@ -128,8 +129,8 @@ struct LigatureSubstFormat1
         hb_set_t new_coverage;
         + hb_zip (this+coverage, hb_iter (ligatureSet) | hb_map (hb_add (this)))
         | hb_filter (glyphset, hb_first)
    -    | hb_filter ([&] (const LigatureSet& _) {
    -      return _.intersects (&glyphset);
    +    | hb_filter ([&] (const LigatureSet& _) {
    +      return _.intersects_lig_glyph (&glyphset);
         }, hb_second)
         | hb_map (hb_first)
         | hb_sink (new_coverage);
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/MultipleSubst.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/MultipleSubst.hh
    index b289175504885..cf3d754e3cc14 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/MultipleSubst.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/MultipleSubst.hh
    @@ -6,14 +6,17 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
     struct MultipleSubst
     {
       protected:
       union {
    -  HBUINT16              format;         /* Format identifier */
    -  MultipleSubstFormat1  format1;
    +  HBUINT16                              format;         /* Format identifier */
    +  MultipleSubstFormat1_2    format1;
    +#ifndef HB_NO_BEYOND_64K
    +  MultipleSubstFormat1_2   format2;
    +#endif
       } u;
     
       public:
    @@ -21,28 +24,34 @@ struct MultipleSubst
       template 
       typename context_t::return_t dispatch (context_t *c, Ts&&... ds) const
       {
    +    if (unlikely (!c->may_dispatch (this, &u.format))) return c->no_dispatch_return_value ();
         TRACE_DISPATCH (this, u.format);
    -    if (unlikely (!c->may_dispatch (this, &u.format))) return_trace (c->no_dispatch_return_value ());
         switch (u.format) {
         case 1: return_trace (c->dispatch (u.format1, std::forward (ds)...));
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return_trace (c->dispatch (u.format2, std::forward (ds)...));
    +#endif
         default:return_trace (c->default_return_value ());
         }
       }
     
    +  template
       bool serialize (hb_serialize_context_t *c,
    -                  hb_sorted_array_t glyphs,
    -                  hb_array_t substitute_len_list,
    -                  hb_array_t substitute_glyphs_list)
    +                  Iterator it)
       {
         TRACE_SERIALIZE (this);
         if (unlikely (!c->extend_min (u.format))) return_trace (false);
         unsigned int format = 1;
         u.format = format;
         switch (u.format) {
    -    case 1: return_trace (u.format1.serialize (c, glyphs, substitute_len_list, substitute_glyphs_list));
    +    case 1: return_trace (u.format1.serialize (c, it));
         default:return_trace (false);
         }
       }
    +
    +  /* TODO subset() should choose format. */
    +
     };
     
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/MultipleSubstFormat1.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/MultipleSubstFormat1.hh
    index 54c6dc847835e..4a9972c29cc51 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/MultipleSubstFormat1.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/MultipleSubstFormat1.hh
    @@ -6,20 +6,21 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
    -struct MultipleSubstFormat1
    +template 
    +struct MultipleSubstFormat1_2
     {
       protected:
       HBUINT16      format;                 /* Format identifier--format = 1 */
    -  Offset16To
    +  typename Types::template OffsetTo
                     coverage;               /* Offset to Coverage table--from
                                              * beginning of Substitution table */
    -  Array16OfOffset16To
    +  Array16Of>>
                     sequence;               /* Array of Sequence tables
                                              * ordered by Coverage Index */
       public:
    -  DEFINE_SIZE_ARRAY (6, sequence);
    +  DEFINE_SIZE_ARRAY (4 + Types::size, sequence);
     
       bool sanitize (hb_sanitize_context_t *c) const
       {
    @@ -39,7 +40,7 @@ struct MultipleSubstFormat1
         | hb_filter (c->parent_active_glyphs (), hb_first)
         | hb_map (hb_second)
         | hb_map (hb_add (this))
    -    | hb_apply ([c] (const Sequence &_) { _.closure (c); })
    +    | hb_apply ([c] (const Sequence &_) { _.closure (c); })
         ;
       }
     
    @@ -51,7 +52,7 @@ struct MultipleSubstFormat1
         + hb_zip (this+coverage, sequence)
         | hb_map (hb_second)
         | hb_map (hb_add (this))
    -    | hb_apply ([c] (const Sequence &_) { _.collect_glyphs (c); })
    +    | hb_apply ([c] (const Sequence &_) { _.collect_glyphs (c); })
         ;
       }
     
    @@ -70,22 +71,31 @@ struct MultipleSubstFormat1
         return_trace ((this+sequence[index]).apply (c));
       }
     
    +  template
       bool serialize (hb_serialize_context_t *c,
    -                  hb_sorted_array_t glyphs,
    -                  hb_array_t substitute_len_list,
    -                  hb_array_t substitute_glyphs_list)
    +                  Iterator it)
       {
         TRACE_SERIALIZE (this);
    +    auto sequences =
    +      + it
    +      | hb_map (hb_second)
    +      ;
    +    auto glyphs =
    +      + it
    +      | hb_map_retains_sorting (hb_first)
    +      ;
         if (unlikely (!c->extend_min (this))) return_trace (false);
    -    if (unlikely (!sequence.serialize (c, glyphs.length))) return_trace (false);
    -    for (unsigned int i = 0; i < glyphs.length; i++)
    +
    +    if (unlikely (!sequence.serialize (c, sequences.length))) return_trace (false);
    +
    +    for (auto& pair : hb_zip (sequences, sequence))
         {
    -      unsigned int substitute_len = substitute_len_list[i];
    -      if (unlikely (!sequence[i]
    -                        .serialize_serialize (c, substitute_glyphs_list.sub_array (0, substitute_len))))
    +      if (unlikely (!pair.second
    +                    .serialize_serialize (c, pair.first)))
             return_trace (false);
    -      substitute_glyphs_list += substitute_len;
         }
    +
         return_trace (coverage.serialize_serialize (c, glyphs));
       }
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ReverseChainSingleSubst.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ReverseChainSingleSubst.hh
    index 435d80fd31043..5ad463fea7927 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ReverseChainSingleSubst.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ReverseChainSingleSubst.hh
    @@ -6,7 +6,7 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
     struct ReverseChainSingleSubst
     {
    @@ -20,8 +20,8 @@ struct ReverseChainSingleSubst
       template 
       typename context_t::return_t dispatch (context_t *c, Ts&&... ds) const
       {
    +    if (unlikely (!c->may_dispatch (this, &u.format))) return c->no_dispatch_return_value ();
         TRACE_DISPATCH (this, u.format);
    -    if (unlikely (!c->may_dispatch (this, &u.format))) return_trace (c->no_dispatch_return_value ());
         switch (u.format) {
         case 1: return_trace (c->dispatch (u.format1, std::forward (ds)...));
         default:return_trace (c->default_return_value ());
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ReverseChainSingleSubstFormat1.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ReverseChainSingleSubstFormat1.hh
    index 7a79a9df25edf..73f222746e5d7 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ReverseChainSingleSubstFormat1.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/ReverseChainSingleSubstFormat1.hh
    @@ -5,7 +5,7 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
     struct ReverseChainSingleSubstFormat1
     {
    @@ -33,10 +33,10 @@ struct ReverseChainSingleSubstFormat1
         TRACE_SANITIZE (this);
         if (!(coverage.sanitize (c, this) && backtrack.sanitize (c, this)))
           return_trace (false);
    -    const Array16OfOffset16To &lookahead = StructAfter> (backtrack);
    +    const auto &lookahead = StructAfter (backtrack);
         if (!lookahead.sanitize (c, this))
           return_trace (false);
    -    const Array16Of &substitute = StructAfter> (lookahead);
    +    const auto &substitute = StructAfter (lookahead);
         return_trace (substitute.sanitize (c));
       }
     
    @@ -45,7 +45,7 @@ struct ReverseChainSingleSubstFormat1
         if (!(this+coverage).intersects (glyphs))
           return false;
     
    -    const Array16OfOffset16To &lookahead = StructAfter> (backtrack);
    +    const auto &lookahead = StructAfter (backtrack);
     
         unsigned int count;
     
    @@ -69,8 +69,8 @@ struct ReverseChainSingleSubstFormat1
       {
         if (!intersects (c->glyphs)) return;
     
    -    const Array16OfOffset16To &lookahead = StructAfter> (backtrack);
    -    const Array16Of &substitute = StructAfter> (lookahead);
    +    const auto &lookahead = StructAfter (backtrack);
    +    const auto &substitute = StructAfter (lookahead);
     
         + hb_zip (this+coverage, substitute)
         | hb_filter (c->parent_active_glyphs (), hb_first)
    @@ -91,12 +91,12 @@ struct ReverseChainSingleSubstFormat1
         for (unsigned int i = 0; i < count; i++)
           if (unlikely (!(this+backtrack[i]).collect_coverage (c->before))) return;
     
    -    const Array16OfOffset16To &lookahead = StructAfter> (backtrack);
    +    const auto &lookahead = StructAfter (backtrack);
         count = lookahead.len;
         for (unsigned int i = 0; i < count; i++)
           if (unlikely (!(this+lookahead[i]).collect_coverage (c->after))) return;
     
    -    const Array16Of &substitute = StructAfter> (lookahead);
    +    const auto &substitute = StructAfter (lookahead);
         count = substitute.len;
         c->output->add_array (substitute.arrayZ, substitute.len);
       }
    @@ -115,8 +115,8 @@ struct ReverseChainSingleSubstFormat1
         unsigned int index = (this+coverage).get_coverage (c->buffer->cur ().codepoint);
         if (likely (index == NOT_COVERED)) return_trace (false);
     
    -    const Array16OfOffset16To &lookahead = StructAfter> (backtrack);
    -    const Array16Of &substitute = StructAfter> (lookahead);
    +    const auto &lookahead = StructAfter (backtrack);
    +    const auto &substitute = StructAfter (lookahead);
     
         if (unlikely (index >= substitute.len)) return_trace (false);
     
    @@ -131,7 +131,23 @@ struct ReverseChainSingleSubstFormat1
                              c->buffer->idx + 1, &end_index))
         {
           c->buffer->unsafe_to_break_from_outbuffer (start_index, end_index);
    +
    +      if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +      {
    +        c->buffer->message (c->font,
    +                            "replacing glyph at %u (reverse chaining substitution)",
    +                            c->buffer->idx);
    +      }
    +
           c->replace_glyph_inplace (substitute[index]);
    +
    +      if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +      {
    +        c->buffer->message (c->font,
    +                            "replaced glyph at %u (reverse chaining substitution)",
    +                            c->buffer->idx);
    +      }
    +
           /* Note: We DON'T decrease buffer->idx.  The main loop does it
            * for us.  This is useful for preventing surprises if someone
            * calls us through a Context lookup. */
    @@ -206,8 +222,8 @@ struct ReverseChainSingleSubstFormat1
         const hb_set_t &glyphset = *c->plan->glyphset_gsub ();
         const hb_map_t &glyph_map = *c->plan->glyph_map;
     
    -    const Array16OfOffset16To &lookahead = StructAfter> (backtrack);
    -    const Array16Of &substitute = StructAfter> (lookahead);
    +    const auto &lookahead = StructAfter (backtrack);
    +    const auto &substitute = StructAfter (lookahead);
     
         auto it =
         + hb_zip (this+coverage, substitute)
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/Sequence.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/Sequence.hh
    index ebd451e6ba31e..62d68160c7bd2 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/Sequence.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/Sequence.hh
    @@ -5,12 +5,13 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
    +template 
     struct Sequence
     {
       protected:
    -  Array16Of
    +  Array16Of
                     substitute;             /* String of GlyphIDs to substitute */
       public:
       DEFINE_SIZE_ARRAY (2, substitute);
    @@ -39,17 +40,58 @@ struct Sequence
          * as a "multiplied" substitution. */
         if (unlikely (count == 1))
         {
    +      if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +      {
    +        c->buffer->sync_so_far ();
    +        c->buffer->message (c->font,
    +                            "replacing glyph at %u (multiple substitution)",
    +                            c->buffer->idx);
    +      }
    +
           c->replace_glyph (substitute.arrayZ[0]);
    +
    +      if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +      {
    +        c->buffer->message (c->font,
    +                            "replaced glyph at %u (multiple subtitution)",
    +                            c->buffer->idx - 1u);
    +      }
    +
           return_trace (true);
         }
         /* Spec disallows this, but Uniscribe allows it.
          * https://github.com/harfbuzz/harfbuzz/issues/253 */
         else if (unlikely (count == 0))
         {
    +      if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +      {
    +        c->buffer->sync_so_far ();
    +        c->buffer->message (c->font,
    +                            "deleting glyph at %u (multiple substitution)",
    +                            c->buffer->idx);
    +      }
    +
           c->buffer->delete_glyph ();
    +
    +      if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +      {
    +        c->buffer->sync_so_far ();
    +        c->buffer->message (c->font,
    +                            "deleted glyph at %u (multiple substitution)",
    +                            c->buffer->idx);
    +      }
    +
           return_trace (true);
         }
     
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->sync_so_far ();
    +      c->buffer->message (c->font,
    +                          "multiplying glyph at %u",
    +                          c->buffer->idx);
    +    }
    +
         unsigned int klass = _hb_glyph_info_is_ligature (&c->buffer->cur()) ?
                              HB_OT_LAYOUT_GLYPH_PROPS_BASE_GLYPH : 0;
         unsigned lig_id = _hb_glyph_info_get_lig_id (&c->buffer->cur());
    @@ -64,6 +106,26 @@ struct Sequence
         }
         c->buffer->skip_glyph ();
     
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->sync_so_far ();
    +
    +      char buf[HB_MAX_CONTEXT_LENGTH * 16] = {0};
    +      char *p = buf;
    +
    +      for (unsigned i = c->buffer->idx - count; i < c->buffer->idx; i++)
    +      {
    +        if (buf < p)
    +          *p++ = ',';
    +        snprintf (p, sizeof(buf) - (p - buf), "%u", i);
    +        p += strlen(p);
    +      }
    +
    +      c->buffer->message (c->font,
    +                          "multiplied glyphs at %s",
    +                          buf);
    +    }
    +
         return_trace (true);
       }
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SingleSubst.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SingleSubst.hh
    index 786428fe453e7..304d1928e2370 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SingleSubst.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SingleSubst.hh
    @@ -7,15 +7,19 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
     struct SingleSubst
     {
       protected:
       union {
    -  HBUINT16              format;         /* Format identifier */
    -  SingleSubstFormat1    format1;
    -  SingleSubstFormat2    format2;
    +  HBUINT16                              format;         /* Format identifier */
    +  SingleSubstFormat1_3      format1;
    +  SingleSubstFormat2_4      format2;
    +#ifndef HB_NO_BEYOND_64K
    +  SingleSubstFormat1_3     format3;
    +  SingleSubstFormat2_4     format4;
    +#endif
       } u;
     
       public:
    @@ -23,11 +27,15 @@ struct SingleSubst
       template 
       typename context_t::return_t dispatch (context_t *c, Ts&&... ds) const
       {
    +    if (unlikely (!c->may_dispatch (this, &u.format))) return c->no_dispatch_return_value ();
         TRACE_DISPATCH (this, u.format);
    -    if (unlikely (!c->may_dispatch (this, &u.format))) return_trace (c->no_dispatch_return_value ());
         switch (u.format) {
         case 1: return_trace (c->dispatch (u.format1, std::forward (ds)...));
         case 2: return_trace (c->dispatch (u.format2, std::forward (ds)...));
    +#ifndef HB_NO_BEYOND_64K
    +    case 3: return_trace (c->dispatch (u.format3, std::forward (ds)...));
    +    case 4: return_trace (c->dispatch (u.format4, std::forward (ds)...));
    +#endif
         default:return_trace (c->default_return_value ());
         }
       }
    @@ -45,11 +53,24 @@ struct SingleSubst
         if (glyphs)
         {
           format = 1;
    +      hb_codepoint_t mask = 0xFFFFu;
    +
    +#ifndef HB_NO_BEYOND_64K
    +       if (+ glyphs
    +           | hb_map_retains_sorting (hb_first)
    +           | hb_filter ([] (hb_codepoint_t gid) { return gid > 0xFFFFu; }))
    +       {
    +         format += 2;
    +         mask = 0xFFFFFFu;
    +       }
    +#endif
    +
           auto get_delta = [=] (hb_codepoint_pair_t _)
    -                       { return (unsigned) (_.second - _.first) & 0xFFFF; };
    +                       { return (unsigned) (_.second - _.first) & mask; };
           delta = get_delta (*glyphs);
    -      if (!hb_all (++(+glyphs), delta, get_delta)) format = 2;
    +      if (!hb_all (++(+glyphs), delta, get_delta)) format += 1;
         }
    +
         u.format = format;
         switch (u.format) {
         case 1: return_trace (u.format1.serialize (c,
    @@ -57,6 +78,13 @@ struct SingleSubst
                                                    | hb_map_retains_sorting (hb_first),
                                                    delta));
         case 2: return_trace (u.format2.serialize (c, glyphs));
    +#ifndef HB_NO_BEYOND_64K
    +    case 3: return_trace (u.format3.serialize (c,
    +                                               + glyphs
    +                                               | hb_map_retains_sorting (hb_first),
    +                                               delta));
    +    case 4: return_trace (u.format4.serialize (c, glyphs));
    +#endif
         default:return_trace (false);
         }
       }
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SingleSubstFormat1.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SingleSubstFormat1.hh
    index 3c6b2954cecd8..268487c5ae770 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SingleSubstFormat1.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SingleSubstFormat1.hh
    @@ -5,27 +5,40 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
    -struct SingleSubstFormat1
    +template 
    +struct SingleSubstFormat1_3
     {
       protected:
       HBUINT16      format;                 /* Format identifier--format = 1 */
    -  Offset16To
    +  typename Types::template OffsetTo
                     coverage;               /* Offset to Coverage table--from
                                              * beginning of Substitution table */
    -  HBUINT16      deltaGlyphID;           /* Add to original GlyphID to get
    +  typename Types::HBUINT
    +                deltaGlyphID;           /* Add to original GlyphID to get
                                              * substitute GlyphID, modulo 0x10000 */
     
       public:
    -  DEFINE_SIZE_STATIC (6);
    +  DEFINE_SIZE_STATIC (2 + 2 * Types::size);
     
       bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
    -    return_trace (coverage.sanitize (c, this) && deltaGlyphID.sanitize (c));
    +    return_trace (c->check_struct (this) &&
    +                  coverage.sanitize (c, this) &&
    +                  /* The coverage  table may use a range to represent a set
    +                   * of glyphs, which means a small number of bytes can
    +                   * generate a large glyph set. Manually modify the
    +                   * sanitizer max ops to take this into account.
    +                   *
    +                   * Note: This check *must* be right after coverage sanitize. */
    +                  c->check_ops ((this + coverage).get_population () >> 1));
       }
     
    +  hb_codepoint_t get_mask () const
    +  { return (1 << (8 * Types::size)) - 1; }
    +
       bool intersects (const hb_set_t *glyphs) const
       { return (this+coverage).intersects (glyphs); }
     
    @@ -34,14 +47,33 @@ struct SingleSubstFormat1
     
       void closure (hb_closure_context_t *c) const
       {
    -    unsigned d = deltaGlyphID;
    -
    -    + hb_iter (this+coverage)
    -    | hb_filter (c->parent_active_glyphs ())
    -    | hb_map ([d] (hb_codepoint_t g) { return (g + d) & 0xFFFFu; })
    +    hb_codepoint_t d = deltaGlyphID;
    +    hb_codepoint_t mask = get_mask ();
    +
    +    /* Help fuzzer avoid this function as much. */
    +    unsigned pop = (this+coverage).get_population ();
    +    if (pop >= mask)
    +      return;
    +
    +    hb_set_t intersection;
    +    (this+coverage).intersect_set (c->parent_active_glyphs (), intersection);
    +
    +    /* In degenerate fuzzer-found fonts, but not real fonts,
    +     * this table can keep adding new glyphs in each round of closure.
    +     * Refuse to close-over, if it maps glyph range to overlapping range. */
    +    hb_codepoint_t min_before = intersection.get_min ();
    +    hb_codepoint_t max_before = intersection.get_max ();
    +    hb_codepoint_t min_after = (min_before + d) & mask;
    +    hb_codepoint_t max_after = (max_before + d) & mask;
    +    if (intersection.get_population () == max_before - min_before + 1 &&
    +        ((min_before <= min_after && min_after <= max_before) ||
    +         (min_before <= max_after && max_after <= max_before)))
    +      return;
    +
    +    + hb_iter (intersection)
    +    | hb_map ([d, mask] (hb_codepoint_t g) { return (g + d) & mask; })
         | hb_sink (c->output)
         ;
    -
       }
     
       void closure_lookups (hb_closure_lookups_context_t *c) const {}
    @@ -49,9 +81,11 @@ struct SingleSubstFormat1
       void collect_glyphs (hb_collect_glyphs_context_t *c) const
       {
         if (unlikely (!(this+coverage).collect_coverage (c->input))) return;
    -    unsigned d = deltaGlyphID;
    +    hb_codepoint_t d = deltaGlyphID;
    +    hb_codepoint_t mask = get_mask ();
    +
         + hb_iter (this+coverage)
    -    | hb_map ([d] (hb_codepoint_t g) { return (g + d) & 0xFFFFu; })
    +    | hb_map ([d, mask] (hb_codepoint_t g) { return (g + d) & mask; })
         | hb_sink (c->output)
         ;
       }
    @@ -61,6 +95,34 @@ struct SingleSubstFormat1
       bool would_apply (hb_would_apply_context_t *c) const
       { return c->len == 1 && (this+coverage).get_coverage (c->glyphs[0]) != NOT_COVERED; }
     
    +  unsigned
    +  get_glyph_alternates (hb_codepoint_t  glyph_id,
    +                        unsigned        start_offset,
    +                        unsigned       *alternate_count  /* IN/OUT.  May be NULL. */,
    +                        hb_codepoint_t *alternate_glyphs /* OUT.     May be NULL. */) const
    +  {
    +    unsigned int index = (this+coverage).get_coverage (glyph_id);
    +    if (likely (index == NOT_COVERED))
    +    {
    +      if (alternate_count)
    +        *alternate_count = 0;
    +      return 0;
    +    }
    +
    +    if (alternate_count && *alternate_count)
    +    {
    +      hb_codepoint_t d = deltaGlyphID;
    +      hb_codepoint_t mask = get_mask ();
    +
    +      glyph_id = (glyph_id + d) & mask;
    +
    +      *alternate_glyphs = glyph_id;
    +      *alternate_count = 1;
    +    }
    +
    +    return 1;
    +  }
    +
       bool apply (hb_ot_apply_context_t *c) const
       {
         TRACE_APPLY (this);
    @@ -68,11 +130,28 @@ struct SingleSubstFormat1
         unsigned int index = (this+coverage).get_coverage (glyph_id);
         if (likely (index == NOT_COVERED)) return_trace (false);
     
    -    /* According to the Adobe Annotated OpenType Suite, result is always
    -     * limited to 16bit. */
    -    glyph_id = (glyph_id + deltaGlyphID) & 0xFFFFu;
    +    hb_codepoint_t d = deltaGlyphID;
    +    hb_codepoint_t mask = get_mask ();
    +
    +    glyph_id = (glyph_id + d) & mask;
    +
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->sync_so_far ();
    +      c->buffer->message (c->font,
    +                          "replacing glyph at %u (single substitution)",
    +                          c->buffer->idx);
    +    }
    +
         c->replace_glyph (glyph_id);
     
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->message (c->font,
    +                          "replaced glyph at %u (single substitution)",
    +                          c->buffer->idx - 1u);
    +    }
    +
         return_trace (true);
       }
     
    @@ -95,14 +174,17 @@ struct SingleSubstFormat1
         const hb_set_t &glyphset = *c->plan->glyphset_gsub ();
         const hb_map_t &glyph_map = *c->plan->glyph_map;
     
    -    hb_codepoint_t delta = deltaGlyphID;
    +    hb_codepoint_t d = deltaGlyphID;
    +    hb_codepoint_t mask = get_mask ();
    +
    +    hb_set_t intersection;
    +    (this+coverage).intersect_set (glyphset, intersection);
     
         auto it =
    -    + hb_iter (this+coverage)
    -    | hb_filter (glyphset)
    -    | hb_map_retains_sorting ([&] (hb_codepoint_t g) {
    +    + hb_iter (intersection)
    +    | hb_map_retains_sorting ([d, mask] (hb_codepoint_t g) {
                                     return hb_codepoint_pair_t (g,
    -                                                            (g + delta) & 0xFFFF); })
    +                                                            (g + d) & mask); })
         | hb_filter (glyphset, hb_second)
         | hb_map_retains_sorting ([&] (hb_codepoint_pair_t p) -> hb_codepoint_pair_t
                                   { return hb_pair (glyph_map[p.first], glyph_map[p.second]); })
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SingleSubstFormat2.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SingleSubstFormat2.hh
    index df75bb52bbc24..518900116710e 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SingleSubstFormat2.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SingleSubstFormat2.hh
    @@ -5,21 +5,22 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
    -struct SingleSubstFormat2
    +template 
    +struct SingleSubstFormat2_4
     {
       protected:
       HBUINT16      format;                 /* Format identifier--format = 2 */
    -  Offset16To
    +  typename Types::template OffsetTo
                     coverage;               /* Offset to Coverage table--from
                                              * beginning of Substitution table */
    -  Array16Of
    +  Array16Of
                     substitute;             /* Array of substitute
                                              * GlyphIDs--ordered by Coverage Index */
     
       public:
    -  DEFINE_SIZE_ARRAY (6, substitute);
    +  DEFINE_SIZE_ARRAY (4 + Types::size, substitute);
     
       bool sanitize (hb_sanitize_context_t *c) const
       {
    @@ -35,12 +36,27 @@ struct SingleSubstFormat2
     
       void closure (hb_closure_context_t *c) const
       {
    -    + hb_zip (this+coverage, substitute)
    -    | hb_filter (c->parent_active_glyphs (), hb_first)
    +    auto &cov = this+coverage;
    +    auto &glyph_set = c->parent_active_glyphs ();
    +
    +    if (substitute.len > glyph_set.get_population () * 4)
    +    {
    +      for (auto g : glyph_set)
    +      {
    +        unsigned i = cov.get_coverage (g);
    +        if (i == NOT_COVERED || i >= substitute.len)
    +          continue;
    +        c->output->add (substitute.arrayZ[i]);
    +      }
    +
    +      return;
    +    }
    +
    +    + hb_zip (cov, substitute)
    +    | hb_filter (glyph_set, hb_first)
         | hb_map (hb_second)
         | hb_sink (c->output)
         ;
    -
       }
     
       void closure_lookups (hb_closure_lookups_context_t *c) const {}
    @@ -59,6 +75,31 @@ struct SingleSubstFormat2
       bool would_apply (hb_would_apply_context_t *c) const
       { return c->len == 1 && (this+coverage).get_coverage (c->glyphs[0]) != NOT_COVERED; }
     
    +  unsigned
    +  get_glyph_alternates (hb_codepoint_t  glyph_id,
    +                        unsigned        start_offset,
    +                        unsigned       *alternate_count  /* IN/OUT.  May be NULL. */,
    +                        hb_codepoint_t *alternate_glyphs /* OUT.     May be NULL. */) const
    +  {
    +    unsigned int index = (this+coverage).get_coverage (glyph_id);
    +    if (likely (index == NOT_COVERED))
    +    {
    +      if (alternate_count)
    +        *alternate_count = 0;
    +      return 0;
    +    }
    +
    +    if (alternate_count && *alternate_count)
    +    {
    +      glyph_id = substitute[index];
    +
    +      *alternate_glyphs = glyph_id;
    +      *alternate_count = 1;
    +    }
    +
    +    return 1;
    +  }
    +
       bool apply (hb_ot_apply_context_t *c) const
       {
         TRACE_APPLY (this);
    @@ -67,8 +108,23 @@ struct SingleSubstFormat2
     
         if (unlikely (index >= substitute.len)) return_trace (false);
     
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->sync_so_far ();
    +      c->buffer->message (c->font,
    +                          "replacing glyph at %u (single substitution)",
    +                          c->buffer->idx);
    +    }
    +
         c->replace_glyph (substitute[index]);
     
    +    if (HB_BUFFER_MESSAGE_MORE && c->buffer->messaging ())
    +    {
    +      c->buffer->message (c->font,
    +                          "replaced glyph at %u (single substitution)",
    +                          c->buffer->idx - 1u);
    +    }
    +
         return_trace (true);
       }
     
    @@ -103,7 +159,7 @@ struct SingleSubstFormat2
         + hb_zip (this+coverage, substitute)
         | hb_filter (glyphset, hb_first)
         | hb_filter (glyphset, hb_second)
    -    | hb_map_retains_sorting ([&] (hb_pair_t p) -> hb_codepoint_pair_t
    +    | hb_map_retains_sorting ([&] (hb_pair_t p) -> hb_codepoint_pair_t
                                   { return hb_pair (glyph_map[p.first], glyph_map[p.second]); })
         ;
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SubstLookup.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SubstLookup.hh
    index 8fb3b550976a2..5796cc3be5787 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SubstLookup.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SubstLookup.hh
    @@ -6,7 +6,7 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
     struct SubstLookup : Lookup
     {
    @@ -25,7 +25,7 @@ struct SubstLookup : Lookup
       {
         unsigned int type = get_type ();
         if (unlikely (type == SubTable::Extension))
    -      return reinterpret_cast (get_subtable (0)).is_reverse ();
    +      return get_subtable (0).u.extension.is_reverse ();
         return lookup_type_is_reverse (type);
       }
     
    @@ -98,10 +98,15 @@ struct SubstLookup : Lookup
           return dispatch (c);
       }
     
    +  template
       bool serialize_single (hb_serialize_context_t *c,
                              uint32_t lookup_props,
    -                         hb_sorted_array_t glyphs,
    -                         hb_array_t substitutes)
    +                         Glyphs glyphs,
    +                         Substitutes substitutes)
       {
         TRACE_SERIALIZE (this);
         if (unlikely (!Lookup::serialize (c, SubTable::Single, lookup_props, 1))) return_trace (false);
    @@ -114,19 +119,16 @@ struct SubstLookup : Lookup
         return_trace (false);
       }
     
    -  bool serialize_multiple (hb_serialize_context_t *c,
    -                           uint32_t lookup_props,
    -                           hb_sorted_array_t glyphs,
    -                           hb_array_t substitute_len_list,
    -                           hb_array_t substitute_glyphs_list)
    +  template
    +  bool serialize (hb_serialize_context_t *c,
    +                  uint32_t lookup_props,
    +                  Iterator it)
       {
         TRACE_SERIALIZE (this);
         if (unlikely (!Lookup::serialize (c, SubTable::Multiple, lookup_props, 1))) return_trace (false);
         if (c->push ()->u.multiple.
    -        serialize (c,
    -                   glyphs,
    -                   substitute_len_list,
    -                   substitute_glyphs_list))
    +        serialize (c, it))
         {
           c->add_link (get_subtables ()[0], c->pop_pack ());
           return_trace (true);
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SubstLookupSubTable.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SubstLookupSubTable.hh
    index 53e963e2a2918..a525fba039971 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SubstLookupSubTable.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/GSUB/SubstLookupSubTable.hh
    @@ -13,7 +13,7 @@
     
     namespace OT {
     namespace Layout {
    -namespace GSUB {
    +namespace GSUB_impl {
     
     struct SubstLookupSubTable
     {
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/Layout/types.hh b/src/java.desktop/share/native/libharfbuzz/OT/Layout/types.hh
    new file mode 100644
    index 0000000000000..6a43403e94b2d
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/Layout/types.hh
    @@ -0,0 +1,66 @@
    +/*
    + * Copyright © 2007,2008,2009  Red Hat, Inc.
    + * Copyright © 2010,2012  Google, Inc.
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + *
    + * Red Hat Author(s): Behdad Esfahbod
    + * Google Author(s): Behdad Esfahbod, Garret Rieger
    + */
    +
    +#ifndef OT_LAYOUT_TYPES_HH
    +#define OT_LAYOUT_TYPES_HH
    +
    +namespace OT {
    +namespace Layout {
    +
    +struct SmallTypes {
    +  static constexpr unsigned size = 2;
    +  using large_int = uint32_t;
    +  using HBUINT = HBUINT16;
    +  using HBGlyphID = HBGlyphID16;
    +  using Offset = Offset16;
    +  template 
    +  using OffsetTo = OT::Offset16To;
    +  template 
    +  using ArrayOf = OT::Array16Of;
    +  template 
    +  using SortedArrayOf = OT::SortedArray16Of;
    +};
    +
    +struct MediumTypes {
    +  static constexpr unsigned size = 3;
    +  using large_int = uint64_t;
    +  using HBUINT = HBUINT24;
    +  using HBGlyphID = HBGlyphID24;
    +  using Offset = Offset24;
    +  template 
    +  using OffsetTo = OT::Offset24To;
    +  template 
    +  using ArrayOf = OT::Array24Of;
    +  template 
    +  using SortedArrayOf = OT::SortedArray24Of;
    +};
    +
    +}
    +}
    +
    +#endif  /* OT_LAYOUT_TYPES_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/glyf/CompositeGlyph.hh b/src/java.desktop/share/native/libharfbuzz/OT/glyf/CompositeGlyph.hh
    index 7ebf64d0d5cd7..94cb61abc07f1 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/glyf/CompositeGlyph.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/glyf/CompositeGlyph.hh
    @@ -3,6 +3,7 @@
     
     
     #include "../../hb-open-type.hh"
    +#include "composite-iter.hh"
     
     
     namespace OT {
    @@ -25,13 +26,20 @@ struct CompositeGlyphRecord
         USE_MY_METRICS              = 0x0200,
         OVERLAP_COMPOUND            = 0x0400,
         SCALED_COMPONENT_OFFSET     = 0x0800,
    -    UNSCALED_COMPONENT_OFFSET   = 0x1000
    +    UNSCALED_COMPONENT_OFFSET   = 0x1000,
    +#ifndef HB_NO_BEYOND_64K
    +    GID_IS_24BIT                = 0x2000
    +#endif
       };
     
       public:
       unsigned int get_size () const
       {
         unsigned int size = min_size;
    +    /* glyphIndex is 24bit instead of 16bit */
    +#ifndef HB_NO_BEYOND_64K
    +    if (flags & GID_IS_24BIT) size += HBGlyphID24::static_size - HBGlyphID16::static_size;
    +#endif
         /* arg1 and 2 are int16 */
         if (flags & ARG_1_AND_2_ARE_WORDS) size += 4;
         /* arg1 and 2 are int8 */
    @@ -60,7 +68,13 @@ struct CompositeGlyphRecord
       bool is_anchored ()       const { return !(flags & ARGS_ARE_XY_VALUES); }
       void get_anchor_points (unsigned int &point1, unsigned int &point2) const
       {
    -    const HBUINT8 *p = &StructAfter (glyphIndex);
    +    const auto *p = &StructAfter (flags);
    +#ifndef HB_NO_BEYOND_64K
    +    if (flags & GID_IS_24BIT)
    +      p += HBGlyphID24::static_size;
    +    else
    +#endif
    +      p += HBGlyphID16::static_size;
         if (flags & ARG_1_AND_2_ARE_WORDS)
         {
           point1 = ((const HBUINT16 *) p)[0];
    @@ -73,36 +87,110 @@ struct CompositeGlyphRecord
         }
       }
     
    -  void transform_points (contour_point_vector_t &points) const
    +  void transform_points (contour_point_vector_t &points,
    +                         const float (&matrix)[4],
    +                         const contour_point_t &trans) const
    +  {
    +    if (scaled_offsets ())
    +    {
    +      points.translate (trans);
    +      points.transform (matrix);
    +    }
    +    else
    +    {
    +      points.transform (matrix);
    +      points.translate (trans);
    +    }
    +  }
    +
    +  bool get_points (contour_point_vector_t &points) const
       {
         float matrix[4];
         contour_point_t trans;
    -    if (get_transformation (matrix, trans))
    +    get_transformation (matrix, trans);
    +    if (unlikely (!points.resize (points.length + 1))) return false;
    +    points[points.length - 1] = trans;
    +    return true;
    +  }
    +
    +  unsigned compile_with_point (const contour_point_t &point,
    +                               char *out) const
    +  {
    +    const HBINT8 *p = &StructAfter (flags);
    +#ifndef HB_NO_BEYOND_64K
    +    if (flags & GID_IS_24BIT)
    +      p += HBGlyphID24::static_size;
    +    else
    +#endif
    +      p += HBGlyphID16::static_size;
    +
    +    unsigned len = get_size ();
    +    unsigned len_before_val = (const char *)p - (const char *)this;
    +    if (flags & ARG_1_AND_2_ARE_WORDS)
    +    {
    +      // no overflow, copy value
    +      hb_memcpy (out, this, len);
    +
    +      HBINT16 *o = reinterpret_cast (out + len_before_val);
    +      o[0] = roundf (point.x);
    +      o[1] = roundf (point.y);
    +    }
    +    else
         {
    -      if (scaled_offsets ())
    +      int new_x = roundf (point.x);
    +      int new_y = roundf (point.y);
    +      if (new_x <= 127 && new_x >= -128 &&
    +          new_y <= 127 && new_y >= -128)
           {
    -        points.translate (trans);
    -        points.transform (matrix);
    +        hb_memcpy (out, this, len);
    +        HBINT8 *o = reinterpret_cast (out + len_before_val);
    +        o[0] = new_x;
    +        o[1] = new_y;
           }
           else
           {
    -        points.transform (matrix);
    -        points.translate (trans);
    +        // new point value has an int8 overflow
    +        hb_memcpy (out, this, len_before_val);
    +
    +        //update flags
    +        CompositeGlyphRecord *o = reinterpret_cast (out);
    +        o->flags = flags | ARG_1_AND_2_ARE_WORDS;
    +        out += len_before_val;
    +
    +        HBINT16 new_value;
    +        new_value = new_x;
    +        hb_memcpy (out, &new_value, HBINT16::static_size);
    +        out += HBINT16::static_size;
    +
    +        new_value = new_y;
    +        hb_memcpy (out, &new_value, HBINT16::static_size);
    +        out += HBINT16::static_size;
    +
    +        hb_memcpy (out, p+2, len - len_before_val - 2);
    +        len += 2;
           }
         }
    +    return len;
       }
     
       protected:
       bool scaled_offsets () const
       { return (flags & (SCALED_COMPONENT_OFFSET | UNSCALED_COMPONENT_OFFSET)) == SCALED_COMPONENT_OFFSET; }
     
    +  public:
       bool get_transformation (float (&matrix)[4], contour_point_t &trans) const
       {
         matrix[0] = matrix[3] = 1.f;
         matrix[1] = matrix[2] = 0.f;
     
    +    const auto *p = &StructAfter (flags);
    +#ifndef HB_NO_BEYOND_64K
    +    if (flags & GID_IS_24BIT)
    +      p += HBGlyphID24::static_size;
    +    else
    +#endif
    +      p += HBGlyphID16::static_size;
         int tx, ty;
    -    const HBINT8 *p = &StructAfter (glyphIndex);
         if (flags & ARG_1_AND_2_ARE_WORDS)
         {
           tx = *(const HBINT16 *) p;
    @@ -144,63 +232,56 @@ struct CompositeGlyphRecord
         return tx || ty;
       }
     
    -  public:
    -  HBUINT16      flags;
    -  HBGlyphID16   glyphIndex;
    -  public:
    -  DEFINE_SIZE_MIN (4);
    -};
    -
    -struct composite_iter_t : hb_iter_with_fallback_t
    -{
    -  typedef const CompositeGlyphRecord *__item_t__;
    -  composite_iter_t (hb_bytes_t glyph_, __item_t__ current_) :
    -      glyph (glyph_), current (nullptr), current_size (0)
    +  hb_codepoint_t get_gid () const
       {
    -    set_current (current_);
    +#ifndef HB_NO_BEYOND_64K
    +    if (flags & GID_IS_24BIT)
    +      return StructAfter (flags);
    +    else
    +#endif
    +      return StructAfter (flags);
       }
    -
    -  composite_iter_t () : glyph (hb_bytes_t ()), current (nullptr), current_size (0) {}
    -
    -  item_t __item__ () const { return *current; }
    -  bool __more__ () const { return current; }
    -  void __next__ ()
    +  void set_gid (hb_codepoint_t gid)
       {
    -    if (!current->has_more ()) { current = nullptr; return; }
    -
    -    set_current (&StructAtOffset (current, current_size));
    +#ifndef HB_NO_BEYOND_64K
    +    if (flags & GID_IS_24BIT)
    +      StructAfter (flags) = gid;
    +    else
    +#endif
    +      /* TODO assert? */
    +      StructAfter (flags) = gid;
       }
    -  composite_iter_t __end__ () const { return composite_iter_t (); }
    -  bool operator != (const composite_iter_t& o) const
    -  { return current != o.current; }
     
    -
    -  void set_current (__item_t__ current_)
    +#ifndef HB_NO_BEYOND_64K
    +  void lower_gid_24_to_16 ()
       {
    -    if (!glyph.check_range (current_, CompositeGlyphRecord::min_size))
    -    {
    -      current = nullptr;
    -      current_size = 0;
    -      return;
    -    }
    -    unsigned size = current_->get_size ();
    -    if (!glyph.check_range (current_, size))
    -    {
    -      current = nullptr;
    -      current_size = 0;
    +    hb_codepoint_t gid = get_gid ();
    +    if (!(flags & GID_IS_24BIT) || gid > 0xFFFFu)
           return;
    -    }
     
    -    current = current_;
    -    current_size = size;
    +    /* Lower the flag and move the rest of the struct down. */
    +
    +    unsigned size = get_size ();
    +    char *end = (char *) this + size;
    +    char *p = &StructAfter (flags);
    +    p += HBGlyphID24::static_size;
    +
    +    flags = flags & ~GID_IS_24BIT;
    +    set_gid (gid);
    +
    +    memmove (p - HBGlyphID24::static_size + HBGlyphID16::static_size, p, end - p);
       }
    +#endif
     
    -  private:
    -  hb_bytes_t glyph;
    -  __item_t__ current;
    -  unsigned current_size;
    +  protected:
    +  HBUINT16      flags;
    +  HBUINT24      pad;
    +  public:
    +  DEFINE_SIZE_MIN (4);
     };
     
    +using composite_iter_t = composite_iter_tmpl;
    +
     struct CompositeGlyph
     {
       const GlyphHeader &header;
    @@ -248,6 +329,66 @@ struct CompositeGlyph
           return;
         glyph_chain.set_overlaps_flag ();
       }
    +
    +  bool compile_bytes_with_deltas (const hb_bytes_t &source_bytes,
    +                                  const contour_point_vector_t &points_with_deltas,
    +                                  hb_bytes_t &dest_bytes /* OUT */)
    +  {
    +    if (source_bytes.length <= GlyphHeader::static_size ||
    +        header.numberOfContours != -1)
    +    {
    +      dest_bytes = hb_bytes_t ();
    +      return true;
    +    }
    +
    +    unsigned source_len = source_bytes.length - GlyphHeader::static_size;
    +
    +    /* try to allocate more memories than source glyph bytes
    +     * in case that there might be an overflow for int8 value
    +     * and we would need to use int16 instead */
    +    char *o = (char *) hb_calloc (source_len * 2, sizeof (char));
    +    if (unlikely (!o)) return false;
    +
    +    const CompositeGlyphRecord *c = reinterpret_cast (source_bytes.arrayZ + GlyphHeader::static_size);
    +    auto it = composite_iter_t (hb_bytes_t ((const char *)c, source_len), c);
    +
    +    char *p = o;
    +    unsigned i = 0, source_comp_len = 0;
    +    for (const auto &component : it)
    +    {
    +      /* last 4 points in points_with_deltas are phantom points and should not be included */
    +      if (i >= points_with_deltas.length - 4) {
    +        free (o);
    +        return false;
    +      }
    +
    +      unsigned comp_len = component.get_size ();
    +      if (component.is_anchored ())
    +      {
    +        hb_memcpy (p, &component, comp_len);
    +        p += comp_len;
    +      }
    +      else
    +      {
    +        unsigned new_len = component.compile_with_point (points_with_deltas[i], p);
    +        p += new_len;
    +      }
    +      i++;
    +      source_comp_len += comp_len;
    +    }
    +
    +    //copy instructions if any
    +    if (source_len > source_comp_len)
    +    {
    +      unsigned instr_len = source_len - source_comp_len;
    +      hb_memcpy (p, (const char *)c + source_comp_len, instr_len);
    +      p += instr_len;
    +    }
    +
    +    unsigned len = p - o;
    +    dest_bytes = hb_bytes_t (o, len);
    +    return true;
    +  }
     };
     
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/glyf/Glyph.hh b/src/java.desktop/share/native/libharfbuzz/OT/glyf/Glyph.hh
    index 3d2095ac2dfb8..1ebaaa3f831e1 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/glyf/Glyph.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/glyf/Glyph.hh
    @@ -7,6 +7,8 @@
     #include "GlyphHeader.hh"
     #include "SimpleGlyph.hh"
     #include "CompositeGlyph.hh"
    +#include "VarCompositeGlyph.hh"
    +#include "coord-setter.hh"
     
     
     namespace OT {
    @@ -27,7 +29,14 @@ enum phantom_point_index_t
     
     struct Glyph
     {
    -  enum glyph_type_t { EMPTY, SIMPLE, COMPOSITE };
    +  enum glyph_type_t {
    +    EMPTY,
    +    SIMPLE,
    +    COMPOSITE,
    +#ifndef HB_NO_VAR_COMPOSITES
    +    VAR_COMPOSITE,
    +#endif
    +  };
     
       public:
       composite_iter_t get_composite_iterator () const
    @@ -35,12 +44,25 @@ struct Glyph
         if (type != COMPOSITE) return composite_iter_t ();
         return CompositeGlyph (*header, bytes).iter ();
       }
    +  var_composite_iter_t get_var_composite_iterator () const
    +  {
    +#ifndef HB_NO_VAR_COMPOSITES
    +    if (type != VAR_COMPOSITE) return var_composite_iter_t ();
    +    return VarCompositeGlyph (*header, bytes).iter ();
    +#else
    +    return var_composite_iter_t ();
    +#endif
    +  }
     
       const hb_bytes_t trim_padding () const
       {
         switch (type) {
    +#ifndef HB_NO_VAR_COMPOSITES
    +    case VAR_COMPOSITE: return VarCompositeGlyph (*header, bytes).trim_padding ();
    +#endif
         case COMPOSITE: return CompositeGlyph (*header, bytes).trim_padding ();
         case SIMPLE:    return SimpleGlyph (*header, bytes).trim_padding ();
    +    case EMPTY:     return bytes;
         default:        return bytes;
         }
       }
    @@ -48,92 +70,318 @@ struct Glyph
       void drop_hints ()
       {
         switch (type) {
    +#ifndef HB_NO_VAR_COMPOSITES
    +    case VAR_COMPOSITE: return; // No hinting
    +#endif
         case COMPOSITE: CompositeGlyph (*header, bytes).drop_hints (); return;
         case SIMPLE:    SimpleGlyph (*header, bytes).drop_hints (); return;
    -    default:        return;
    +    case EMPTY:     return;
         }
       }
     
       void set_overlaps_flag ()
       {
         switch (type) {
    +#ifndef HB_NO_VAR_COMPOSITES
    +    case VAR_COMPOSITE: return; // No overlaps flag
    +#endif
         case COMPOSITE: CompositeGlyph (*header, bytes).set_overlaps_flag (); return;
         case SIMPLE:    SimpleGlyph (*header, bytes).set_overlaps_flag (); return;
    -    default:        return;
    +    case EMPTY:     return;
         }
       }
     
       void drop_hints_bytes (hb_bytes_t &dest_start, hb_bytes_t &dest_end) const
       {
         switch (type) {
    +#ifndef HB_NO_VAR_COMPOSITES
    +    case VAR_COMPOSITE: return; // No hinting
    +#endif
         case COMPOSITE: CompositeGlyph (*header, bytes).drop_hints_bytes (dest_start); return;
         case SIMPLE:    SimpleGlyph (*header, bytes).drop_hints_bytes (dest_start, dest_end); return;
    -    default:        return;
    +    case EMPTY:     return;
    +    }
    +  }
    +
    +  void update_mtx (const hb_subset_plan_t *plan,
    +                   int xMin, int xMax,
    +                   int yMin, int yMax,
    +                   const contour_point_vector_t &all_points) const
    +  {
    +    hb_codepoint_t new_gid = 0;
    +    if (!plan->new_gid_for_old_gid (gid, &new_gid))
    +      return;
    +
    +    if (type != EMPTY)
    +    {
    +      plan->bounds_width_map.set (new_gid, xMax - xMin);
    +      plan->bounds_height_map.set (new_gid, yMax - yMin);
    +    }
    +
    +    unsigned len = all_points.length;
    +    float leftSideX = all_points[len - 4].x;
    +    float rightSideX = all_points[len - 3].x;
    +    float topSideY = all_points[len - 2].y;
    +    float bottomSideY = all_points[len - 1].y;
    +
    +    signed hori_aw = roundf (rightSideX - leftSideX);
    +    if (hori_aw < 0) hori_aw = 0;
    +    int lsb = roundf (xMin - leftSideX);
    +    plan->hmtx_map.set (new_gid, hb_pair ((unsigned) hori_aw, lsb));
    +    //flag value should be computed using non-empty glyphs
    +    if (type != EMPTY && lsb != xMin)
    +      plan->head_maxp_info.allXMinIsLsb = false;
    +
    +    signed vert_aw = roundf (topSideY - bottomSideY);
    +    if (vert_aw < 0) vert_aw = 0;
    +    int tsb = roundf (topSideY - yMax);
    +    plan->vmtx_map.set (new_gid, hb_pair ((unsigned) vert_aw, tsb));
    +  }
    +
    +  bool compile_header_bytes (const hb_subset_plan_t *plan,
    +                             const contour_point_vector_t &all_points,
    +                             hb_bytes_t &dest_bytes /* OUT */) const
    +  {
    +    GlyphHeader *glyph_header = nullptr;
    +    if (!plan->pinned_at_default && type != EMPTY && all_points.length >= 4)
    +    {
    +      glyph_header = (GlyphHeader *) hb_calloc (1, GlyphHeader::static_size);
    +      if (unlikely (!glyph_header)) return false;
    +    }
    +
    +    float xMin = 0, xMax = 0;
    +    float yMin = 0, yMax = 0;
    +    if (all_points.length > 4)
    +    {
    +      xMin = xMax = all_points[0].x;
    +      yMin = yMax = all_points[0].y;
    +    }
    +
    +    for (unsigned i = 1; i < all_points.length - 4; i++)
    +    {
    +      float x = all_points[i].x;
    +      float y = all_points[i].y;
    +      xMin = hb_min (xMin, x);
    +      xMax = hb_max (xMax, x);
    +      yMin = hb_min (yMin, y);
    +      yMax = hb_max (yMax, y);
    +    }
    +
    +    update_mtx (plan, roundf (xMin), roundf (xMax), roundf (yMin), roundf (yMax), all_points);
    +
    +    int rounded_xMin = roundf (xMin);
    +    int rounded_xMax = roundf (xMax);
    +    int rounded_yMin = roundf (yMin);
    +    int rounded_yMax = roundf (yMax);
    +
    +    if (type != EMPTY)
    +    {
    +      plan->head_maxp_info.xMin = hb_min (plan->head_maxp_info.xMin, rounded_xMin);
    +      plan->head_maxp_info.yMin = hb_min (plan->head_maxp_info.yMin, rounded_yMin);
    +      plan->head_maxp_info.xMax = hb_max (plan->head_maxp_info.xMax, rounded_xMax);
    +      plan->head_maxp_info.yMax = hb_max (plan->head_maxp_info.yMax, rounded_yMax);
    +    }
    +
    +    /* when pinned at default, no need to compile glyph header
    +     * and for empty glyphs: all_points only include phantom points.
    +     * just update metrics and then return */
    +    if (!glyph_header)
    +      return true;
    +
    +    glyph_header->numberOfContours = header->numberOfContours;
    +
    +    glyph_header->xMin = rounded_xMin;
    +    glyph_header->yMin = rounded_yMin;
    +    glyph_header->xMax = rounded_xMax;
    +    glyph_header->yMax = rounded_yMax;
    +
    +    dest_bytes = hb_bytes_t ((const char *)glyph_header, GlyphHeader::static_size);
    +    return true;
    +  }
    +
    +  bool compile_bytes_with_deltas (const hb_subset_plan_t *plan,
    +                                  hb_font_t *font,
    +                                  const glyf_accelerator_t &glyf,
    +                                  hb_bytes_t &dest_start,  /* IN/OUT */
    +                                  hb_bytes_t &dest_end /* OUT */)
    +  {
    +    contour_point_vector_t all_points, points_with_deltas;
    +    unsigned composite_contours = 0;
    +    head_maxp_info_t *head_maxp_info_p = &plan->head_maxp_info;
    +    unsigned *composite_contours_p = &composite_contours;
    +
    +    // don't compute head/maxp values when glyph has no contours(type is EMPTY)
    +    // also ignore .notdef glyph when --notdef-outline is not enabled
    +    if (type == EMPTY ||
    +        (gid == 0 && !(plan->flags & HB_SUBSET_FLAGS_NOTDEF_OUTLINE)))
    +    {
    +      head_maxp_info_p = nullptr;
    +      composite_contours_p = nullptr;
    +    }
    +
    +    if (!get_points (font, glyf, all_points, &points_with_deltas, head_maxp_info_p, composite_contours_p, false, false))
    +      return false;
    +
    +    // .notdef, set type to empty so we only update metrics and don't compile bytes for
    +    // it
    +    if (gid == 0 &&
    +        !(plan->flags & HB_SUBSET_FLAGS_NOTDEF_OUTLINE))
    +    {
    +      type = EMPTY;
    +      dest_start = hb_bytes_t ();
    +      dest_end = hb_bytes_t ();
    +    }
    +
    +    //dont compile bytes when pinned at default, just recalculate bounds
    +    if (!plan->pinned_at_default)
    +    {
    +      switch (type)
    +      {
    +#ifndef HB_NO_VAR_COMPOSITES
    +      case VAR_COMPOSITE:
    +        // TODO
    +        dest_end = hb_bytes_t ();
    +        break;
    +#endif
    +
    +      case COMPOSITE:
    +        if (!CompositeGlyph (*header, bytes).compile_bytes_with_deltas (dest_start,
    +                                                                        points_with_deltas,
    +                                                                        dest_end))
    +          return false;
    +        break;
    +      case SIMPLE:
    +        if (!SimpleGlyph (*header, bytes).compile_bytes_with_deltas (all_points,
    +                                                                     plan->flags & HB_SUBSET_FLAGS_NO_HINTING,
    +                                                                     dest_end))
    +          return false;
    +        break;
    +      case EMPTY:
    +        /* set empty bytes for empty glyph
    +         * do not use source glyph's pointers */
    +        dest_start = hb_bytes_t ();
    +        dest_end = hb_bytes_t ();
    +        break;
    +      }
         }
    +
    +    if (!compile_header_bytes (plan, all_points, dest_start))
    +    {
    +      dest_end.fini ();
    +      return false;
    +    }
    +    return true;
       }
     
    +
       /* Note: Recursively calls itself.
        * all_points includes phantom points
        */
       template 
       bool get_points (hb_font_t *font, const accelerator_t &glyf_accelerator,
                        contour_point_vector_t &all_points /* OUT */,
    +                   contour_point_vector_t *points_with_deltas = nullptr, /* OUT */
    +                   head_maxp_info_t * head_maxp_info = nullptr, /* OUT */
    +                   unsigned *composite_contours = nullptr, /* OUT */
    +                   bool shift_points_hori = true,
    +                   bool use_my_metrics = true,
                        bool phantom_only = false,
    -                   unsigned int depth = 0) const
    +                   hb_array_t coords = hb_array_t (),
    +                   unsigned int depth = 0,
    +                   unsigned *edge_count = nullptr) const
       {
         if (unlikely (depth > HB_MAX_NESTING_LEVEL)) return false;
    +    unsigned stack_edge_count = 0;
    +    if (!edge_count) edge_count = &stack_edge_count;
    +    if (unlikely (*edge_count > HB_GLYF_MAX_EDGE_COUNT)) return false;
    +    (*edge_count)++;
    +
    +    if (head_maxp_info)
    +    {
    +      head_maxp_info->maxComponentDepth = hb_max (head_maxp_info->maxComponentDepth, depth);
    +    }
    +
    +    if (!coords)
    +      coords = hb_array (font->coords, font->num_coords);
    +
         contour_point_vector_t stack_points;
         bool inplace = type == SIMPLE && all_points.length == 0;
         /* Load into all_points if it's empty, as an optimization. */
         contour_point_vector_t &points = inplace ? all_points : stack_points;
     
         switch (type) {
    +    case SIMPLE:
    +      if (depth == 0 && head_maxp_info)
    +        head_maxp_info->maxContours = hb_max (head_maxp_info->maxContours, (unsigned) header->numberOfContours);
    +      if (depth > 0 && composite_contours)
    +        *composite_contours += (unsigned) header->numberOfContours;
    +      if (unlikely (!SimpleGlyph (*header, bytes).get_contour_points (points, phantom_only)))
    +        return false;
    +      break;
         case COMPOSITE:
         {
    -      /* pseudo component points for each component in composite glyph */
    -      unsigned num_points = hb_len (CompositeGlyph (*header, bytes).iter ());
    -      if (unlikely (!points.resize (num_points))) return false;
    +      for (auto &item : get_composite_iterator ())
    +        if (unlikely (!item.get_points (points))) return false;
           break;
         }
    -    case SIMPLE:
    -      if (unlikely (!SimpleGlyph (*header, bytes).get_contour_points (points, phantom_only)))
    -        return false;
    +#ifndef HB_NO_VAR_COMPOSITES
    +    case VAR_COMPOSITE:
    +    {
    +      for (auto &item : get_var_composite_iterator ())
    +        if (unlikely (!item.get_points (points))) return false;
    +    }
    +#endif
    +    case EMPTY:
           break;
         }
     
         /* Init phantom points */
         if (unlikely (!points.resize (points.length + PHANTOM_COUNT))) return false;
    -    hb_array_t phantoms = points.sub_array (points.length - PHANTOM_COUNT, PHANTOM_COUNT);
    +    hb_array_t phantoms = points.as_array ().sub_array (points.length - PHANTOM_COUNT, PHANTOM_COUNT);
         {
    -      int h_delta = (int) header->xMin -
    -                    glyf_accelerator.hmtx->get_side_bearing (gid);
    +      int lsb = 0;
    +      int h_delta = glyf_accelerator.hmtx->get_leading_bearing_without_var_unscaled (gid, &lsb) ?
    +                    (int) header->xMin - lsb : 0;
    +      HB_UNUSED int tsb = 0;
           int v_orig  = (int) header->yMax +
     #ifndef HB_NO_VERTICAL
    -                    glyf_accelerator.vmtx->get_side_bearing (gid)
    +                    ((void) glyf_accelerator.vmtx->get_leading_bearing_without_var_unscaled (gid, &tsb), tsb)
     #else
                         0
     #endif
                         ;
    -      unsigned h_adv = glyf_accelerator.hmtx->get_advance (gid);
    +      unsigned h_adv = glyf_accelerator.hmtx->get_advance_without_var_unscaled (gid);
           unsigned v_adv =
     #ifndef HB_NO_VERTICAL
    -                       glyf_accelerator.vmtx->get_advance (gid)
    +                       glyf_accelerator.vmtx->get_advance_without_var_unscaled (gid)
     #else
                            - font->face->get_upem ()
     #endif
                            ;
           phantoms[PHANTOM_LEFT].x = h_delta;
    -      phantoms[PHANTOM_RIGHT].x = h_adv + h_delta;
    +      phantoms[PHANTOM_RIGHT].x = (int) h_adv + h_delta;
           phantoms[PHANTOM_TOP].y = v_orig;
           phantoms[PHANTOM_BOTTOM].y = v_orig - (int) v_adv;
         }
     
     #ifndef HB_NO_VAR
    -    glyf_accelerator.gvar->apply_deltas_to_points (gid, font, points.as_array ());
    +    glyf_accelerator.gvar->apply_deltas_to_points (gid,
    +                                                   coords,
    +                                                   points.as_array ());
     #endif
     
    +    // mainly used by CompositeGlyph calculating new X/Y offset value so no need to extend it
    +    // with child glyphs' points
    +    if (points_with_deltas != nullptr && depth == 0 && type == COMPOSITE)
    +    {
    +      if (unlikely (!points_with_deltas->resize (points.length))) return false;
    +      points_with_deltas->copy_vector (points);
    +    }
    +
         switch (type) {
         case SIMPLE:
    +      if (depth == 0 && head_maxp_info)
    +        head_maxp_info->maxPoints = hb_max (head_maxp_info->maxPoints, points.length - 4);
           if (!inplace)
             all_points.extend (points.as_array ());
           break;
    @@ -144,21 +392,32 @@ struct Glyph
           for (auto &item : get_composite_iterator ())
           {
             comp_points.reset ();
    -        if (unlikely (!glyf_accelerator.glyph_for_gid (item.glyphIndex)
    -                                       .get_points (font, glyf_accelerator, comp_points,
    -                                                    phantom_only, depth + 1)))
    +        if (unlikely (!glyf_accelerator.glyph_for_gid (item.get_gid ())
    +                                       .get_points (font,
    +                                                    glyf_accelerator,
    +                                                    comp_points,
    +                                                    points_with_deltas,
    +                                                    head_maxp_info,
    +                                                    composite_contours,
    +                                                    shift_points_hori,
    +                                                    use_my_metrics,
    +                                                    phantom_only,
    +                                                    coords,
    +                                                    depth + 1,
    +                                                    edge_count)))
               return false;
     
             /* Copy phantom points from component if USE_MY_METRICS flag set */
    -        if (item.is_use_my_metrics ())
    +        if (use_my_metrics && item.is_use_my_metrics ())
               for (unsigned int i = 0; i < PHANTOM_COUNT; i++)
                 phantoms[i] = comp_points[comp_points.length - PHANTOM_COUNT + i];
     
    -        /* Apply component transformation & translation */
    -        item.transform_points (comp_points);
    +        float matrix[4];
    +        contour_point_t default_trans;
    +        item.get_transformation (matrix, default_trans);
     
    -        /* Apply translation from gvar */
    -        comp_points.translate (points[comp_index]);
    +        /* Apply component transformation & translation (with deltas applied) */
    +        item.transform_points (comp_points, matrix, points[comp_index]);
     
             if (item.is_anchored ())
             {
    @@ -174,18 +433,81 @@ struct Glyph
               }
             }
     
    -        all_points.extend (comp_points.sub_array (0, comp_points.length - PHANTOM_COUNT));
    +        all_points.extend (comp_points.as_array ().sub_array (0, comp_points.length - PHANTOM_COUNT));
    +
    +        if (all_points.length > HB_GLYF_MAX_POINTS)
    +          return false;
     
             comp_index++;
           }
     
    +      if (head_maxp_info && depth == 0)
    +      {
    +        if (composite_contours)
    +          head_maxp_info->maxCompositeContours = hb_max (head_maxp_info->maxCompositeContours, *composite_contours);
    +        head_maxp_info->maxCompositePoints = hb_max (head_maxp_info->maxCompositePoints, all_points.length);
    +        head_maxp_info->maxComponentElements = hb_max (head_maxp_info->maxComponentElements, comp_index);
    +      }
    +      all_points.extend (phantoms);
    +    } break;
    +#ifndef HB_NO_VAR_COMPOSITES
    +    case VAR_COMPOSITE:
    +    {
    +      contour_point_vector_t comp_points;
    +      hb_array_t points_left = points.as_array ();
    +      for (auto &item : get_var_composite_iterator ())
    +      {
    +        unsigned item_num_points = item.get_num_points ();
    +        hb_array_t record_points = points_left.sub_array (0, item_num_points);
    +
    +        comp_points.reset ();
    +
    +        auto component_coords = coords;
    +        if (item.is_reset_unspecified_axes ())
    +          component_coords = hb_array ();
    +
    +        coord_setter_t coord_setter (component_coords);
    +        item.set_variations (coord_setter, record_points);
    +
    +        if (unlikely (!glyf_accelerator.glyph_for_gid (item.get_gid ())
    +                                       .get_points (font,
    +                                                    glyf_accelerator,
    +                                                    comp_points,
    +                                                    points_with_deltas,
    +                                                    head_maxp_info,
    +                                                    nullptr,
    +                                                    shift_points_hori,
    +                                                    use_my_metrics,
    +                                                    phantom_only,
    +                                                    coord_setter.get_coords (),
    +                                                    depth + 1,
    +                                                    edge_count)))
    +          return false;
    +
    +        /* Apply component transformation */
    +        item.transform_points (record_points, comp_points);
    +
    +        /* Copy phantom points from component if USE_MY_METRICS flag set */
    +        if (use_my_metrics && item.is_use_my_metrics ())
    +          for (unsigned int i = 0; i < PHANTOM_COUNT; i++)
    +            phantoms[i] = comp_points[comp_points.length - PHANTOM_COUNT + i];
    +
    +        all_points.extend (comp_points.as_array ().sub_array (0, comp_points.length - PHANTOM_COUNT));
    +
    +        if (all_points.length > HB_GLYF_MAX_POINTS)
    +          return false;
    +
    +        points_left += item_num_points;
    +      }
           all_points.extend (phantoms);
         } break;
    -    default:
    +#endif
    +    case EMPTY:
           all_points.extend (phantoms);
    +      break;
         }
     
    -    if (depth == 0) /* Apply at top level */
    +    if (depth == 0 && shift_points_hori) /* Apply at top level */
         {
           /* Undocumented rasterizer behavior:
            * Shift points horizontally by the updated left side bearing
    @@ -198,23 +520,34 @@ struct Glyph
         return !all_points.in_error ();
       }
     
    -  bool get_extents (hb_font_t *font, const glyf_accelerator_t &glyf_accelerator,
    -                    hb_glyph_extents_t *extents) const
    +  bool get_extents_without_var_scaled (hb_font_t *font, const glyf_accelerator_t &glyf_accelerator,
    +                                       hb_glyph_extents_t *extents) const
       {
         if (type == EMPTY) return true; /* Empty glyph; zero extents. */
    -    return header->get_extents (font, glyf_accelerator, gid, extents);
    +    return header->get_extents_without_var_scaled (font, glyf_accelerator, gid, extents);
       }
     
       hb_bytes_t get_bytes () const { return bytes; }
    -
    -  Glyph (hb_bytes_t bytes_ = hb_bytes_t (),
    -         hb_codepoint_t gid_ = (hb_codepoint_t) -1) : bytes (bytes_),
    -                                                      header (bytes.as ()),
    -                                                      gid (gid_)
    +  glyph_type_t get_type () const { return type; }
    +  const GlyphHeader *get_header () const { return header; }
    +
    +  Glyph () : bytes (),
    +             header (bytes.as ()),
    +             gid (-1),
    +             type(EMPTY)
    +  {}
    +
    +  Glyph (hb_bytes_t bytes_,
    +         hb_codepoint_t gid_ = (unsigned) -1) : bytes (bytes_),
    +                                                header (bytes.as ()),
    +                                                gid (gid_)
       {
         int num_contours = header->numberOfContours;
         if (unlikely (num_contours == 0)) type = EMPTY;
         else if (num_contours > 0) type = SIMPLE;
    +#ifndef HB_NO_VAR_COMPOSITES
    +    else if (num_contours == -2) type = VAR_COMPOSITE;
    +#endif
         else type = COMPOSITE; /* negative numbers */
       }
     
    @@ -222,7 +555,7 @@ struct Glyph
       hb_bytes_t bytes;
       const GlyphHeader *header;
       hb_codepoint_t gid;
    -  unsigned type;
    +  glyph_type_t type;
     };
     
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/glyf/GlyphHeader.hh b/src/java.desktop/share/native/libharfbuzz/OT/glyf/GlyphHeader.hh
    index a2a83fe198a5c..72c99f88cddb4 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/glyf/GlyphHeader.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/glyf/GlyphHeader.hh
    @@ -14,15 +14,19 @@ struct GlyphHeader
       bool has_data () const { return numberOfContours; }
     
       template 
    -  bool get_extents (hb_font_t *font, const accelerator_t &glyf_accelerator,
    -                    hb_codepoint_t gid, hb_glyph_extents_t *extents) const
    +  bool get_extents_without_var_scaled (hb_font_t *font, const accelerator_t &glyf_accelerator,
    +                                       hb_codepoint_t gid, hb_glyph_extents_t *extents) const
       {
         /* Undocumented rasterizer behavior: shift glyph to the left by (lsb - xMin), i.e., xMin = lsb */
         /* extents->x_bearing = hb_min (glyph_header.xMin, glyph_header.xMax); */
    -    extents->x_bearing = font->em_scale_x (glyf_accelerator.hmtx->get_side_bearing (gid));
    -    extents->y_bearing = font->em_scale_y (hb_max (yMin, yMax));
    -    extents->width     = font->em_scale_x (hb_max (xMin, xMax) - hb_min (xMin, xMax));
    -    extents->height    = font->em_scale_y (hb_min (yMin, yMax) - hb_max (yMin, yMax));
    +    int lsb = hb_min (xMin, xMax);
    +    (void) glyf_accelerator.hmtx->get_leading_bearing_without_var_unscaled (gid, &lsb);
    +    extents->x_bearing = lsb;
    +    extents->y_bearing = hb_max (yMin, yMax);
    +    extents->width     = hb_max (xMin, xMax) - hb_min (xMin, xMax);
    +    extents->height    = hb_min (yMin, yMax) - hb_max (yMin, yMax);
    +
    +    font->scale_glyph_extents (extents);
     
         return true;
       }
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/glyf/SimpleGlyph.hh b/src/java.desktop/share/native/libharfbuzz/OT/glyf/SimpleGlyph.hh
    index 4b74967928bdb..bed9fc81d817a 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/glyf/SimpleGlyph.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/glyf/SimpleGlyph.hh
    @@ -20,7 +20,7 @@ struct SimpleGlyph
         FLAG_X_SAME         = 0x10,
         FLAG_Y_SAME         = 0x20,
         FLAG_OVERLAP_SIMPLE = 0x40,
    -    FLAG_RESERVED2      = 0x80
    +    FLAG_CUBIC          = 0x80
       };
     
       const GlyphHeader &header;
    @@ -34,6 +34,11 @@ struct SimpleGlyph
       unsigned int length (unsigned int instruction_len) const
       { return instruction_len_offset () + 2 + instruction_len; }
     
    +  bool has_instructions_length () const
    +  {
    +    return instruction_len_offset () + 2 <= bytes.length;
    +  }
    +
       unsigned int instructions_length () const
       {
         unsigned int instruction_length_offset = instruction_len_offset ();
    @@ -94,6 +99,7 @@ struct SimpleGlyph
       /* zero instruction length */
       void drop_hints ()
       {
    +    if (!has_instructions_length ()) return;
         GlyphHeader &glyph_header = const_cast (header);
         (HBUINT16 &) StructAtOffset (&glyph_header, instruction_len_offset ()) = 0;
       }
    @@ -132,8 +138,8 @@ struct SimpleGlyph
             if (unlikely (p + 1 > end)) return false;
             unsigned int repeat_count = *p++;
             unsigned stop = hb_min (i + repeat_count, count);
    -        for (; i < stop;)
    -          points_.arrayZ[i++].flag = flag;
    +        for (; i < stop; i++)
    +          points_.arrayZ[i].flag = flag;
           }
         }
         return true;
    @@ -184,7 +190,7 @@ struct SimpleGlyph
         if (unlikely (!bytes.check_range (&endPtsOfContours[num_contours]))) return false;
         unsigned int num_points = endPtsOfContours[num_contours - 1] + 1;
     
    -    points_.alloc (num_points + 4); // Allocate for phantom points, to avoid a possible copy
    +    points_.alloc (num_points + 4, true); // Allocate for phantom points, to avoid a possible copy
         if (!points_.resize (num_points)) return false;
         if (phantom_only) return true;
     
    @@ -206,6 +212,129 @@ struct SimpleGlyph
             && read_points (p, points_, end, &contour_point_t::y,
                             FLAG_Y_SHORT, FLAG_Y_SAME);
       }
    +
    +  static void encode_coord (int value,
    +                            uint8_t &flag,
    +                            const simple_glyph_flag_t short_flag,
    +                            const simple_glyph_flag_t same_flag,
    +                            hb_vector_t &coords /* OUT */)
    +  {
    +    if (value == 0)
    +    {
    +      flag |= same_flag;
    +    }
    +    else if (value >= -255 && value <= 255)
    +    {
    +      flag |= short_flag;
    +      if (value > 0) flag |= same_flag;
    +      else value = -value;
    +
    +      coords.arrayZ[coords.length++] = (uint8_t) value;
    +    }
    +    else
    +    {
    +      int16_t val = value;
    +      coords.arrayZ[coords.length++] = val >> 8;
    +      coords.arrayZ[coords.length++] = val & 0xff;
    +    }
    +  }
    +
    +  static void encode_flag (uint8_t &flag,
    +                           uint8_t &repeat,
    +                           uint8_t lastflag,
    +                           hb_vector_t &flags /* OUT */)
    +  {
    +    if (flag == lastflag && repeat != 255)
    +    {
    +      repeat++;
    +      if (repeat == 1)
    +      {
    +        /* We know there's room. */
    +        flags.arrayZ[flags.length++] = flag;
    +      }
    +      else
    +      {
    +        unsigned len = flags.length;
    +        flags.arrayZ[len-2] = flag | FLAG_REPEAT;
    +        flags.arrayZ[len-1] = repeat;
    +      }
    +    }
    +    else
    +    {
    +      repeat = 0;
    +      flags.push (flag);
    +    }
    +  }
    +
    +  bool compile_bytes_with_deltas (const contour_point_vector_t &all_points,
    +                                  bool no_hinting,
    +                                  hb_bytes_t &dest_bytes /* OUT */)
    +  {
    +    if (header.numberOfContours == 0 || all_points.length <= 4)
    +    {
    +      dest_bytes = hb_bytes_t ();
    +      return true;
    +    }
    +    unsigned num_points = all_points.length - 4;
    +
    +    hb_vector_t flags, x_coords, y_coords;
    +    if (unlikely (!flags.alloc (num_points, true))) return false;
    +    if (unlikely (!x_coords.alloc (2*num_points, true))) return false;
    +    if (unlikely (!y_coords.alloc (2*num_points, true))) return false;
    +
    +    uint8_t lastflag = 255, repeat = 0;
    +    int prev_x = 0, prev_y = 0;
    +
    +    for (unsigned i = 0; i < num_points; i++)
    +    {
    +      uint8_t flag = all_points.arrayZ[i].flag;
    +      flag &= FLAG_ON_CURVE + FLAG_OVERLAP_SIMPLE;
    +
    +      int cur_x = roundf (all_points.arrayZ[i].x);
    +      int cur_y = roundf (all_points.arrayZ[i].y);
    +      encode_coord (cur_x - prev_x, flag, FLAG_X_SHORT, FLAG_X_SAME, x_coords);
    +      encode_coord (cur_y - prev_y, flag, FLAG_Y_SHORT, FLAG_Y_SAME, y_coords);
    +      encode_flag (flag, repeat, lastflag, flags);
    +
    +      prev_x = cur_x;
    +      prev_y = cur_y;
    +      lastflag = flag;
    +    }
    +
    +    unsigned len_before_instrs = 2 * header.numberOfContours + 2;
    +    unsigned len_instrs = instructions_length ();
    +    unsigned total_len = len_before_instrs + flags.length + x_coords.length + y_coords.length;
    +
    +    if (!no_hinting)
    +      total_len += len_instrs;
    +
    +    char *p = (char *) hb_malloc (total_len);
    +    if (unlikely (!p)) return false;
    +
    +    const char *src = bytes.arrayZ + GlyphHeader::static_size;
    +    char *cur = p;
    +    hb_memcpy (p, src, len_before_instrs);
    +
    +    cur += len_before_instrs;
    +    src += len_before_instrs;
    +
    +    if (!no_hinting)
    +    {
    +      hb_memcpy (cur, src, len_instrs);
    +      cur += len_instrs;
    +    }
    +
    +    hb_memcpy (cur, flags.arrayZ, flags.length);
    +    cur += flags.length;
    +
    +    hb_memcpy (cur, x_coords.arrayZ, x_coords.length);
    +    cur += x_coords.length;
    +
    +    hb_memcpy (cur, y_coords.arrayZ, y_coords.length);
    +
    +    dest_bytes = hb_bytes_t (p, total_len);
    +    return true;
    +  }
     };
     
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/glyf/SubsetGlyph.hh b/src/java.desktop/share/native/libharfbuzz/OT/glyf/SubsetGlyph.hh
    index 8106a80338e6f..d6ce5be07bbc4 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/glyf/SubsetGlyph.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/glyf/SubsetGlyph.hh
    @@ -6,27 +6,35 @@
     
     
     namespace OT {
    +
    +struct glyf_accelerator_t;
    +
     namespace glyf_impl {
     
     
     struct SubsetGlyph
     {
    -  hb_codepoint_t new_gid;
       hb_codepoint_t old_gid;
       Glyph source_glyph;
       hb_bytes_t dest_start;  /* region of source_glyph to copy first */
       hb_bytes_t dest_end;    /* region of source_glyph to copy second */
    +  bool allocated;
     
       bool serialize (hb_serialize_context_t *c,
                       bool use_short_loca,
    -                  const hb_subset_plan_t *plan) const
    +                  const hb_subset_plan_t *plan)
       {
         TRACE_SERIALIZE (this);
     
         hb_bytes_t dest_glyph = dest_start.copy (c);
    -    dest_glyph = hb_bytes_t (&dest_glyph, dest_glyph.length + dest_end.copy (c).length);
    +    hb_bytes_t end_copy = dest_end.copy (c);
    +    if (!end_copy.arrayZ || !dest_glyph.arrayZ) {
    +      return false;
    +    }
    +
    +    dest_glyph = hb_bytes_t (&dest_glyph, dest_glyph.length + end_copy.length);
         unsigned int pad_length = use_short_loca ? padding () : 0;
    -    DEBUG_MSG (SUBSET, nullptr, "serialize %d byte glyph, width %d pad %d", dest_glyph.length, dest_glyph.length + pad_length, pad_length);
    +    DEBUG_MSG (SUBSET, nullptr, "serialize %u byte glyph, width %u pad %u", dest_glyph.length, dest_glyph.length + pad_length, pad_length);
     
         HBUINT8 pad;
         pad = 0;
    @@ -38,13 +46,68 @@ struct SubsetGlyph
     
         if (unlikely (!dest_glyph.length)) return_trace (true);
     
    -    /* update components gids */
    +    /* update components gids. */
         for (auto &_ : Glyph (dest_glyph).get_composite_iterator ())
         {
           hb_codepoint_t new_gid;
    -      if (plan->new_gid_for_old_gid (_.glyphIndex, &new_gid))
    -        const_cast (_).glyphIndex = new_gid;
    +      if (plan->new_gid_for_old_gid (_.get_gid(), &new_gid))
    +        const_cast (_).set_gid (new_gid);
         }
    +#ifndef HB_NO_VAR_COMPOSITES
    +    for (auto &_ : Glyph (dest_glyph).get_var_composite_iterator ())
    +    {
    +      hb_codepoint_t new_gid;
    +      if (plan->new_gid_for_old_gid (_.get_gid(), &new_gid))
    +        const_cast (_).set_gid (new_gid);
    +    }
    +#endif
    +
    +#ifndef HB_NO_BEYOND_64K
    +    auto it = Glyph (dest_glyph).get_composite_iterator ();
    +    if (it)
    +    {
    +      /* lower GID24 to GID16 in components if possible.
    +       *
    +       * TODO: VarComposite. Not as critical, since VarComposite supports
    +       * gid24 from the first version. */
    +      char *p = it ? (char *) &*it : nullptr;
    +      char *q = p;
    +      const char *end = dest_glyph.arrayZ + dest_glyph.length;
    +      while (it)
    +      {
    +        auto &rec = const_cast (*it);
    +        ++it;
    +
    +        q += rec.get_size ();
    +
    +        rec.lower_gid_24_to_16 ();
    +
    +        unsigned size = rec.get_size ();
    +
    +        memmove (p, &rec, size);
    +
    +        p += size;
    +      }
    +      memmove (p, q, end - q);
    +      p += end - q;
    +
    +      /* We want to shorten the glyph, but we can't do that without
    +       * updating the length in the loca table, which is already
    +       * written out :-(.  So we just fill the rest of the glyph with
    +       * harmless instructions, since that's what they will be
    +       * interpreted as.
    +       *
    +       * Should move the lowering to _populate_subset_glyphs() to
    +       * fix this issue. */
    +
    +      hb_memset (p, 0x7A /* TrueType instruction ROFF; harmless */, end - p);
    +      p += end - p;
    +      dest_glyph = hb_bytes_t (dest_glyph.arrayZ, p - (char *) dest_glyph.arrayZ);
    +
    +      // TODO: Padding; & trim serialized bytes.
    +      // TODO: Update length in loca. Ugh.
    +    }
    +#endif
     
         if (plan->flags & HB_SUBSET_FLAGS_NO_HINTING)
           Glyph (dest_glyph).drop_hints ();
    @@ -55,6 +118,23 @@ struct SubsetGlyph
         return_trace (true);
       }
     
    +  bool compile_bytes_with_deltas (const hb_subset_plan_t *plan,
    +                                  hb_font_t *font,
    +                                  const glyf_accelerator_t &glyf)
    +  {
    +    allocated = source_glyph.compile_bytes_with_deltas (plan, font, glyf, dest_start, dest_end);
    +    return allocated;
    +  }
    +
    +  void free_compiled_bytes ()
    +  {
    +    if (likely (allocated)) {
    +      allocated = false;
    +      dest_start.fini ();
    +      dest_end.fini ();
    +    }
    +  }
    +
       void drop_hints_bytes ()
       { source_glyph.drop_hints_bytes (dest_start, dest_end); }
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/glyf/VarCompositeGlyph.hh b/src/java.desktop/share/native/libharfbuzz/OT/glyf/VarCompositeGlyph.hh
    new file mode 100644
    index 0000000000000..f2dcb065e263d
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/glyf/VarCompositeGlyph.hh
    @@ -0,0 +1,371 @@
    +#ifndef OT_GLYF_VARCOMPOSITEGLYPH_HH
    +#define OT_GLYF_VARCOMPOSITEGLYPH_HH
    +
    +
    +#include "../../hb-open-type.hh"
    +#include "coord-setter.hh"
    +
    +
    +namespace OT {
    +namespace glyf_impl {
    +
    +
    +struct VarCompositeGlyphRecord
    +{
    +  protected:
    +  enum var_composite_glyph_flag_t
    +  {
    +    USE_MY_METRICS              = 0x0001,
    +    AXIS_INDICES_ARE_SHORT      = 0x0002,
    +    UNIFORM_SCALE               = 0x0004,
    +    HAVE_TRANSLATE_X            = 0x0008,
    +    HAVE_TRANSLATE_Y            = 0x0010,
    +    HAVE_ROTATION               = 0x0020,
    +    HAVE_SCALE_X                = 0x0040,
    +    HAVE_SCALE_Y                = 0x0080,
    +    HAVE_SKEW_X                 = 0x0100,
    +    HAVE_SKEW_Y                 = 0x0200,
    +    HAVE_TCENTER_X              = 0x0400,
    +    HAVE_TCENTER_Y              = 0x0800,
    +    GID_IS_24BIT                = 0x1000,
    +    AXES_HAVE_VARIATION         = 0x2000,
    +    RESET_UNSPECIFIED_AXES      = 0x4000,
    +  };
    +
    +  public:
    +
    +  unsigned int get_size () const
    +  {
    +    unsigned int size = min_size;
    +
    +    unsigned axis_width = (flags & AXIS_INDICES_ARE_SHORT) ? 4 : 3;
    +    size += numAxes * axis_width;
    +
    +    // gid
    +    size += 2;
    +    if (flags & GID_IS_24BIT)           size += 1;
    +
    +    if (flags & HAVE_TRANSLATE_X)       size += 2;
    +    if (flags & HAVE_TRANSLATE_Y)       size += 2;
    +    if (flags & HAVE_ROTATION)          size += 2;
    +    if (flags & HAVE_SCALE_X)           size += 2;
    +    if (flags & HAVE_SCALE_Y)           size += 2;
    +    if (flags & HAVE_SKEW_X)            size += 2;
    +    if (flags & HAVE_SKEW_Y)            size += 2;
    +    if (flags & HAVE_TCENTER_X)         size += 2;
    +    if (flags & HAVE_TCENTER_Y)         size += 2;
    +
    +    return size;
    +  }
    +
    +  bool has_more () const { return true; }
    +
    +  bool is_use_my_metrics () const { return flags & USE_MY_METRICS; }
    +  bool is_reset_unspecified_axes () const { return flags & RESET_UNSPECIFIED_AXES; }
    +
    +  hb_codepoint_t get_gid () const
    +  {
    +    if (flags & GID_IS_24BIT)
    +      return StructAfter (numAxes);
    +    else
    +      return StructAfter (numAxes);
    +  }
    +
    +  void set_gid (hb_codepoint_t gid)
    +  {
    +    if (flags & GID_IS_24BIT)
    +      StructAfter (numAxes) = gid;
    +    else
    +      StructAfter (numAxes) = gid;
    +  }
    +
    +  unsigned get_numAxes () const
    +  {
    +    return numAxes;
    +  }
    +
    +  unsigned get_num_points () const
    +  {
    +    unsigned num = 0;
    +    if (flags & AXES_HAVE_VARIATION)                    num += numAxes;
    +    if (flags & (HAVE_TRANSLATE_X | HAVE_TRANSLATE_Y))  num++;
    +    if (flags & HAVE_ROTATION)                          num++;
    +    if (flags & (HAVE_SCALE_X | HAVE_SCALE_Y))          num++;
    +    if (flags & (HAVE_SKEW_X | HAVE_SKEW_Y))            num++;
    +    if (flags & (HAVE_TCENTER_X | HAVE_TCENTER_Y))      num++;
    +    return num;
    +  }
    +
    +  void transform_points (hb_array_t record_points,
    +                         contour_point_vector_t &points) const
    +  {
    +    float matrix[4];
    +    contour_point_t trans;
    +
    +    get_transformation_from_points (record_points, matrix, trans);
    +
    +    points.transform (matrix);
    +    points.translate (trans);
    +  }
    +
    +  static inline void transform (float (&matrix)[4], contour_point_t &trans,
    +                                float (other)[6])
    +  {
    +    // https://github.com/fonttools/fonttools/blob/f66ee05f71c8b57b5f519ee975e95edcd1466e14/Lib/fontTools/misc/transform.py#L268
    +    float xx1 = other[0];
    +    float xy1 = other[1];
    +    float yx1 = other[2];
    +    float yy1 = other[3];
    +    float dx1 = other[4];
    +    float dy1 = other[5];
    +    float xx2 = matrix[0];
    +    float xy2 = matrix[1];
    +    float yx2 = matrix[2];
    +    float yy2 = matrix[3];
    +    float dx2 = trans.x;
    +    float dy2 = trans.y;
    +
    +    matrix[0] = xx1*xx2 + xy1*yx2;
    +    matrix[1] = xx1*xy2 + xy1*yy2;
    +    matrix[2] = yx1*xx2 + yy1*yx2;
    +    matrix[3] = yx1*xy2 + yy1*yy2;
    +    trans.x = xx2*dx1 + yx2*dy1 + dx2;
    +    trans.y = xy2*dx1 + yy2*dy1 + dy2;
    +  }
    +
    +  static void translate (float (&matrix)[4], contour_point_t &trans,
    +                         float translateX, float translateY)
    +  {
    +    // https://github.com/fonttools/fonttools/blob/f66ee05f71c8b57b5f519ee975e95edcd1466e14/Lib/fontTools/misc/transform.py#L213
    +    float other[6] = {1.f, 0.f, 0.f, 1.f, translateX, translateY};
    +    transform (matrix, trans, other);
    +  }
    +
    +  static void scale (float (&matrix)[4], contour_point_t &trans,
    +                     float scaleX, float scaleY)
    +  {
    +    // https://github.com/fonttools/fonttools/blob/f66ee05f71c8b57b5f519ee975e95edcd1466e14/Lib/fontTools/misc/transform.py#L224
    +    float other[6] = {scaleX, 0.f, 0.f, scaleY, 0.f, 0.f};
    +    transform (matrix, trans, other);
    +  }
    +
    +  static void rotate (float (&matrix)[4], contour_point_t &trans,
    +                      float rotation)
    +  {
    +    // https://github.com/fonttools/fonttools/blob/f66ee05f71c8b57b5f519ee975e95edcd1466e14/Lib/fontTools/misc/transform.py#L240
    +    rotation = rotation * HB_PI;
    +    float c = cosf (rotation);
    +    float s = sinf (rotation);
    +    float other[6] = {c, s, -s, c, 0.f, 0.f};
    +    transform (matrix, trans, other);
    +  }
    +
    +  static void skew (float (&matrix)[4], contour_point_t &trans,
    +                    float skewX, float skewY)
    +  {
    +    // https://github.com/fonttools/fonttools/blob/f66ee05f71c8b57b5f519ee975e95edcd1466e14/Lib/fontTools/misc/transform.py#L255
    +    skewX = skewX * HB_PI;
    +    skewY = skewY * HB_PI;
    +    float other[6] = {1.f, tanf (skewY), tanf (skewX), 1.f, 0.f, 0.f};
    +    transform (matrix, trans, other);
    +  }
    +
    +  bool get_points (contour_point_vector_t &points) const
    +  {
    +    float translateX = 0.f;
    +    float translateY = 0.f;
    +    float rotation = 0.f;
    +    float scaleX = 1.f * (1 << 10);
    +    float scaleY = 1.f * (1 << 10);
    +    float skewX = 0.f;
    +    float skewY = 0.f;
    +    float tCenterX = 0.f;
    +    float tCenterY = 0.f;
    +
    +    unsigned num_points = get_num_points ();
    +
    +    if (unlikely (!points.resize (points.length + num_points))) return false;
    +
    +    unsigned axis_width = (flags & AXIS_INDICES_ARE_SHORT) ? 2 : 1;
    +    unsigned axes_size = numAxes * axis_width;
    +
    +    const F2DOT14 *q = (const F2DOT14 *) (axes_size +
    +                                          (flags & GID_IS_24BIT ? 3 : 2) +
    +                                          &StructAfter (numAxes));
    +
    +    hb_array_t rec_points = points.as_array ().sub_array (points.length - num_points);
    +
    +    unsigned count = numAxes;
    +    if (flags & AXES_HAVE_VARIATION)
    +    {
    +      for (unsigned i = 0; i < count; i++)
    +        rec_points[i].x = q++->to_int ();
    +      rec_points += count;
    +    }
    +    else
    +      q += count;
    +
    +    const HBUINT16 *p = (const HBUINT16 *) q;
    +
    +    if (flags & HAVE_TRANSLATE_X)       translateX = * (const FWORD *) p++;
    +    if (flags & HAVE_TRANSLATE_Y)       translateY = * (const FWORD *) p++;
    +    if (flags & HAVE_ROTATION)          rotation = ((const F4DOT12 *) p++)->to_int ();
    +    if (flags & HAVE_SCALE_X)           scaleX = ((const F6DOT10 *) p++)->to_int ();
    +    if (flags & HAVE_SCALE_Y)           scaleY = ((const F6DOT10 *) p++)->to_int ();
    +    if (flags & HAVE_SKEW_X)            skewX = ((const F4DOT12 *) p++)->to_int ();
    +    if (flags & HAVE_SKEW_Y)            skewY = ((const F4DOT12 *) p++)->to_int ();
    +    if (flags & HAVE_TCENTER_X)         tCenterX = * (const FWORD *) p++;
    +    if (flags & HAVE_TCENTER_Y)         tCenterY = * (const FWORD *) p++;
    +
    +    if ((flags & UNIFORM_SCALE) && !(flags & HAVE_SCALE_Y))
    +      scaleY = scaleX;
    +
    +    if (flags & (HAVE_TRANSLATE_X | HAVE_TRANSLATE_Y))
    +    {
    +      rec_points[0].x = translateX;
    +      rec_points[0].y = translateY;
    +      rec_points++;
    +    }
    +    if (flags & HAVE_ROTATION)
    +    {
    +      rec_points[0].x = rotation;
    +      rec_points++;
    +    }
    +    if (flags & (HAVE_SCALE_X | HAVE_SCALE_Y))
    +    {
    +      rec_points[0].x = scaleX;
    +      rec_points[0].y = scaleY;
    +      rec_points++;
    +    }
    +    if (flags & (HAVE_SKEW_X | HAVE_SKEW_Y))
    +    {
    +      rec_points[0].x = skewX;
    +      rec_points[0].y = skewY;
    +      rec_points++;
    +    }
    +    if (flags & (HAVE_TCENTER_X | HAVE_TCENTER_Y))
    +    {
    +      rec_points[0].x = tCenterX;
    +      rec_points[0].y = tCenterY;
    +      rec_points++;
    +    }
    +    assert (!rec_points);
    +
    +    return true;
    +  }
    +
    +  void get_transformation_from_points (hb_array_t rec_points,
    +                                       float (&matrix)[4], contour_point_t &trans) const
    +  {
    +    if (flags & AXES_HAVE_VARIATION)
    +      rec_points += numAxes;
    +
    +    matrix[0] = matrix[3] = 1.f;
    +    matrix[1] = matrix[2] = 0.f;
    +    trans.init (0.f, 0.f);
    +
    +    float translateX = 0.f;
    +    float translateY = 0.f;
    +    float rotation = 0.f;
    +    float scaleX = 1.f;
    +    float scaleY = 1.f;
    +    float skewX = 0.f;
    +    float skewY = 0.f;
    +    float tCenterX = 0.f;
    +    float tCenterY = 0.f;
    +
    +    if (flags & (HAVE_TRANSLATE_X | HAVE_TRANSLATE_Y))
    +    {
    +      translateX = rec_points[0].x;
    +      translateY = rec_points[0].y;
    +      rec_points++;
    +    }
    +    if (flags & HAVE_ROTATION)
    +    {
    +      rotation = rec_points[0].x / (1 << 12);
    +      rec_points++;
    +    }
    +    if (flags & (HAVE_SCALE_X | HAVE_SCALE_Y))
    +    {
    +      scaleX = rec_points[0].x / (1 << 10);
    +      scaleY = rec_points[0].y / (1 << 10);
    +      rec_points++;
    +    }
    +    if (flags & (HAVE_SKEW_X | HAVE_SKEW_Y))
    +    {
    +      skewX = rec_points[0].x / (1 << 12);
    +      skewY = rec_points[0].y / (1 << 12);
    +      rec_points++;
    +    }
    +    if (flags & (HAVE_TCENTER_X | HAVE_TCENTER_Y))
    +    {
    +      tCenterX = rec_points[0].x;
    +      tCenterY = rec_points[0].y;
    +      rec_points++;
    +    }
    +    assert (!rec_points);
    +
    +    translate (matrix, trans, translateX + tCenterX, translateY + tCenterY);
    +    rotate (matrix, trans, rotation);
    +    scale (matrix, trans, scaleX, scaleY);
    +    skew (matrix, trans, -skewX, skewY);
    +    translate (matrix, trans, -tCenterX, -tCenterY);
    +  }
    +
    +  void set_variations (coord_setter_t &setter,
    +                       hb_array_t rec_points) const
    +  {
    +    bool have_variations = flags & AXES_HAVE_VARIATION;
    +    unsigned axis_width = (flags & AXIS_INDICES_ARE_SHORT) ? 2 : 1;
    +
    +    const HBUINT8  *p = (const HBUINT8 *)  (((HBUINT8 *) &numAxes) + numAxes.static_size + (flags & GID_IS_24BIT ? 3 : 2));
    +    const HBUINT16 *q = (const HBUINT16 *) (((HBUINT8 *) &numAxes) + numAxes.static_size + (flags & GID_IS_24BIT ? 3 : 2));
    +
    +    const F2DOT14 *a = (const F2DOT14 *) ((HBUINT8 *) (axis_width == 1 ? (p + numAxes) : (HBUINT8 *) (q + numAxes)));
    +
    +    unsigned count = numAxes;
    +    for (unsigned i = 0; i < count; i++)
    +    {
    +      unsigned axis_index = axis_width == 1 ? (unsigned) *p++ : (unsigned) *q++;
    +
    +      signed v = have_variations ? rec_points[i].x : a++->to_int ();
    +
    +      v = hb_clamp (v, -(1<<14), (1<<14));
    +      setter[axis_index] = v;
    +    }
    +  }
    +
    +  protected:
    +  HBUINT16      flags;
    +  HBUINT8       numAxes;
    +  public:
    +  DEFINE_SIZE_MIN (3);
    +};
    +
    +using var_composite_iter_t = composite_iter_tmpl;
    +
    +struct VarCompositeGlyph
    +{
    +  const GlyphHeader &header;
    +  hb_bytes_t bytes;
    +  VarCompositeGlyph (const GlyphHeader &header_, hb_bytes_t bytes_) :
    +    header (header_), bytes (bytes_) {}
    +
    +  var_composite_iter_t iter () const
    +  { return var_composite_iter_t (bytes, &StructAfter (header)); }
    +
    +  const hb_bytes_t trim_padding () const
    +  {
    +    unsigned length = GlyphHeader::static_size;
    +    for (auto &comp : iter ())
    +      length += comp.get_size ();
    +    return bytes.sub_array (0, length);
    +  }
    +};
    +
    +
    +} /* namespace glyf_impl */
    +} /* namespace OT */
    +
    +
    +#endif /* OT_GLYF_VARCOMPOSITEGLYPH_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/glyf/composite-iter.hh b/src/java.desktop/share/native/libharfbuzz/OT/glyf/composite-iter.hh
    new file mode 100644
    index 0000000000000..21e6b4ee9622b
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/glyf/composite-iter.hh
    @@ -0,0 +1,68 @@
    +#ifndef OT_GLYF_COMPOSITE_ITER_HH
    +#define OT_GLYF_COMPOSITE_ITER_HH
    +
    +
    +#include "../../hb.hh"
    +
    +
    +namespace OT {
    +namespace glyf_impl {
    +
    +
    +template 
    +struct composite_iter_tmpl : hb_iter_with_fallback_t,
    +                                                     const CompositeGlyphRecord &>
    +{
    +  typedef const CompositeGlyphRecord *__item_t__;
    +  composite_iter_tmpl (hb_bytes_t glyph_, __item_t__ current_) :
    +      glyph (glyph_), current (nullptr), current_size (0)
    +  {
    +    set_current (current_);
    +  }
    +
    +  composite_iter_tmpl () : glyph (hb_bytes_t ()), current (nullptr), current_size (0) {}
    +
    +  const CompositeGlyphRecord & __item__ () const { return *current; }
    +  bool __more__ () const { return current; }
    +  void __next__ ()
    +  {
    +    if (!current->has_more ()) { current = nullptr; return; }
    +
    +    set_current (&StructAtOffset (current, current_size));
    +  }
    +  composite_iter_tmpl __end__ () const { return composite_iter_tmpl (); }
    +  bool operator != (const composite_iter_tmpl& o) const
    +  { return current != o.current; }
    +
    +
    +  void set_current (__item_t__ current_)
    +  {
    +    if (!glyph.check_range (current_, CompositeGlyphRecord::min_size))
    +    {
    +      current = nullptr;
    +      current_size = 0;
    +      return;
    +    }
    +    unsigned size = current_->get_size ();
    +    if (!glyph.check_range (current_, size))
    +    {
    +      current = nullptr;
    +      current_size = 0;
    +      return;
    +    }
    +
    +    current = current_;
    +    current_size = size;
    +  }
    +
    +  private:
    +  hb_bytes_t glyph;
    +  __item_t__ current;
    +  unsigned current_size;
    +};
    +
    +
    +} /* namespace glyf_impl */
    +} /* namespace OT */
    +
    +#endif /* OT_GLYF_COMPOSITE_ITER_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/glyf/coord-setter.hh b/src/java.desktop/share/native/libharfbuzz/OT/glyf/coord-setter.hh
    new file mode 100644
    index 0000000000000..df64ed5af7d2e
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/glyf/coord-setter.hh
    @@ -0,0 +1,34 @@
    +#ifndef OT_GLYF_COORD_SETTER_HH
    +#define OT_GLYF_COORD_SETTER_HH
    +
    +
    +#include "../../hb.hh"
    +
    +
    +namespace OT {
    +namespace glyf_impl {
    +
    +
    +struct coord_setter_t
    +{
    +  coord_setter_t (hb_array_t coords) :
    +    coords (coords) {}
    +
    +  int& operator [] (unsigned idx)
    +  {
    +    if (coords.length < idx + 1)
    +      coords.resize (idx + 1);
    +    return coords[idx];
    +  }
    +
    +  hb_array_t get_coords ()
    +  { return coords.as_array (); }
    +
    +  hb_vector_t coords;
    +};
    +
    +
    +} /* namespace glyf_impl */
    +} /* namespace OT */
    +
    +#endif /* OT_GLYF_COORD_SETTER_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/glyf/glyf-helpers.hh b/src/java.desktop/share/native/libharfbuzz/OT/glyf/glyf-helpers.hh
    index 1fad84ce831e8..18e2d92d0f418 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/glyf/glyf-helpers.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/glyf/glyf-helpers.hh
    @@ -16,7 +16,7 @@ template
     static void
    -_write_loca (IteratorIn it, bool short_offsets, IteratorOut dest)
    +_write_loca (IteratorIn&& it, bool short_offsets, IteratorOut&& dest)
     {
       unsigned right_shift = short_offsets ? 1 : 0;
       unsigned int offset = 0;
    @@ -25,7 +25,7 @@ _write_loca (IteratorIn it, bool short_offsets, IteratorOut dest)
       | hb_map ([=, &offset] (unsigned int padded_size)
                 {
                   offset += padded_size;
    -              DEBUG_MSG (SUBSET, nullptr, "loca entry offset %d", offset);
    +              DEBUG_MSG (SUBSET, nullptr, "loca entry offset %u", offset);
                   return offset >> right_shift;
                 })
       | hb_sink (dest)
    @@ -44,6 +44,20 @@ _add_head_and_set_loca_version (hb_subset_plan_t *plan, bool use_short_loca)
     
       head *head_prime = (head *) hb_blob_get_data_writable (head_prime_blob, nullptr);
       head_prime->indexToLocFormat = use_short_loca ? 0 : 1;
    +  if (plan->normalized_coords)
    +  {
    +    head_prime->xMin = plan->head_maxp_info.xMin;
    +    head_prime->xMax = plan->head_maxp_info.xMax;
    +    head_prime->yMin = plan->head_maxp_info.yMin;
    +    head_prime->yMax = plan->head_maxp_info.yMax;
    +
    +    unsigned orig_flag = head_prime->flags;
    +    if (plan->head_maxp_info.allXMinIsLsb)
    +      orig_flag |= 1 << 1;
    +    else
    +      orig_flag &= ~(1 << 1);
    +    head_prime->flags = orig_flag;
    +  }
       bool success = plan->add_table (HB_OT_TAG_head, head_prime_blob);
     
       hb_blob_destroy (head_prime_blob);
    @@ -61,7 +75,7 @@ _add_loca_and_head (hb_subset_plan_t * plan, Iterator padded_offsets, bool use_s
     
       if (unlikely (!loca_prime_data)) return false;
     
    -  DEBUG_MSG (SUBSET, nullptr, "loca entry_size %d num_offsets %d size %d",
    +  DEBUG_MSG (SUBSET, nullptr, "loca entry_size %u num_offsets %u size %u",
                  entry_size, num_offsets, entry_size * num_offsets);
     
       if (use_short_loca)
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/glyf/glyf.hh b/src/java.desktop/share/native/libharfbuzz/OT/glyf/glyf.hh
    index caaf2d1f2eb34..d2a93a56d8502 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/glyf/glyf.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/glyf/glyf.hh
    @@ -7,6 +7,7 @@
     #include "../../hb-ot-hmtx-table.hh"
     #include "../../hb-ot-var-gvar-table.hh"
     #include "../../hb-draw.hh"
    +#include "../../hb-paint.hh"
     
     #include "glyf-helpers.hh"
     #include "Glyph.hh"
    @@ -24,13 +25,18 @@ namespace OT {
      */
     #define HB_OT_TAG_glyf HB_TAG('g','l','y','f')
     
    -
     struct glyf
     {
       friend struct glyf_accelerator_t;
     
       static constexpr hb_tag_t tableTag = HB_OT_TAG_glyf;
     
    +  static bool has_valid_glyf_format(const hb_face_t* face)
    +  {
    +    const OT::head &head = *face->table.head;
    +    return head.indexToLocFormat <= 1 && head.glyphDataFormat <= 1;
    +  }
    +
       bool sanitize (hb_sanitize_context_t *c HB_UNUSED) const
       {
         TRACE_SANITIZE (this);
    @@ -46,8 +52,11 @@ struct glyf
                       const hb_subset_plan_t *plan)
       {
         TRACE_SERIALIZE (this);
    +
         unsigned init_len = c->length ();
    -    for (const auto &_ : it) _.serialize (c, use_short_loca, plan);
    +    for (auto &_ : it)
    +      if (unlikely (!_.serialize (c, use_short_loca, plan)))
    +        return false;
     
         /* As a special case when all glyph in the font are empty, add a zero byte
          * to the table, so that OTS doesn’t reject it, and to make the table work
    @@ -69,39 +78,83 @@ struct glyf
       {
         TRACE_SUBSET (this);
     
    +    if (!has_valid_glyf_format (c->plan->source)) {
    +      // glyf format is unknown don't attempt to subset it.
    +      DEBUG_MSG (SUBSET, nullptr,
    +                 "unkown glyf format, dropping from subset.");
    +      return_trace (false);
    +    }
    +
         glyf *glyf_prime = c->serializer->start_embed  ();
         if (unlikely (!c->serializer->check_success (glyf_prime))) return_trace (false);
     
    +    hb_font_t *font = nullptr;
    +    if (c->plan->normalized_coords)
    +    {
    +      font = _create_font_for_instancing (c->plan);
    +      if (unlikely (!font)) return false;
    +    }
    +
    +    hb_vector_t padded_offsets;
    +    unsigned num_glyphs = c->plan->num_output_glyphs ();
    +    if (unlikely (!padded_offsets.resize (num_glyphs)))
    +    {
    +      hb_font_destroy (font);
    +      return false;
    +    }
    +
         hb_vector_t glyphs;
    -    _populate_subset_glyphs (c->plan, &glyphs);
    +    if (!_populate_subset_glyphs (c->plan, font, glyphs))
    +    {
    +      hb_font_destroy (font);
    +      return false;
    +    }
     
    -    auto padded_offsets =
    -    + hb_iter (glyphs)
    -    | hb_map (&glyf_impl::SubsetGlyph::padded_size)
    -    ;
    +    if (font)
    +      hb_font_destroy (font);
     
    -    unsigned max_offset = + padded_offsets | hb_reduce (hb_add, 0);
    -    bool use_short_loca = max_offset < 0x1FFFF;
    +    unsigned max_offset = 0;
    +    for (unsigned i = 0; i < num_glyphs; i++)
    +    {
    +      padded_offsets[i] = glyphs[i].padded_size ();
    +      max_offset += padded_offsets[i];
    +    }
     
    +    bool use_short_loca = false;
    +    if (likely (!c->plan->force_long_loca))
    +      use_short_loca = max_offset < 0x1FFFF;
     
    -    glyf_prime->serialize (c->serializer, hb_iter (glyphs), use_short_loca, c->plan);
         if (!use_short_loca) {
    -      padded_offsets =
    -          + hb_iter (glyphs)
    -          | hb_map (&glyf_impl::SubsetGlyph::length)
    -          ;
    +      for (unsigned i = 0; i < num_glyphs; i++)
    +        padded_offsets[i] = glyphs[i].length ();
         }
     
    +    bool result = glyf_prime->serialize (c->serializer, glyphs.writer (), use_short_loca, c->plan);
    +    if (c->plan->normalized_coords && !c->plan->pinned_at_default)
    +      _free_compiled_subset_glyphs (glyphs);
    +
    +    if (!result) return false;
     
         if (unlikely (c->serializer->in_error ())) return_trace (false);
    +
         return_trace (c->serializer->check_success (glyf_impl::_add_loca_and_head (c->plan,
    -                                                                               padded_offsets,
    +                                                                               padded_offsets.iter (),
                                                                                    use_short_loca)));
       }
     
    -  void
    +  bool
       _populate_subset_glyphs (const hb_subset_plan_t   *plan,
    -                           hb_vector_t *glyphs /* OUT */) const;
    +                           hb_font_t                *font,
    +                           hb_vector_t &glyphs /* OUT */) const;
    +
    +  hb_font_t *
    +  _create_font_for_instancing (const hb_subset_plan_t *plan) const;
    +
    +  void _free_compiled_subset_glyphs (hb_vector_t &glyphs) const
    +  {
    +    for (unsigned i = 0; i < glyphs.length; i++)
    +      glyphs[i].free_compiled_bytes ();
    +  }
     
       protected:
       UnsizedArrayOf
    @@ -128,7 +181,7 @@ struct glyf_accelerator_t
         vmtx = nullptr;
     #endif
         const OT::head &head = *face->table.head;
    -    if (head.indexToLocFormat > 1 || head.glyphDataFormat > 0)
    +    if (!glyf::has_valid_glyf_format (face))
           /* Unknown format.  Leave num_glyphs=0, that takes care of disabling us. */
           return;
         short_offset = 0 == head.indexToLocFormat;
    @@ -166,7 +219,7 @@ struct glyf_accelerator_t
         contour_point_vector_t all_points;
     
         bool phantom_only = !consumer.is_consuming_contour_points ();
    -    if (unlikely (!glyph_for_gid (gid).get_points (font, *this, all_points, phantom_only)))
    +    if (unlikely (!glyph_for_gid (gid).get_points (font, *this, all_points, nullptr, nullptr, nullptr, true, true, phantom_only)))
           return false;
     
         if (consumer.is_consuming_contour_points ())
    @@ -188,12 +241,15 @@ struct glyf_accelerator_t
         return true;
       }
     
    +  public:
    +
     #ifndef HB_NO_VAR
       struct points_aggregator_t
       {
         hb_font_t *font;
         hb_glyph_extents_t *extents;
         contour_point_t *phantoms;
    +    bool scaled;
     
         struct contour_bounds_t
         {
    @@ -209,7 +265,7 @@ struct glyf_accelerator_t
     
           bool empty () const { return (min_x >= max_x) || (min_y >= max_y); }
     
    -      void get_extents (hb_font_t *font, hb_glyph_extents_t *extents)
    +      void get_extents (hb_font_t *font, hb_glyph_extents_t *extents, bool scaled)
           {
             if (unlikely (empty ()))
             {
    @@ -219,49 +275,54 @@ struct glyf_accelerator_t
               extents->y_bearing = 0;
               return;
             }
    -        extents->x_bearing = font->em_scalef_x (min_x);
    -        extents->width = font->em_scalef_x (max_x) - extents->x_bearing;
    -        extents->y_bearing = font->em_scalef_y (max_y);
    -        extents->height = font->em_scalef_y (min_y) - extents->y_bearing;
    +        {
    +          extents->x_bearing = roundf (min_x);
    +          extents->width = roundf (max_x - extents->x_bearing);
    +          extents->y_bearing = roundf (max_y);
    +          extents->height = roundf (min_y - extents->y_bearing);
    +
    +          if (scaled)
    +            font->scale_glyph_extents (extents);
    +        }
           }
     
           protected:
           float min_x, min_y, max_x, max_y;
         } bounds;
     
    -    points_aggregator_t (hb_font_t *font_, hb_glyph_extents_t *extents_, contour_point_t *phantoms_)
    +    points_aggregator_t (hb_font_t *font_, hb_glyph_extents_t *extents_, contour_point_t *phantoms_, bool scaled_)
         {
           font = font_;
           extents = extents_;
           phantoms = phantoms_;
    +      scaled = scaled_;
           if (extents) bounds = contour_bounds_t ();
         }
     
         void consume_point (const contour_point_t &point) { bounds.add (point); }
    -    void points_end () { bounds.get_extents (font, extents); }
    +    void points_end () { bounds.get_extents (font, extents, scaled); }
     
         bool is_consuming_contour_points () { return extents; }
         contour_point_t *get_phantoms_sink () { return phantoms; }
       };
     
    -  public:
       unsigned
    -  get_advance_var (hb_font_t *font, hb_codepoint_t gid, bool is_vertical) const
    +  get_advance_with_var_unscaled (hb_font_t *font, hb_codepoint_t gid, bool is_vertical) const
       {
         if (unlikely (gid >= num_glyphs)) return 0;
     
         bool success = false;
     
         contour_point_t phantoms[glyf_impl::PHANTOM_COUNT];
    -    if (likely (font->num_coords == gvar->get_axis_count ()))
    -      success = get_points (font, gid, points_aggregator_t (font, nullptr, phantoms));
    +    if (font->num_coords)
    +      success = get_points (font, gid, points_aggregator_t (font, nullptr, phantoms, false));
     
         if (unlikely (!success))
           return
     #ifndef HB_NO_VERTICAL
    -        is_vertical ? vmtx->get_advance (gid) :
    +        is_vertical ? vmtx->get_advance_without_var_unscaled (gid) :
     #endif
    -        hmtx->get_advance (gid);
    +        hmtx->get_advance_without_var_unscaled (gid);
     
         float result = is_vertical
                      ? phantoms[glyf_impl::PHANTOM_TOP].y - phantoms[glyf_impl::PHANTOM_BOTTOM].y
    @@ -269,26 +330,32 @@ struct glyf_accelerator_t
         return hb_clamp (roundf (result), 0.f, (float) UINT_MAX / 2);
       }
     
    -  int get_side_bearing_var (hb_font_t *font, hb_codepoint_t gid, bool is_vertical) const
    +  bool get_leading_bearing_with_var_unscaled (hb_font_t *font, hb_codepoint_t gid, bool is_vertical, int *lsb) const
       {
    -    if (unlikely (gid >= num_glyphs)) return 0;
    +    if (unlikely (gid >= num_glyphs)) return false;
     
         hb_glyph_extents_t extents;
     
         contour_point_t phantoms[glyf_impl::PHANTOM_COUNT];
    -    if (unlikely (!get_points (font, gid, points_aggregator_t (font, &extents, phantoms))))
    -      return
    -#ifndef HB_NO_VERTICAL
    -        is_vertical ? vmtx->get_side_bearing (gid) :
    -#endif
    -        hmtx->get_side_bearing (gid);
    +    if (unlikely (!get_points (font, gid, points_aggregator_t (font, &extents, phantoms, false))))
    +      return false;
     
    -    return is_vertical
    -         ? ceilf (phantoms[glyf_impl::PHANTOM_TOP].y) - extents.y_bearing
    -         : floorf (phantoms[glyf_impl::PHANTOM_LEFT].x);
    +    *lsb = is_vertical
    +         ? roundf (phantoms[glyf_impl::PHANTOM_TOP].y) - extents.y_bearing
    +         : roundf (phantoms[glyf_impl::PHANTOM_LEFT].x);
    +    return true;
       }
     #endif
     
    +  bool get_leading_bearing_without_var_unscaled (hb_codepoint_t gid, bool is_vertical, int *lsb) const
    +  {
    +    if (unlikely (gid >= num_glyphs)) return false;
    +    if (is_vertical) return false; // TODO Humm, what to do here?
    +
    +    *lsb = glyph_for_gid (gid).get_header ()->xMin;
    +    return true;
    +  }
    +
       public:
       bool get_extents (hb_font_t *font, hb_codepoint_t gid, hb_glyph_extents_t *extents) const
       {
    @@ -296,9 +363,18 @@ struct glyf_accelerator_t
     
     #ifndef HB_NO_VAR
         if (font->num_coords)
    -      return get_points (font, gid, points_aggregator_t (font, extents, nullptr));
    +      return get_points (font, gid, points_aggregator_t (font, extents, nullptr, true));
     #endif
    -    return glyph_for_gid (gid).get_extents (font, *this, extents);
    +    return glyph_for_gid (gid).get_extents_without_var_scaled (font, *this, extents);
    +  }
    +
    +  bool paint_glyph (hb_font_t *font, hb_codepoint_t gid, hb_paint_funcs_t *funcs, void *data, hb_color_t foreground) const
    +  {
    +    funcs->push_clip_glyph (data, gid, font);
    +    funcs->color (data, true, foreground);
    +    funcs->pop_clip (data);
    +
    +    return true;
       }
     
       const glyf_impl::Glyph
    @@ -349,37 +425,77 @@ struct glyf_accelerator_t
     };
     
     
    -inline void
    +inline bool
     glyf::_populate_subset_glyphs (const hb_subset_plan_t   *plan,
    -                               hb_vector_t *glyphs /* OUT */) const
    +                               hb_font_t *font,
    +                               hb_vector_t& glyphs /* OUT */) const
     {
       OT::glyf_accelerator_t glyf (plan->source);
    +  unsigned num_glyphs = plan->num_output_glyphs ();
    +  if (!glyphs.resize (num_glyphs)) return false;
     
    -  + hb_range (plan->num_output_glyphs ())
    -  | hb_map ([&] (hb_codepoint_t new_gid)
    -        {
    -          glyf_impl::SubsetGlyph subset_glyph = {0};
    -          subset_glyph.new_gid = new_gid;
    -
    -          /* should never fail: all old gids should be mapped */
    -          if (!plan->old_gid_for_new_gid (new_gid, &subset_glyph.old_gid))
    -            return subset_glyph;
    -
    -          if (new_gid == 0 &&
    -              !(plan->flags & HB_SUBSET_FLAGS_NOTDEF_OUTLINE))
    -            subset_glyph.source_glyph = glyf_impl::Glyph ();
    -          else
    -            subset_glyph.source_glyph = glyf.glyph_for_gid (subset_glyph.old_gid, true);
    -          if (plan->flags & HB_SUBSET_FLAGS_NO_HINTING)
    -            subset_glyph.drop_hints_bytes ();
    -          else
    -            subset_glyph.dest_start = subset_glyph.source_glyph.get_bytes ();
    -          return subset_glyph;
    -        })
    -  | hb_sink (glyphs)
    -  ;
    +  for (auto p : plan->glyph_map->iter ())
    +  {
    +    unsigned new_gid = p.second;
    +    glyf_impl::SubsetGlyph& subset_glyph = glyphs.arrayZ[new_gid];
    +    subset_glyph.old_gid = p.first;
    +
    +    if (unlikely (new_gid == 0 &&
    +                  !(plan->flags & HB_SUBSET_FLAGS_NOTDEF_OUTLINE)) &&
    +                  !plan->normalized_coords)
    +      subset_glyph.source_glyph = glyf_impl::Glyph ();
    +    else
    +    {
    +      /* If plan has an accelerator, the preprocessing step already trimmed glyphs.
    +       * Don't trim them again! */
    +      subset_glyph.source_glyph = glyf.glyph_for_gid (subset_glyph.old_gid, !plan->accelerator);
    +    }
    +
    +    if (plan->flags & HB_SUBSET_FLAGS_NO_HINTING)
    +      subset_glyph.drop_hints_bytes ();
    +    else
    +      subset_glyph.dest_start = subset_glyph.source_glyph.get_bytes ();
    +
    +    if (font)
    +    {
    +      if (unlikely (!subset_glyph.compile_bytes_with_deltas (plan, font, glyf)))
    +      {
    +        // when pinned at default, only bounds are updated, thus no need to free
    +        if (!plan->pinned_at_default)
    +          _free_compiled_subset_glyphs (glyphs);
    +        return false;
    +      }
    +    }
    +  }
    +  return true;
     }
     
    +inline hb_font_t *
    +glyf::_create_font_for_instancing (const hb_subset_plan_t *plan) const
    +{
    +  hb_font_t *font = hb_font_create (plan->source);
    +  if (unlikely (font == hb_font_get_empty ())) return nullptr;
    +
    +  hb_vector_t vars;
    +  if (unlikely (!vars.alloc (plan->user_axes_location.get_population (), true)))
    +  {
    +    hb_font_destroy (font);
    +    return nullptr;
    +  }
    +
    +  for (auto _ : plan->user_axes_location)
    +  {
    +    hb_variation_t var;
    +    var.tag = _.first;
    +    var.value = _.second;
    +    vars.push (var);
    +  }
    +
    +#ifndef HB_NO_VAR
    +  hb_font_set_variations (font, vars.arrayZ, plan->user_axes_location.get_population ());
    +#endif
    +  return font;
    +}
     
     
     } /* namespace OT */
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/glyf/path-builder.hh b/src/java.desktop/share/native/libharfbuzz/OT/glyf/path-builder.hh
    index 853c7ed1eaa1c..6a476204f1040 100644
    --- a/src/java.desktop/share/native/libharfbuzz/OT/glyf/path-builder.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/glyf/path-builder.hh
    @@ -26,22 +26,29 @@ struct path_builder_t
     
         optional_point_t lerp (optional_point_t p, float t)
         { return optional_point_t (x + t * (p.x - x), y + t * (p.y - y)); }
    -  } first_oncurve, first_offcurve, last_offcurve;
    +  } first_oncurve, first_offcurve, first_offcurve2, last_offcurve, last_offcurve2;
     
       path_builder_t (hb_font_t *font_, hb_draw_session_t &draw_session_)
       {
         font = font_;
         draw_session = &draw_session_;
    -    first_oncurve = first_offcurve = last_offcurve = optional_point_t ();
    +    first_oncurve = first_offcurve = first_offcurve2 = last_offcurve = last_offcurve2 = optional_point_t ();
       }
     
       /* based on https://github.com/RazrFalcon/ttf-parser/blob/4f32821/src/glyf.rs#L287
          See also:
          * https://developer.apple.com/fonts/TrueType-Reference-Manual/RM01/Chap1.html
    -     * https://stackoverflow.com/a/20772557 */
    +     * https://stackoverflow.com/a/20772557
    +     *
    +     * Cubic support added. */
       void consume_point (const contour_point_t &point)
       {
         bool is_on_curve = point.flag & glyf_impl::SimpleGlyph::FLAG_ON_CURVE;
    +#ifdef HB_NO_CUBIC_GLYF
    +    bool is_cubic = false;
    +#else
    +    bool is_cubic = !is_on_curve && (point.flag & glyf_impl::SimpleGlyph::FLAG_CUBIC);
    +#endif
         optional_point_t p (font->em_fscalef_x (point.x), font->em_fscalef_y (point.y));
         if (!first_oncurve)
         {
    @@ -52,7 +59,12 @@ struct path_builder_t
           }
           else
           {
    -        if (first_offcurve)
    +        if (is_cubic && !first_offcurve2)
    +        {
    +          first_offcurve2 = first_offcurve;
    +          first_offcurve = p;
    +        }
    +        else if (first_offcurve)
             {
               optional_point_t mid = first_offcurve.lerp (p, .5f);
               first_oncurve = mid;
    @@ -69,16 +81,41 @@ struct path_builder_t
           {
             if (is_on_curve)
             {
    -          draw_session->quadratic_to (last_offcurve.x, last_offcurve.y,
    -                                     p.x, p.y);
    +          if (last_offcurve2)
    +          {
    +            draw_session->cubic_to (last_offcurve2.x, last_offcurve2.y,
    +                                    last_offcurve.x, last_offcurve.y,
    +                                    p.x, p.y);
    +            last_offcurve2 = optional_point_t ();
    +          }
    +          else
    +            draw_session->quadratic_to (last_offcurve.x, last_offcurve.y,
    +                                       p.x, p.y);
               last_offcurve = optional_point_t ();
             }
             else
             {
    -          optional_point_t mid = last_offcurve.lerp (p, .5f);
    -          draw_session->quadratic_to (last_offcurve.x, last_offcurve.y,
    -                                     mid.x, mid.y);
    -          last_offcurve = p;
    +          if (is_cubic && !last_offcurve2)
    +          {
    +            last_offcurve2 = last_offcurve;
    +            last_offcurve = p;
    +          }
    +          else
    +          {
    +            optional_point_t mid = last_offcurve.lerp (p, .5f);
    +
    +            if (is_cubic)
    +            {
    +              draw_session->cubic_to (last_offcurve2.x, last_offcurve2.y,
    +                                      last_offcurve.x, last_offcurve.y,
    +                                      mid.x, mid.y);
    +              last_offcurve2 = optional_point_t ();
    +            }
    +            else
    +              draw_session->quadratic_to (last_offcurve.x, last_offcurve.y,
    +                                         mid.x, mid.y);
    +            last_offcurve = p;
    +          }
             }
           }
           else
    @@ -94,19 +131,40 @@ struct path_builder_t
         {
           if (first_offcurve && last_offcurve)
           {
    -        optional_point_t mid = last_offcurve.lerp (first_offcurve, .5f);
    -        draw_session->quadratic_to (last_offcurve.x, last_offcurve.y,
    -                                   mid.x, mid.y);
    +        optional_point_t mid = last_offcurve.lerp (first_offcurve2 ?
    +                                                   first_offcurve2 :
    +                                                   first_offcurve, .5f);
    +        if (last_offcurve2)
    +          draw_session->cubic_to (last_offcurve2.x, last_offcurve2.y,
    +                                  last_offcurve.x, last_offcurve.y,
    +                                  mid.x, mid.y);
    +        else
    +          draw_session->quadratic_to (last_offcurve.x, last_offcurve.y,
    +                                     mid.x, mid.y);
             last_offcurve = optional_point_t ();
    -        /* now check the rest */
           }
    +      /* now check the rest */
     
           if (first_offcurve && first_oncurve)
    -        draw_session->quadratic_to (first_offcurve.x, first_offcurve.y,
    -                                   first_oncurve.x, first_oncurve.y);
    +      {
    +        if (first_offcurve2)
    +          draw_session->cubic_to (first_offcurve2.x, first_offcurve2.y,
    +                                  first_offcurve.x, first_offcurve.y,
    +                                  first_oncurve.x, first_oncurve.y);
    +        else
    +          draw_session->quadratic_to (first_offcurve.x, first_offcurve.y,
    +                                     first_oncurve.x, first_oncurve.y);
    +      }
           else if (last_offcurve && first_oncurve)
    -        draw_session->quadratic_to (last_offcurve.x, last_offcurve.y,
    -                                   first_oncurve.x, first_oncurve.y);
    +      {
    +        if (last_offcurve2)
    +          draw_session->cubic_to (last_offcurve2.x, last_offcurve2.y,
    +                                  last_offcurve.x, last_offcurve.y,
    +                                  first_oncurve.x, first_oncurve.y);
    +        else
    +          draw_session->quadratic_to (last_offcurve.x, last_offcurve.y,
    +                                     first_oncurve.x, first_oncurve.y);
    +      }
           else if (first_oncurve)
             draw_session->line_to (first_oncurve.x, first_oncurve.y);
           else if (first_offcurve)
    @@ -117,7 +175,7 @@ struct path_builder_t
           }
     
           /* Getting ready for the next contour */
    -      first_oncurve = first_offcurve = last_offcurve = optional_point_t ();
    +      first_oncurve = first_offcurve = last_offcurve = last_offcurve2 = optional_point_t ();
           draw_session->close_path ();
         }
       }
    diff --git a/src/java.desktop/share/native/libharfbuzz/OT/name/name.hh b/src/java.desktop/share/native/libharfbuzz/OT/name/name.hh
    new file mode 100644
    index 0000000000000..15ff7a8bdb75c
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/OT/name/name.hh
    @@ -0,0 +1,589 @@
    +/*
    + * Copyright © 2011,2012  Google, Inc.
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + *
    + * Google Author(s): Behdad Esfahbod
    + */
    +
    +#ifndef OT_NAME_NAME_HH
    +#define OT_NAME_NAME_HH
    +
    +#include "../../hb-open-type.hh"
    +#include "../../hb-ot-name-language.hh"
    +#include "../../hb-aat-layout.hh"
    +#include "../../hb-utf.hh"
    +
    +
    +namespace OT {
    +
    +template 
    +inline unsigned int
    +hb_ot_name_convert_utf (hb_bytes_t                       bytes,
    +                        unsigned int                    *text_size /* IN/OUT */,
    +                        typename out_utf_t::codepoint_t *text /* OUT */)
    +{
    +  unsigned int src_len = bytes.length / sizeof (typename in_utf_t::codepoint_t);
    +  const typename in_utf_t::codepoint_t *src = (const typename in_utf_t::codepoint_t *) bytes.arrayZ;
    +  const typename in_utf_t::codepoint_t *src_end = src + src_len;
    +
    +  typename out_utf_t::codepoint_t *dst = text;
    +
    +  hb_codepoint_t unicode;
    +  const hb_codepoint_t replacement = HB_BUFFER_REPLACEMENT_CODEPOINT_DEFAULT;
    +
    +  if (text_size && *text_size)
    +  {
    +    (*text_size)--; /* Save room for NUL-termination. */
    +    const typename out_utf_t::codepoint_t *dst_end = text + *text_size;
    +
    +    while (src < src_end && dst < dst_end)
    +    {
    +      const typename in_utf_t::codepoint_t *src_next = in_utf_t::next (src, src_end, &unicode, replacement);
    +      typename out_utf_t::codepoint_t *dst_next = out_utf_t::encode (dst, dst_end, unicode);
    +      if (dst_next == dst)
    +        break; /* Out-of-room. */
    +
    +      dst = dst_next;
    +      src = src_next;
    +    }
    +
    +    *text_size = dst - text;
    +    *dst = 0; /* NUL-terminate. */
    +  }
    +
    +  /* Accumulate length of rest. */
    +  unsigned int dst_len = dst - text;
    +  while (src < src_end)
    +  {
    +    src = in_utf_t::next (src, src_end, &unicode, replacement);
    +    dst_len += out_utf_t::encode_len (unicode);
    +  }
    +  return dst_len;
    +}
    +
    +#define entry_score var.u16[0]
    +#define entry_index var.u16[1]
    +
    +
    +/*
    + * name -- Naming
    + * https://docs.microsoft.com/en-us/typography/opentype/spec/name
    + */
    +#define HB_OT_TAG_name HB_TAG('n','a','m','e')
    +
    +#define UNSUPPORTED     42
    +
    +struct NameRecord
    +{
    +  hb_language_t language (hb_face_t *face) const
    +  {
    +#ifndef HB_NO_OT_NAME_LANGUAGE
    +    unsigned int p = platformID;
    +    unsigned int l = languageID;
    +
    +    if (p == 3)
    +      return _hb_ot_name_language_for_ms_code (l);
    +
    +    if (p == 1)
    +      return _hb_ot_name_language_for_mac_code (l);
    +
    +#ifndef HB_NO_OT_NAME_LANGUAGE_AAT
    +    if (p == 0)
    +      return face->table.ltag->get_language (l);
    +#endif
    +
    +#endif
    +    return HB_LANGUAGE_INVALID;
    +  }
    +
    +  uint16_t score () const
    +  {
    +    /* Same order as in cmap::find_best_subtable(). */
    +    unsigned int p = platformID;
    +    unsigned int e = encodingID;
    +
    +    /* 32-bit. */
    +    if (p == 3 && e == 10) return 0;
    +    if (p == 0 && e ==  6) return 1;
    +    if (p == 0 && e ==  4) return 2;
    +
    +    /* 16-bit. */
    +    if (p == 3 && e ==  1) return 3;
    +    if (p == 0 && e ==  3) return 4;
    +    if (p == 0 && e ==  2) return 5;
    +    if (p == 0 && e ==  1) return 6;
    +    if (p == 0 && e ==  0) return 7;
    +
    +    /* Symbol. */
    +    if (p == 3 && e ==  0) return 8;
    +
    +    /* We treat all Mac Latin names as ASCII only. */
    +    if (p == 1 && e ==  0) return 10; /* 10 is magic number :| */
    +
    +    return UNSUPPORTED;
    +  }
    +
    +  NameRecord* copy (hb_serialize_context_t *c, const void *base
    +#ifdef HB_EXPERIMENTAL_API
    +                    , const hb_hashmap_t *name_table_overrides
    +#endif
    +                    ) const
    +  {
    +    TRACE_SERIALIZE (this);
    +    HB_UNUSED auto snap = c->snapshot ();
    +    auto *out = c->embed (this);
    +    if (unlikely (!out)) return_trace (nullptr);
    +#ifdef HB_EXPERIMENTAL_API
    +    hb_ot_name_record_ids_t record_ids (platformID, encodingID, languageID, nameID);
    +    hb_bytes_t* name_bytes;
    +
    +    if (name_table_overrides->has (record_ids, &name_bytes)) {
    +      hb_bytes_t encoded_bytes = *name_bytes;
    +      char *name_str_utf16_be = nullptr;
    +
    +      if (platformID != 1)
    +      {
    +        unsigned text_size = hb_ot_name_convert_utf (*name_bytes, nullptr, nullptr);
    +
    +        text_size++; // needs to consider NULL terminator for use in hb_ot_name_convert_utf()
    +        unsigned byte_len = text_size * hb_utf16_be_t::codepoint_t::static_size;
    +        name_str_utf16_be = (char *) hb_calloc (byte_len, 1);
    +        if (!name_str_utf16_be)
    +        {
    +          c->revert (snap);
    +          return_trace (nullptr);
    +        }
    +        hb_ot_name_convert_utf (*name_bytes, &text_size,
    +                                                          (hb_utf16_be_t::codepoint_t *) name_str_utf16_be);
    +
    +        unsigned encoded_byte_len = text_size * hb_utf16_be_t::codepoint_t::static_size;
    +        if (!encoded_byte_len || !c->check_assign (out->length, encoded_byte_len, HB_SERIALIZE_ERROR_INT_OVERFLOW)) {
    +          c->revert (snap);
    +          hb_free (name_str_utf16_be);
    +          return_trace (nullptr);
    +        }
    +
    +        encoded_bytes = hb_bytes_t (name_str_utf16_be, encoded_byte_len);
    +      }
    +      else
    +      {
    +        // mac platform, copy the UTF-8 string(all ascii characters) as is
    +        if (!c->check_assign (out->length, encoded_bytes.length, HB_SERIALIZE_ERROR_INT_OVERFLOW)) {
    +          c->revert (snap);
    +          return_trace (nullptr);
    +        }
    +      }
    +
    +      out->offset = 0;
    +      c->push ();
    +      encoded_bytes.copy (c);
    +      c->add_link (out->offset, c->pop_pack (), hb_serialize_context_t::Tail, 0);
    +      hb_free (name_str_utf16_be);
    +    }
    +    else
    +#endif
    +    {
    +      out->offset.serialize_copy (c, offset, base, 0, hb_serialize_context_t::Tail, length);
    +    }
    +    return_trace (out);
    +  }
    +
    +  bool isUnicode () const
    +  {
    +    unsigned int p = platformID;
    +    unsigned int e = encodingID;
    +
    +    return (p == 0 ||
    +            (p == 3 && (e == 0 || e == 1 || e == 10)));
    +  }
    +
    +  static int cmp (const void *pa, const void *pb)
    +  {
    +    const NameRecord *a = (const NameRecord *)pa;
    +    const NameRecord *b = (const NameRecord *)pb;
    +
    +    if (a->platformID != b->platformID)
    +      return a->platformID - b->platformID;
    +
    +    if (a->encodingID != b->encodingID)
    +      return a->encodingID - b->encodingID;
    +
    +    if (a->languageID != b->languageID)
    +      return a->languageID - b->languageID;
    +
    +    if (a->nameID != b->nameID)
    +      return a->nameID - b->nameID;
    +
    +    if (a->length != b->length)
    +      return a->length - b->length;
    +
    +    return 0;
    +  }
    +
    +  bool sanitize (hb_sanitize_context_t *c, const void *base) const
    +  {
    +    TRACE_SANITIZE (this);
    +    return_trace (c->check_struct (this) && offset.sanitize (c, base, length));
    +  }
    +
    +  HBUINT16      platformID;     /* Platform ID. */
    +  HBUINT16      encodingID;     /* Platform-specific encoding ID. */
    +  HBUINT16      languageID;     /* Language ID. */
    +  HBUINT16      nameID;         /* Name ID. */
    +  HBUINT16      length;         /* String length (in bytes). */
    +  NNOffset16To>
    +                offset;         /* String offset from start of storage area (in bytes). */
    +  public:
    +  DEFINE_SIZE_STATIC (12);
    +};
    +
    +static int
    +_hb_ot_name_entry_cmp_key (const void *pa, const void *pb, bool exact)
    +{
    +  const hb_ot_name_entry_t *a = (const hb_ot_name_entry_t *) pa;
    +  const hb_ot_name_entry_t *b = (const hb_ot_name_entry_t *) pb;
    +
    +  /* Compare by name_id, then language. */
    +
    +  if (a->name_id != b->name_id)
    +    return a->name_id - b->name_id;
    +
    +  if (a->language == b->language) return 0;
    +  if (!a->language) return -1;
    +  if (!b->language) return +1;
    +
    +  const char *astr = hb_language_to_string (a->language);
    +  const char *bstr = hb_language_to_string (b->language);
    +
    +  signed c = strcmp (astr, bstr);
    +
    +  // 'a' is the user request, and 'b' is string in the font.
    +  // If eg. user asks for "en-us" and font has "en", approve.
    +  if (!exact && c &&
    +      hb_language_matches (b->language, a->language))
    +    return 0;
    +
    +  return c;
    +}
    +
    +static int
    +_hb_ot_name_entry_cmp (const void *pa, const void *pb)
    +{
    +  /* Compare by name_id, then language, then score, then index. */
    +
    +  int v = _hb_ot_name_entry_cmp_key (pa, pb, true);
    +  if (v)
    +    return v;
    +
    +  const hb_ot_name_entry_t *a = (const hb_ot_name_entry_t *) pa;
    +  const hb_ot_name_entry_t *b = (const hb_ot_name_entry_t *) pb;
    +
    +  if (a->entry_score != b->entry_score)
    +    return a->entry_score - b->entry_score;
    +
    +  if (a->entry_index != b->entry_index)
    +    return a->entry_index - b->entry_index;
    +
    +  return 0;
    +}
    +
    +struct name
    +{
    +  static constexpr hb_tag_t tableTag = HB_OT_TAG_name;
    +
    +  unsigned int get_size () const
    +  { return min_size + count * nameRecordZ.item_size; }
    +
    +  template 
    +  bool serialize (hb_serialize_context_t *c,
    +                  Iterator it,
    +                  const void *src_string_pool
    +#ifdef HB_EXPERIMENTAL_API
    +                  , const hb_vector_t& insert_name_records
    +                  , const hb_hashmap_t *name_table_overrides
    +#endif
    +                  )
    +  {
    +    TRACE_SERIALIZE (this);
    +
    +    if (unlikely (!c->extend_min ((*this))))  return_trace (false);
    +
    +    unsigned total_count = it.len ()
    +#ifdef HB_EXPERIMENTAL_API
    +        + insert_name_records.length
    +#endif
    +        ;
    +    this->format = 0;
    +    if (!c->check_assign (this->count, total_count, HB_SERIALIZE_ERROR_INT_OVERFLOW))
    +      return false;
    +
    +    NameRecord *name_records = (NameRecord *) hb_calloc (total_count, NameRecord::static_size);
    +    if (unlikely (!name_records)) return_trace (false);
    +
    +    hb_array_t records (name_records, total_count);
    +
    +    for (const NameRecord& record : it)
    +    {
    +      hb_memcpy (name_records, &record, NameRecord::static_size);
    +      name_records++;
    +    }
    +
    +#ifdef HB_EXPERIMENTAL_API
    +    for (unsigned i = 0; i < insert_name_records.length; i++)
    +    {
    +      const hb_ot_name_record_ids_t& ids = insert_name_records[i];
    +      NameRecord record;
    +      record.platformID = ids.platform_id;
    +      record.encodingID = ids.encoding_id;
    +      record.languageID = ids.language_id;
    +      record.nameID = ids.name_id;
    +      record.length = 0; // handled in NameRecord copy()
    +      record.offset = 0;
    +      memcpy (name_records, &record, NameRecord::static_size);
    +      name_records++;
    +    }
    +#endif
    +
    +    records.qsort ();
    +
    +    c->copy_all (records,
    +                 src_string_pool
    +#ifdef HB_EXPERIMENTAL_API
    +                 , name_table_overrides
    +#endif
    +                 );
    +    hb_free (records.arrayZ);
    +
    +
    +    if (unlikely (c->ran_out_of_room ())) return_trace (false);
    +
    +    this->stringOffset = c->length ();
    +
    +    return_trace (true);
    +  }
    +
    +  bool subset (hb_subset_context_t *c) const
    +  {
    +    TRACE_SUBSET (this);
    +
    +    name *name_prime = c->serializer->start_embed ();
    +    if (unlikely (!name_prime)) return_trace (false);
    +
    +#ifdef HB_EXPERIMENTAL_API
    +    const hb_hashmap_t *name_table_overrides =
    +        &c->plan->name_table_overrides;
    +#endif
    +
    +    auto it =
    +    + nameRecordZ.as_array (count)
    +    | hb_filter (c->plan->name_ids, &NameRecord::nameID)
    +    | hb_filter (c->plan->name_languages, &NameRecord::languageID)
    +    | hb_filter ([&] (const NameRecord& namerecord) {
    +      return
    +          (c->plan->flags & HB_SUBSET_FLAGS_NAME_LEGACY)
    +          || namerecord.isUnicode ();
    +    })
    +#ifdef HB_EXPERIMENTAL_API
    +    | hb_filter ([&] (const NameRecord& namerecord) {
    +      if (name_table_overrides->is_empty ())
    +        return true;
    +      hb_ot_name_record_ids_t rec_ids (namerecord.platformID,
    +                                       namerecord.encodingID,
    +                                       namerecord.languageID,
    +                                       namerecord.nameID);
    +
    +      hb_bytes_t *p;
    +      if (name_table_overrides->has (rec_ids, &p) &&
    +          (*p).length == 0)
    +        return false;
    +      return true;
    +    })
    +#endif
    +    ;
    +
    +#ifdef HB_EXPERIMENTAL_API
    +    hb_hashmap_t retained_name_record_ids;
    +    for (const NameRecord& rec : it)
    +    {
    +      hb_ot_name_record_ids_t rec_ids (rec.platformID,
    +                                       rec.encodingID,
    +                                       rec.languageID,
    +                                       rec.nameID);
    +      retained_name_record_ids.set (rec_ids, 1);
    +    }
    +
    +    hb_vector_t insert_name_records;
    +    if (!name_table_overrides->is_empty ())
    +    {
    +      if (unlikely (!insert_name_records.alloc (name_table_overrides->get_population (), true)))
    +        return_trace (false);
    +      for (const auto& record_ids : name_table_overrides->keys ())
    +      {
    +        if (name_table_overrides->get (record_ids).length == 0)
    +          continue;
    +        if (retained_name_record_ids.has (record_ids))
    +          continue;
    +        insert_name_records.push (record_ids);
    +      }
    +    }
    +#endif
    +
    +    return (name_prime->serialize (c->serializer, it,
    +                                   std::addressof (this + stringOffset)
    +#ifdef HB_EXPERIMENTAL_API
    +                                   , insert_name_records
    +                                   , name_table_overrides
    +#endif
    +                                   ));
    +  }
    +
    +  bool sanitize_records (hb_sanitize_context_t *c) const
    +  {
    +    TRACE_SANITIZE (this);
    +    const void *string_pool = (this+stringOffset).arrayZ;
    +    return_trace (nameRecordZ.sanitize (c, count, string_pool));
    +  }
    +
    +  bool sanitize (hb_sanitize_context_t *c) const
    +  {
    +    TRACE_SANITIZE (this);
    +    return_trace (c->check_struct (this) &&
    +                  likely (format == 0 || format == 1) &&
    +                  c->check_array (nameRecordZ.arrayZ, count) &&
    +                  c->check_range (this, stringOffset) &&
    +                  sanitize_records (c));
    +  }
    +
    +  struct accelerator_t
    +  {
    +    accelerator_t (hb_face_t *face)
    +    {
    +      this->table = hb_sanitize_context_t ().reference_table (face);
    +      assert (this->table.get_length () >= this->table->stringOffset);
    +      this->pool = (const char *) (const void *) (this->table+this->table->stringOffset);
    +      this->pool_len = this->table.get_length () - this->table->stringOffset;
    +      const hb_array_t all_names (this->table->nameRecordZ.arrayZ,
    +                                                    this->table->count);
    +
    +      this->names.alloc (all_names.length, true);
    +
    +      for (unsigned int i = 0; i < all_names.length; i++)
    +      {
    +        hb_ot_name_entry_t *entry = this->names.push ();
    +
    +        entry->name_id = all_names[i].nameID;
    +        entry->language = all_names[i].language (face);
    +        entry->entry_score =  all_names[i].score ();
    +        entry->entry_index = i;
    +      }
    +
    +      this->names.qsort (_hb_ot_name_entry_cmp);
    +      /* Walk and pick best only for each name_id,language pair,
    +       * while dropping unsupported encodings. */
    +      unsigned int j = 0;
    +      for (unsigned int i = 0; i < this->names.length; i++)
    +      {
    +        if (this->names[i].entry_score == UNSUPPORTED ||
    +            this->names[i].language == HB_LANGUAGE_INVALID)
    +          continue;
    +        if (i &&
    +            this->names[i - 1].name_id  == this->names[i].name_id &&
    +            this->names[i - 1].language == this->names[i].language)
    +          continue;
    +        this->names[j++] = this->names[i];
    +      }
    +      this->names.resize (j);
    +    }
    +    ~accelerator_t ()
    +    {
    +      this->table.destroy ();
    +    }
    +
    +    int get_index (hb_ot_name_id_t  name_id,
    +                   hb_language_t    language,
    +                   unsigned int    *width=nullptr) const
    +    {
    +      const hb_ot_name_entry_t key = {name_id, {0}, language};
    +      const hb_ot_name_entry_t *entry = hb_bsearch (key, (const hb_ot_name_entry_t *) this->names,
    +                                                    this->names.length,
    +                                                    sizeof (hb_ot_name_entry_t),
    +                                                    _hb_ot_name_entry_cmp_key,
    +                                                    true);
    +
    +      if (!entry)
    +      {
    +        entry = hb_bsearch (key, (const hb_ot_name_entry_t *) this->names,
    +                            this->names.length,
    +                            sizeof (hb_ot_name_entry_t),
    +                            _hb_ot_name_entry_cmp_key,
    +                            false);
    +      }
    +
    +      if (!entry)
    +        return -1;
    +
    +      if (width)
    +        *width = entry->entry_score < 10 ? 2 : 1;
    +
    +      return entry->entry_index;
    +    }
    +
    +    hb_bytes_t get_name (unsigned int idx) const
    +    {
    +      const hb_array_t all_names (table->nameRecordZ.arrayZ, table->count);
    +      const NameRecord &record = all_names[idx];
    +      const hb_bytes_t string_pool (pool, pool_len);
    +      return string_pool.sub_array (record.offset, record.length);
    +    }
    +
    +    private:
    +    const char *pool;
    +    unsigned int pool_len;
    +    public:
    +    hb_blob_ptr_t table;
    +    hb_vector_t names;
    +  };
    +
    +  public:
    +  /* We only implement format 0 for now. */
    +  HBUINT16      format;         /* Format selector (=0/1). */
    +  HBUINT16      count;          /* Number of name records. */
    +  NNOffset16To>
    +                stringOffset;   /* Offset to start of string storage (from start of table). */
    +  UnsizedArrayOf
    +                nameRecordZ;    /* The name records where count is the number of records. */
    +  public:
    +  DEFINE_SIZE_ARRAY (6, nameRecordZ);
    +};
    +
    +#undef entry_index
    +#undef entry_score
    +
    +struct name_accelerator_t : name::accelerator_t {
    +  name_accelerator_t (hb_face_t *face) : name::accelerator_t (face) {}
    +};
    +
    +} /* namespace OT */
    +
    +
    +#endif /* OT_NAME_NAME_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/UPDATING.txt b/src/java.desktop/share/native/libharfbuzz/UPDATING.txt
    index 34434ddef04fc..6b4e7ccc4fef2 100644
    --- a/src/java.desktop/share/native/libharfbuzz/UPDATING.txt
    +++ b/src/java.desktop/share/native/libharfbuzz/UPDATING.txt
    @@ -1,53 +1,119 @@
     Tips and tasks when updating harfbuzz sources to a newer version.
     -----------------------------------------------------------------
     
    +STEP 1: UPDATING FILES
    +----------------------
    +Download and unzip the latest version from https://github.com/harfbuzz/harfbuzz/releases
     We only use files from the src directory and even then only the ones we need.
     So just C++ include and source files and only the ones needed for the library,
    -and even then just the ones we use. Do NOT just copy everything.
    +and even then just the ones we use.
    +IMPORTANT! DO NOT just copy everything.
     
    -So one way to update is to
    -
    -- copy over from the updated harfbuzz the exact same files we already have
    -- it isn't a flat directory so watch out for that
    -- any that are no longer available (copy fails) we remove but these may come
    -  back later if they were actually renamed
    -- look for files in the destination that were NOT updated - perhaps they
    -  are gone in the upstream - or renamed. Remove them if they are really
    -  obsolete, or add their replacements/renames.
    -- iterate over : build and see what new file is missing that causes a build failure
    -- when this is done we have something buildable
    -- make sure it builds on all supported platforms.
    -- Harfbuzz is not modular so it is not easy,
    +- Harfbuzz is not modular, so the update is not a straightforward process.
     - The main thing is we do NOT want any
        * "test" programs (test in the name, have a main are clues)
        * support for (eg) GLib, DirectWrite, Graphite, GDI, ICU, Uniscribe
        * aggregators like harfbuzz.cc - since it includes things from the above
          as well as hb-ft.cc which we specifically exclude in the Makefile
    -  * but we do use core text support on macOS.
    -  * I really wish that "src" were just library source but I expect the authors
    +   * but we do use core text support on macOS.
    +   * I really wish that "src" were just library source, but I expect the authors
         have their reasons.
     
    -- we do not apply any header file changes so this is not an issue
    +So one way to update is to
    +
    +- copy over from the updated harfbuzz the exact same files we already have
    +- it isn't a flat directory so watch out for that.
    +
    +- For files that are no longer available (for which copy fails), we remove such files,
    +  but these may come back later if they were actually renamed.
    +
    +- look for files in the destination that were NOT updated - perhaps they
    +  are removed or renamed in the upstream. Remove them if they are really
    +  obsolete, or add their replacements/renames.
    +  In IntelliJ IDE:
    +    Newly added files are shown in RED
    +    Modified in BLUE
    +    NOT Updated in WHITE
    +  This feature might be helpful to keep track of new, modified and unchanged files.
    +
    +
    +STEP 2: BUILD CHANGES INCREMENTALLY
    +-----------------------------------
    +- iterate over : build and see what new file is missing that causes a build failure.
    +  Sometimes just running a build does not show up any failures due to stale files.
    +  Clean followed by build would be helpful in this situation.
    +
    +- You might run into compiler warnings that are treated as errors or the requirement
    +  to set certain compiler flags if the build fails on a specific platform.
    +  Check "COMPILER WARNINGS AND SETTING FLAGS" section for more details.
    +
    +- when this is done we have something buildable, make sure it builds
    +  on all supported platforms.
    +
    +
    +STEP 3: COMPILER WARNINGS AND SETTING FLAGS
    +-------------------------------------------
    +- Update make parameters in Awt2DLibraries.gmk
    +  Since we don't use configure we need to manually specify the options
    +  we need in the Harfbuzz section of Awt2DLibraries.gmk.
    +  As well as adding new options, we may need to clean up obsolete options.
    +  Note there may be platform variations in the flags.
    +
    +- As with other 3rd party libs we do not fix the code to eliminate compiler
    +  warnings unless they are critical and clearly avoiding a bug. Even then
    +  we'd report it upstream and apply the patch once it is made available.
    +  The usual practice is do just disable the warnings.
    +
    +
    +STEP 4: UPDATING .md FILE
    +-------------------------
    +- we do not apply any header file changes so this is not an issue.
    +
     - verify the license text is unchanged (extra steps are needed if it is) and update
    -   src/java.desktop/share/legal/harfbuzz.md with the new version
    +  src/java.desktop/share/legal/harfbuzz.md with the new version.
    +
    +
    +STEP 5: REPLACE TABS & REMOVE TRAILING SPACES
    +---------------------------------------------
     - clean up trailing white space and tabs to follow jcheck rules.
       Use "expand" and "sed" to remove tabs and trailing white space from the
       imported sources.
    -- test using all the automated jtreg tests on all platforms
    -- do manual verification of Arabic, Hebrew, Thai, Indic against previous releases. 
    -  Look for manual related layout jtreg tests and run on Windows,Linux and Mac.
    +  To clean up the extra spaces and tabs run the following script at
    +  each folder level within libharfbuzz.
    +
    +  shopt -s nullglob
    +  for f in *.c *.h *.cc *.hh;
    +      do
    +          # replace tabs with spaces
    +          expand ${f} > ${f}.tmp;
    +          mv ${f}.tmp $f;
    +
    +          # fix line endings to LF
    +          sed -e 's/\r$//g' ${f} > ${f}.tmp;
    +          mv ${f}.tmp $f;
    +
    +          # remove trailing spaces
    +          sed -e 's/[ ]* $//g' ${f} > ${f}.tmp;
    +          mv ${f}.tmp $f;
    +      done
    +
    +
    +STEP 6: TESTING
    +---------------
    +- test using all the automated jtreg tests on all platforms.
    +
    +- do MANUAL verification of Arabic, Hebrew, Thai, Indic against previous releases.
    +  Look for manual related layout jtreg tests (test/jdk/java/awt/font/TextLayout)
    +  and run on Windows,Linux and Mac.
       Use Font2DTest set to TextLayout and check the above languages. Probably
       not going to see layout problems a code point at a time but it needs to
       be checked.
     
    -- Update make parameters as needed
    -  Since we don't use configure we need to manually specify the options
    -  we need in the harfbuzz section of Awt2DLibraries.gmk.
    -  As well as adding new options, we may need to clean up obsolete options.
    -  Note there may be platform variations in the flags.
    +  Different unicode combinations can be checked using Font2DTest.
    +  Run Font2DTest, select 'UserText' option for 'Text to use'.
    +  Paste unicodes of different languages (Arabic, Hebrew, Thai, Indic)
    +  and compare the glyphs with previous versions.
    +  It should look the same in both cases.
     
    -- As with other 3rd party libs we do not fix the code to eliminate compiler
    -  warnings unless they are critical and clearly avoiding a bug. Even then
    -  we'd report it upstream. The usual practice is do just disable the warnings
     
    -- Update THIS UPDATING.txt file too if it is outdated.
    +- FINALLY, Do update THIS UPDATING.txt file too if it is outdated.
    diff --git a/src/java.desktop/share/native/libharfbuzz/graph/classdef-graph.hh b/src/java.desktop/share/native/libharfbuzz/graph/classdef-graph.hh
    new file mode 100644
    index 0000000000000..c2e24a7067874
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/graph/classdef-graph.hh
    @@ -0,0 +1,216 @@
    +/*
    + * Copyright © 2022  Google, Inc.
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + *
    + * Google Author(s): Garret Rieger
    + */
    +
    +#include "graph.hh"
    +#include "../hb-ot-layout-common.hh"
    +
    +#ifndef GRAPH_CLASSDEF_GRAPH_HH
    +#define GRAPH_CLASSDEF_GRAPH_HH
    +
    +namespace graph {
    +
    +struct ClassDefFormat1 : public OT::ClassDefFormat1_3
    +{
    +  bool sanitize (graph_t::vertex_t& vertex) const
    +  {
    +    int64_t vertex_len = vertex.obj.tail - vertex.obj.head;
    +    constexpr unsigned min_size = OT::ClassDefFormat1_3::min_size;
    +    if (vertex_len < min_size) return false;
    +    return vertex_len >= min_size + classValue.get_size () - classValue.len.get_size ();
    +  }
    +};
    +
    +struct ClassDefFormat2 : public OT::ClassDefFormat2_4
    +{
    +  bool sanitize (graph_t::vertex_t& vertex) const
    +  {
    +    int64_t vertex_len = vertex.obj.tail - vertex.obj.head;
    +    constexpr unsigned min_size = OT::ClassDefFormat2_4::min_size;
    +    if (vertex_len < min_size) return false;
    +    return vertex_len >= min_size + rangeRecord.get_size () - rangeRecord.len.get_size ();
    +  }
    +};
    +
    +struct ClassDef : public OT::ClassDef
    +{
    +  template
    +  static bool add_class_def (gsubgpos_graph_context_t& c,
    +                             unsigned parent_id,
    +                             unsigned link_position,
    +                             It glyph_and_class,
    +                             unsigned max_size)
    +  {
    +    unsigned class_def_prime_id = c.graph.new_node (nullptr, nullptr);
    +    auto& class_def_prime_vertex = c.graph.vertices_[class_def_prime_id];
    +    if (!make_class_def (c, glyph_and_class, class_def_prime_id, max_size))
    +      return false;
    +
    +    auto* class_def_link = c.graph.vertices_[parent_id].obj.real_links.push ();
    +    class_def_link->width = SmallTypes::size;
    +    class_def_link->objidx = class_def_prime_id;
    +    class_def_link->position = link_position;
    +    class_def_prime_vertex.parents.push (parent_id);
    +
    +    return true;
    +  }
    +
    +  template
    +  static bool make_class_def (gsubgpos_graph_context_t& c,
    +                              It glyph_and_class,
    +                              unsigned dest_obj,
    +                              unsigned max_size)
    +  {
    +    char* buffer = (char*) hb_calloc (1, max_size);
    +    hb_serialize_context_t serializer (buffer, max_size);
    +    OT::ClassDef_serialize (&serializer, glyph_and_class);
    +    serializer.end_serialize ();
    +    if (serializer.in_error ())
    +    {
    +      hb_free (buffer);
    +      return false;
    +    }
    +
    +    hb_bytes_t class_def_copy = serializer.copy_bytes ();
    +    c.add_buffer ((char *) class_def_copy.arrayZ); // Give ownership to the context, it will cleanup the buffer.
    +
    +    auto& obj = c.graph.vertices_[dest_obj].obj;
    +    obj.head = (char *) class_def_copy.arrayZ;
    +    obj.tail = obj.head + class_def_copy.length;
    +
    +    hb_free (buffer);
    +    return true;
    +  }
    +
    +  bool sanitize (graph_t::vertex_t& vertex) const
    +  {
    +    int64_t vertex_len = vertex.obj.tail - vertex.obj.head;
    +    if (vertex_len < OT::ClassDef::min_size) return false;
    +    switch (u.format)
    +    {
    +    case 1: return ((ClassDefFormat1*)this)->sanitize (vertex);
    +    case 2: return ((ClassDefFormat2*)this)->sanitize (vertex);
    +#ifndef HB_NO_BEYOND_64K
    +    // Not currently supported
    +    case 3:
    +    case 4:
    +#endif
    +    default: return false;
    +    }
    +  }
    +};
    +
    +
    +struct class_def_size_estimator_t
    +{
    +  template
    +  class_def_size_estimator_t (It glyph_and_class)
    +      : gids_consecutive (true), num_ranges_per_class (), glyphs_per_class ()
    +  {
    +    unsigned last_gid = (unsigned) -1;
    +    for (auto p : + glyph_and_class)
    +    {
    +      unsigned gid = p.first;
    +      unsigned klass = p.second;
    +
    +      if (last_gid != (unsigned) -1 && gid != last_gid + 1)
    +        gids_consecutive = false;
    +      last_gid = gid;
    +
    +      hb_set_t* glyphs;
    +      if (glyphs_per_class.has (klass, &glyphs) && glyphs) {
    +        glyphs->add (gid);
    +        continue;
    +      }
    +
    +      hb_set_t new_glyphs;
    +      new_glyphs.add (gid);
    +      glyphs_per_class.set (klass, std::move (new_glyphs));
    +    }
    +
    +    if (in_error ()) return;
    +
    +    for (unsigned klass : glyphs_per_class.keys ())
    +    {
    +      if (!klass) continue; // class 0 doesn't get encoded.
    +
    +      const hb_set_t& glyphs = glyphs_per_class.get (klass);
    +      hb_codepoint_t start = HB_SET_VALUE_INVALID;
    +      hb_codepoint_t end = HB_SET_VALUE_INVALID;
    +
    +      unsigned count = 0;
    +      while (glyphs.next_range (&start, &end))
    +        count++;
    +
    +      num_ranges_per_class.set (klass, count);
    +    }
    +  }
    +
    +  // Incremental increase in the Coverage and ClassDef table size
    +  // (worst case) if all glyphs associated with 'klass' were added.
    +  unsigned incremental_coverage_size (unsigned klass) const
    +  {
    +    // Coverage takes 2 bytes per glyph worst case,
    +    return 2 * glyphs_per_class.get (klass).get_population ();
    +  }
    +
    +  // Incremental increase in the Coverage and ClassDef table size
    +  // (worst case) if all glyphs associated with 'klass' were added.
    +  unsigned incremental_class_def_size (unsigned klass) const
    +  {
    +    // ClassDef takes 6 bytes per range
    +    unsigned class_def_2_size = 6 * num_ranges_per_class.get (klass);
    +    if (gids_consecutive)
    +    {
    +      // ClassDef1 takes 2 bytes per glyph, but only can be used
    +      // when gids are consecutive.
    +      return hb_min (2 * glyphs_per_class.get (klass).get_population (), class_def_2_size);
    +    }
    +
    +    return class_def_2_size;
    +  }
    +
    +  bool in_error ()
    +  {
    +    if (num_ranges_per_class.in_error ()) return true;
    +    if (glyphs_per_class.in_error ()) return true;
    +
    +    for (const hb_set_t& s : glyphs_per_class.values ())
    +    {
    +      if (s.in_error ()) return true;
    +    }
    +    return false;
    +  }
    +
    + private:
    +  bool gids_consecutive;
    +  hb_hashmap_t num_ranges_per_class;
    +  hb_hashmap_t glyphs_per_class;
    +};
    +
    +
    +}
    +
    +#endif  // GRAPH_CLASSDEF_GRAPH_HH
    diff --git a/src/java.desktop/share/native/libharfbuzz/graph/coverage-graph.hh b/src/java.desktop/share/native/libharfbuzz/graph/coverage-graph.hh
    new file mode 100644
    index 0000000000000..49d09363156a9
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/graph/coverage-graph.hh
    @@ -0,0 +1,152 @@
    +/*
    + * Copyright © 2022  Google, Inc.
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + *
    + * Google Author(s): Garret Rieger
    + */
    +
    +#include "graph.hh"
    +#include "../OT/Layout/Common/Coverage.hh"
    +
    +#ifndef GRAPH_COVERAGE_GRAPH_HH
    +#define GRAPH_COVERAGE_GRAPH_HH
    +
    +namespace graph {
    +
    +struct CoverageFormat1 : public OT::Layout::Common::CoverageFormat1_3
    +{
    +  bool sanitize (graph_t::vertex_t& vertex) const
    +  {
    +    int64_t vertex_len = vertex.obj.tail - vertex.obj.head;
    +    constexpr unsigned min_size = OT::Layout::Common::CoverageFormat1_3::min_size;
    +    if (vertex_len < min_size) return false;
    +    return vertex_len >= min_size + glyphArray.get_size () - glyphArray.len.get_size ();
    +  }
    +};
    +
    +struct CoverageFormat2 : public OT::Layout::Common::CoverageFormat2_4
    +{
    +  bool sanitize (graph_t::vertex_t& vertex) const
    +  {
    +    int64_t vertex_len = vertex.obj.tail - vertex.obj.head;
    +    constexpr unsigned min_size = OT::Layout::Common::CoverageFormat2_4::min_size;
    +    if (vertex_len < min_size) return false;
    +    return vertex_len >= min_size + rangeRecord.get_size () - rangeRecord.len.get_size ();
    +  }
    +};
    +
    +struct Coverage : public OT::Layout::Common::Coverage
    +{
    +  static Coverage* clone_coverage (gsubgpos_graph_context_t& c,
    +                                   unsigned coverage_id,
    +                                   unsigned new_parent_id,
    +                                   unsigned link_position,
    +                                   unsigned start, unsigned end)
    +
    +  {
    +    unsigned coverage_size = c.graph.vertices_[coverage_id].table_size ();
    +    auto& coverage_v = c.graph.vertices_[coverage_id];
    +    Coverage* coverage_table = (Coverage*) coverage_v.obj.head;
    +    if (!coverage_table || !coverage_table->sanitize (coverage_v))
    +      return nullptr;
    +
    +    auto new_coverage =
    +        + hb_zip (coverage_table->iter (), hb_range ())
    +        | hb_filter ([&] (hb_pair_t p) {
    +          return p.second >= start && p.second < end;
    +        })
    +        | hb_map_retains_sorting (hb_first)
    +        ;
    +
    +    return add_coverage (c, new_parent_id, link_position, new_coverage, coverage_size);
    +  }
    +
    +  template
    +  static Coverage* add_coverage (gsubgpos_graph_context_t& c,
    +                                 unsigned parent_id,
    +                                 unsigned link_position,
    +                                 It glyphs,
    +                                 unsigned max_size)
    +  {
    +    unsigned coverage_prime_id = c.graph.new_node (nullptr, nullptr);
    +    auto& coverage_prime_vertex = c.graph.vertices_[coverage_prime_id];
    +    if (!make_coverage (c, glyphs, coverage_prime_id, max_size))
    +      return nullptr;
    +
    +    auto* coverage_link = c.graph.vertices_[parent_id].obj.real_links.push ();
    +    coverage_link->width = SmallTypes::size;
    +    coverage_link->objidx = coverage_prime_id;
    +    coverage_link->position = link_position;
    +    coverage_prime_vertex.parents.push (parent_id);
    +
    +    return (Coverage*) coverage_prime_vertex.obj.head;
    +  }
    +
    +  template
    +  static bool make_coverage (gsubgpos_graph_context_t& c,
    +                             It glyphs,
    +                             unsigned dest_obj,
    +                             unsigned max_size)
    +  {
    +    char* buffer = (char*) hb_calloc (1, max_size);
    +    hb_serialize_context_t serializer (buffer, max_size);
    +    OT::Layout::Common::Coverage_serialize (&serializer, glyphs);
    +    serializer.end_serialize ();
    +    if (serializer.in_error ())
    +    {
    +      hb_free (buffer);
    +      return false;
    +    }
    +
    +    hb_bytes_t coverage_copy = serializer.copy_bytes ();
    +    c.add_buffer ((char *) coverage_copy.arrayZ); // Give ownership to the context, it will cleanup the buffer.
    +
    +    auto& obj = c.graph.vertices_[dest_obj].obj;
    +    obj.head = (char *) coverage_copy.arrayZ;
    +    obj.tail = obj.head + coverage_copy.length;
    +
    +    hb_free (buffer);
    +    return true;
    +  }
    +
    +  bool sanitize (graph_t::vertex_t& vertex) const
    +  {
    +    int64_t vertex_len = vertex.obj.tail - vertex.obj.head;
    +    if (vertex_len < OT::Layout::Common::Coverage::min_size) return false;
    +    switch (u.format)
    +    {
    +    case 1: return ((CoverageFormat1*)this)->sanitize (vertex);
    +    case 2: return ((CoverageFormat2*)this)->sanitize (vertex);
    +#ifndef HB_NO_BEYOND_64K
    +    // Not currently supported
    +    case 3:
    +    case 4:
    +#endif
    +    default: return false;
    +    }
    +  }
    +};
    +
    +
    +}
    +
    +#endif  // GRAPH_COVERAGE_GRAPH_HH
    diff --git a/src/java.desktop/share/native/libharfbuzz/graph/graph.hh b/src/java.desktop/share/native/libharfbuzz/graph/graph.hh
    index 52ca9dd142e23..38ca5db096186 100644
    --- a/src/java.desktop/share/native/libharfbuzz/graph/graph.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/graph/graph.hh
    @@ -24,6 +24,10 @@
      * Google Author(s): Garret Rieger
      */
     
    +#include "../hb-set.hh"
    +#include "../hb-priority-queue.hh"
    +#include "../hb-serialize.hh"
    +
     #ifndef GRAPH_GRAPH_HH
     #define GRAPH_GRAPH_HH
     
    @@ -45,6 +49,95 @@ struct graph_t
         unsigned end = 0;
         unsigned priority = 0;
     
    +
    +    bool link_positions_valid (unsigned num_objects, bool removed_nil)
    +    {
    +      hb_set_t assigned_bytes;
    +      for (const auto& l : obj.real_links)
    +      {
    +        if (l.objidx >= num_objects
    +            || (removed_nil && !l.objidx))
    +        {
    +          DEBUG_MSG (SUBSET_REPACK, nullptr,
    +                     "Invalid graph. Invalid object index.");
    +          return false;
    +        }
    +
    +        unsigned start = l.position;
    +        unsigned end = start + l.width - 1;
    +
    +        if (unlikely (l.width < 2 || l.width > 4))
    +        {
    +          DEBUG_MSG (SUBSET_REPACK, nullptr,
    +                     "Invalid graph. Invalid link width.");
    +          return false;
    +        }
    +
    +        if (unlikely (end >= table_size ()))
    +        {
    +          DEBUG_MSG (SUBSET_REPACK, nullptr,
    +                     "Invalid graph. Link position is out of bounds.");
    +          return false;
    +        }
    +
    +        if (unlikely (assigned_bytes.intersects (start, end)))
    +        {
    +          DEBUG_MSG (SUBSET_REPACK, nullptr,
    +                     "Invalid graph. Found offsets whose positions overlap.");
    +          return false;
    +        }
    +
    +        assigned_bytes.add_range (start, end);
    +      }
    +
    +      return !assigned_bytes.in_error ();
    +    }
    +
    +    void normalize ()
    +    {
    +      obj.real_links.qsort ();
    +      for (auto& l : obj.real_links)
    +      {
    +        for (unsigned i = 0; i < l.width; i++)
    +        {
    +          obj.head[l.position + i] = 0;
    +        }
    +      }
    +    }
    +
    +    bool equals (const vertex_t& other,
    +                 const graph_t& graph,
    +                 const graph_t& other_graph,
    +                 unsigned depth) const
    +    {
    +      if (!(as_bytes () == other.as_bytes ()))
    +      {
    +        DEBUG_MSG (SUBSET_REPACK, nullptr,
    +                   "vertex [%lu] bytes != [%lu] bytes, depth = %u",
    +                   (unsigned long) table_size (),
    +                   (unsigned long) other.table_size (),
    +                   depth);
    +
    +        auto a = as_bytes ();
    +        auto b = other.as_bytes ();
    +        while (a || b)
    +        {
    +          DEBUG_MSG (SUBSET_REPACK, nullptr,
    +                     "  0x%x %s 0x%x", (unsigned) *a, (*a == *b) ? "==" : "!=", (unsigned) *b);
    +          a++;
    +          b++;
    +        }
    +        return false;
    +      }
    +
    +      return links_equal (obj.real_links, other.obj.real_links, graph, other_graph, depth);
    +    }
    +
    +    hb_bytes_t as_bytes () const
    +    {
    +      return hb_bytes_t (obj.head, table_size ());
    +    }
    +
         friend void swap (vertex_t& a, vertex_t& b)
         {
           hb_swap (a.obj, b.obj);
    @@ -56,6 +149,18 @@ struct graph_t
           hb_swap (a.priority, b.priority);
         }
     
    +    hb_hashmap_t
    +    position_to_index_map () const
    +    {
    +      hb_hashmap_t result;
    +
    +      for (const auto& l : obj.real_links) {
    +        result.set (l.position, l.objidx);
    +      }
    +
    +      return result;
    +    }
    +
         bool is_shared () const
         {
           return parents.length > 1;
    @@ -71,11 +176,27 @@ struct graph_t
           for (unsigned i = 0; i < parents.length; i++)
           {
             if (parents[i] != parent_index) continue;
    -        parents.remove (i);
    +        parents.remove_unordered (i);
             break;
           }
         }
     
    +    void remove_real_link (unsigned child_index, const void* offset)
    +    {
    +      for (unsigned i = 0; i < obj.real_links.length; i++)
    +      {
    +        auto& link = obj.real_links.arrayZ[i];
    +        if (link.objidx != child_index)
    +          continue;
    +
    +        if ((obj.head + link.position) != offset)
    +          continue;
    +
    +        obj.real_links.remove_unordered (i);
    +        return;
    +      }
    +    }
    +
         void remap_parents (const hb_vector_t& id_map)
         {
           for (unsigned i = 0; i < parents.length; i++)
    @@ -107,6 +228,10 @@ struct graph_t
           return priority >= 3;
         }
     
    +    size_t table_size () const {
    +      return obj.tail - obj.head;
    +    }
    +
         int64_t modified_distance (unsigned order) const
         {
           // TODO(garretrieger): once priority is high enough, should try
    @@ -131,6 +256,57 @@ struct graph_t
     
           return -table_size;
         }
    +
    +   private:
    +    bool links_equal (const hb_vector_t& this_links,
    +                      const hb_vector_t& other_links,
    +                      const graph_t& graph,
    +                      const graph_t& other_graph,
    +                      unsigned depth) const
    +    {
    +      auto a = this_links.iter ();
    +      auto b = other_links.iter ();
    +
    +      while (a && b)
    +      {
    +        const auto& link_a = *a;
    +        const auto& link_b = *b;
    +
    +        if (link_a.width != link_b.width ||
    +            link_a.is_signed != link_b.is_signed ||
    +            link_a.whence != link_b.whence ||
    +            link_a.position != link_b.position ||
    +            link_a.bias != link_b.bias)
    +          return false;
    +
    +        if (!graph.vertices_[link_a.objidx].equals (
    +                other_graph.vertices_[link_b.objidx], graph, other_graph, depth + 1))
    +          return false;
    +
    +        a++;
    +        b++;
    +      }
    +
    +      if (bool (a) != bool (b))
    +        return false;
    +
    +      return true;
    +    }
    +  };
    +
    +  template 
    +  struct vertex_and_table_t
    +  {
    +    vertex_and_table_t () : index (0), vertex (nullptr), table (nullptr)
    +    {}
    +
    +    unsigned index;
    +    vertex_t* vertex;
    +    T* table;
    +
    +    operator bool () {
    +       return table && vertex;
    +    }
       };
     
       /*
    @@ -145,7 +321,8 @@ struct graph_t
           : parents_invalid (true),
             distance_invalid (true),
             positions_invalid (true),
    -        successful (true)
    +        successful (true),
    +        buffers ()
       {
         num_roots_for_space_.push (1);
         bool removed_nil = false;
    @@ -153,8 +330,6 @@ struct graph_t
         vertices_scratch_.alloc (objects.length);
         for (unsigned i = 0; i < objects.length; i++)
         {
    -      // TODO(grieger): check all links point to valid objects.
    -
           // If this graph came from a serialization buffer object 0 is the
           // nil object. We don't need it for our purposes here so drop it.
           if (i == 0 && !objects[i])
    @@ -166,6 +341,9 @@ struct graph_t
           vertex_t* v = vertices_.push ();
           if (check_success (!vertices_.in_error ()))
             v->obj = *objects[i];
    +
    +      check_success (v->link_positions_valid (objects.length, removed_nil));
    +
           if (!removed_nil) continue;
           // Fix indices to account for removed nil object.
           for (auto& l : v->obj.all_links_writer ()) {
    @@ -177,6 +355,20 @@ struct graph_t
       ~graph_t ()
       {
         vertices_.fini ();
    +    for (char* b : buffers)
    +      hb_free (b);
    +  }
    +
    +  bool operator== (const graph_t& other) const
    +  {
    +    return root ().equals (other.root (), *this, other, 0);
    +  }
    +
    +  // Sorts links of all objects in a consistent manner and zeroes all offsets.
    +  void normalize ()
    +  {
    +    for (auto& v : vertices_.writer ())
    +      v.normalize ();
       }
     
       bool in_error () const
    @@ -199,11 +391,43 @@ struct graph_t
         return vertices_.length - 1;
       }
     
    -  const hb_serialize_context_t::object_t& object(unsigned i) const
    +  const hb_serialize_context_t::object_t& object (unsigned i) const
       {
         return vertices_[i].obj;
       }
     
    +  void add_buffer (char* buffer)
    +  {
    +    buffers.push (buffer);
    +  }
    +
    +  /*
    +   * Adds a 16 bit link from parent_id to child_id
    +   */
    +  template
    +  void add_link (T* offset,
    +                 unsigned parent_id,
    +                 unsigned child_id)
    +  {
    +    auto& v = vertices_[parent_id];
    +    auto* link = v.obj.real_links.push ();
    +    link->width = 2;
    +    link->objidx = child_id;
    +    link->position = (char*) offset - (char*) v.obj.head;
    +    vertices_[child_id].parents.push (parent_id);
    +  }
    +
    +  /*
    +   * Generates a new topological sorting of graph ordered by the shortest
    +   * distance to each node if positions are marked as invalid.
    +   */
    +  void sort_shortest_distance_if_needed ()
    +  {
    +    if (!positions_invalid) return;
    +    sort_shortest_distance ();
    +  }
    +
    +
       /*
        * Generates a new topological sorting of graph ordered by the shortest
        * distance to each node.
    @@ -239,6 +463,13 @@ struct graph_t
           hb_swap (sorted_graph[new_id], vertices_[next_id]);
           const vertex_t& next = sorted_graph[new_id];
     
    +      if (unlikely (!check_success(new_id >= 0))) {
    +        // We are out of ids. Which means we've visited a node more than once.
    +        // This graph contains a cycle which is not allowed.
    +        DEBUG_MSG (SUBSET_REPACK, nullptr, "Invalid graph. Contains cycle.");
    +        return;
    +      }
    +
           id_map[next_id] = new_id--;
     
           for (const auto& link : next.obj.all_links ()) {
    @@ -256,44 +487,152 @@ struct graph_t
     
         check_success (!queue.in_error ());
         check_success (!sorted_graph.in_error ());
    -    if (!check_success (new_id == -1))
    -      print_orphaned_nodes ();
     
         remap_all_obj_indices (id_map, &sorted_graph);
    -
         hb_swap (vertices_, sorted_graph);
    +
    +    if (!check_success (new_id == -1))
    +      print_orphaned_nodes ();
       }
     
       /*
    -   * Assign unique space numbers to each connected subgraph of 32 bit offset(s).
    +   * Finds the set of nodes (placed into roots) that should be assigned unique spaces.
    +   * More specifically this looks for the top most 24 bit or 32 bit links in the graph.
    +   * Some special casing is done that is specific to the layout of GSUB/GPOS tables.
        */
    -  bool assign_32bit_spaces ()
    +  void find_space_roots (hb_set_t& visited, hb_set_t& roots)
       {
    -    unsigned root_index = root_idx ();
    -    hb_set_t visited;
    -    hb_set_t roots;
    -    for (unsigned i = 0; i <= root_index; i++)
    +    int root_index = (int) root_idx ();
    +    for (int i = root_index; i >= 0; i--)
         {
    +      if (visited.has (i)) continue;
    +
           // Only real links can form 32 bit spaces
           for (auto& l : vertices_[i].obj.real_links)
           {
    -        if (l.width == 4 && !l.is_signed)
    +        if (l.is_signed || l.width < 3)
    +          continue;
    +
    +        if (i == root_index && l.width == 3)
    +          // Ignore 24bit links from the root node, this skips past the single 24bit
    +          // pointer to the lookup list.
    +          continue;
    +
    +        if (l.width == 3)
             {
    -          roots.add (l.objidx);
    -          find_subgraph (l.objidx, visited);
    +          // A 24bit offset forms a root, unless there is 32bit offsets somewhere
    +          // in it's subgraph, then those become the roots instead. This is to make sure
    +          // that extension subtables beneath a 24bit lookup become the spaces instead
    +          // of the offset to the lookup.
    +          hb_set_t sub_roots;
    +          find_32bit_roots (l.objidx, sub_roots);
    +          if (sub_roots) {
    +            for (unsigned sub_root_idx : sub_roots) {
    +              roots.add (sub_root_idx);
    +              find_subgraph (sub_root_idx, visited);
    +            }
    +            continue;
    +          }
             }
    +
    +        roots.add (l.objidx);
    +        find_subgraph (l.objidx, visited);
           }
         }
    +  }
     
    -    // Mark everything not in the subgraphs of 32 bit roots as visited.
    -    // This prevents 32 bit subgraphs from being connected via nodes not in the 32 bit subgraphs.
    +  template 
    +  vertex_and_table_t as_table (unsigned parent, const void* offset, Ts... ds)
    +  {
    +    return as_table_from_index (index_for_offset (parent, offset), std::forward(ds)...);
    +  }
    +
    +  template 
    +  vertex_and_table_t as_mutable_table (unsigned parent, const void* offset, Ts... ds)
    +  {
    +    return as_table_from_index (mutable_index_for_offset (parent, offset), std::forward(ds)...);
    +  }
    +
    +  template 
    +  vertex_and_table_t as_table_from_index (unsigned index, Ts... ds)
    +  {
    +    if (index >= vertices_.length)
    +      return vertex_and_table_t ();
    +
    +    vertex_and_table_t r;
    +    r.vertex = &vertices_[index];
    +    r.table = (T*) r.vertex->obj.head;
    +    r.index = index;
    +    if (!r.table)
    +      return vertex_and_table_t ();
    +
    +    if (!r.table->sanitize (*(r.vertex), std::forward(ds)...))
    +      return vertex_and_table_t ();
    +
    +    return r;
    +  }
    +
    +  // Finds the object id of the object pointed to by the offset at 'offset'
    +  // within object[node_idx].
    +  unsigned index_for_offset (unsigned node_idx, const void* offset) const
    +  {
    +    const auto& node = object (node_idx);
    +    if (offset < node.head || offset >= node.tail) return -1;
    +
    +    unsigned length = node.real_links.length;
    +    for (unsigned i = 0; i < length; i++)
    +    {
    +      // Use direct access for increased performance, this is a hot method.
    +      const auto& link = node.real_links.arrayZ[i];
    +      if (offset != node.head + link.position)
    +        continue;
    +      return link.objidx;
    +    }
    +
    +    return -1;
    +  }
    +
    +  // Finds the object id of the object pointed to by the offset at 'offset'
    +  // within object[node_idx]. Ensures that the returned object is safe to mutate.
    +  // That is, if the original child object is shared by parents other than node_idx
    +  // it will be duplicated and the duplicate will be returned instead.
    +  unsigned mutable_index_for_offset (unsigned node_idx, const void* offset)
    +  {
    +    unsigned child_idx = index_for_offset (node_idx, offset);
    +    auto& child = vertices_[child_idx];
    +    for (unsigned p : child.parents)
    +    {
    +      if (p != node_idx) {
    +        return duplicate (node_idx, child_idx);
    +      }
    +    }
    +
    +    return child_idx;
    +  }
    +
    +
    +  /*
    +   * Assign unique space numbers to each connected subgraph of 24 bit and/or 32 bit offset(s).
    +   * Currently, this is implemented specifically tailored to the structure of a GPOS/GSUB
    +   * (including with 24bit offsets) table.
    +   */
    +  bool assign_spaces ()
    +  {
    +    update_parents ();
    +
    +    hb_set_t visited;
    +    hb_set_t roots;
    +    find_space_roots (visited, roots);
    +
    +    // Mark everything not in the subgraphs of the roots as visited. This prevents
    +    // subgraphs from being connected via nodes not in those subgraphs.
         visited.invert ();
     
         if (!roots) return false;
     
         while (roots)
         {
    -      unsigned next = HB_SET_VALUE_INVALID;
    +      uint32_t next = HB_SET_VALUE_INVALID;
           if (unlikely (!check_success (!roots.in_error ()))) break;
           if (!roots.next (&next)) break;
     
    @@ -361,6 +700,9 @@ struct graph_t
           }
         }
     
    +    if (in_error ())
    +      return false;
    +
         if (!made_changes)
           return false;
     
    @@ -374,8 +716,8 @@ struct graph_t
     
         auto new_subgraph =
             + subgraph.keys ()
    -        | hb_map([&] (unsigned node_idx) {
    -          const unsigned *v;
    +        | hb_map([&] (uint32_t node_idx) {
    +          const uint32_t *v;
               if (index_map.has (node_idx, &v)) return *v;
               return node_idx;
             })
    @@ -385,10 +727,10 @@ struct graph_t
         remap_obj_indices (index_map, parents.iter (), true);
     
         // Update roots set with new indices as needed.
    -    unsigned next = HB_SET_VALUE_INVALID;
    +    uint32_t next = HB_SET_VALUE_INVALID;
         while (roots.next (&next))
         {
    -      const unsigned *v;
    +      const uint32_t *v;
           if (index_map.has (next, &v))
           {
             roots.del (next);
    @@ -403,7 +745,7 @@ struct graph_t
       {
         for (const auto& link : vertices_[node_idx].obj.all_links ())
         {
    -      const unsigned *v;
    +      const uint32_t *v;
           if (subgraph.has (link.objidx, &v))
           {
             subgraph.set (link.objidx, *v + 1);
    @@ -422,6 +764,68 @@ struct graph_t
           find_subgraph (link.objidx, subgraph);
       }
     
    +  size_t find_subgraph_size (unsigned node_idx, hb_set_t& subgraph, unsigned max_depth = -1)
    +  {
    +    if (subgraph.has (node_idx)) return 0;
    +    subgraph.add (node_idx);
    +
    +    const auto& o = vertices_[node_idx].obj;
    +    size_t size = o.tail - o.head;
    +    if (max_depth == 0)
    +      return size;
    +
    +    for (const auto& link : o.all_links ())
    +      size += find_subgraph_size (link.objidx, subgraph, max_depth - 1);
    +    return size;
    +  }
    +
    +  /*
    +   * Finds the topmost children of 32bit offsets in the subgraph starting
    +   * at node_idx. Found indices are placed into 'found'.
    +   */
    +  void find_32bit_roots (unsigned node_idx, hb_set_t& found)
    +  {
    +    for (const auto& link : vertices_[node_idx].obj.all_links ())
    +    {
    +      if (!link.is_signed && link.width == 4) {
    +        found.add (link.objidx);
    +        continue;
    +      }
    +      find_32bit_roots (link.objidx, found);
    +    }
    +  }
    +
    +  /*
    +   * Moves the child of old_parent_idx pointed to by old_offset to a new
    +   * vertex at the new_offset.
    +   */
    +  template
    +  void move_child (unsigned old_parent_idx,
    +                   const O* old_offset,
    +                   unsigned new_parent_idx,
    +                   const O* new_offset)
    +  {
    +    distance_invalid = true;
    +    positions_invalid = true;
    +
    +    auto& old_v = vertices_[old_parent_idx];
    +    auto& new_v = vertices_[new_parent_idx];
    +
    +    unsigned child_id = index_for_offset (old_parent_idx,
    +                                          old_offset);
    +
    +    auto* new_link = new_v.obj.real_links.push ();
    +    new_link->width = O::static_size;
    +    new_link->objidx = child_id;
    +    new_link->position = (const char*) new_offset - (const char*) new_v.obj.head;
    +
    +    auto& child = vertices_[child_id];
    +    child.parents.push (new_parent_idx);
    +
    +    old_v.remove_real_link (child_id, old_offset);
    +    child.remove_parent (old_parent_idx);
    +  }
    +
       /*
        * duplicates all nodes in the subgraph reachable from node_idx. Does not re-assign
        * links. index_map is updated with mappings from old id to new id. If a duplication has already
    @@ -432,7 +836,11 @@ struct graph_t
         if (index_map.has (node_idx))
           return;
     
    -    index_map.set (node_idx, duplicate (node_idx));
    +    unsigned clone_idx = duplicate (node_idx);
    +    if (!check_success (clone_idx != (unsigned) -1))
    +      return;
    +
    +    index_map.set (node_idx, clone_idx);
         for (const auto& l : object (node_idx).all_links ()) {
           duplicate_subgraph (l.objidx, index_map);
         }
    @@ -490,7 +898,20 @@ struct graph_t
        * parent to the clone. The copy is a shallow copy, objects
        * linked from child are not duplicated.
        */
    -  bool duplicate (unsigned parent_idx, unsigned child_idx)
    +  unsigned duplicate_if_shared (unsigned parent_idx, unsigned child_idx)
    +  {
    +    unsigned new_idx = duplicate (parent_idx, child_idx);
    +    if (new_idx == (unsigned) -1) return child_idx;
    +    return new_idx;
    +  }
    +
    +
    +  /*
    +   * Creates a copy of child and re-assigns the link from
    +   * parent to the clone. The copy is a shallow copy, objects
    +   * linked from child are not duplicated.
    +   */
    +  unsigned duplicate (unsigned parent_idx, unsigned child_idx)
       {
         update_parents ();
     
    @@ -504,12 +925,12 @@ struct graph_t
         {
           // Can't duplicate this node, doing so would orphan the original one as all remaining links
           // to child are from parent.
    -      DEBUG_MSG (SUBSET_REPACK, nullptr, "  Not duplicating %d => %d",
    +      DEBUG_MSG (SUBSET_REPACK, nullptr, "  Not duplicating %u => %u",
                      parent_idx, child_idx);
    -      return false;
    +      return -1;
         }
     
    -    DEBUG_MSG (SUBSET_REPACK, nullptr, "  Duplicating %d => %d",
    +    DEBUG_MSG (SUBSET_REPACK, nullptr, "  Duplicating %u => %u",
                    parent_idx, child_idx);
     
         unsigned clone_idx = duplicate (child_idx);
    @@ -526,7 +947,40 @@ struct graph_t
           reassign_link (l, parent_idx, clone_idx);
         }
     
    -    return true;
    +    return clone_idx;
    +  }
    +
    +
    +  /*
    +   * Adds a new node to the graph, not connected to anything.
    +   */
    +  unsigned new_node (char* head, char* tail)
    +  {
    +    positions_invalid = true;
    +    distance_invalid = true;
    +
    +    auto* clone = vertices_.push ();
    +    if (vertices_.in_error ()) {
    +      return -1;
    +    }
    +
    +    clone->obj.head = head;
    +    clone->obj.tail = tail;
    +    clone->distance = 0;
    +    clone->space = 0;
    +
    +    unsigned clone_idx = vertices_.length - 2;
    +
    +    // The last object is the root of the graph, so swap back the root to the end.
    +    // The root's obj idx does change, however since it's root nothing else refers to it.
    +    // all other obj idx's will be unaffected.
    +    hb_swap (vertices_[vertices_.length - 2], *clone);
    +
    +    // Since the root moved, update the parents arrays of all children on the root.
    +    for (const auto& l : root ().obj.all_links ())
    +      vertices_[l.objidx].remap_parent (root_idx () - 1, root_idx ());
    +
    +    return clone_idx;
       }
     
       /*
    @@ -534,7 +988,7 @@ struct graph_t
        */
       bool raise_childrens_priority (unsigned parent_idx)
       {
    -    DEBUG_MSG (SUBSET_REPACK, nullptr, "  Raising priority of all children of %d",
    +    DEBUG_MSG (SUBSET_REPACK, nullptr, "  Raising priority of all children of %u",
                    parent_idx);
         // This operation doesn't change ordering until a sort is run, so no need
         // to invalidate positions. It does not change graph structure so no need
    @@ -546,6 +1000,72 @@ struct graph_t
         return made_change;
       }
     
    +  bool is_fully_connected ()
    +  {
    +    update_parents();
    +
    +    if (root().parents)
    +      // Root cannot have parents.
    +      return false;
    +
    +    for (unsigned i = 0; i < root_idx (); i++)
    +    {
    +      if (!vertices_[i].parents)
    +        return false;
    +    }
    +    return true;
    +  }
    +
    +#if 0
    +  /*
    +   * Saves the current graph to a packed binary format which the repacker fuzzer takes
    +   * as a seed.
    +   */
    +  void save_fuzzer_seed (hb_tag_t tag) const
    +  {
    +    FILE* f = fopen ("./repacker_fuzzer_seed", "w");
    +    fwrite ((void*) &tag, sizeof (tag), 1, f);
    +
    +    uint16_t num_objects = vertices_.length;
    +    fwrite ((void*) &num_objects, sizeof (num_objects), 1, f);
    +
    +    for (const auto& v : vertices_)
    +    {
    +      uint16_t blob_size = v.table_size ();
    +      fwrite ((void*) &blob_size, sizeof (blob_size), 1, f);
    +      fwrite ((const void*) v.obj.head, blob_size, 1, f);
    +    }
    +
    +    uint16_t link_count = 0;
    +    for (const auto& v : vertices_)
    +      link_count += v.obj.real_links.length;
    +
    +    fwrite ((void*) &link_count, sizeof (link_count), 1, f);
    +
    +    typedef struct
    +    {
    +      uint16_t parent;
    +      uint16_t child;
    +      uint16_t position;
    +      uint8_t width;
    +    } link_t;
    +
    +    for (unsigned i = 0; i < vertices_.length; i++)
    +    {
    +      for (const auto& l : vertices_[i].obj.real_links)
    +      {
    +        link_t link {
    +          (uint16_t) i, (uint16_t) l.objidx,
    +          (uint16_t) l.position, (uint8_t) l.width
    +        };
    +        fwrite ((void*) &link, sizeof (link), 1, f);
    +      }
    +    }
    +
    +    fclose (f);
    +  }
    +#endif
    +
       void print_orphaned_nodes ()
       {
         if (!DEBUG_ENABLED(SUBSET_REPACK)) return;
    @@ -554,6 +1074,10 @@ struct graph_t
         parents_invalid = true;
         update_parents();
     
    +    if (root().parents) {
    +      DEBUG_MSG (SUBSET_REPACK, nullptr, "Root node has incoming edges.");
    +    }
    +
         for (unsigned i = 0; i < root_idx (); i++)
         {
           const auto& v = vertices_[i];
    @@ -622,7 +1146,7 @@ struct graph_t
      private:
     
       /*
    -   * Returns the numbers of incoming edges that are 32bits wide.
    +   * Returns the numbers of incoming edges that are 24 or 32 bits wide.
        */
       unsigned wide_parents (unsigned node_idx, hb_set_t& parents) const
       {
    @@ -636,7 +1160,9 @@ struct graph_t
           // Only real links can be wide
           for (const auto& l : vertices_[p].obj.real_links)
           {
    -        if (l.objidx == node_idx && l.width == 4 && !l.is_signed)
    +        if (l.objidx == node_idx
    +            && (l.width == 3 || l.width == 4)
    +            && !l.is_signed)
             {
               count++;
               parents.add (p);
    @@ -668,6 +1194,11 @@ struct graph_t
           }
         }
     
    +    for (unsigned i = 0; i < vertices_.length; i++)
    +      // parents arrays must be accurate or downstream operations like cycle detection
    +      // and sorting won't work correctly.
    +      check_success (!vertices_[i].parents.in_error ());
    +
         parents_invalid = false;
       }
     
    @@ -786,7 +1317,7 @@ struct graph_t
         {
           for (auto& link : vertices_[i].obj.all_links_writer ())
           {
    -        const unsigned *v;
    +        const uint32_t *v;
             if (!id_map.has (link.objidx, &v)) continue;
             if (only_wide && !(link.width == 4 && !link.is_signed)) continue;
     
    @@ -853,6 +1384,7 @@ struct graph_t
       bool positions_invalid;
       bool successful;
       hb_vector_t num_roots_for_space_;
    +  hb_vector_t buffers;
     };
     
     }
    diff --git a/src/java.desktop/share/native/libharfbuzz/graph/gsubgpos-context.cc b/src/java.desktop/share/native/libharfbuzz/graph/gsubgpos-context.cc
    new file mode 100644
    index 0000000000000..b2044426d46d2
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/graph/gsubgpos-context.cc
    @@ -0,0 +1,70 @@
    +/*
    + * Copyright © 2022  Google, Inc.
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + *
    + * Google Author(s): Garret Rieger
    + */
    +
    +#include "gsubgpos-graph.hh"
    +
    +namespace graph {
    +
    +gsubgpos_graph_context_t::gsubgpos_graph_context_t (hb_tag_t table_tag_,
    +                                                    graph_t& graph_)
    +    : table_tag (table_tag_),
    +      graph (graph_),
    +      lookup_list_index (0),
    +      lookups ()
    +{
    +  if (table_tag_ != HB_OT_TAG_GPOS
    +      &&  table_tag_ != HB_OT_TAG_GSUB)
    +    return;
    +
    +  GSTAR* gstar = graph::GSTAR::graph_to_gstar (graph_);
    +  if (gstar) {
    +    gstar->find_lookups (graph, lookups);
    +    lookup_list_index = gstar->get_lookup_list_index (graph_);
    +  }
    +}
    +
    +unsigned gsubgpos_graph_context_t::create_node (unsigned size)
    +{
    +  char* buffer = (char*) hb_calloc (1, size);
    +  if (!buffer)
    +    return -1;
    +
    +  add_buffer (buffer);
    +
    +  return graph.new_node (buffer, buffer + size);
    +}
    +
    +unsigned gsubgpos_graph_context_t::num_non_ext_subtables ()  {
    +  unsigned count = 0;
    +  for (auto l : lookups.values ())
    +  {
    +    if (l->is_extension (table_tag)) continue;
    +    count += l->number_of_subtables ();
    +  }
    +  return count;
    +}
    +
    +}
    diff --git a/src/java.desktop/share/native/libharfbuzz/graph/gsubgpos-context.hh b/src/java.desktop/share/native/libharfbuzz/graph/gsubgpos-context.hh
    new file mode 100644
    index 0000000000000..9fe9662e64508
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/graph/gsubgpos-context.hh
    @@ -0,0 +1,61 @@
    +/*
    + * Copyright © 2022  Google, Inc.
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + *
    + * Google Author(s): Garret Rieger
    + */
    +
    +#include "graph.hh"
    +#include "../hb-ot-layout-gsubgpos.hh"
    +
    +#ifndef GRAPH_GSUBGPOS_CONTEXT_HH
    +#define GRAPH_GSUBGPOS_CONTEXT_HH
    +
    +namespace graph {
    +
    +struct Lookup;
    +
    +struct gsubgpos_graph_context_t
    +{
    +  hb_tag_t table_tag;
    +  graph_t& graph;
    +  unsigned lookup_list_index;
    +  hb_hashmap_t lookups;
    +
    +
    +  HB_INTERNAL gsubgpos_graph_context_t (hb_tag_t table_tag_,
    +                                        graph_t& graph_);
    +
    +  HB_INTERNAL unsigned create_node (unsigned size);
    +
    +  void add_buffer (char* buffer)
    +  {
    +    graph.add_buffer (buffer);
    +  }
    +
    + private:
    +  HB_INTERNAL unsigned num_non_ext_subtables ();
    +};
    +
    +}
    +
    +#endif  // GRAPH_GSUBGPOS_CONTEXT
    diff --git a/src/java.desktop/share/native/libharfbuzz/graph/gsubgpos-graph.hh b/src/java.desktop/share/native/libharfbuzz/graph/gsubgpos-graph.hh
    new file mode 100644
    index 0000000000000..c170638409f82
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/graph/gsubgpos-graph.hh
    @@ -0,0 +1,414 @@
    +/*
    + * Copyright © 2022  Google, Inc.
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + *
    + * Google Author(s): Garret Rieger
    + */
    +
    +#include "graph.hh"
    +#include "../hb-ot-layout-gsubgpos.hh"
    +#include "../OT/Layout/GSUB/ExtensionSubst.hh"
    +#include "gsubgpos-context.hh"
    +#include "pairpos-graph.hh"
    +#include "markbasepos-graph.hh"
    +
    +#ifndef GRAPH_GSUBGPOS_GRAPH_HH
    +#define GRAPH_GSUBGPOS_GRAPH_HH
    +
    +namespace graph {
    +
    +struct Lookup;
    +
    +template
    +struct ExtensionFormat1 : public OT::ExtensionFormat1
    +{
    +  void reset(unsigned type)
    +  {
    +    this->format = 1;
    +    this->extensionLookupType = type;
    +    this->extensionOffset = 0;
    +  }
    +
    +  bool sanitize (graph_t::vertex_t& vertex) const
    +  {
    +    int64_t vertex_len = vertex.obj.tail - vertex.obj.head;
    +    return vertex_len >= OT::ExtensionFormat1::static_size;
    +  }
    +
    +  unsigned get_lookup_type () const
    +  {
    +    return this->extensionLookupType;
    +  }
    +
    +  unsigned get_subtable_index (graph_t& graph, unsigned this_index) const
    +  {
    +    return graph.index_for_offset (this_index, &this->extensionOffset);
    +  }
    +};
    +
    +struct Lookup : public OT::Lookup
    +{
    +  unsigned number_of_subtables () const
    +  {
    +    return subTable.len;
    +  }
    +
    +  bool sanitize (graph_t::vertex_t& vertex) const
    +  {
    +    int64_t vertex_len = vertex.obj.tail - vertex.obj.head;
    +    if (vertex_len < OT::Lookup::min_size) return false;
    +    return vertex_len >= this->get_size ();
    +  }
    +
    +  bool is_extension (hb_tag_t table_tag) const
    +  {
    +    return lookupType == extension_type (table_tag);
    +  }
    +
    +  bool make_extension (gsubgpos_graph_context_t& c,
    +                       unsigned this_index)
    +  {
    +    unsigned type = lookupType;
    +    unsigned ext_type = extension_type (c.table_tag);
    +    if (!ext_type || is_extension (c.table_tag))
    +    {
    +      // NOOP
    +      return true;
    +    }
    +
    +    DEBUG_MSG (SUBSET_REPACK, nullptr,
    +               "Promoting lookup type %u (obj %u) to extension.",
    +               type,
    +               this_index);
    +
    +    for (unsigned i = 0; i < subTable.len; i++)
    +    {
    +      unsigned subtable_index = c.graph.index_for_offset (this_index, &subTable[i]);
    +      if (!make_subtable_extension (c,
    +                                    this_index,
    +                                    subtable_index))
    +        return false;
    +    }
    +
    +    lookupType = ext_type;
    +    return true;
    +  }
    +
    +  bool split_subtables_if_needed (gsubgpos_graph_context_t& c,
    +                                  unsigned this_index)
    +  {
    +    unsigned type = lookupType;
    +    bool is_ext = is_extension (c.table_tag);
    +
    +    if (c.table_tag != HB_OT_TAG_GPOS)
    +      return true;
    +
    +    if (!is_ext &&
    +        type != OT::Layout::GPOS_impl::PosLookupSubTable::Type::Pair &&
    +        type != OT::Layout::GPOS_impl::PosLookupSubTable::Type::MarkBase)
    +      return true;
    +
    +    hb_vector_t>> all_new_subtables;
    +    for (unsigned i = 0; i < subTable.len; i++)
    +    {
    +      unsigned subtable_index = c.graph.index_for_offset (this_index, &subTable[i]);
    +      unsigned parent_index = this_index;
    +      if (is_ext) {
    +        unsigned ext_subtable_index = subtable_index;
    +        parent_index = ext_subtable_index;
    +        ExtensionFormat1* extension =
    +            (ExtensionFormat1*)
    +            c.graph.object (ext_subtable_index).head;
    +        if (!extension || !extension->sanitize (c.graph.vertices_[ext_subtable_index]))
    +          continue;
    +
    +        subtable_index = extension->get_subtable_index (c.graph, ext_subtable_index);
    +        type = extension->get_lookup_type ();
    +        if (type != OT::Layout::GPOS_impl::PosLookupSubTable::Type::Pair
    +            && type != OT::Layout::GPOS_impl::PosLookupSubTable::Type::MarkBase)
    +          continue;
    +      }
    +
    +      hb_vector_t new_sub_tables;
    +      switch (type)
    +      {
    +      case 2:
    +        new_sub_tables = split_subtable (c, parent_index, subtable_index); break;
    +      case 4:
    +        new_sub_tables = split_subtable (c, parent_index, subtable_index); break;
    +      default:
    +        break;
    +      }
    +      if (new_sub_tables.in_error ()) return false;
    +      if (!new_sub_tables) continue;
    +      hb_pair_t>* entry = all_new_subtables.push ();
    +      entry->first = i;
    +      entry->second = std::move (new_sub_tables);
    +    }
    +
    +    if (all_new_subtables) {
    +      add_sub_tables (c, this_index, type, all_new_subtables);
    +    }
    +
    +    return true;
    +  }
    +
    +  template
    +  hb_vector_t split_subtable (gsubgpos_graph_context_t& c,
    +                                        unsigned parent_idx,
    +                                        unsigned objidx)
    +  {
    +    T* sub_table = (T*) c.graph.object (objidx).head;
    +    if (!sub_table || !sub_table->sanitize (c.graph.vertices_[objidx]))
    +      return hb_vector_t ();
    +
    +    return sub_table->split_subtables (c, parent_idx, objidx);
    +  }
    +
    +  void add_sub_tables (gsubgpos_graph_context_t& c,
    +                       unsigned this_index,
    +                       unsigned type,
    +                       hb_vector_t>>& subtable_ids)
    +  {
    +    bool is_ext = is_extension (c.table_tag);
    +    auto& v = c.graph.vertices_[this_index];
    +    fix_existing_subtable_links (c, this_index, subtable_ids);
    +
    +    unsigned new_subtable_count = 0;
    +    for (const auto& p : subtable_ids)
    +      new_subtable_count += p.second.length;
    +
    +    size_t new_size = v.table_size ()
    +                      + new_subtable_count * OT::Offset16::static_size;
    +    char* buffer = (char*) hb_calloc (1, new_size);
    +    c.add_buffer (buffer);
    +    hb_memcpy (buffer, v.obj.head, v.table_size());
    +
    +    v.obj.head = buffer;
    +    v.obj.tail = buffer + new_size;
    +
    +    Lookup* new_lookup = (Lookup*) buffer;
    +
    +    unsigned shift = 0;
    +    new_lookup->subTable.len = subTable.len + new_subtable_count;
    +    for (const auto& p : subtable_ids)
    +    {
    +      unsigned offset_index = p.first + shift + 1;
    +      shift += p.second.length;
    +
    +      for (unsigned subtable_id : p.second)
    +      {
    +        if (is_ext)
    +        {
    +          unsigned ext_id = create_extension_subtable (c, subtable_id, type);
    +          c.graph.vertices_[subtable_id].parents.push (ext_id);
    +          subtable_id = ext_id;
    +        }
    +
    +        auto* link = v.obj.real_links.push ();
    +        link->width = 2;
    +        link->objidx = subtable_id;
    +        link->position = (char*) &new_lookup->subTable[offset_index++] -
    +                         (char*) new_lookup;
    +        c.graph.vertices_[subtable_id].parents.push (this_index);
    +      }
    +    }
    +
    +    // Repacker sort order depends on link order, which we've messed up so resort it.
    +    v.obj.real_links.qsort ();
    +
    +    // The head location of the lookup has changed, invalidating the lookups map entry
    +    // in the context. Update the map.
    +    c.lookups.set (this_index, new_lookup);
    +  }
    +
    +  void fix_existing_subtable_links (gsubgpos_graph_context_t& c,
    +                                    unsigned this_index,
    +                                    hb_vector_t>>& subtable_ids)
    +  {
    +    auto& v = c.graph.vertices_[this_index];
    +    Lookup* lookup = (Lookup*) v.obj.head;
    +
    +    unsigned shift = 0;
    +    for (const auto& p : subtable_ids)
    +    {
    +      unsigned insert_index = p.first + shift;
    +      unsigned pos_offset = p.second.length * OT::Offset16::static_size;
    +      unsigned insert_offset = (char*) &lookup->subTable[insert_index] - (char*) lookup;
    +      shift += p.second.length;
    +
    +      for (auto& l : v.obj.all_links_writer ())
    +      {
    +        if (l.position > insert_offset) l.position += pos_offset;
    +      }
    +    }
    +  }
    +
    +  unsigned create_extension_subtable (gsubgpos_graph_context_t& c,
    +                                      unsigned subtable_index,
    +                                      unsigned type)
    +  {
    +    unsigned extension_size = OT::ExtensionFormat1::static_size;
    +
    +    unsigned ext_index = c.create_node (extension_size);
    +    if (ext_index == (unsigned) -1)
    +      return -1;
    +
    +    auto& ext_vertex = c.graph.vertices_[ext_index];
    +    ExtensionFormat1* extension =
    +        (ExtensionFormat1*) ext_vertex.obj.head;
    +    extension->reset (type);
    +
    +    // Make extension point at the subtable.
    +    auto* l = ext_vertex.obj.real_links.push ();
    +
    +    l->width = 4;
    +    l->objidx = subtable_index;
    +    l->position = 4;
    +
    +    return ext_index;
    +  }
    +
    +  bool make_subtable_extension (gsubgpos_graph_context_t& c,
    +                                unsigned lookup_index,
    +                                unsigned subtable_index)
    +  {
    +    unsigned type = lookupType;
    +
    +    unsigned ext_index = create_extension_subtable(c, subtable_index, type);
    +    if (ext_index == (unsigned) -1)
    +      return false;
    +
    +    auto& lookup_vertex = c.graph.vertices_[lookup_index];
    +    for (auto& l : lookup_vertex.obj.real_links.writer ())
    +    {
    +      if (l.objidx == subtable_index)
    +        // Change lookup to point at the extension.
    +        l.objidx = ext_index;
    +    }
    +
    +    // Make extension point at the subtable.
    +    auto& ext_vertex = c.graph.vertices_[ext_index];
    +    auto& subtable_vertex = c.graph.vertices_[subtable_index];
    +    ext_vertex.parents.push (lookup_index);
    +    subtable_vertex.remap_parent (lookup_index, ext_index);
    +
    +    return true;
    +  }
    +
    + private:
    +  unsigned extension_type (hb_tag_t table_tag) const
    +  {
    +    switch (table_tag)
    +    {
    +    case HB_OT_TAG_GPOS: return 9;
    +    case HB_OT_TAG_GSUB: return 7;
    +    default: return 0;
    +    }
    +  }
    +};
    +
    +template 
    +struct LookupList : public OT::LookupList
    +{
    +  bool sanitize (const graph_t::vertex_t& vertex) const
    +  {
    +    int64_t vertex_len = vertex.obj.tail - vertex.obj.head;
    +    if (vertex_len < OT::LookupList::min_size) return false;
    +    return vertex_len >= OT::LookupList::item_size * this->len;
    +  }
    +};
    +
    +struct GSTAR : public OT::GSUBGPOS
    +{
    +  static GSTAR* graph_to_gstar (graph_t& graph)
    +  {
    +    const auto& r = graph.root ();
    +
    +    GSTAR* gstar = (GSTAR*) r.obj.head;
    +    if (!gstar || !gstar->sanitize (r))
    +      return nullptr;
    +
    +    return gstar;
    +  }
    +
    +  const void* get_lookup_list_field_offset () const
    +  {
    +    switch (u.version.major) {
    +    case 1: return u.version1.get_lookup_list_offset ();
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: return u.version2.get_lookup_list_offset ();
    +#endif
    +    default: return 0;
    +    }
    +  }
    +
    +  bool sanitize (const graph_t::vertex_t& vertex)
    +  {
    +    int64_t len = vertex.obj.tail - vertex.obj.head;
    +    if (len < OT::GSUBGPOS::min_size) return false;
    +    return len >= get_size ();
    +  }
    +
    +  void find_lookups (graph_t& graph,
    +                     hb_hashmap_t& lookups /* OUT */)
    +  {
    +    switch (u.version.major) {
    +      case 1: find_lookups (graph, lookups); break;
    +#ifndef HB_NO_BEYOND_64K
    +      case 2: find_lookups (graph, lookups); break;
    +#endif
    +    }
    +  }
    +
    +  unsigned get_lookup_list_index (graph_t& graph)
    +  {
    +    return graph.index_for_offset (graph.root_idx (),
    +                                   get_lookup_list_field_offset());
    +  }
    +
    +  template
    +  void find_lookups (graph_t& graph,
    +                     hb_hashmap_t& lookups /* OUT */)
    +  {
    +    unsigned lookup_list_idx = get_lookup_list_index (graph);
    +    const LookupList* lookupList =
    +        (const LookupList*) graph.object (lookup_list_idx).head;
    +    if (!lookupList || !lookupList->sanitize (graph.vertices_[lookup_list_idx]))
    +      return;
    +
    +    for (unsigned i = 0; i < lookupList->len; i++)
    +    {
    +      unsigned lookup_idx = graph.index_for_offset (lookup_list_idx, &(lookupList->arrayZ[i]));
    +      Lookup* lookup = (Lookup*) graph.object (lookup_idx).head;
    +      if (!lookup || !lookup->sanitize (graph.vertices_[lookup_idx])) continue;
    +      lookups.set (lookup_idx, lookup);
    +    }
    +  }
    +};
    +
    +
    +
    +
    +}
    +
    +#endif  /* GRAPH_GSUBGPOS_GRAPH_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/graph/markbasepos-graph.hh b/src/java.desktop/share/native/libharfbuzz/graph/markbasepos-graph.hh
    new file mode 100644
    index 0000000000000..84ef5f71b93cf
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/graph/markbasepos-graph.hh
    @@ -0,0 +1,510 @@
    +/*
    + * Copyright © 2022  Google, Inc.
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + *
    + * Google Author(s): Garret Rieger
    + */
    +
    +#ifndef GRAPH_MARKBASEPOS_GRAPH_HH
    +#define GRAPH_MARKBASEPOS_GRAPH_HH
    +
    +#include "split-helpers.hh"
    +#include "coverage-graph.hh"
    +#include "../OT/Layout/GPOS/MarkBasePos.hh"
    +#include "../OT/Layout/GPOS/PosLookupSubTable.hh"
    +
    +namespace graph {
    +
    +struct AnchorMatrix : public OT::Layout::GPOS_impl::AnchorMatrix
    +{
    +  bool sanitize (graph_t::vertex_t& vertex, unsigned class_count) const
    +  {
    +    int64_t vertex_len = vertex.obj.tail - vertex.obj.head;
    +    if (vertex_len < AnchorMatrix::min_size) return false;
    +
    +    return vertex_len >= AnchorMatrix::min_size +
    +        OT::Offset16::static_size * class_count * this->rows;
    +  }
    +
    +  bool shrink (gsubgpos_graph_context_t& c,
    +               unsigned this_index,
    +               unsigned old_class_count,
    +               unsigned new_class_count)
    +  {
    +    if (new_class_count >= old_class_count) return false;
    +    auto& o = c.graph.vertices_[this_index].obj;
    +    unsigned base_count = rows;
    +    o.tail = o.head +
    +             AnchorMatrix::min_size +
    +             OT::Offset16::static_size * base_count * new_class_count;
    +
    +    // Reposition links into the new indexing scheme.
    +    for (auto& link : o.real_links.writer ())
    +    {
    +      unsigned index = (link.position - 2) / 2;
    +      unsigned base = index / old_class_count;
    +      unsigned klass = index % old_class_count;
    +      if (klass >= new_class_count)
    +        // should have already been removed
    +        return false;
    +
    +      unsigned new_index = base * new_class_count + klass;
    +
    +      link.position = (char*) &(this->matrixZ[new_index]) - (char*) this;
    +    }
    +
    +    return true;
    +  }
    +
    +  unsigned clone (gsubgpos_graph_context_t& c,
    +                  unsigned this_index,
    +                  unsigned start,
    +                  unsigned end,
    +                  unsigned class_count)
    +  {
    +    unsigned base_count = rows;
    +    unsigned new_class_count = end - start;
    +    unsigned size = AnchorMatrix::min_size +
    +                    OT::Offset16::static_size * new_class_count * rows;
    +    unsigned prime_id = c.create_node (size);
    +    if (prime_id == (unsigned) -1) return -1;
    +    AnchorMatrix* prime = (AnchorMatrix*) c.graph.object (prime_id).head;
    +    prime->rows = base_count;
    +
    +    auto& o = c.graph.vertices_[this_index].obj;
    +    int num_links = o.real_links.length;
    +    for (int i = 0; i < num_links; i++)
    +    {
    +      const auto& link = o.real_links[i];
    +      unsigned old_index = (link.position - 2) / OT::Offset16::static_size;
    +      unsigned klass = old_index % class_count;
    +      if (klass < start || klass >= end) continue;
    +
    +      unsigned base = old_index / class_count;
    +      unsigned new_klass = klass - start;
    +      unsigned new_index = base * new_class_count + new_klass;
    +
    +
    +      unsigned child_idx = link.objidx;
    +      c.graph.add_link (&(prime->matrixZ[new_index]),
    +                        prime_id,
    +                        child_idx);
    +
    +      auto& child = c.graph.vertices_[child_idx];
    +      child.remove_parent (this_index);
    +
    +      o.real_links.remove_unordered (i);
    +      num_links--;
    +      i--;
    +    }
    +
    +    return prime_id;
    +  }
    +};
    +
    +struct MarkArray : public OT::Layout::GPOS_impl::MarkArray
    +{
    +  bool sanitize (graph_t::vertex_t& vertex) const
    +  {
    +    int64_t vertex_len = vertex.obj.tail - vertex.obj.head;
    +    unsigned min_size = MarkArray::min_size;
    +    if (vertex_len < min_size) return false;
    +
    +    return vertex_len >= get_size ();
    +  }
    +
    +  bool shrink (gsubgpos_graph_context_t& c,
    +               const hb_hashmap_t& mark_array_links,
    +               unsigned this_index,
    +               unsigned new_class_count)
    +  {
    +    auto& o = c.graph.vertices_[this_index].obj;
    +    for (const auto& link : o.real_links)
    +      c.graph.vertices_[link.objidx].remove_parent (this_index);
    +    o.real_links.reset ();
    +
    +    unsigned new_index = 0;
    +    for (const auto& record : this->iter ())
    +    {
    +      unsigned klass = record.klass;
    +      if (klass >= new_class_count) continue;
    +
    +      (*this)[new_index].klass = klass;
    +      unsigned position = (char*) &record.markAnchor - (char*) this;
    +      unsigned* objidx;
    +      if (!mark_array_links.has (position, &objidx))
    +      {
    +        new_index++;
    +        continue;
    +      }
    +
    +      c.graph.add_link (&(*this)[new_index].markAnchor, this_index, *objidx);
    +      new_index++;
    +    }
    +
    +    this->len = new_index;
    +    o.tail = o.head + MarkArray::min_size +
    +             OT::Layout::GPOS_impl::MarkRecord::static_size * new_index;
    +    return true;
    +  }
    +
    +  unsigned clone (gsubgpos_graph_context_t& c,
    +                  unsigned this_index,
    +                  const hb_hashmap_t& pos_to_index,
    +                  hb_set_t& marks,
    +                  unsigned start_class)
    +  {
    +    unsigned size = MarkArray::min_size +
    +                    OT::Layout::GPOS_impl::MarkRecord::static_size *
    +                    marks.get_population ();
    +    unsigned prime_id = c.create_node (size);
    +    if (prime_id == (unsigned) -1) return -1;
    +    MarkArray* prime = (MarkArray*) c.graph.object (prime_id).head;
    +    prime->len = marks.get_population ();
    +
    +
    +    unsigned i = 0;
    +    for (hb_codepoint_t mark : marks)
    +    {
    +      (*prime)[i].klass = (*this)[mark].klass - start_class;
    +      unsigned offset_pos = (char*) &((*this)[mark].markAnchor) - (char*) this;
    +      unsigned* anchor_index;
    +      if (pos_to_index.has (offset_pos, &anchor_index))
    +        c.graph.move_child (this_index,
    +                            &((*this)[mark].markAnchor),
    +                            prime_id,
    +                            &((*prime)[i].markAnchor));
    +
    +      i++;
    +    }
    +
    +    return prime_id;
    +  }
    +};
    +
    +struct MarkBasePosFormat1 : public OT::Layout::GPOS_impl::MarkBasePosFormat1_2
    +{
    +  bool sanitize (graph_t::vertex_t& vertex) const
    +  {
    +    int64_t vertex_len = vertex.obj.tail - vertex.obj.head;
    +    return vertex_len >= MarkBasePosFormat1::static_size;
    +  }
    +
    +  hb_vector_t split_subtables (gsubgpos_graph_context_t& c,
    +                                         unsigned parent_index,
    +                                         unsigned this_index)
    +  {
    +    hb_set_t visited;
    +
    +    const unsigned base_coverage_id = c.graph.index_for_offset (this_index, &baseCoverage);
    +    const unsigned base_size =
    +        OT::Layout::GPOS_impl::PairPosFormat1_3::min_size +
    +        MarkArray::min_size +
    +        AnchorMatrix::min_size +
    +        c.graph.vertices_[base_coverage_id].table_size ();
    +
    +    hb_vector_t class_to_info = get_class_info (c, this_index);
    +
    +    unsigned class_count = classCount;
    +    auto base_array = c.graph.as_table (this_index,
    +                                                      &baseArray,
    +                                                      class_count);
    +    if (!base_array) return hb_vector_t ();
    +    unsigned base_count = base_array.table->rows;
    +
    +    unsigned partial_coverage_size = 4;
    +    unsigned accumulated = base_size;
    +    hb_vector_t split_points;
    +
    +    for (unsigned klass = 0; klass < class_count; klass++)
    +    {
    +      class_info_t& info = class_to_info[klass];
    +      partial_coverage_size += OT::HBUINT16::static_size * info.marks.get_population ();
    +      unsigned accumulated_delta =
    +          OT::Layout::GPOS_impl::MarkRecord::static_size * info.marks.get_population () +
    +          OT::Offset16::static_size * base_count;
    +
    +      for (unsigned objidx : info.child_indices)
    +        accumulated_delta += c.graph.find_subgraph_size (objidx, visited);
    +
    +      accumulated += accumulated_delta;
    +      unsigned total = accumulated + partial_coverage_size;
    +
    +      if (total >= (1 << 16))
    +      {
    +        split_points.push (klass);
    +        accumulated = base_size + accumulated_delta;
    +        partial_coverage_size = 4 + OT::HBUINT16::static_size * info.marks.get_population ();
    +        visited.clear (); // node sharing isn't allowed between splits.
    +      }
    +    }
    +
    +
    +    const unsigned mark_array_id = c.graph.index_for_offset (this_index, &markArray);
    +    split_context_t split_context {
    +      c,
    +      this,
    +      c.graph.duplicate_if_shared (parent_index, this_index),
    +      std::move (class_to_info),
    +      c.graph.vertices_[mark_array_id].position_to_index_map (),
    +    };
    +
    +    return actuate_subtable_split (split_context, split_points);
    +  }
    +
    + private:
    +
    +  struct class_info_t {
    +    hb_set_t marks;
    +    hb_vector_t child_indices;
    +  };
    +
    +  struct split_context_t {
    +    gsubgpos_graph_context_t& c;
    +    MarkBasePosFormat1* thiz;
    +    unsigned this_index;
    +    hb_vector_t class_to_info;
    +    hb_hashmap_t mark_array_links;
    +
    +    hb_set_t marks_for (unsigned start, unsigned end)
    +    {
    +      hb_set_t marks;
    +      for (unsigned klass = start; klass < end; klass++)
    +      {
    +        + class_to_info[klass].marks.iter ()
    +        | hb_sink (marks)
    +        ;
    +      }
    +      return marks;
    +    }
    +
    +    unsigned original_count ()
    +    {
    +      return thiz->classCount;
    +    }
    +
    +    unsigned clone_range (unsigned start, unsigned end)
    +    {
    +      return thiz->clone_range (*this, this->this_index, start, end);
    +    }
    +
    +    bool shrink (unsigned count)
    +    {
    +      return thiz->shrink (*this, this->this_index, count);
    +    }
    +  };
    +
    +  hb_vector_t get_class_info (gsubgpos_graph_context_t& c,
    +                                            unsigned this_index)
    +  {
    +    hb_vector_t class_to_info;
    +
    +    unsigned class_count= classCount;
    +    class_to_info.resize (class_count);
    +
    +    auto mark_array = c.graph.as_table (this_index, &markArray);
    +    if (!mark_array) return hb_vector_t ();
    +    unsigned mark_count = mark_array.table->len;
    +    for (unsigned mark = 0; mark < mark_count; mark++)
    +    {
    +      unsigned klass = (*mark_array.table)[mark].get_class ();
    +      class_to_info[klass].marks.add (mark);
    +    }
    +
    +    for (const auto& link : mark_array.vertex->obj.real_links)
    +    {
    +      unsigned mark = (link.position - 2) /
    +                     OT::Layout::GPOS_impl::MarkRecord::static_size;
    +      unsigned klass = (*mark_array.table)[mark].get_class ();
    +      class_to_info[klass].child_indices.push (link.objidx);
    +    }
    +
    +    unsigned base_array_id =
    +        c.graph.index_for_offset (this_index, &baseArray);
    +    auto& base_array_v = c.graph.vertices_[base_array_id];
    +
    +    for (const auto& link : base_array_v.obj.real_links)
    +    {
    +      unsigned index = (link.position - 2) / OT::Offset16::static_size;
    +      unsigned klass = index % class_count;
    +      class_to_info[klass].child_indices.push (link.objidx);
    +    }
    +
    +    return class_to_info;
    +  }
    +
    +  bool shrink (split_context_t& sc,
    +               unsigned this_index,
    +               unsigned count)
    +  {
    +    DEBUG_MSG (SUBSET_REPACK, nullptr,
    +               "  Shrinking MarkBasePosFormat1 (%u) to [0, %u).",
    +               this_index,
    +               count);
    +
    +    unsigned old_count = classCount;
    +    if (count >= old_count)
    +      return true;
    +
    +    classCount = count;
    +
    +    auto mark_coverage = sc.c.graph.as_mutable_table (this_index,
    +                                                                &markCoverage);
    +    if (!mark_coverage) return false;
    +    hb_set_t marks = sc.marks_for (0, count);
    +    auto new_coverage =
    +        + hb_enumerate (mark_coverage.table->iter ())
    +        | hb_filter (marks, hb_first)
    +        | hb_map_retains_sorting (hb_second)
    +        ;
    +    if (!Coverage::make_coverage (sc.c, + new_coverage,
    +                                  mark_coverage.index,
    +                                  4 + 2 * marks.get_population ()))
    +      return false;
    +
    +
    +    auto base_array = sc.c.graph.as_mutable_table (this_index,
    +                                                                 &baseArray,
    +                                                                 old_count);
    +    if (!base_array || !base_array.table->shrink (sc.c,
    +                                                  base_array.index,
    +                                                  old_count,
    +                                                  count))
    +      return false;
    +
    +    auto mark_array = sc.c.graph.as_mutable_table (this_index,
    +                                                              &markArray);
    +    if (!mark_array || !mark_array.table->shrink (sc.c,
    +                                                  sc.mark_array_links,
    +                                                  mark_array.index,
    +                                                  count))
    +      return false;
    +
    +    return true;
    +  }
    +
    +  // Create a new MarkBasePos that has all of the data for classes from [start, end).
    +  unsigned clone_range (split_context_t& sc,
    +                        unsigned this_index,
    +                        unsigned start, unsigned end) const
    +  {
    +    DEBUG_MSG (SUBSET_REPACK, nullptr,
    +               "  Cloning MarkBasePosFormat1 (%u) range [%u, %u).", this_index, start, end);
    +
    +    graph_t& graph = sc.c.graph;
    +    unsigned prime_size = OT::Layout::GPOS_impl::MarkBasePosFormat1_2::static_size;
    +
    +    unsigned prime_id = sc.c.create_node (prime_size);
    +    if (prime_id == (unsigned) -1) return -1;
    +
    +    MarkBasePosFormat1* prime = (MarkBasePosFormat1*) graph.object (prime_id).head;
    +    prime->format = this->format;
    +    unsigned new_class_count = end - start;
    +    prime->classCount = new_class_count;
    +
    +    unsigned base_coverage_id =
    +        graph.index_for_offset (sc.this_index, &baseCoverage);
    +    graph.add_link (&(prime->baseCoverage), prime_id, base_coverage_id);
    +    graph.duplicate (prime_id, base_coverage_id);
    +
    +    auto mark_coverage = sc.c.graph.as_table (this_index,
    +                                                        &markCoverage);
    +    if (!mark_coverage) return false;
    +    hb_set_t marks = sc.marks_for (start, end);
    +    auto new_coverage =
    +        + hb_enumerate (mark_coverage.table->iter ())
    +        | hb_filter (marks, hb_first)
    +        | hb_map_retains_sorting (hb_second)
    +        ;
    +    if (!Coverage::add_coverage (sc.c,
    +                                 prime_id,
    +                                 2,
    +                                 + new_coverage,
    +                                 marks.get_population () * 2 + 4))
    +      return -1;
    +
    +    auto mark_array =
    +        graph.as_table  (sc.this_index, &markArray);
    +    if (!mark_array) return -1;
    +    unsigned new_mark_array =
    +        mark_array.table->clone (sc.c,
    +                                 mark_array.index,
    +                                 sc.mark_array_links,
    +                                 marks,
    +                                 start);
    +    graph.add_link (&(prime->markArray), prime_id, new_mark_array);
    +
    +    unsigned class_count = classCount;
    +    auto base_array =
    +        graph.as_table (sc.this_index, &baseArray, class_count);
    +    if (!base_array) return -1;
    +    unsigned new_base_array =
    +        base_array.table->clone (sc.c,
    +                                 base_array.index,
    +                                 start, end, this->classCount);
    +    graph.add_link (&(prime->baseArray), prime_id, new_base_array);
    +
    +    return prime_id;
    +  }
    +};
    +
    +
    +struct MarkBasePos : public OT::Layout::GPOS_impl::MarkBasePos
    +{
    +  hb_vector_t split_subtables (gsubgpos_graph_context_t& c,
    +                                         unsigned parent_index,
    +                                         unsigned this_index)
    +  {
    +    switch (u.format) {
    +    case 1:
    +      return ((MarkBasePosFormat1*)(&u.format1))->split_subtables (c, parent_index, this_index);
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: HB_FALLTHROUGH;
    +      // Don't split 24bit PairPos's.
    +#endif
    +    default:
    +      return hb_vector_t ();
    +    }
    +  }
    +
    +  bool sanitize (graph_t::vertex_t& vertex) const
    +  {
    +    int64_t vertex_len = vertex.obj.tail - vertex.obj.head;
    +    if (vertex_len < u.format.get_size ()) return false;
    +
    +    switch (u.format) {
    +    case 1:
    +      return ((MarkBasePosFormat1*)(&u.format1))->sanitize (vertex);
    +#ifndef HB_NO_BEYOND_64K
    +    case 2: HB_FALLTHROUGH;
    +#endif
    +    default:
    +      // We don't handle format 3 and 4 here.
    +      return false;
    +    }
    +  }
    +};
    +
    +
    +}
    +
    +#endif  // GRAPH_MARKBASEPOS_GRAPH_HH
    diff --git a/src/java.desktop/share/native/libharfbuzz/graph/pairpos-graph.hh b/src/java.desktop/share/native/libharfbuzz/graph/pairpos-graph.hh
    new file mode 100644
    index 0000000000000..1c13eb24f9458
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/graph/pairpos-graph.hh
    @@ -0,0 +1,647 @@
    +/*
    + * Copyright © 2022  Google, Inc.
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + *
    + * Google Author(s): Garret Rieger
    + */
    +
    +#ifndef GRAPH_PAIRPOS_GRAPH_HH
    +#define GRAPH_PAIRPOS_GRAPH_HH
    +
    +#include "split-helpers.hh"
    +#include "coverage-graph.hh"
    +#include "classdef-graph.hh"
    +#include "../OT/Layout/GPOS/PairPos.hh"
    +#include "../OT/Layout/GPOS/PosLookupSubTable.hh"
    +
    +namespace graph {
    +
    +struct PairPosFormat1 : public OT::Layout::GPOS_impl::PairPosFormat1_3
    +{
    +  bool sanitize (graph_t::vertex_t& vertex) const
    +  {
    +    int64_t vertex_len = vertex.obj.tail - vertex.obj.head;
    +    unsigned min_size = OT::Layout::GPOS_impl::PairPosFormat1_3::min_size;
    +    if (vertex_len < min_size) return false;
    +
    +    return vertex_len >=
    +        min_size + pairSet.get_size () - pairSet.len.get_size();
    +  }
    +
    +  hb_vector_t split_subtables (gsubgpos_graph_context_t& c,
    +                                         unsigned parent_index,
    +                                         unsigned this_index)
    +  {
    +    hb_set_t visited;
    +
    +    const unsigned coverage_id = c.graph.index_for_offset (this_index, &coverage);
    +    const unsigned coverage_size = c.graph.vertices_[coverage_id].table_size ();
    +    const unsigned base_size = OT::Layout::GPOS_impl::PairPosFormat1_3::min_size;
    +
    +    unsigned partial_coverage_size = 4;
    +    unsigned accumulated = base_size;
    +    hb_vector_t split_points;
    +    for (unsigned i = 0; i < pairSet.len; i++)
    +    {
    +      unsigned pair_set_index = pair_set_graph_index (c, this_index, i);
    +      unsigned accumulated_delta =
    +          c.graph.find_subgraph_size (pair_set_index, visited) +
    +          SmallTypes::size; // for PairSet offset.
    +      partial_coverage_size += OT::HBUINT16::static_size;
    +
    +      accumulated += accumulated_delta;
    +      unsigned total = accumulated + hb_min (partial_coverage_size, coverage_size);
    +
    +      if (total >= (1 << 16))
    +      {
    +        split_points.push (i);
    +        accumulated = base_size + accumulated_delta;
    +        partial_coverage_size = 6;
    +        visited.clear (); // node sharing isn't allowed between splits.
    +      }
    +    }
    +
    +    split_context_t split_context {
    +      c,
    +      this,
    +      c.graph.duplicate_if_shared (parent_index, this_index),
    +    };
    +
    +    return actuate_subtable_split (split_context, split_points);
    +  }
    +
    + private:
    +
    +  struct split_context_t {
    +    gsubgpos_graph_context_t& c;
    +    PairPosFormat1* thiz;
    +    unsigned this_index;
    +
    +    unsigned original_count ()
    +    {
    +      return thiz->pairSet.len;
    +    }
    +
    +    unsigned clone_range (unsigned start, unsigned end)
    +    {
    +      return thiz->clone_range (this->c, this->this_index, start, end);
    +    }
    +
    +    bool shrink (unsigned count)
    +    {
    +      return thiz->shrink (this->c, this->this_index, count);
    +    }
    +  };
    +
    +  bool shrink (gsubgpos_graph_context_t& c,
    +               unsigned this_index,
    +               unsigned count)
    +  {
    +    DEBUG_MSG (SUBSET_REPACK, nullptr,
    +               "  Shrinking PairPosFormat1 (%u) to [0, %u).",
    +               this_index,
    +               count);
    +    unsigned old_count = pairSet.len;
    +    if (count >= old_count)
    +      return true;
    +
    +    pairSet.len = count;
    +    c.graph.vertices_[this_index].obj.tail -= (old_count - count) * SmallTypes::size;
    +
    +    auto coverage = c.graph.as_mutable_table (this_index, &this->coverage);
    +    if (!coverage) return false;
    +
    +    unsigned coverage_size = coverage.vertex->table_size ();
    +    auto new_coverage =
    +        + hb_zip (coverage.table->iter (), hb_range ())
    +        | hb_filter ([&] (hb_pair_t p) {
    +          return p.second < count;
    +        })
    +        | hb_map_retains_sorting (hb_first)
    +        ;
    +
    +    return Coverage::make_coverage (c, new_coverage, coverage.index, coverage_size);
    +  }
    +
    +  // Create a new PairPos including PairSet's from start (inclusive) to end (exclusive).
    +  // Returns object id of the new object.
    +  unsigned clone_range (gsubgpos_graph_context_t& c,
    +                        unsigned this_index,
    +                        unsigned start, unsigned end) const
    +  {
    +    DEBUG_MSG (SUBSET_REPACK, nullptr,
    +               "  Cloning PairPosFormat1 (%u) range [%u, %u).", this_index, start, end);
    +
    +    unsigned num_pair_sets = end - start;
    +    unsigned prime_size = OT::Layout::GPOS_impl::PairPosFormat1_3::min_size
    +                          + num_pair_sets * SmallTypes::size;
    +
    +    unsigned pair_pos_prime_id = c.create_node (prime_size);
    +    if (pair_pos_prime_id == (unsigned) -1) return -1;
    +
    +    PairPosFormat1* pair_pos_prime = (PairPosFormat1*) c.graph.object (pair_pos_prime_id).head;
    +    pair_pos_prime->format = this->format;
    +    pair_pos_prime->valueFormat[0] = this->valueFormat[0];
    +    pair_pos_prime->valueFormat[1] = this->valueFormat[1];
    +    pair_pos_prime->pairSet.len = num_pair_sets;
    +
    +    for (unsigned i = start; i < end; i++)
    +    {
    +      c.graph.move_child<> (this_index,
    +                            &pairSet[i],
    +                            pair_pos_prime_id,
    +                            &pair_pos_prime->pairSet[i - start]);
    +    }
    +
    +    unsigned coverage_id = c.graph.index_for_offset (this_index, &coverage);
    +    if (!Coverage::clone_coverage (c,
    +                                   coverage_id,
    +                                   pair_pos_prime_id,
    +                                   2,
    +                                   start, end))
    +      return -1;
    +
    +    return pair_pos_prime_id;
    +  }
    +
    +
    +
    +  unsigned pair_set_graph_index (gsubgpos_graph_context_t& c, unsigned this_index, unsigned i) const
    +  {
    +    return c.graph.index_for_offset (this_index, &pairSet[i]);
    +  }
    +};
    +
    +struct PairPosFormat2 : public OT::Layout::GPOS_impl::PairPosFormat2_4
    +{
    +  bool sanitize (graph_t::vertex_t& vertex) const
    +  {
    +    size_t vertex_len = vertex.table_size ();
    +    unsigned min_size = OT::Layout::GPOS_impl::PairPosFormat2_4::min_size;
    +    if (vertex_len < min_size) return false;
    +
    +    const unsigned class1_count = class1Count;
    +    return vertex_len >=
    +        min_size + class1_count * get_class1_record_size ();
    +  }
    +
    +  hb_vector_t split_subtables (gsubgpos_graph_context_t& c,
    +                                         unsigned parent_index,
    +                                         unsigned this_index)
    +  {
    +    const unsigned base_size = OT::Layout::GPOS_impl::PairPosFormat2_4::min_size;
    +    const unsigned class_def_2_size = size_of (c, this_index, &classDef2);
    +    const Coverage* coverage = get_coverage (c, this_index);
    +    const ClassDef* class_def_1 = get_class_def_1 (c, this_index);
    +    auto gid_and_class =
    +        + coverage->iter ()
    +        | hb_map_retains_sorting ([&] (hb_codepoint_t gid) {
    +          return hb_pair_t (gid, class_def_1->get_class (gid));
    +        })
    +        ;
    +    class_def_size_estimator_t estimator (gid_and_class);
    +
    +    const unsigned class1_count = class1Count;
    +    const unsigned class2_count = class2Count;
    +    const unsigned class1_record_size = get_class1_record_size ();
    +
    +    const unsigned value_1_len = valueFormat1.get_len ();
    +    const unsigned value_2_len = valueFormat2.get_len ();
    +    const unsigned total_value_len = value_1_len + value_2_len;
    +
    +    unsigned accumulated = base_size;
    +    unsigned coverage_size = 4;
    +    unsigned class_def_1_size = 4;
    +    unsigned max_coverage_size = coverage_size;
    +    unsigned max_class_def_1_size = class_def_1_size;
    +
    +    hb_vector_t split_points;
    +
    +    hb_hashmap_t device_tables = get_all_device_tables (c, this_index);
    +    hb_vector_t format1_device_table_indices = valueFormat1.get_device_table_indices ();
    +    hb_vector_t format2_device_table_indices = valueFormat2.get_device_table_indices ();
    +    bool has_device_tables = bool(format1_device_table_indices) || bool(format2_device_table_indices);
    +
    +    hb_set_t visited;
    +    for (unsigned i = 0; i < class1_count; i++)
    +    {
    +      unsigned accumulated_delta = class1_record_size;
    +      coverage_size += estimator.incremental_coverage_size (i);
    +      class_def_1_size += estimator.incremental_class_def_size (i);
    +      max_coverage_size = hb_max (max_coverage_size, coverage_size);
    +      max_class_def_1_size = hb_max (max_class_def_1_size, class_def_1_size);
    +
    +      if (has_device_tables) {
    +        for (unsigned j = 0; j < class2_count; j++)
    +        {
    +          unsigned value1_index = total_value_len * (class2_count * i + j);
    +          unsigned value2_index = value1_index + value_1_len;
    +          accumulated_delta += size_of_value_record_children (c,
    +                                                        device_tables,
    +                                                        format1_device_table_indices,
    +                                                        value1_index,
    +                                                        visited);
    +          accumulated_delta += size_of_value_record_children (c,
    +                                                        device_tables,
    +                                                        format2_device_table_indices,
    +                                                        value2_index,
    +                                                        visited);
    +        }
    +      }
    +
    +      accumulated += accumulated_delta;
    +      unsigned total = accumulated
    +                       + coverage_size + class_def_1_size + class_def_2_size
    +                       // The largest object will pack last and can exceed the size limit.
    +                       - hb_max (hb_max (coverage_size, class_def_1_size), class_def_2_size);
    +      if (total >= (1 << 16))
    +      {
    +        split_points.push (i);
    +        // split does not include i, so add the size for i when we reset the size counters.
    +        accumulated = base_size + accumulated_delta;
    +        coverage_size = 4 + estimator.incremental_coverage_size (i);
    +        class_def_1_size = 4 + estimator.incremental_class_def_size (i);
    +        visited.clear (); // node sharing isn't allowed between splits.
    +      }
    +    }
    +
    +    split_context_t split_context {
    +      c,
    +      this,
    +      c.graph.duplicate_if_shared (parent_index, this_index),
    +      class1_record_size,
    +      total_value_len,
    +      value_1_len,
    +      value_2_len,
    +      max_coverage_size,
    +      max_class_def_1_size,
    +      device_tables,
    +      format1_device_table_indices,
    +      format2_device_table_indices
    +    };
    +
    +    return actuate_subtable_split (split_context, split_points);
    +  }
    + private:
    +
    +  struct split_context_t
    +  {
    +    gsubgpos_graph_context_t& c;
    +    PairPosFormat2* thiz;
    +    unsigned this_index;
    +    unsigned class1_record_size;
    +    unsigned value_record_len;
    +    unsigned value1_record_len;
    +    unsigned value2_record_len;
    +    unsigned max_coverage_size;
    +    unsigned max_class_def_size;
    +
    +    const hb_hashmap_t& device_tables;
    +    const hb_vector_t& format1_device_table_indices;
    +    const hb_vector_t& format2_device_table_indices;
    +
    +    unsigned original_count ()
    +    {
    +      return thiz->class1Count;
    +    }
    +
    +    unsigned clone_range (unsigned start, unsigned end)
    +    {
    +      return thiz->clone_range (*this, start, end);
    +    }
    +
    +    bool shrink (unsigned count)
    +    {
    +      return thiz->shrink (*this, count);
    +    }
    +  };
    +
    +  size_t get_class1_record_size () const
    +  {
    +    const size_t class2_count = class2Count;
    +    return
    +        class2_count * (valueFormat1.get_size () + valueFormat2.get_size ());
    +  }
    +
    +  unsigned clone_range (split_context_t& split_context,
    +                        unsigned start, unsigned end) const
    +  {
    +    DEBUG_MSG (SUBSET_REPACK, nullptr,
    +               "  Cloning PairPosFormat2 (%u) range [%u, %u).", split_context.this_index, start, end);
    +
    +    graph_t& graph = split_context.c.graph;
    +
    +    unsigned num_records = end - start;
    +    unsigned prime_size = OT::Layout::GPOS_impl::PairPosFormat2_4::min_size
    +                          + num_records * split_context.class1_record_size;
    +
    +    unsigned pair_pos_prime_id = split_context.c.create_node (prime_size);
    +    if (pair_pos_prime_id == (unsigned) -1) return -1;
    +
    +    PairPosFormat2* pair_pos_prime =
    +        (PairPosFormat2*) graph.object (pair_pos_prime_id).head;
    +    pair_pos_prime->format = this->format;
    +    pair_pos_prime->valueFormat1 = this->valueFormat1;
    +    pair_pos_prime->valueFormat2 = this->valueFormat2;
    +    pair_pos_prime->class1Count = num_records;
    +    pair_pos_prime->class2Count = this->class2Count;
    +    clone_class1_records (split_context,
    +                          pair_pos_prime_id,
    +                          start,
    +                          end);
    +
    +    unsigned coverage_id =
    +        graph.index_for_offset (split_context.this_index, &coverage);
    +    unsigned class_def_1_id =
    +        graph.index_for_offset (split_context.this_index, &classDef1);
    +    auto& coverage_v = graph.vertices_[coverage_id];
    +    auto& class_def_1_v = graph.vertices_[class_def_1_id];
    +    Coverage* coverage_table = (Coverage*) coverage_v.obj.head;
    +    ClassDef* class_def_1_table = (ClassDef*) class_def_1_v.obj.head;
    +    if (!coverage_table
    +        || !coverage_table->sanitize (coverage_v)
    +        || !class_def_1_table
    +        || !class_def_1_table->sanitize (class_def_1_v))
    +      return -1;
    +
    +    auto klass_map =
    +    + coverage_table->iter ()
    +    | hb_map_retains_sorting ([&] (hb_codepoint_t gid) {
    +      return hb_pair_t (gid, class_def_1_table->get_class (gid));
    +    })
    +    | hb_filter ([&] (hb_codepoint_t klass) {
    +      return klass >= start && klass < end;
    +    }, hb_second)
    +    | hb_map_retains_sorting ([&] (hb_pair_t gid_and_class) {
    +      // Classes must be from 0...N so subtract start
    +      return hb_pair_t (gid_and_class.first, gid_and_class.second - start);
    +    })
    +    ;
    +
    +    if (!Coverage::add_coverage (split_context.c,
    +                                 pair_pos_prime_id,
    +                                 2,
    +                                 + klass_map | hb_map_retains_sorting (hb_first),
    +                                 split_context.max_coverage_size))
    +      return -1;
    +
    +    // classDef1
    +    if (!ClassDef::add_class_def (split_context.c,
    +                                  pair_pos_prime_id,
    +                                  8,
    +                                  + klass_map,
    +                                  split_context.max_class_def_size))
    +      return -1;
    +
    +    // classDef2
    +    unsigned class_def_2_id =
    +        graph.index_for_offset (split_context.this_index, &classDef2);
    +    auto* class_def_link = graph.vertices_[pair_pos_prime_id].obj.real_links.push ();
    +    class_def_link->width = SmallTypes::size;
    +    class_def_link->objidx = class_def_2_id;
    +    class_def_link->position = 10;
    +    graph.vertices_[class_def_2_id].parents.push (pair_pos_prime_id);
    +    graph.duplicate (pair_pos_prime_id, class_def_2_id);
    +
    +    return pair_pos_prime_id;
    +  }
    +
    +  void clone_class1_records (split_context_t& split_context,
    +                             unsigned pair_pos_prime_id,
    +                             unsigned start, unsigned end) const
    +  {
    +    PairPosFormat2* pair_pos_prime =
    +        (PairPosFormat2*) split_context.c.graph.object (pair_pos_prime_id).head;
    +
    +    char* start_addr = ((char*)&values[0]) + start * split_context.class1_record_size;
    +    unsigned num_records = end - start;
    +    hb_memcpy (&pair_pos_prime->values[0],
    +            start_addr,
    +            num_records * split_context.class1_record_size);
    +
    +    if (!split_context.format1_device_table_indices
    +        && !split_context.format2_device_table_indices)
    +      // No device tables to move over.
    +      return;
    +
    +    unsigned class2_count = class2Count;
    +    for (unsigned i = start; i < end; i++)
    +    {
    +      for (unsigned j = 0; j < class2_count; j++)
    +      {
    +        unsigned value1_index = split_context.value_record_len * (class2_count * i + j);
    +        unsigned value2_index = value1_index + split_context.value1_record_len;
    +
    +        unsigned new_value1_index = split_context.value_record_len * (class2_count * (i - start) + j);
    +        unsigned new_value2_index = new_value1_index + split_context.value1_record_len;
    +
    +        transfer_device_tables (split_context,
    +                                pair_pos_prime_id,
    +                                split_context.format1_device_table_indices,
    +                                value1_index,
    +                                new_value1_index);
    +
    +        transfer_device_tables (split_context,
    +                                pair_pos_prime_id,
    +                                split_context.format2_device_table_indices,
    +                                value2_index,
    +                                new_value2_index);
    +      }
    +    }
    +  }
    +
    +  void transfer_device_tables (split_context_t& split_context,
    +                               unsigned pair_pos_prime_id,
    +                               const hb_vector_t& device_table_indices,
    +                               unsigned old_value_record_index,
    +                               unsigned new_value_record_index) const
    +  {
    +    PairPosFormat2* pair_pos_prime =
    +        (PairPosFormat2*) split_context.c.graph.object (pair_pos_prime_id).head;
    +
    +    for (unsigned i : device_table_indices)
    +    {
    +      OT::Offset16* record = (OT::Offset16*) &values[old_value_record_index + i];
    +      unsigned record_position = ((char*) record) - ((char*) this);
    +      if (!split_context.device_tables.has (record_position)) continue;
    +
    +      split_context.c.graph.move_child (
    +          split_context.this_index,
    +          record,
    +          pair_pos_prime_id,
    +          (OT::Offset16*) &pair_pos_prime->values[new_value_record_index + i]);
    +    }
    +  }
    +
    +  bool shrink (split_context_t& split_context,
    +               unsigned count)
    +  {
    +    DEBUG_MSG (SUBSET_REPACK, nullptr,
    +               "  Shrinking PairPosFormat2 (%u) to [0, %u).",
    +               split_context.this_index,
    +               count);
    +    unsigned old_count = class1Count;
    +    if (count >= old_count)
    +      return true;
    +
    +    graph_t& graph = split_context.c.graph;
    +    class1Count = count;
    +    graph.vertices_[split_context.this_index].obj.tail -=
    +        (old_count - count) * split_context.class1_record_size;
    +
    +    auto coverage =
    +        graph.as_mutable_table (split_context.this_index, &this->coverage);
    +    if (!coverage) return false;
    +
    +    auto class_def_1 =
    +        graph.as_mutable_table (split_context.this_index, &classDef1);
    +    if (!class_def_1) return false;
    +
    +    auto klass_map =
    +    + coverage.table->iter ()
    +    | hb_map_retains_sorting ([&] (hb_codepoint_t gid) {
    +      return hb_pair_t (gid, class_def_1.table->get_class (gid));
    +    })
    +    | hb_filter ([&] (hb_codepoint_t klass) {
    +      return klass < count;
    +    }, hb_second)
    +    ;
    +
    +    auto new_coverage = + klass_map | hb_map_retains_sorting (hb_first);
    +    if (!Coverage::make_coverage (split_context.c,
    +                                  + new_coverage,
    +                                  coverage.index,
    +                                  // existing ranges my not be kept, worst case size is a format 1
    +                                  // coverage table.
    +                                  4 + new_coverage.len() * 2))
    +      return false;
    +
    +    return ClassDef::make_class_def (split_context.c,
    +                                     + klass_map,
    +                                     class_def_1.index,
    +                                     class_def_1.vertex->table_size ());
    +  }
    +
    +  hb_hashmap_t
    +  get_all_device_tables (gsubgpos_graph_context_t& c,
    +                         unsigned this_index) const
    +  {
    +    const auto& v = c.graph.vertices_[this_index];
    +    return v.position_to_index_map ();
    +  }
    +
    +  const Coverage* get_coverage (gsubgpos_graph_context_t& c,
    +                          unsigned this_index) const
    +  {
    +    unsigned coverage_id = c.graph.index_for_offset (this_index, &coverage);
    +    auto& coverage_v = c.graph.vertices_[coverage_id];
    +
    +    Coverage* coverage_table = (Coverage*) coverage_v.obj.head;
    +    if (!coverage_table || !coverage_table->sanitize (coverage_v))
    +      return &Null(Coverage);
    +    return coverage_table;
    +  }
    +
    +  const ClassDef* get_class_def_1 (gsubgpos_graph_context_t& c,
    +                                   unsigned this_index) const
    +  {
    +    unsigned class_def_1_id = c.graph.index_for_offset (this_index, &classDef1);
    +    auto& class_def_1_v = c.graph.vertices_[class_def_1_id];
    +
    +    ClassDef* class_def_1_table = (ClassDef*) class_def_1_v.obj.head;
    +    if (!class_def_1_table || !class_def_1_table->sanitize (class_def_1_v))
    +      return &Null(ClassDef);
    +    return class_def_1_table;
    +  }
    +
    +  unsigned size_of_value_record_children (gsubgpos_graph_context_t& c,
    +                                          const hb_hashmap_t& device_tables,
    +                                          const hb_vector_t device_table_indices,
    +                                          unsigned value_record_index,
    +                                          hb_set_t& visited)
    +  {
    +    unsigned size = 0;
    +    for (unsigned i : device_table_indices)
    +    {
    +      OT::Layout::GPOS_impl::Value* record = &values[value_record_index + i];
    +      unsigned record_position = ((char*) record) - ((char*) this);
    +      unsigned* obj_idx;
    +      if (!device_tables.has (record_position, &obj_idx)) continue;
    +      size += c.graph.find_subgraph_size (*obj_idx, visited);
    +    }
    +    return size;
    +  }
    +
    +  unsigned size_of (gsubgpos_graph_context_t& c,
    +                    unsigned this_index,
    +                    const void* offset) const
    +  {
    +    const unsigned id = c.graph.index_for_offset (this_index, offset);
    +    return c.graph.vertices_[id].table_size ();
    +  }
    +};
    +
    +struct PairPos : public OT::Layout::GPOS_impl::PairPos
    +{
    +  hb_vector_t split_subtables (gsubgpos_graph_context_t& c,
    +                                         unsigned parent_index,
    +                                         unsigned this_index)
    +  {
    +    switch (u.format) {
    +    case 1:
    +      return ((PairPosFormat1*)(&u.format1))->split_subtables (c, parent_index, this_index);
    +    case 2:
    +      return ((PairPosFormat2*)(&u.format2))->split_subtables (c, parent_index, this_index);
    +#ifndef HB_NO_BEYOND_64K
    +    case 3: HB_FALLTHROUGH;
    +    case 4: HB_FALLTHROUGH;
    +      // Don't split 24bit PairPos's.
    +#endif
    +    default:
    +      return hb_vector_t ();
    +    }
    +  }
    +
    +  bool sanitize (graph_t::vertex_t& vertex) const
    +  {
    +    int64_t vertex_len = vertex.obj.tail - vertex.obj.head;
    +    if (vertex_len < u.format.get_size ()) return false;
    +
    +    switch (u.format) {
    +    case 1:
    +      return ((PairPosFormat1*)(&u.format1))->sanitize (vertex);
    +    case 2:
    +      return ((PairPosFormat2*)(&u.format2))->sanitize (vertex);
    +#ifndef HB_NO_BEYOND_64K
    +    case 3: HB_FALLTHROUGH;
    +    case 4: HB_FALLTHROUGH;
    +#endif
    +    default:
    +      // We don't handle format 3 and 4 here.
    +      return false;
    +    }
    +  }
    +};
    +
    +}
    +
    +#endif  // GRAPH_PAIRPOS_GRAPH_HH
    diff --git a/src/java.desktop/share/native/libharfbuzz/graph/serialize.hh b/src/java.desktop/share/native/libharfbuzz/graph/serialize.hh
    index ecc6cc5aea2b5..040fd1de5fd5b 100644
    --- a/src/java.desktop/share/native/libharfbuzz/graph/serialize.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/graph/serialize.hh
    @@ -33,6 +33,23 @@ struct overflow_record_t
     {
       unsigned parent;
       unsigned child;
    +
    +  bool operator != (const overflow_record_t o) const
    +  { return !(*this == o); }
    +
    +  inline bool operator == (const overflow_record_t& o) const
    +  {
    +    return parent == o.parent &&
    +        child == o.child;
    +  }
    +
    +  inline uint32_t hash () const
    +  {
    +    uint32_t current = 0;
    +    current = current * 31 + hb_hash (parent);
    +    current = current * 31 + hb_hash (child);
    +    return current;
    +  }
     };
     
     inline
    @@ -94,6 +111,7 @@ will_overflow (graph_t& graph,
       if (overflows) overflows->resize (0);
       graph.update_positions ();
     
    +  hb_hashmap_t record_set;
       const auto& vertices = graph.vertices_;
       for (int parent_idx = vertices.length - 1; parent_idx >= 0; parent_idx--)
       {
    @@ -109,7 +127,10 @@ will_overflow (graph_t& graph,
           overflow_record_t r;
           r.parent = parent_idx;
           r.child = link.objidx;
    +      if (record_set.has(&r)) continue; // don't keep duplicate overflows.
    +
           overflows->push (r);
    +      record_set.set(&r, true);
         }
       }
     
    @@ -132,8 +153,8 @@ void print_overflows (graph_t& graph,
         const auto& child = graph.vertices_[o.child];
         DEBUG_MSG (SUBSET_REPACK, nullptr,
                    "  overflow from "
    -               "%4d (%4d in, %4d out, space %2d) => "
    -               "%4d (%4d in, %4d out, space %2d)",
    +               "%4u (%4u in, %4u out, space %2u) => "
    +               "%4u (%4u in, %4u out, space %2u)",
                    o.parent,
                    parent.incoming_edges (),
                    parent.obj.real_links.length + parent.obj.virtual_links.length,
    @@ -144,7 +165,7 @@ void print_overflows (graph_t& graph,
                    graph.space_for (o.child));
       }
       if (overflows.length > 10) {
    -    DEBUG_MSG (SUBSET_REPACK, nullptr, "  ... plus %d more overflows.", overflows.length - 10);
    +    DEBUG_MSG (SUBSET_REPACK, nullptr, "  ... plus %u more overflows.", overflows.length - 10);
       }
     }
     
    @@ -223,7 +244,7 @@ inline hb_blob_t* serialize (const graph_t& graph)
           return nullptr;
         }
     
    -    memcpy (start, vertices[i].obj.head, size);
    +    hb_memcpy (start, vertices[i].obj.head, size);
     
         // Only real links needs to be serialized.
         for (const auto& link : vertices[i].obj.real_links)
    diff --git a/src/java.desktop/share/native/libharfbuzz/graph/split-helpers.hh b/src/java.desktop/share/native/libharfbuzz/graph/split-helpers.hh
    new file mode 100644
    index 0000000000000..61fd7c2d2ff65
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/graph/split-helpers.hh
    @@ -0,0 +1,69 @@
    +/*
    + * Copyright © 2022  Google, Inc.
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + *
    + * Google Author(s): Garret Rieger
    + */
    +
    +#ifndef GRAPH_SPLIT_HELPERS_HH
    +#define GRAPH_SPLIT_HELPERS_HH
    +
    +namespace graph {
    +
    +template
    +HB_INTERNAL
    +hb_vector_t actuate_subtable_split (Context& split_context,
    +                                              const hb_vector_t& split_points)
    +{
    +  hb_vector_t new_objects;
    +  if (!split_points)
    +    return new_objects;
    +
    +  for (unsigned i = 0; i < split_points.length; i++)
    +  {
    +    unsigned start = split_points[i];
    +    unsigned end = (i < split_points.length - 1)
    +                   ? split_points[i + 1]
    +                   : split_context.original_count ();
    +    unsigned id = split_context.clone_range (start, end);
    +
    +    if (id == (unsigned) -1)
    +    {
    +      new_objects.reset ();
    +      new_objects.allocated = -1; // mark error
    +      return new_objects;
    +    }
    +    new_objects.push (id);
    +  }
    +
    +  if (!split_context.shrink (split_points[0]))
    +  {
    +    new_objects.reset ();
    +    new_objects.allocated = -1; // mark error
    +  }
    +
    +  return new_objects;
    +}
    +
    +}
    +
    +#endif  // GRAPH_SPLIT_HELPERS_HH
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-bsln-table.hh b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-bsln-table.hh
    index ed376b9efc74b..24d53e224c788 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-bsln-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-bsln-table.hh
    @@ -42,7 +42,7 @@ struct BaselineTableFormat0Part
       bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
    -    return_trace (likely (c->check_struct (this)));
    +    return_trace (c->check_struct (this));
       }
     
       protected:
    @@ -78,7 +78,7 @@ struct BaselineTableFormat2Part
       bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
    -    return_trace (likely (c->check_struct (this)));
    +    return_trace (c->check_struct (this));
       }
     
       protected:
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-common.hh b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-common.hh
    index b8c9b992fb886..8230cba7c94b1 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-common.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-common.hh
    @@ -28,6 +28,7 @@
     #define HB_AAT_LAYOUT_COMMON_HH
     
     #include "hb-aat-layout.hh"
    +#include "hb-aat-map.hh"
     #include "hb-open-type.hh"
     
     namespace OT {
    @@ -39,6 +40,43 @@ namespace AAT {
     using namespace OT;
     
     
    +struct ankr;
    +
    +struct hb_aat_apply_context_t :
    +       hb_dispatch_context_t
    +{
    +  const char *get_name () { return "APPLY"; }
    +  template 
    +  return_t dispatch (const T &obj) { return obj.apply (this); }
    +  static return_t default_return_value () { return false; }
    +  bool stop_sublookup_iteration (return_t r) const { return r; }
    +
    +  const hb_ot_shape_plan_t *plan;
    +  hb_font_t *font;
    +  hb_face_t *face;
    +  hb_buffer_t *buffer;
    +  hb_sanitize_context_t sanitizer;
    +  const ankr *ankr_table;
    +  const OT::GDEF *gdef_table;
    +  const hb_sorted_vector_t *range_flags = nullptr;
    +  hb_mask_t subtable_flags = 0;
    +
    +  /* Unused. For debug tracing only. */
    +  unsigned int lookup_index;
    +
    +  HB_INTERNAL hb_aat_apply_context_t (const hb_ot_shape_plan_t *plan_,
    +                                      hb_font_t *font_,
    +                                      hb_buffer_t *buffer_,
    +                                      hb_blob_t *blob = const_cast (&Null (hb_blob_t)));
    +
    +  HB_INTERNAL ~hb_aat_apply_context_t ();
    +
    +  HB_INTERNAL void set_ankr_table (const AAT::ankr *ankr_table_);
    +
    +  void set_lookup_index (unsigned int i) { lookup_index = i; }
    +};
    +
    +
     /*
      * Lookup Table
      */
    @@ -415,18 +453,7 @@ struct Lookup
       public:
       DEFINE_SIZE_UNION (2, format);
     };
    -/* Lookup 0 has unbounded size (dependant on num_glyphs).  So we need to defined
    - * special NULL objects for Lookup<> objects, but since it's template our macros
    - * don't work.  So we have to hand-code them here.  UGLY. */
    -} /* Close namespace. */
    -/* Ugly hand-coded null objects for template Lookup<> :(. */
    -extern HB_INTERNAL const unsigned char _hb_Null_AAT_Lookup[2];
    -template 
    -struct Null> {
    -  static AAT::Lookup const & get_null ()
    -  { return *reinterpret_cast *> (_hb_Null_AAT_Lookup); }
    -};
    -namespace AAT {
    +DECLARE_NULL_NAMESPACE_BYTES_TEMPLATE1 (AAT, Lookup, 2);
     
     enum { DELETED_GLYPH = 0xFFFF };
     
    @@ -437,7 +464,8 @@ enum { DELETED_GLYPH = 0xFFFF };
     template 
     struct Entry
     {
    -  bool sanitize (hb_sanitize_context_t *c, unsigned int count) const
    +  // This does seem like it's ever called.
    +  bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
         /* Note, we don't recurse-sanitize data because we don't access it.
    @@ -465,7 +493,8 @@ struct Entry
     template <>
     struct Entry
     {
    -  bool sanitize (hb_sanitize_context_t *c, unsigned int count /*XXX Unused?*/) const
    +  // This does seem like it's ever called.
    +  bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
         return_trace (c->check_struct (this));
    @@ -681,6 +710,13 @@ struct ObsoleteTypes
                                          const void *base,
                                          const T *array)
       {
    +    /* https://github.com/harfbuzz/harfbuzz/issues/3483 */
    +    /* If offset is less than base, return an offset that would
    +     * result in an address half a 32bit address-space away,
    +     * to make sure sanitize fails even on 32bit builds. */
    +    if (unlikely (offset < unsigned ((const char *) array - (const char *) base)))
    +      return INT_MAX / T::static_size;
    +
         /* https://github.com/harfbuzz/harfbuzz/issues/2816 */
         return (offset - unsigned ((const char *) array - (const char *) base)) / T::static_size;
       }
    @@ -744,16 +780,44 @@ struct StateTableDriver
                   num_glyphs (face_->get_num_glyphs ()) {}
     
       template 
    -  void drive (context_t *c)
    +  void drive (context_t *c, hb_aat_apply_context_t *ac)
       {
         if (!c->in_place)
           buffer->clear_output ();
     
         int state = StateTableT::STATE_START_OF_TEXT;
    +    // If there's only one range, we already checked the flag.
    +    auto *last_range = ac->range_flags && (ac->range_flags->length > 1) ? &(*ac->range_flags)[0] : nullptr;
         for (buffer->idx = 0; buffer->successful;)
         {
    +      /* This block is copied in NoncontextualSubtable::apply. Keep in sync. */
    +      if (last_range)
    +      {
    +        auto *range = last_range;
    +        if (buffer->idx < buffer->len)
    +        {
    +          unsigned cluster = buffer->cur().cluster;
    +          while (cluster < range->cluster_first)
    +            range--;
    +          while (cluster > range->cluster_last)
    +            range++;
    +
    +
    +          last_range = range;
    +        }
    +        if (!(range->flags & ac->subtable_flags))
    +        {
    +          if (buffer->idx == buffer->len || unlikely (!buffer->successful))
    +            break;
    +
    +          state = StateTableT::STATE_START_OF_TEXT;
    +          (void) buffer->next_glyph ();
    +          continue;
    +        }
    +      }
    +
           unsigned int klass = buffer->idx < buffer->len ?
    -                           machine.get_class (buffer->info[buffer->idx].codepoint, num_glyphs) :
    +                           machine.get_class (buffer->cur().codepoint, num_glyphs) :
                                (unsigned) StateTableT::CLASS_END_OF_TEXT;
           DEBUG_MSG (APPLY, nullptr, "c%u at %u", klass, buffer->idx);
           const EntryT &entry = machine.get_entry (state, klass);
    @@ -849,41 +913,6 @@ struct StateTableDriver
     };
     
     
    -struct ankr;
    -
    -struct hb_aat_apply_context_t :
    -       hb_dispatch_context_t
    -{
    -  const char *get_name () { return "APPLY"; }
    -  template 
    -  return_t dispatch (const T &obj) { return obj.apply (this); }
    -  static return_t default_return_value () { return false; }
    -  bool stop_sublookup_iteration (return_t r) const { return r; }
    -
    -  const hb_ot_shape_plan_t *plan;
    -  hb_font_t *font;
    -  hb_face_t *face;
    -  hb_buffer_t *buffer;
    -  hb_sanitize_context_t sanitizer;
    -  const ankr *ankr_table;
    -  const OT::GDEF *gdef_table;
    -
    -  /* Unused. For debug tracing only. */
    -  unsigned int lookup_index;
    -
    -  HB_INTERNAL hb_aat_apply_context_t (const hb_ot_shape_plan_t *plan_,
    -                                      hb_font_t *font_,
    -                                      hb_buffer_t *buffer_,
    -                                      hb_blob_t *blob = const_cast (&Null (hb_blob_t)));
    -
    -  HB_INTERNAL ~hb_aat_apply_context_t ();
    -
    -  HB_INTERNAL void set_ankr_table (const AAT::ankr *ankr_table_);
    -
    -  void set_lookup_index (unsigned int i) { lookup_index = i; }
    -};
    -
    -
     } /* namespace AAT */
     
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-feat-table.hh b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-feat-table.hh
    index 527bfe45b35da..04260a946168a 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-feat-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-feat-table.hh
    @@ -62,7 +62,7 @@ struct SettingName
       bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
    -    return_trace (likely (c->check_struct (this)));
    +    return_trace (c->check_struct (this));
       }
     
       protected:
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-just-table.hh b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-just-table.hh
    index 4b1319ac31ca7..0588310b53b78 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-just-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-just-table.hh
    @@ -48,7 +48,7 @@ struct ActionSubrecordHeader
       bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
    -    return_trace (likely (c->check_struct (this)));
    +    return_trace (c->check_struct (this));
       }
     
       HBUINT16      actionClass;    /* The JustClass value associated with this
    @@ -65,14 +65,14 @@ struct DecompositionAction
       bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
    -    return_trace (likely (c->check_struct (this)));
    +    return_trace (c->check_struct (this));
       }
     
       ActionSubrecordHeader
                     header;
    -  HBFixed       lowerLimit;     /* If the distance factor is less than this value,
    +  F16DOT16      lowerLimit;     /* If the distance factor is less than this value,
                                      * then the ligature is decomposed. */
    -  HBFixed       upperLimit;     /* If the distance factor is greater than this value,
    +  F16DOT16      upperLimit;     /* If the distance factor is greater than this value,
                                      * then the ligature is decomposed. */
       HBUINT16      order;          /* Numerical order in which this ligature will
                                      * be decomposed; you may want infrequent ligatures
    @@ -112,13 +112,13 @@ struct ConditionalAddGlyphAction
       bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
    -    return_trace (likely (c->check_struct (this)));
    +    return_trace (c->check_struct (this));
       }
     
       protected:
       ActionSubrecordHeader
                     header;
    -  HBFixed       substThreshold; /* Distance growth factor (in ems) at which
    +  F16DOT16      substThreshold; /* Distance growth factor (in ems) at which
                                      * this glyph is replaced and the growth factor
                                      * recalculated. */
       HBGlyphID16   addGlyph;       /* Glyph to be added as kashida. If this value is
    @@ -137,7 +137,7 @@ struct DuctileGlyphAction
       bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
    -    return_trace (likely (c->check_struct (this)));
    +    return_trace (c->check_struct (this));
       }
     
       protected:
    @@ -146,13 +146,13 @@ struct DuctileGlyphAction
       HBUINT32      variationAxis;  /* The 4-byte tag identifying the ductile axis.
                                      * This would normally be 0x64756374 ('duct'),
                                      * but you may use any axis the font contains. */
    -  HBFixed       minimumLimit;   /* The lowest value for the ductility axis that
    +  F16DOT16      minimumLimit;   /* The lowest value for the ductility axis that
                                      * still yields an acceptable appearance. Normally
                                      * this will be 1.0. */
    -  HBFixed       noStretchValue; /* This is the default value that corresponds to
    +  F16DOT16      noStretchValue; /* This is the default value that corresponds to
                                      * no change in appearance. Normally, this will
                                      * be 1.0. */
    -  HBFixed       maximumLimit;   /* The highest value for the ductility axis that
    +  F16DOT16      maximumLimit;   /* The highest value for the ductility axis that
                                      * still yields an acceptable appearance. */
       public:
       DEFINE_SIZE_STATIC (22);
    @@ -163,7 +163,7 @@ struct RepeatedAddGlyphAction
       bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
    -    return_trace (likely (c->check_struct (this)));
    +    return_trace (c->check_struct (this));
       }
     
       protected:
    @@ -271,14 +271,14 @@ struct JustWidthDeltaEntry
       };
     
       protected:
    -  HBFixed       beforeGrowLimit;/* The ratio by which the advance width of the
    +  F16DOT16      beforeGrowLimit;/* The ratio by which the advance width of the
                                      * glyph is permitted to grow on the left or top side. */
    -  HBFixed       beforeShrinkLimit;
    +  F16DOT16      beforeShrinkLimit;
                                     /* The ratio by which the advance width of the
                                      * glyph is permitted to shrink on the left or top side. */
    -  HBFixed       afterGrowLimit; /* The ratio by which the advance width of the glyph
    +  F16DOT16      afterGrowLimit; /* The ratio by which the advance width of the glyph
                                      * is permitted to shrink on the left or top side. */
    -  HBFixed       afterShrinkLimit;
    +  F16DOT16      afterShrinkLimit;
                                     /* The ratio by which the advance width of the glyph
                                      * is at most permitted to shrink on the right or
                                      * bottom side. */
    @@ -294,7 +294,7 @@ struct WidthDeltaPair
       bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
    -    return_trace (likely (c->check_struct (this)));
    +    return_trace (c->check_struct (this));
       }
     
       protected:
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-kerx-table.hh b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-kerx-table.hh
    index 51cb106bb08b1..2bc7b03ce896f 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-kerx-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-kerx-table.hh
    @@ -350,7 +350,7 @@ struct KerxSubTableFormat1
         driver_context_t dc (this, c);
     
         StateTableDriver driver (machine, c->buffer, c->font->face);
    -    driver.drive (&dc);
    +    driver.drive (&dc, c);
     
         return_trace (true);
       }
    @@ -594,7 +594,7 @@ struct KerxSubTableFormat4
         driver_context_t dc (this, c);
     
         StateTableDriver driver (machine, c->buffer, c->font->face);
    -    driver.drive (&dc);
    +    driver.drive (&dc, c);
     
         return_trace (true);
       }
    @@ -751,7 +751,7 @@ struct KerxSubTableHeader
       bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
    -    return_trace (likely (c->check_struct (this)));
    +    return_trace (c->check_struct (this));
       }
     
       public:
    @@ -869,6 +869,8 @@ struct KerxTable
     
       bool apply (AAT::hb_aat_apply_context_t *c) const
       {
    +    c->buffer->unsafe_to_concat ();
    +
         typedef typename T::SubTable SubTable;
     
         bool ret = false;
    @@ -889,7 +891,7 @@ struct KerxTable
           reverse = bool (st->u.header.coverage & st->u.header.Backwards) !=
                     HB_DIRECTION_IS_BACKWARD (c->buffer->props.direction);
     
    -      if (!c->buffer->message (c->font, "start subtable %d", c->lookup_index))
    +      if (!c->buffer->message (c->font, "start subtable %u", c->lookup_index))
             goto skip;
     
           if (!seenCrossStream &&
    @@ -921,7 +923,7 @@ struct KerxTable
           if (reverse)
             c->buffer->reverse ();
     
    -      (void) c->buffer->message (c->font, "end subtable %d", c->lookup_index);
    +      (void) c->buffer->message (c->font, "end subtable %u", c->lookup_index);
     
         skip:
           st = &StructAfter (*st);
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-morx-table.hh b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-morx-table.hh
    index 889784e7d5264..81e126d5eb11a 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-morx-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-morx-table.hh
    @@ -131,14 +131,14 @@ struct RearrangementSubtable
               hb_glyph_info_t *info = buffer->info;
               hb_glyph_info_t buf[4];
     
    -          memcpy (buf, info + start, l * sizeof (buf[0]));
    -          memcpy (buf + 2, info + end - r, r * sizeof (buf[0]));
    +          hb_memcpy (buf, info + start, l * sizeof (buf[0]));
    +          hb_memcpy (buf + 2, info + end - r, r * sizeof (buf[0]));
     
               if (l != r)
                 memmove (info + start + r, info + start + l, (end - start - l - r) * sizeof (buf[0]));
     
    -          memcpy (info + start, buf + 2, r * sizeof (buf[0]));
    -          memcpy (info + end - l, buf, l * sizeof (buf[0]));
    +          hb_memcpy (info + start, buf + 2, r * sizeof (buf[0]));
    +          hb_memcpy (info + end - l, buf, l * sizeof (buf[0]));
               if (reverse_l)
               {
                 buf[0] = info[end - 1];
    @@ -169,7 +169,7 @@ struct RearrangementSubtable
         driver_context_t dc (this);
     
         StateTableDriver driver (machine, c->buffer, c->face);
    -    driver.drive (&dc);
    +    driver.drive (&dc, c);
     
         return_trace (dc.ret);
       }
    @@ -325,7 +325,7 @@ struct ContextualSubtable
         driver_context_t dc (this, c);
     
         StateTableDriver driver (machine, c->buffer, c->face);
    -    driver.drive (&dc);
    +    driver.drive (&dc, c);
     
         return_trace (dc.ret);
       }
    @@ -525,7 +525,7 @@ struct LigatureSubtable
               if (unlikely (!componentData.sanitize (&c->sanitizer))) break;
               ligature_idx += componentData;
     
    -          DEBUG_MSG (APPLY, nullptr, "Action store %u last %u",
    +          DEBUG_MSG (APPLY, nullptr, "Action store %d last %d",
                          bool (action & LigActionStore),
                          bool (action & LigActionLast));
               if (action & (LigActionStore | LigActionLast))
    @@ -577,7 +577,7 @@ struct LigatureSubtable
         driver_context_t dc (this, c);
     
         StateTableDriver driver (machine, c->buffer, c->face);
    -    driver.drive (&dc);
    +    driver.drive (&dc, c);
     
         return_trace (dc.ret);
       }
    @@ -618,8 +618,27 @@ struct NoncontextualSubtable
     
         hb_glyph_info_t *info = c->buffer->info;
         unsigned int count = c->buffer->len;
    +    // If there's only one range, we already checked the flag.
    +    auto *last_range = c->range_flags && (c->range_flags->length > 1) ? &(*c->range_flags)[0] : nullptr;
         for (unsigned int i = 0; i < count; i++)
         {
    +      /* This block copied from StateTableDriver::drive. Keep in sync. */
    +      if (last_range)
    +      {
    +        auto *range = last_range;
    +        {
    +          unsigned cluster = info[i].cluster;
    +          while (cluster < range->cluster_first)
    +            range--;
    +          while (cluster > range->cluster_last)
    +            range++;
    +
    +          last_range = range;
    +        }
    +        if (!(range->flags & c->subtable_flags))
    +          continue;
    +      }
    +
           const HBGlyphID16 *replacement = substitute.get_value (info[i].codepoint, num_glyphs);
           if (replacement)
           {
    @@ -820,7 +839,7 @@ struct InsertionSubtable
         driver_context_t dc (this, c);
     
         StateTableDriver driver (machine, c->buffer, c->face);
    -    driver.drive (&dc);
    +    driver.drive (&dc, c);
     
         return_trace (dc.ret);
       }
    @@ -968,7 +987,7 @@ struct Chain
             // Check whether this type/setting pair was requested in the map, and if so, apply its flags.
             // (The search here only looks at the type and setting fields of feature_info_t.)
             hb_aat_map_builder_t::feature_info_t info = { type, setting, false, 0 };
    -        if (map->features.bsearch (info))
    +        if (map->current_features.bsearch (info))
             {
               flags &= feature.disableFlags;
               flags |= feature.enableFlags;
    @@ -980,13 +999,21 @@ struct Chain
               setting = HB_AAT_LAYOUT_FEATURE_SELECTOR_LOWER_CASE_SMALL_CAPS;
               goto retry;
             }
    +#ifndef HB_NO_AAT
    +        else if (type == HB_AAT_LAYOUT_FEATURE_TYPE_LANGUAGE_TAG_TYPE && setting &&
    +                 /* TODO: Rudimentary language matching. */
    +                 hb_language_matches (map->face->table.ltag->get_language (setting - 1), map->props.language))
    +        {
    +          flags &= feature.disableFlags;
    +          flags |= feature.enableFlags;
    +        }
    +#endif
           }
         }
         return flags;
       }
     
    -  void apply (hb_aat_apply_context_t *c,
    -              hb_mask_t flags) const
    +  void apply (hb_aat_apply_context_t *c) const
       {
         const ChainSubtable *subtable = &StructAfter> (featureZ.as_array (featureCount));
         unsigned int count = subtableCount;
    @@ -994,8 +1021,10 @@ struct Chain
         {
           bool reverse;
     
    -      if (!(subtable->subFeatureFlags & flags))
    +      if (hb_none (hb_iter (c->range_flags) |
    +                   hb_map ([&subtable] (const hb_aat_map_t::range_flags_t _) -> bool { return subtable->subFeatureFlags & (_.flags); })))
             goto skip;
    +      c->subtable_flags = subtable->subFeatureFlags;
     
           if (!(subtable->get_coverage() & ChainSubtable::AllDirections) &&
               HB_DIRECTION_IS_VERTICAL (c->buffer->props.direction) !=
    @@ -1034,7 +1063,7 @@ struct Chain
                     bool (subtable->get_coverage () & ChainSubtable::Backwards) !=
                     HB_DIRECTION_IS_BACKWARD (c->buffer->props.direction);
     
    -      if (!c->buffer->message (c->font, "start chainsubtable %d", c->lookup_index))
    +      if (!c->buffer->message (c->font, "start chainsubtable %u", c->lookup_index))
             goto skip;
     
           if (reverse)
    @@ -1045,7 +1074,7 @@ struct Chain
           if (reverse)
             c->buffer->reverse ();
     
    -      (void) c->buffer->message (c->font, "end chainsubtable %d", c->lookup_index);
    +      (void) c->buffer->message (c->font, "end chainsubtable %u", c->lookup_index);
     
           if (unlikely (!c->buffer->successful)) return;
     
    @@ -1111,22 +1140,31 @@ struct mortmorx
       {
         const Chain *chain = &firstChain;
         unsigned int count = chainCount;
    +    if (unlikely (!map->chain_flags.resize (count)))
    +      return;
         for (unsigned int i = 0; i < count; i++)
         {
    -      map->chain_flags.push (chain->compile_flags (mapper));
    +      map->chain_flags[i].push (hb_aat_map_t::range_flags_t {chain->compile_flags (mapper),
    +                                                             mapper->range_first,
    +                                                             mapper->range_last});
           chain = &StructAfter> (*chain);
         }
       }
     
    -  void apply (hb_aat_apply_context_t *c) const
    +  void apply (hb_aat_apply_context_t *c,
    +              const hb_aat_map_t &map) const
       {
         if (unlikely (!c->buffer->successful)) return;
    +
    +    c->buffer->unsafe_to_concat ();
    +
         c->set_lookup_index (0);
         const Chain *chain = &firstChain;
         unsigned int count = chainCount;
         for (unsigned int i = 0; i < count; i++)
         {
    -      chain->apply (c, c->plan->aat_map.chain_flags[i]);
    +      c->range_flags = &map.chain_flags[i];
    +      chain->apply (c);
           if (unlikely (!c->buffer->successful)) return;
           chain = &StructAfter> (*chain);
         }
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-opbd-table.hh b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-opbd-table.hh
    index 3054c76b96b47..959382f356fb0 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-opbd-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-opbd-table.hh
    @@ -42,7 +42,7 @@ struct OpticalBounds
       bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
    -    return_trace (likely (c->check_struct (this)));
    +    return_trace (c->check_struct (this));
       }
     
       FWORD         leftSide;
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-trak-table.hh b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-trak-table.hh
    index d99a53b14c438..cb53128349888 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-trak-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout-trak-table.hh
    @@ -62,7 +62,7 @@ struct TrackTableEntry
       }
     
       protected:
    -  HBFixed       track;          /* Track value for this record. */
    +  F16DOT16      track;          /* Track value for this record. */
       NameID        trackNameID;    /* The 'name' table index for this track.
                                      * (a short word or phrase like "loose"
                                      * or "very tight") */
    @@ -82,7 +82,7 @@ struct TrackData
                             const void *base) const
       {
         unsigned int sizes = nSizes;
    -    hb_array_t size_table ((base+sizeTable).arrayZ, sizes);
    +    hb_array_t size_table ((base+sizeTable).arrayZ, sizes);
     
         float s0 = size_table[idx].to_float ();
         float s1 = size_table[idx + 1].to_float ();
    @@ -120,7 +120,7 @@ struct TrackData
         if (!sizes) return 0.;
         if (sizes == 1) return trackTableEntry->get_value (base, 0, sizes);
     
    -    hb_array_t size_table ((base+sizeTable).arrayZ, sizes);
    +    hb_array_t size_table ((base+sizeTable).arrayZ, sizes);
         unsigned int size_index;
         for (size_index = 0; size_index < sizes - 1; size_index++)
           if (size_table[size_index].to_float () >= ptem)
    @@ -141,7 +141,7 @@ struct TrackData
       protected:
       HBUINT16      nTracks;        /* Number of separate tracks included in this table. */
       HBUINT16      nSizes;         /* Number of point sizes included in this table. */
    -  NNOffset32To>
    +  NNOffset32To>
                     sizeTable;      /* Offset from start of the tracking table to
                                      * Array[nSizes] of size values.. */
       UnsizedArrayOf
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout.cc b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout.cc
    index a9a9ce6b3261d..b0afbdfbb0453 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout.cc
    @@ -131,6 +131,7 @@ static const hb_aat_feature_mapping_t feature_mappings[] =
       {HB_TAG ('p','n','u','m'), HB_AAT_LAYOUT_FEATURE_TYPE_NUMBER_SPACING,          HB_AAT_LAYOUT_FEATURE_SELECTOR_PROPORTIONAL_NUMBERS,           (hb_aat_layout_feature_selector_t) 4},
       {HB_TAG ('p','w','i','d'), HB_AAT_LAYOUT_FEATURE_TYPE_TEXT_SPACING,            HB_AAT_LAYOUT_FEATURE_SELECTOR_PROPORTIONAL_TEXT,              (hb_aat_layout_feature_selector_t) 7},
       {HB_TAG ('q','w','i','d'), HB_AAT_LAYOUT_FEATURE_TYPE_TEXT_SPACING,            HB_AAT_LAYOUT_FEATURE_SELECTOR_QUARTER_WIDTH_TEXT,             (hb_aat_layout_feature_selector_t) 7},
    +  {HB_TAG ('r','l','i','g'), HB_AAT_LAYOUT_FEATURE_TYPE_LIGATURES,               HB_AAT_LAYOUT_FEATURE_SELECTOR_REQUIRED_LIGATURES_ON,          HB_AAT_LAYOUT_FEATURE_SELECTOR_REQUIRED_LIGATURES_OFF},
       {HB_TAG ('r','u','b','y'), HB_AAT_LAYOUT_FEATURE_TYPE_RUBY_KANA,               HB_AAT_LAYOUT_FEATURE_SELECTOR_RUBY_KANA_ON,                   HB_AAT_LAYOUT_FEATURE_SELECTOR_RUBY_KANA_OFF},
       {HB_TAG ('s','i','n','f'), HB_AAT_LAYOUT_FEATURE_TYPE_VERTICAL_POSITION,       HB_AAT_LAYOUT_FEATURE_SELECTOR_SCIENTIFIC_INFERIORS,           HB_AAT_LAYOUT_FEATURE_SELECTOR_NORMAL_POSITION},
       {HB_TAG ('s','m','c','p'), HB_AAT_LAYOUT_FEATURE_TYPE_LOWER_CASE,              HB_AAT_LAYOUT_FEATURE_SELECTOR_LOWER_CASE_SMALL_CAPS,          HB_AAT_LAYOUT_FEATURE_SELECTOR_DEFAULT_LOWER_CASE},
    @@ -229,7 +230,7 @@ hb_aat_layout_compile_map (const hb_aat_map_builder_t *mapper,
      *
      * Note: does not examine the `GSUB` table.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 2.3.0
      */
    @@ -243,15 +244,23 @@ hb_aat_layout_has_substitution (hb_face_t *face)
     void
     hb_aat_layout_substitute (const hb_ot_shape_plan_t *plan,
                               hb_font_t *font,
    -                          hb_buffer_t *buffer)
    +                          hb_buffer_t *buffer,
    +                          const hb_feature_t *features,
    +                          unsigned num_features)
     {
    +  hb_aat_map_builder_t builder (font->face, plan->props);
    +  for (unsigned i = 0; i < num_features; i++)
    +    builder.add_feature (features[i]);
    +  hb_aat_map_t map;
    +  builder.compile (map);
    +
       hb_blob_t *morx_blob = font->face->table.morx.get_blob ();
       const AAT::morx& morx = *morx_blob->as ();
       if (morx.has_data ())
       {
         AAT::hb_aat_apply_context_t c (plan, font, buffer, morx_blob);
         if (!buffer->message (font, "start table morx")) return;
    -    morx.apply (&c);
    +    morx.apply (&c, map);
         (void) buffer->message (font, "end table morx");
         return;
       }
    @@ -262,7 +271,7 @@ hb_aat_layout_substitute (const hb_ot_shape_plan_t *plan,
       {
         AAT::hb_aat_apply_context_t c (plan, font, buffer, mort_blob);
         if (!buffer->message (font, "start table mort")) return;
    -    mort.apply (&c);
    +    mort.apply (&c, map);
         (void) buffer->message (font, "end table mort");
         return;
       }
    @@ -288,7 +297,7 @@ is_deleted_glyph (const hb_glyph_info_t *info)
     void
     hb_aat_layout_remove_deleted_glyphs (hb_buffer_t *buffer)
     {
    -  hb_ot_layout_delete_glyphs_inplace (buffer, is_deleted_glyph);
    +  buffer->delete_glyphs_inplace (is_deleted_glyph);
     }
     
     /**
    @@ -300,7 +309,7 @@ hb_aat_layout_remove_deleted_glyphs (hb_buffer_t *buffer)
      *
      * Note: does not examine the `GPOS` table.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 2.3.0
      */
    @@ -333,7 +342,7 @@ hb_aat_layout_position (const hb_ot_shape_plan_t *plan,
      * Tests whether the specified face includes any tracking information
      * in the `trak` table.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 2.3.0
      */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout.hh b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout.hh
    index 1a95507ccee5e..36bb0ef07ddb0 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-aat-layout.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-aat-layout.hh
    @@ -53,7 +53,9 @@ hb_aat_layout_compile_map (const hb_aat_map_builder_t *mapper,
     HB_INTERNAL void
     hb_aat_layout_substitute (const hb_ot_shape_plan_t *plan,
                               hb_font_t *font,
    -                          hb_buffer_t *buffer);
    +                          hb_buffer_t *buffer,
    +                          const hb_feature_t *features,
    +                          unsigned num_features);
     
     HB_INTERNAL void
     hb_aat_layout_zero_width_deleted_glyphs (hb_buffer_t *buffer);
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-aat-map.cc b/src/java.desktop/share/native/libharfbuzz/hb-aat-map.cc
    index ad3eff7935fe2..122f29dc6816f 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-aat-map.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-aat-map.cc
    @@ -36,27 +36,29 @@
     #include "hb-aat-layout-feat-table.hh"
     
     
    -void hb_aat_map_builder_t::add_feature (hb_tag_t tag, unsigned value)
    +void hb_aat_map_builder_t::add_feature (const hb_feature_t &feature)
     {
       if (!face->table.feat->has_data ()) return;
     
    -  if (tag == HB_TAG ('a','a','l','t'))
    +  if (feature.tag == HB_TAG ('a','a','l','t'))
       {
         if (!face->table.feat->exposes_feature (HB_AAT_LAYOUT_FEATURE_TYPE_CHARACTER_ALTERNATIVES))
           return;
    -    feature_info_t *info = features.push();
    -    info->type = HB_AAT_LAYOUT_FEATURE_TYPE_CHARACTER_ALTERNATIVES;
    -    info->setting = (hb_aat_layout_feature_selector_t) value;
    -    info->seq = features.length;
    -    info->is_exclusive = true;
    +    feature_range_t *range = features.push();
    +    range->start = feature.start;
    +    range->end = feature.end;
    +    range->info.type = HB_AAT_LAYOUT_FEATURE_TYPE_CHARACTER_ALTERNATIVES;
    +    range->info.setting = (hb_aat_layout_feature_selector_t) feature.value;
    +    range->info.seq = features.length;
    +    range->info.is_exclusive = true;
         return;
       }
     
    -  const hb_aat_feature_mapping_t *mapping = hb_aat_layout_find_feature_mapping (tag);
    +  const hb_aat_feature_mapping_t *mapping = hb_aat_layout_find_feature_mapping (feature.tag);
       if (!mapping) return;
     
    -  const AAT::FeatureName* feature = &face->table.feat->get_feature (mapping->aatFeatureType);
    -  if (!feature->has_data ())
    +  const AAT::FeatureName* feature_name = &face->table.feat->get_feature (mapping->aatFeatureType);
    +  if (!feature_name->has_data ())
       {
         /* Special case: Chain::compile_flags will fall back to the deprecated version of
          * small-caps if necessary, so we need to check for that possibility.
    @@ -64,38 +66,106 @@ void hb_aat_map_builder_t::add_feature (hb_tag_t tag, unsigned value)
         if (mapping->aatFeatureType == HB_AAT_LAYOUT_FEATURE_TYPE_LOWER_CASE &&
             mapping->selectorToEnable == HB_AAT_LAYOUT_FEATURE_SELECTOR_LOWER_CASE_SMALL_CAPS)
         {
    -      feature = &face->table.feat->get_feature (HB_AAT_LAYOUT_FEATURE_TYPE_LETTER_CASE);
    -      if (!feature->has_data ()) return;
    +      feature_name = &face->table.feat->get_feature (HB_AAT_LAYOUT_FEATURE_TYPE_LETTER_CASE);
    +      if (!feature_name->has_data ()) return;
         }
         else return;
       }
     
    -  feature_info_t *info = features.push();
    -  info->type = mapping->aatFeatureType;
    -  info->setting = value ? mapping->selectorToEnable : mapping->selectorToDisable;
    -  info->seq = features.length;
    -  info->is_exclusive = feature->is_exclusive ();
    +  feature_range_t *range = features.push();
    +  range->start = feature.start;
    +  range->end = feature.end;
    +  range->info.type = mapping->aatFeatureType;
    +  range->info.setting = feature.value ? mapping->selectorToEnable : mapping->selectorToDisable;
    +  range->info.seq = features.length;
    +  range->info.is_exclusive = feature_name->is_exclusive ();
     }
     
     void
     hb_aat_map_builder_t::compile (hb_aat_map_t  &m)
     {
    -  /* Sort features and merge duplicates */
    -  if (features.length)
    +  /* Compute active features per range, and compile each. */
    +
    +  /* Sort features by start/end events. */
    +  hb_vector_t feature_events;
    +  for (unsigned int i = 0; i < features.length; i++)
    +  {
    +    auto &feature = features[i];
    +
    +    if (features[i].start == features[i].end)
    +      continue;
    +
    +    feature_event_t *event;
    +
    +    event = feature_events.push ();
    +    event->index = features[i].start;
    +    event->start = true;
    +    event->feature = feature.info;
    +
    +    event = feature_events.push ();
    +    event->index = features[i].end;
    +    event->start = false;
    +    event->feature = feature.info;
    +  }
    +  feature_events.qsort ();
    +  /* Add a strategic final event. */
    +  {
    +    feature_info_t feature;
    +    feature.seq = features.length + 1;
    +
    +    feature_event_t *event = feature_events.push ();
    +    event->index = -1; /* This value does magic. */
    +    event->start = false;
    +    event->feature = feature;
    +  }
    +
    +  /* Scan events and save features for each range. */
    +  hb_sorted_vector_t active_features;
    +  unsigned int last_index = 0;
    +  for (unsigned int i = 0; i < feature_events.length; i++)
       {
    -    features.qsort ();
    -    unsigned int j = 0;
    -    for (unsigned int i = 1; i < features.length; i++)
    -      if (features[i].type != features[j].type ||
    -          /* Nonexclusive feature selectors come in even/odd pairs to turn a setting on/off
    -           * respectively, so we mask out the low-order bit when checking for "duplicates"
    -           * (selectors referring to the same feature setting) here. */
    -          (!features[i].is_exclusive && ((features[i].setting & ~1) != (features[j].setting & ~1))))
    -        features[++j] = features[i];
    -    features.shrink (j + 1);
    +    feature_event_t *event = &feature_events[i];
    +
    +    if (event->index != last_index)
    +    {
    +      /* Save a snapshot of active features and the range. */
    +
    +      /* Sort features and merge duplicates */
    +      current_features = active_features;
    +      range_first = last_index;
    +      range_last = event->index - 1;
    +      if (current_features.length)
    +      {
    +        current_features.qsort ();
    +        unsigned int j = 0;
    +        for (unsigned int i = 1; i < current_features.length; i++)
    +          if (current_features[i].type != current_features[j].type ||
    +              /* Nonexclusive feature selectors come in even/odd pairs to turn a setting on/off
    +               * respectively, so we mask out the low-order bit when checking for "duplicates"
    +               * (selectors referring to the same feature setting) here. */
    +              (!current_features[i].is_exclusive && ((current_features[i].setting & ~1) != (current_features[j].setting & ~1))))
    +            current_features[++j] = current_features[i];
    +        current_features.shrink (j + 1);
    +      }
    +
    +      hb_aat_layout_compile_map (this, &m);
    +
    +      last_index = event->index;
    +    }
    +
    +    if (event->start)
    +    {
    +      active_features.push (event->feature);
    +    } else {
    +      feature_info_t *feature = active_features.lsearch (event->feature);
    +      if (feature)
    +        active_features.remove_ordered (feature - active_features.arrayZ);
    +    }
       }
     
    -  hb_aat_layout_compile_map (this, &m);
    +  for (auto &chain_flags : m.chain_flags)
    +    // With our above setup this value is one less than desired; adjust it.
    +    chain_flags.tail().cluster_last = HB_FEATURE_GLOBAL_END;
     }
     
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-aat-map.hh b/src/java.desktop/share/native/libharfbuzz/hb-aat-map.hh
    index ce30daa608451..b21f5bacb9e34 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-aat-map.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-aat-map.hh
    @@ -35,16 +35,15 @@ struct hb_aat_map_t
       friend struct hb_aat_map_builder_t;
     
       public:
    -
    -  void init ()
    +  struct range_flags_t
       {
    -    memset (this, 0, sizeof (*this));
    -    chain_flags.init ();
    -  }
    -  void fini () { chain_flags.fini (); }
    +    hb_mask_t flags;
    +    unsigned cluster_first;
    +    unsigned cluster_last; // end - 1
    +  };
     
       public:
    -  hb_vector_t chain_flags;
    +  hb_vector_t> chain_flags;
     };
     
     struct hb_aat_map_builder_t
    @@ -52,10 +51,11 @@ struct hb_aat_map_builder_t
       public:
     
       HB_INTERNAL hb_aat_map_builder_t (hb_face_t *face_,
    -                                    const hb_segment_properties_t *props_ HB_UNUSED) :
    -                                      face (face_) {}
    +                                    const hb_segment_properties_t props_) :
    +                                      face (face_),
    +                                      props (props_) {}
     
    -  HB_INTERNAL void add_feature (hb_tag_t tag, unsigned int value=1);
    +  HB_INTERNAL void add_feature (const hb_feature_t &feature);
     
       HB_INTERNAL void compile (hb_aat_map_t  &m);
     
    @@ -77,7 +77,7 @@ struct hb_aat_map_builder_t
                 return (a->seq < b->seq ? -1 : a->seq > b->seq ? 1 : 0);
         }
     
    -    /* compares type & setting only, not is_exclusive flag or seq number */
    +    /* compares type & setting only */
         int cmp (const feature_info_t& f) const
         {
           return (f.type != type) ? (f.type < type ? -1 : 1) :
    @@ -85,11 +85,38 @@ struct hb_aat_map_builder_t
         }
       };
     
    +  struct feature_range_t
    +  {
    +    feature_info_t info;
    +    unsigned start;
    +    unsigned end;
    +  };
    +
    +  private:
    +  struct feature_event_t
    +  {
    +    unsigned int index;
    +    bool start;
    +    feature_info_t feature;
    +
    +    HB_INTERNAL static int cmp (const void *pa, const void *pb) {
    +      const feature_event_t *a = (const feature_event_t *) pa;
    +      const feature_event_t *b = (const feature_event_t *) pb;
    +      return a->index < b->index ? -1 : a->index > b->index ? 1 :
    +             a->start < b->start ? -1 : a->start > b->start ? 1 :
    +             feature_info_t::cmp (&a->feature, &b->feature);
    +    }
    +  };
    +
       public:
       hb_face_t *face;
    +  hb_segment_properties_t props;
     
       public:
    -  hb_sorted_vector_t features;
    +  hb_sorted_vector_t features;
    +  hb_sorted_vector_t current_features;
    +  unsigned range_first = HB_FEATURE_GLOBAL_START;
    +  unsigned range_last = HB_FEATURE_GLOBAL_END;
     };
     
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-algs.hh b/src/java.desktop/share/native/libharfbuzz/hb-algs.hh
    index 162601b380d36..e2b970f968f14 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-algs.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-algs.hh
    @@ -109,15 +109,17 @@ struct BEInt
       struct __attribute__((packed)) packed_uint16_t { uint16_t v; };
       constexpr operator Type () const
       {
    -#if ((defined(__GNUC__) && __GNUC__ >= 5) || defined(__clang__)) && \
    +#if defined(__OPTIMIZE__) && !defined(HB_NO_PACKED) && \
         defined(__BYTE_ORDER) && \
    -    (__BYTE_ORDER == __LITTLE_ENDIAN || __BYTE_ORDER == __BIG_ENDIAN)
    +    (__BYTE_ORDER == __BIG_ENDIAN || \
    +     (__BYTE_ORDER == __LITTLE_ENDIAN && \
    +      hb_has_builtin(__builtin_bswap16)))
         /* Spoon-feed the compiler a big-endian integer with alignment 1.
          * https://github.com/harfbuzz/harfbuzz/pull/1398 */
     #if __BYTE_ORDER == __LITTLE_ENDIAN
    -    return __builtin_bswap16 (((packed_uint16_t *) this)->v);
    +    return __builtin_bswap16 (((packed_uint16_t *) v)->v);
     #else /* __BYTE_ORDER == __BIG_ENDIAN */
    -    return ((packed_uint16_t *) this)->v;
    +    return ((packed_uint16_t *) v)->v;
     #endif
     #else
         return (v[0] <<  8)
    @@ -153,15 +155,17 @@ struct BEInt
     
       struct __attribute__((packed)) packed_uint32_t { uint32_t v; };
       constexpr operator Type () const {
    -#if ((defined(__GNUC__) && __GNUC__ >= 5) || defined(__clang__)) && \
    +#if defined(__OPTIMIZE__) && !defined(HB_NO_PACKED) && \
         defined(__BYTE_ORDER) && \
    -    (__BYTE_ORDER == __LITTLE_ENDIAN || __BYTE_ORDER == __BIG_ENDIAN)
    +    (__BYTE_ORDER == __BIG_ENDIAN || \
    +     (__BYTE_ORDER == __LITTLE_ENDIAN && \
    +      hb_has_builtin(__builtin_bswap32)))
         /* Spoon-feed the compiler a big-endian integer with alignment 1.
          * https://github.com/harfbuzz/harfbuzz/pull/1398 */
     #if __BYTE_ORDER == __LITTLE_ENDIAN
    -    return __builtin_bswap32 (((packed_uint32_t *) this)->v);
    +    return __builtin_bswap32 (((packed_uint32_t *) v)->v);
     #else /* __BYTE_ORDER == __BIG_ENDIAN */
    -    return ((packed_uint32_t *) this)->v;
    +    return ((packed_uint32_t *) v)->v;
     #endif
     #else
         return (v[0] << 24)
    @@ -234,17 +238,6 @@ struct
       template  constexpr auto
       impl (const T& v, hb_priority<1>) const HB_RETURN (uint32_t, hb_deref (v).hash ())
     
    -  template  constexpr uint32_t
    -  impl (const hb::shared_ptr& v, hb_priority<1>) const
    -  {
    -    return v.get () ? v.get ()->hash () : 0;
    -  }
    -  template  constexpr uint32_t
    -  impl (const hb::unique_ptr& v, hb_priority<1>) const
    -  {
    -    return v.get () ? v.get ()->hash () : 0;
    -  }
    -
       template  constexpr auto
       impl (const T& v, hb_priority<0>) const HB_RETURN (uint32_t, std::hash>{} (hb_deref (v)))
     
    @@ -493,6 +486,17 @@ struct
     }
     HB_FUNCOBJ (hb_equal);
     
    +struct
    +{
    +  template  void
    +  operator () (T& a, T& b) const
    +  {
    +    using std::swap; // allow ADL
    +    swap (a, b);
    +  }
    +}
    +HB_FUNCOBJ (hb_swap);
    +
     
     template 
     struct hb_pair_t
    @@ -505,7 +509,7 @@ struct hb_pair_t
                 hb_enable_if (std::is_default_constructible::value &&
                               std::is_default_constructible::value)>
       hb_pair_t () : first (), second () {}
    -  hb_pair_t (T1 a, T2 b) : first (a), second (b) {}
    +  hb_pair_t (T1 a, T2 b) : first (std::forward (a)), second (std::forward (b)) {}
     
       template  (const pair_t& o) const { return first > o.first || (first == o.first && second > o.second); }
       bool operator <= (const pair_t& o) const { return !(*this > o); }
     
    +  static int cmp (const void *pa, const void *pb)
    +  {
    +    pair_t *a = (pair_t *) pa;
    +    pair_t *b = (pair_t *) pb;
    +
    +    if (a->first < b->first) return -1;
    +    if (a->first > b->first) return +1;
    +    if (a->second < b->second) return -1;
    +    if (a->second > b->second) return +1;
    +    return 0;
    +  }
    +
    +  friend void swap (hb_pair_t& a, hb_pair_t& b)
    +  {
    +    hb_swap (a.first, b.first);
    +    hb_swap (a.second, b.second);
    +  }
    +
    +
       T1 first;
       T2 second;
     };
    -#define hb_pair_t(T1,T2) hb_pair_t
     template  static inline hb_pair_t
     hb_pair (T1&& a, T2&& b) { return hb_pair_t (a, b); }
     
    @@ -551,14 +573,14 @@ struct
     {
       template  constexpr auto
       operator () (T&& a, T2&& b) const HB_AUTO_RETURN
    -  (a <= b ? std::forward (a) : std::forward (b))
    +  (a <= b ? a : b)
     }
     HB_FUNCOBJ (hb_min);
     struct
     {
       template  constexpr auto
       operator () (T&& a, T2&& b) const HB_AUTO_RETURN
    -  (a >= b ? std::forward (a) : std::forward (b))
    +  (a >= b ? a : b)
     }
     HB_FUNCOBJ (hb_max);
     struct
    @@ -569,17 +591,6 @@ struct
     }
     HB_FUNCOBJ (hb_clamp);
     
    -struct
    -{
    -  template  void
    -  operator () (T& a, T& b) const
    -  {
    -    using std::swap; // allow ADL
    -    swap (a, b);
    -  }
    -}
    -HB_FUNCOBJ (hb_swap);
    -
     /*
      * Bithacks.
      */
    @@ -589,13 +600,17 @@ template 
     static inline unsigned int
     hb_popcount (T v)
     {
    -#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)
    +#if hb_has_builtin(__builtin_popcount)
       if (sizeof (T) <= sizeof (unsigned int))
         return __builtin_popcount (v);
    +#endif
     
    +#if hb_has_builtin(__builtin_popcountl)
       if (sizeof (T) <= sizeof (unsigned long))
         return __builtin_popcountl (v);
    +#endif
     
    +#if hb_has_builtin(__builtin_popcountll)
       if (sizeof (T) <= sizeof (unsigned long long))
         return __builtin_popcountll (v);
     #endif
    @@ -632,13 +647,17 @@ hb_bit_storage (T v)
     {
       if (unlikely (!v)) return 0;
     
    -#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)
    +#if hb_has_builtin(__builtin_clz)
       if (sizeof (T) <= sizeof (unsigned int))
         return sizeof (unsigned int) * 8 - __builtin_clz (v);
    +#endif
     
    +#if hb_has_builtin(__builtin_clzl)
       if (sizeof (T) <= sizeof (unsigned long))
         return sizeof (unsigned long) * 8 - __builtin_clzl (v);
    +#endif
     
    +#if hb_has_builtin(__builtin_clzll)
       if (sizeof (T) <= sizeof (unsigned long long))
         return sizeof (unsigned long long) * 8 - __builtin_clzll (v);
     #endif
    @@ -706,13 +725,17 @@ hb_ctz (T v)
     {
       if (unlikely (!v)) return 8 * sizeof (T);
     
    -#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)
    +#if hb_has_builtin(__builtin_ctz)
       if (sizeof (T) <= sizeof (unsigned int))
         return __builtin_ctz (v);
    +#endif
     
    +#if hb_has_builtin(__builtin_ctzl)
       if (sizeof (T) <= sizeof (unsigned long))
         return __builtin_ctzl (v);
    +#endif
     
    +#if hb_has_builtin(__builtin_ctzll)
       if (sizeof (T) <= sizeof (unsigned long long))
         return __builtin_ctzll (v);
     #endif
    @@ -848,19 +871,14 @@ hb_in_range (T u, T lo, T hi)
       return (T)(u - lo) <= (T)(hi - lo);
     }
     template  static inline bool
    -hb_in_ranges (T u, T lo1, T hi1, T lo2, T hi2)
    +hb_in_ranges (T u, T lo1, T hi1)
     {
    -  return hb_in_range (u, lo1, hi1) || hb_in_range (u, lo2, hi2);
    +  return hb_in_range (u, lo1, hi1);
     }
    -template  static inline bool
    -hb_in_ranges (T u, T lo1, T hi1, T lo2, T hi2, T lo3, T hi3)
    -{
    -  return hb_in_range (u, lo1, hi1) || hb_in_range (u, lo2, hi2) || hb_in_range (u, lo3, hi3);
    -}
    -template  static inline bool
    -hb_in_ranges (T u, T lo1, T hi1, T lo2, T hi2, T lo3, T hi3, T lo4, T hi4)
    +template  static inline bool
    +hb_in_ranges (T u, T lo1, T hi1, Ts... ds)
     {
    -  return hb_in_range (u, lo1, hi1) || hb_in_range (u, lo2, hi2) || hb_in_range (u, lo3, hi3) || hb_in_range (u, lo4, hi4);
    +  return hb_in_range (u, lo1, hi1) || hb_in_ranges (u, ds...);
     }
     
     
    @@ -868,10 +886,18 @@ hb_in_ranges (T u, T lo1, T hi1, T lo2, T hi2, T lo3, T hi3, T lo4, T hi4)
      * Overflow checking.
      */
     
    -/* Consider __builtin_mul_overflow use here also */
     static inline bool
    -hb_unsigned_mul_overflows (unsigned int count, unsigned int size)
    +hb_unsigned_mul_overflows (unsigned int count, unsigned int size, unsigned *result = nullptr)
     {
    +#if hb_has_builtin(__builtin_mul_overflow)
    +  unsigned stack_result;
    +  if (!result)
    +    result = &stack_result;
    +  return __builtin_mul_overflow (count, size, result);
    +#endif
    +
    +  if (result)
    +    *result = count * size;
       return (size > 0) && (count >= ((unsigned int) -1) / size);
     }
     
    @@ -972,7 +998,7 @@ void hb_qsort(void *base, size_t nel, size_t width,
                   [void *arg]);
     */
     
    -#define SORT_R_SWAP(a,b,tmp) ((tmp) = (a), (a) = (b), (b) = (tmp))
    +#define SORT_R_SWAP(a,b,tmp) ((void) ((tmp) = (a)), (void) ((a) = (b)), (b) = (tmp))
     
     /* swap a and b */
     /* a and b must not be equal! */
    @@ -1163,9 +1189,12 @@ hb_qsort (void *base, size_t nel, size_t width,
     }
     
     
    -template  static inline void
    -hb_stable_sort (T *array, unsigned int len, int(*compar)(const T2 *, const T2 *), T3 *array2)
    +template  static inline void
    +hb_stable_sort (T *array, unsigned int len, int(*compar)(const T2 *, const T2 *), T3 *array2 = nullptr)
     {
    +  static_assert (hb_is_trivially_copy_assignable (T), "");
    +  static_assert (hb_is_trivially_copy_assignable (T3), "");
    +
       for (unsigned int i = 1; i < len; i++)
       {
         unsigned int j = i;
    @@ -1188,12 +1217,6 @@ hb_stable_sort (T *array, unsigned int len, int(*compar)(const T2 *, const T2 *)
       }
     }
     
    -template  static inline void
    -hb_stable_sort (T *array, unsigned int len, int(*compar)(const T *, const T *))
    -{
    -  hb_stable_sort (array, len, compar, (int *) nullptr);
    -}
    -
     static inline hb_bool_t
     hb_codepoint_parse (const char *s, unsigned int len, int base, hb_codepoint_t *out)
     {
    @@ -1321,47 +1344,62 @@ struct
     HB_FUNCOBJ (hb_dec);
     
     
    -/* Compiler-assisted vectorization. */
    -
    -/* Type behaving similar to vectorized vars defined using __attribute__((vector_size(...))),
    - * basically a fixed-size bitset. */
    -template 
    -struct hb_vector_size_t
    +/* Adapted from kurbo implementation with extra parameters added,
    + * and finding for a particular range instead of 0.
    + *
    + * For documentation and implementation see:
    + *
    + * [ITP method]: https://en.wikipedia.org/wiki/ITP_Method
    + * [An Enhancement of the Bisection Method Average Performance Preserving Minmax Optimality]: https://dl.acm.org/doi/10.1145/3423597
    + * https://docs.rs/kurbo/0.8.1/kurbo/common/fn.solve_itp.html
    + * https://github.com/linebender/kurbo/blob/fd839c25ea0c98576c7ce5789305822675a89938/src/common.rs#L162-L248
    + */
    +template 
    +double solve_itp (func_t f,
    +                  double a, double b,
    +                  double epsilon,
    +                  double min_y, double max_y,
    +                  double &ya, double &yb, double &y)
     {
    -  elt_t& operator [] (unsigned int i) { return v[i]; }
    -  const elt_t& operator [] (unsigned int i) const { return v[i]; }
    -
    -  void clear (unsigned char v = 0) { memset (this, v, sizeof (*this)); }
    -
    -  template 
    -  hb_vector_size_t process (const Op& op) const
    +  unsigned n1_2 = (unsigned) (hb_max (ceil (log2 ((b - a) / epsilon)) - 1.0, 0.0));
    +  const unsigned n0 = 1; // Hardwired
    +  const double k1 = 0.2 / (b - a); // Hardwired.
    +  unsigned nmax = n0 + n1_2;
    +  double scaled_epsilon = epsilon * double (1llu << nmax);
    +  double _2_epsilon = 2.0 * epsilon;
    +  while (b - a > _2_epsilon)
       {
    -    hb_vector_size_t r;
    -    for (unsigned int i = 0; i < ARRAY_LENGTH (v); i++)
    -      r.v[i] = op (v[i]);
    -    return r;
    -  }
    -  template 
    -  hb_vector_size_t process (const Op& op, const hb_vector_size_t &o) const
    -  {
    -    hb_vector_size_t r;
    -    for (unsigned int i = 0; i < ARRAY_LENGTH (v); i++)
    -      r.v[i] = op (v[i], o.v[i]);
    -    return r;
    +    double x1_2 = 0.5 * (a + b);
    +    double r = scaled_epsilon - 0.5 * (b - a);
    +    double xf = (yb * a - ya * b) / (yb - ya);
    +    double sigma = x1_2 - xf;
    +    double b_a = b - a;
    +    // This has k2 = 2 hardwired for efficiency.
    +    double b_a_k2 = b_a * b_a;
    +    double delta = k1 * b_a_k2;
    +    int sigma_sign = sigma >= 0 ? +1 : -1;
    +    double xt = delta <= fabs (x1_2 - xf) ? xf + delta * sigma_sign : x1_2;
    +    double xitp = fabs (xt - x1_2) <= r ? xt : x1_2 - r * sigma_sign;
    +    double yitp = f (xitp);
    +    if (yitp > max_y)
    +    {
    +      b = xitp;
    +      yb = yitp;
    +    }
    +    else if (yitp < min_y)
    +    {
    +      a = xitp;
    +      ya = yitp;
    +    }
    +    else
    +    {
    +      y = yitp;
    +      return xitp;
    +    }
    +    scaled_epsilon *= 0.5;
       }
    -  hb_vector_size_t operator | (const hb_vector_size_t &o) const
    -  { return process (hb_bitwise_or, o); }
    -  hb_vector_size_t operator & (const hb_vector_size_t &o) const
    -  { return process (hb_bitwise_and, o); }
    -  hb_vector_size_t operator ^ (const hb_vector_size_t &o) const
    -  { return process (hb_bitwise_xor, o); }
    -  hb_vector_size_t operator ~ () const
    -  { return process (hb_bitwise_neg); }
    -
    -  private:
    -  static_assert (0 == byte_size % sizeof (elt_t), "");
    -  elt_t v[byte_size / sizeof (elt_t)];
    -};
    +  return 0.5 * (a + b);
    +}
     
     
     #endif /* HB_ALGS_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-array.hh b/src/java.desktop/share/native/libharfbuzz/hb-array.hh
    index 0a92392d319e0..08b25987061ac 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-array.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-array.hh
    @@ -100,10 +100,17 @@ struct hb_array_t : hb_iter_with_fallback_t, Type&>
       /* Ouch. The operator== compares the contents of the array.  For range-based for loops,
        * it's best if we can just compare arrayZ, though comparing contents is still fast,
        * but also would require that Type has operator==.  As such, we optimize this operator
    -   * for range-based for loop and just compare arrayZ.  No need to compare length, as we
    -   * assume we're only compared to .end(). */
    +   * for range-based for loop and just compare arrayZ and length.
    +   *
    +   * The above comment is outdated now because we implemented separate begin/end to
    +   * objects that were using hb_array_t for range-based loop before. */
       bool operator != (const hb_array_t& o) const
    -  { return arrayZ != o.arrayZ; }
    +  { return this->arrayZ != o.arrayZ || this->length != o.length; }
    +
    +  /* Faster range-based for loop without bounds-check. */
    +  Type *begin () const { return arrayZ; }
    +  Type *end () const { return arrayZ + length; }
    +
     
       /* Extra operators.
        */
    @@ -113,11 +120,11 @@ struct hb_array_t : hb_iter_with_fallback_t, Type&>
     
       HB_INTERNAL bool operator == (const hb_array_t &o) const;
     
    -  uint32_t hash () const {
    +  uint32_t hash () const
    +  {
         uint32_t current = 0;
    -    for (unsigned int i = 0; i < this->length; i++) {
    -      current = current * 31 + hb_hash (this->arrayZ[i]);
    -    }
    +    for (auto &v : *this)
    +      current = current * 31 + hb_hash (v);
         return current;
       }
     
    @@ -185,23 +192,18 @@ struct hb_array_t : hb_iter_with_fallback_t, Type&>
     
       hb_sorted_array_t qsort (int (*cmp_)(const void*, const void*))
       {
    +    //static_assert (hb_enable_if (hb_is_trivially_copy_assignable(Type)), "");
         if (likely (length))
           hb_qsort (arrayZ, length, this->get_item_size (), cmp_);
         return hb_sorted_array_t (*this);
       }
       hb_sorted_array_t qsort ()
       {
    +    //static_assert (hb_enable_if (hb_is_trivially_copy_assignable(Type)), "");
         if (likely (length))
           hb_qsort (arrayZ, length, this->get_item_size (), Type::cmp);
         return hb_sorted_array_t (*this);
       }
    -  void qsort (unsigned int start, unsigned int end)
    -  {
    -    end = hb_min (end, length);
    -    assert (start <= end);
    -    if (likely (start < end))
    -      hb_qsort (arrayZ + start, end - start, this->get_item_size (), Type::cmp);
    -  }
     
       /*
        * Other methods.
    @@ -220,11 +222,8 @@ struct hb_array_t : hb_iter_with_fallback_t, Type&>
         if (end < start + 2)
           return;
     
    -    for (unsigned lhs = start, rhs = end - 1; lhs < rhs; lhs++, rhs--) {
    -      Type temp = arrayZ[rhs];
    -      arrayZ[rhs] = arrayZ[lhs];
    -      arrayZ[lhs] = temp;
    -    }
    +    for (unsigned lhs = start, rhs = end - 1; lhs < rhs; lhs++, rhs--)
    +      hb_swap (arrayZ[rhs], arrayZ[lhs]);
       }
     
       hb_array_t sub_array (unsigned int start_offset = 0, unsigned int *seg_count = nullptr /* IN/OUT */) const
    @@ -266,17 +265,31 @@ struct hb_array_t : hb_iter_with_fallback_t, Type&>
       void fini ()
       { hb_free ((void *) arrayZ); arrayZ = nullptr; length = 0; }
     
    -  template 
    +  template )))>
       hb_array_t copy (hb_serialize_context_t *c) const
       {
         TRACE_SERIALIZE (this);
         auto* out = c->start_embed (arrayZ);
    -    if (unlikely (!c->extend_size (out, get_size ()))) return_trace (hb_array_t ());
    +    if (unlikely (!c->extend_size (out, get_size (), false))) return_trace (hb_array_t ());
         for (unsigned i = 0; i < length; i++)
           out[i] = arrayZ[i]; /* TODO: add version that calls c->copy() */
         return_trace (hb_array_t (out, length));
       }
     
    +  template ))>
    +  hb_array_t copy (hb_serialize_context_t *c) const
    +  {
    +    TRACE_SERIALIZE (this);
    +    auto* out = c->start_embed (arrayZ);
    +    if (unlikely (!c->extend_size (out, get_size (), false))) return_trace (hb_array_t ());
    +    hb_memcpy (out, arrayZ, get_size ());
    +    return_trace (hb_array_t (out, length));
    +  }
    +
       template 
       bool sanitize (hb_sanitize_context_t *c) const
       { return c->check_array (arrayZ, length); }
    @@ -291,6 +304,9 @@ struct hb_array_t : hb_iter_with_fallback_t, Type&>
       unsigned int backwards_length = 0;
     };
     template  inline hb_array_t
    +hb_array ()
    +{ return hb_array_t (); }
    +template  inline hb_array_t
     hb_array (T *array, unsigned int length)
     { return hb_array_t (array, length); }
     template  inline hb_array_t
    @@ -299,8 +315,8 @@ hb_array (T (&array_)[length_])
     
     template 
     struct hb_sorted_array_t :
    -        hb_iter_t, Type&>,
    -        hb_array_t
    +        hb_array_t,
    +        hb_iter_t, Type&>
     {
       typedef hb_iter_t iter_base_t;
       HB_ITER_USING (iter_base_t);
    @@ -320,17 +336,24 @@ struct hb_sorted_array_t :
       template 
       constexpr hb_sorted_array_t (const hb_array_t &o) :
    -    hb_iter_t (),
    -    hb_array_t (o) {}
    +    hb_array_t (o),
    +    hb_iter_t () {}
       template 
       hb_sorted_array_t& operator = (const hb_array_t &o)
       { hb_array_t (*this) = o; return *this; }
     
       /* Iterator implementation. */
    +
    +  /* See comment in hb_array_of::operator != */
       bool operator != (const hb_sorted_array_t& o) const
       { return this->arrayZ != o.arrayZ || this->length != o.length; }
     
    +  /* Faster range-based for loop without bounds-check. */
    +  Type *begin () const { return this->arrayZ; }
    +  Type *end () const { return this->arrayZ + this->length; }
    +
    +
       hb_sorted_array_t sub_array (unsigned int start_offset, unsigned int *seg_count /* IN/OUT */) const
       { return hb_sorted_array_t (((const hb_array_t *) (this))->sub_array (start_offset, seg_count)); }
       hb_sorted_array_t sub_array (unsigned int start_offset, unsigned int seg_count) const
    @@ -344,7 +367,7 @@ struct hb_sorted_array_t :
         unsigned int i;
         return bfind (x, &i) ? &this->arrayZ[i] : not_found;
       }
    -  template 
    +  template 
       const Type *bsearch (const T &x, const Type *not_found = nullptr) const
       {
         unsigned int i;
    @@ -391,7 +414,7 @@ struct hb_sorted_array_t :
                                 this->length,
                                 sizeof (Type),
                                 _hb_cmp_method,
    -                            ds...);
    +                            std::forward (ds)...);
       }
     };
     template  inline hb_sorted_array_t
    @@ -423,18 +446,42 @@ inline bool hb_array_t::operator == (const hb_array_t
    -inline uint32_t hb_array_t::hash () const {
    +inline uint32_t hb_array_t::hash () const
    +{
       uint32_t current = 0;
    -  for (unsigned int i = 0; i < this->length; i++)
    -    current = current * 31 + (uint32_t) (this->arrayZ[i] * 2654435761u);
    +  unsigned i = 0;
    +
    +#if defined(__OPTIMIZE__) && !defined(HB_NO_PACKED) && \
    +    ((defined(__GNUC__) && __GNUC__ >= 5) || defined(__clang__))
    +  struct __attribute__((packed)) packed_uint32_t { uint32_t v; };
    +  for (; i + 4 <= this->length; i += 4)
    +    current = current * 31 + hb_hash ((uint32_t) ((packed_uint32_t *) &this->arrayZ[i])->v);
    +#endif
    +
    +  for (; i < this->length; i++)
    +    current = current * 31 + hb_hash (this->arrayZ[i]);
       return current;
     }
    +
     template <>
    -inline uint32_t hb_array_t::hash () const {
    +inline uint32_t hb_array_t::hash () const
    +{
       uint32_t current = 0;
    -  for (unsigned int i = 0; i < this->length; i++)
    -    current = current * 31 + (uint32_t) (this->arrayZ[i] * 2654435761u);
    +  unsigned i = 0;
    +
    +#if defined(__OPTIMIZE__) && !defined(HB_NO_PACKED) && \
    +    ((defined(__GNUC__) && __GNUC__ >= 5) || defined(__clang__))
    +  struct __attribute__((packed)) packed_uint32_t { uint32_t v; };
    +  for (; i + 4 <= this->length; i += 4)
    +    current = current * 31 + hb_hash ((uint32_t) ((packed_uint32_t *) &this->arrayZ[i])->v);
    +#endif
    +
    +  for (; i < this->length; i++)
    +    current = current * 31 + hb_hash (this->arrayZ[i]);
       return current;
     }
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-atomic.hh b/src/java.desktop/share/native/libharfbuzz/hb-atomic.hh
    index 1267bc4fe20bc..57e94761e800e 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-atomic.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-atomic.hh
    @@ -84,11 +84,11 @@ _hb_atomic_ptr_impl_cmplexch (const void **P, const void *O_, const void *N)
     #define _hb_memory_r_barrier()                  std::atomic_thread_fence(std::memory_order_acquire)
     #define _hb_memory_w_barrier()                  std::atomic_thread_fence(std::memory_order_release)
     
    -#define hb_atomic_int_impl_add(AI, V)           (reinterpret_cast *> (AI)->fetch_add ((V), std::memory_order_acq_rel))
    -#define hb_atomic_int_impl_set_relaxed(AI, V)   (reinterpret_cast *> (AI)->store ((V), std::memory_order_relaxed))
    -#define hb_atomic_int_impl_set(AI, V)           (reinterpret_cast *> (AI)->store ((V), std::memory_order_release))
    -#define hb_atomic_int_impl_get_relaxed(AI)      (reinterpret_cast const *> (AI)->load (std::memory_order_relaxed))
    -#define hb_atomic_int_impl_get(AI)              (reinterpret_cast const *> (AI)->load (std::memory_order_acquire))
    +#define hb_atomic_int_impl_add(AI, V)           (reinterpret_cast::type> *> (AI)->fetch_add ((V), std::memory_order_acq_rel))
    +#define hb_atomic_int_impl_set_relaxed(AI, V)   (reinterpret_cast::type> *> (AI)->store ((V), std::memory_order_relaxed))
    +#define hb_atomic_int_impl_set(AI, V)           (reinterpret_cast::type> *> (AI)->store ((V), std::memory_order_release))
    +#define hb_atomic_int_impl_get_relaxed(AI)      (reinterpret_cast::type> const *> (AI)->load (std::memory_order_relaxed))
    +#define hb_atomic_int_impl_get(AI)              (reinterpret_cast::type> const *> (AI)->load (std::memory_order_acquire))
     
     #define hb_atomic_ptr_impl_set_relaxed(P, V)    (reinterpret_cast *> (P)->store ((V), std::memory_order_relaxed))
     #define hb_atomic_ptr_impl_get_relaxed(P)       (reinterpret_cast const *> (P)->load (std::memory_order_relaxed))
    @@ -111,6 +111,24 @@ _hb_atomic_ptr_impl_cmplexch (const void **P, const void *O_, const void *N)
     #endif
     
     
    +/* This should never be disabled, even under HB_NO_MT.
    + * except that MSVC gives me an internal compiler error, so disabled there.
    + *
    + * https://github.com/harfbuzz/harfbuzz/pull/4119
    + */
    +#ifndef _hb_compiler_memory_r_barrier
    +#if defined(__ATOMIC_ACQUIRE) // gcc-like
    +#define _hb_compiler_memory_r_barrier() asm volatile("": : :"memory")
    +#elif !defined(_MSC_VER)
    +#include 
    +#define _hb_compiler_memory_r_barrier() std::atomic_signal_fence (std::memory_order_acquire)
    +#else
    +#define _hb_compiler_memory_r_barrier() do {} while (0)
    +#endif
    +#endif
    +
    +
    +
     #ifndef _hb_memory_r_barrier
     #define _hb_memory_r_barrier()                  _hb_memory_barrier ()
     #endif
    @@ -132,24 +150,47 @@ _hb_atomic_ptr_impl_cmplexch (const void **P, const void *O_, const void *N)
     #endif
     #ifndef hb_atomic_int_impl_set
     inline void hb_atomic_int_impl_set (int *AI, int v)     { _hb_memory_w_barrier (); *AI = v; }
    +inline void hb_atomic_int_impl_set (short *AI, short v) { _hb_memory_w_barrier (); *AI = v; }
     #endif
     #ifndef hb_atomic_int_impl_get
     inline int hb_atomic_int_impl_get (const int *AI)       { int v = *AI; _hb_memory_r_barrier (); return v; }
    +inline short hb_atomic_int_impl_get (const short *AI)   { short v = *AI; _hb_memory_r_barrier (); return v; }
     #endif
     #ifndef hb_atomic_ptr_impl_get
     inline void *hb_atomic_ptr_impl_get (void ** const P)   { void *v = *P; _hb_memory_r_barrier (); return v; }
     #endif
     
     
    +struct hb_atomic_short_t
    +{
    +  hb_atomic_short_t () = default;
    +  constexpr hb_atomic_short_t (short v) : v (v) {}
    +
    +  hb_atomic_short_t& operator = (short v_) { set_relaxed (v_); return *this; }
    +  operator short () const { return get_relaxed (); }
    +
    +  void set_relaxed (short v_) { hb_atomic_int_impl_set_relaxed (&v, v_); }
    +  void set_release (short v_) { hb_atomic_int_impl_set (&v, v_); }
    +  short get_relaxed () const { return hb_atomic_int_impl_get_relaxed (&v); }
    +  short get_acquire () const { return hb_atomic_int_impl_get (&v); }
    +  short inc () { return hb_atomic_int_impl_add (&v,  1); }
    +  short dec () { return hb_atomic_int_impl_add (&v, -1); }
    +
    +  short v = 0;
    +};
    +
     struct hb_atomic_int_t
     {
       hb_atomic_int_t () = default;
       constexpr hb_atomic_int_t (int v) : v (v) {}
     
    +  hb_atomic_int_t& operator = (int v_) { set_relaxed (v_); return *this; }
    +  operator int () const { return get_relaxed (); }
    +
       void set_relaxed (int v_) { hb_atomic_int_impl_set_relaxed (&v, v_); }
    -  void set (int v_) { hb_atomic_int_impl_set (&v, v_); }
    +  void set_release (int v_) { hb_atomic_int_impl_set (&v, v_); }
       int get_relaxed () const { return hb_atomic_int_impl_get_relaxed (&v); }
    -  int get () const { return hb_atomic_int_impl_get (&v); }
    +  int get_acquire () const { return hb_atomic_int_impl_get (&v); }
       int inc () { return hb_atomic_int_impl_add (&v,  1); }
       int dec () { return hb_atomic_int_impl_add (&v, -1); }
     
    @@ -167,11 +208,11 @@ struct hb_atomic_ptr_t
       void init (T* v_ = nullptr) { set_relaxed (v_); }
       void set_relaxed (T* v_) { hb_atomic_ptr_impl_set_relaxed (&v, v_); }
       T *get_relaxed () const { return (T *) hb_atomic_ptr_impl_get_relaxed (&v); }
    -  T *get () const { return (T *) hb_atomic_ptr_impl_get ((void **) &v); }
    +  T *get_acquire () const { return (T *) hb_atomic_ptr_impl_get ((void **) &v); }
       bool cmpexch (const T *old, T *new_) const { return hb_atomic_ptr_impl_cmpexch ((void **) &v, (void *) old, (void *) new_); }
     
    -  T * operator -> () const                    { return get (); }
    -  template  operator C * () const { return get (); }
    +  T * operator -> () const                    { return get_acquire (); }
    +  template  operator C * () const { return get_acquire (); }
     
       T *v = nullptr;
     };
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-bimap.hh b/src/java.desktop/share/native/libharfbuzz/hb-bimap.hh
    index 8e8c988716d68..9edefd9710657 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-bimap.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-bimap.hh
    @@ -83,9 +83,15 @@ struct hb_bimap_t
     
       unsigned int get_population () const { return forw_map.get_population (); }
     
    +
       protected:
       hb_map_t  forw_map;
       hb_map_t  back_map;
    +
    +  public:
    +  auto keys () const HB_AUTO_RETURN (+ forw_map.keys())
    +  auto values () const HB_AUTO_RETURN (+ forw_map.values())
    +  auto iter () const HB_AUTO_RETURN (+ forw_map.iter())
     };
     
     /* Inremental bimap: only lhs is given, rhs is incrementally assigned */
    @@ -108,6 +114,9 @@ struct hb_inc_bimap_t : hb_bimap_t
       hb_codepoint_t skip ()
       { return next_value++; }
     
    +  hb_codepoint_t skip (unsigned count)
    +  { return next_value += count; }
    +
       hb_codepoint_t get_next_value () const
       { return next_value; }
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-bit-page.hh b/src/java.desktop/share/native/libharfbuzz/hb-bit-page.hh
    index 5629ddc4cea56..81e2a4997bd63 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-bit-page.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-bit-page.hh
    @@ -30,32 +30,89 @@
     
     #include "hb.hh"
     
    +
    +/* Compiler-assisted vectorization. */
    +
    +/* Type behaving similar to vectorized vars defined using __attribute__((vector_size(...))),
    + * basically a fixed-size bitset. We can't use the compiler type because hb_vector_t cannot
    + * guarantee alignment requirements. */
    +template 
    +struct hb_vector_size_t
    +{
    +  elt_t& operator [] (unsigned int i) { return v[i]; }
    +  const elt_t& operator [] (unsigned int i) const { return v[i]; }
    +
    +  void init0 ()
    +  {
    +    for (unsigned int i = 0; i < ARRAY_LENGTH (v); i++)
    +      v[i] = 0;
    +  }
    +  void init1 ()
    +  {
    +    for (unsigned int i = 0; i < ARRAY_LENGTH (v); i++)
    +      v[i] = (elt_t) -1;
    +  }
    +
    +  template 
    +  hb_vector_size_t process (const Op& op) const
    +  {
    +    hb_vector_size_t r;
    +    for (unsigned int i = 0; i < ARRAY_LENGTH (v); i++)
    +      r.v[i] = op (v[i]);
    +    return r;
    +  }
    +  template 
    +  hb_vector_size_t process (const Op& op, const hb_vector_size_t &o) const
    +  {
    +    hb_vector_size_t r;
    +    for (unsigned int i = 0; i < ARRAY_LENGTH (v); i++)
    +      r.v[i] = op (v[i], o.v[i]);
    +    return r;
    +  }
    +  hb_vector_size_t operator | (const hb_vector_size_t &o) const
    +  { return process (hb_bitwise_or, o); }
    +  hb_vector_size_t operator & (const hb_vector_size_t &o) const
    +  { return process (hb_bitwise_and, o); }
    +  hb_vector_size_t operator ^ (const hb_vector_size_t &o) const
    +  { return process (hb_bitwise_xor, o); }
    +  hb_vector_size_t operator ~ () const
    +  { return process (hb_bitwise_neg); }
    +
    +  hb_array_t iter () const
    +  { return hb_array (v); }
    +
    +  private:
    +  static_assert (0 == byte_size % sizeof (elt_t), "");
    +  elt_t v[byte_size / sizeof (elt_t)];
    +};
    +
    +
     struct hb_bit_page_t
     {
    -  void init0 () { v.clear (); }
    -  void init1 () { v.clear (0xFF); }
    +  void init0 () { v.init0 (); }
    +  void init1 () { v.init1 (); }
     
    -  constexpr unsigned len () const
    +  static inline constexpr unsigned len ()
       { return ARRAY_LENGTH_CONST (v); }
     
       bool is_empty () const
       {
    -    for (unsigned i = 0; i < len (); i++)
    -      if (v[i])
    -        return false;
    -    return true;
    +    return
    +    + hb_iter (v)
    +    | hb_none
    +    ;
       }
       uint32_t hash () const
       {
    -    uint32_t h = 0;
    -    for (unsigned i = 0; i < len (); i++)
    -      h = h * 31 + hb_hash (v[i]);
    -    return h;
    +    return
    +    + hb_iter (v)
    +    | hb_reduce ([] (uint32_t h, const elt_t &_) { return h * 31 + hb_hash (_); }, (uint32_t) 0u)
    +    ;
       }
     
       void add (hb_codepoint_t g) { elt (g) |= mask (g); }
       void del (hb_codepoint_t g) { elt (g) &= ~mask (g); }
    -  void set (hb_codepoint_t g, bool v) { if (v) add (g); else del (g); }
    +  void set (hb_codepoint_t g, bool value) { if (value) add (g); else del (g); }
       bool get (hb_codepoint_t g) const { return elt (g) & mask (g); }
     
       void add_range (hb_codepoint_t a, hb_codepoint_t b)
    @@ -69,7 +126,7 @@ struct hb_bit_page_t
           *la |= ~(mask (a) - 1);
           la++;
     
    -      memset (la, 0xff, (char *) lb - (char *) la);
    +      hb_memset (la, 0xff, (char *) lb - (char *) la);
     
           *lb |= ((mask (b) << 1) - 1);
         }
    @@ -85,7 +142,7 @@ struct hb_bit_page_t
           *la &= mask (a) - 1;
           la++;
     
    -      memset (la, 0, (char *) lb - (char *) la);
    +      hb_memset (la, 0, (char *) lb - (char *) la);
     
           *lb &= ~((mask (b) << 1) - 1);
         }
    @@ -101,13 +158,13 @@ struct hb_bit_page_t
                           hb_codepoint_t *p,
                           unsigned int    size) const
       {
    -    unsigned int start_v = start_value >> ELT_BITS_LOG_2;
    +    unsigned int start_v = start_value / ELT_BITS;
         unsigned int start_bit = start_value & ELT_MASK;
         unsigned int count = 0;
         for (unsigned i = start_v; i < len () && count < size; i++)
         {
           elt_t bits = v[i];
    -      uint32_t v_base = base | (i << ELT_BITS_LOG_2);
    +      uint32_t v_base = base | (i * ELT_BITS);
           for (unsigned int j = start_bit; j < ELT_BITS && count < size; j++)
           {
             if ((elt_t(1) << j) & bits) {
    @@ -132,13 +189,13 @@ struct hb_bit_page_t
                                    unsigned int    size,
                                    hb_codepoint_t *next_value) const
       {
    -    unsigned int start_v = start_value >> ELT_BITS_LOG_2;
    +    unsigned int start_v = start_value / ELT_BITS;
         unsigned int start_bit = start_value & ELT_MASK;
         unsigned int count = 0;
         for (unsigned i = start_v; i < len () && count < size; i++)
         {
           elt_t bits = v[i];
    -      uint32_t v_offset = i << ELT_BITS_LOG_2;
    +      uint32_t v_offset = i * ELT_BITS;
           for (unsigned int j = start_bit; j < ELT_BITS && count < size; j++)
           {
             if ((elt_t(1) << j) & bits)
    @@ -161,7 +218,10 @@ struct hb_bit_page_t
     
       bool is_equal (const hb_bit_page_t &other) const
       {
    -    return 0 == hb_memcmp (&v, &other.v, sizeof (v));
    +    for (unsigned i = 0; i < len (); i++)
    +      if (v[i] != other.v[i])
    +        return false;
    +    return true;
       }
       bool is_subset (const hb_bit_page_t &larger_page) const
       {
    @@ -173,10 +233,10 @@ struct hb_bit_page_t
     
       unsigned int get_population () const
       {
    -    unsigned int pop = 0;
    -    for (unsigned int i = 0; i < len (); i++)
    -      pop += hb_popcount (v[i]);
    -    return pop;
    +    return
    +    + hb_iter (v)
    +    | hb_reduce ([] (unsigned pop, const elt_t &_) { return pop + hb_popcount (_); }, 0u)
    +    ;
       }
     
       bool next (hb_codepoint_t *codepoint) const
    @@ -250,10 +310,10 @@ struct hb_bit_page_t
       static constexpr hb_codepoint_t INVALID = HB_SET_VALUE_INVALID;
     
       typedef unsigned long long elt_t;
    -  static constexpr unsigned PAGE_BITS = 512;
    -  static_assert ((PAGE_BITS & ((PAGE_BITS) - 1)) == 0, "");
    -  static constexpr unsigned PAGE_BITS_LOG_2 = 9;
    +  static constexpr unsigned PAGE_BITS_LOG_2 = 9; // 512 bits
    +  static constexpr unsigned PAGE_BITS = 1 << PAGE_BITS_LOG_2;
       static_assert (1 << PAGE_BITS_LOG_2 == PAGE_BITS, "");
    +  static_assert ((PAGE_BITS & ((PAGE_BITS) - 1)) == 0, "");
       static constexpr unsigned PAGE_BITMASK = PAGE_BITS - 1;
     
       static unsigned int elt_get_min (const elt_t &elt) { return hb_ctz (elt); }
    @@ -262,8 +322,6 @@ struct hb_bit_page_t
       typedef hb_vector_size_t vector_t;
     
       static constexpr unsigned ELT_BITS = sizeof (elt_t) * 8;
    -  static constexpr unsigned ELT_BITS_LOG_2 = 6;
    -  static_assert (1 << ELT_BITS_LOG_2 == ELT_BITS, "");
       static constexpr unsigned ELT_MASK = ELT_BITS - 1;
     
       static constexpr unsigned BITS = sizeof (vector_t) * 8;
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-bit-set-invertible.hh b/src/java.desktop/share/native/libharfbuzz/hb-bit-set-invertible.hh
    index 9562ca36a3241..bf5a0b446defd 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-bit-set-invertible.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-bit-set-invertible.hh
    @@ -74,6 +74,11 @@ struct hb_bit_set_invertible_t
           inverted = !inverted;
       }
     
    +  bool is_inverted () const
    +  {
    +    return inverted;
    +  }
    +
       bool is_empty () const
       {
         hb_codepoint_t v = INVALID;
    @@ -123,10 +128,8 @@ struct hb_bit_set_invertible_t
       bool get (hb_codepoint_t g) const { return s.get (g) ^ inverted; }
     
       /* Has interface. */
    -  static constexpr bool SENTINEL = false;
    -  typedef bool value_t;
    -  value_t operator [] (hb_codepoint_t k) const { return get (k); }
    -  bool has (hb_codepoint_t k) const { return (*this)[k] != SENTINEL; }
    +  bool operator [] (hb_codepoint_t k) const { return get (k); }
    +  bool has (hb_codepoint_t k) const { return (*this)[k]; }
       /* Predicate. */
       bool operator () (hb_codepoint_t k) const { return has (k); }
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-bit-set.hh b/src/java.desktop/share/native/libharfbuzz/hb-bit-set.hh
    index be244835a1527..31ee52f60964c 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-bit-set.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-bit-set.hh
    @@ -38,7 +38,7 @@ struct hb_bit_set_t
       hb_bit_set_t () = default;
       ~hb_bit_set_t () = default;
     
    -  hb_bit_set_t (const hb_bit_set_t& other) : hb_bit_set_t () { set (other); }
    +  hb_bit_set_t (const hb_bit_set_t& other) : hb_bit_set_t () { set (other, true); }
       hb_bit_set_t ( hb_bit_set_t&& other) : hb_bit_set_t () { hb_swap (*this, other); }
       hb_bit_set_t& operator= (const hb_bit_set_t& other) { set (other); return *this; }
       hb_bit_set_t& operator= (hb_bit_set_t&& other) { hb_swap (*this, other); return *this; }
    @@ -56,7 +56,7 @@ struct hb_bit_set_t
       {
         successful = true;
         population = 0;
    -    last_page_lookup.set_relaxed (0);
    +    last_page_lookup = 0;
         page_map.init ();
         pages.init ();
       }
    @@ -85,12 +85,16 @@ struct hb_bit_set_t
       void err () { if (successful) successful = false; } /* TODO Remove */
       bool in_error () const { return !successful; }
     
    -  bool resize (unsigned int count)
    +  bool resize (unsigned int count, bool clear = true, bool exact_size = false)
       {
         if (unlikely (!successful)) return false;
    -    if (unlikely (!pages.resize (count) || !page_map.resize (count)))
    +
    +    if (pages.length == 0 && count == 1)
    +      exact_size = true; // Most sets are small and local
    +
    +    if (unlikely (!pages.resize (count, clear, exact_size) || !page_map.resize (count, clear, exact_size)))
         {
    -      pages.resize (page_map.length);
    +      pages.resize (page_map.length, clear, exact_size);
           successful = false;
           return false;
         }
    @@ -190,7 +194,7 @@ struct hb_bit_set_t
           unsigned int end = major_start (m + 1);
           do
           {
    -        if (v || page) /* The v check is to optimize out the page check if v is true. */
    +        if (g != INVALID && (v || page)) /* The v check is to optimize out the page check if v is true. */
               page->set (g, v);
     
             array = &StructAtOffsetUnaligned (array, stride);
    @@ -234,7 +238,7 @@ struct hb_bit_set_t
             if (g < last_g) return false;
             last_g = g;
     
    -        if (v || page) /* The v check is to optimize out the page check if v is true. */
    +        if (g != INVALID && (v || page)) /* The v check is to optimize out the page check if v is true. */
               page->add (g);
     
             array = &StructAtOffsetUnaligned (array, stride);
    @@ -330,10 +334,8 @@ struct hb_bit_set_t
       }
     
       /* Has interface. */
    -  static constexpr bool SENTINEL = false;
    -  typedef bool value_t;
    -  value_t operator [] (hb_codepoint_t k) const { return get (k); }
    -  bool has (hb_codepoint_t k) const { return (*this)[k] != SENTINEL; }
    +  bool operator [] (hb_codepoint_t k) const { return get (k); }
    +  bool has (hb_codepoint_t k) const { return (*this)[k]; }
       /* Predicate. */
       bool operator () (hb_codepoint_t k) const { return has (k); }
     
    @@ -348,11 +350,11 @@ struct hb_bit_set_t
         hb_codepoint_t c = first - 1;
         return next (&c) && c <= last;
       }
    -  void set (const hb_bit_set_t &other)
    +  void set (const hb_bit_set_t &other, bool exact_size = false)
       {
         if (unlikely (!successful)) return;
         unsigned int count = other.pages.length;
    -    if (unlikely (!resize (count)))
    +    if (unlikely (!resize (count, false, exact_size)))
           return;
         population = other.population;
     
    @@ -391,7 +393,7 @@ struct hb_bit_set_t
       bool is_subset (const hb_bit_set_t &larger_set) const
       {
         if (has_population () && larger_set.has_population () &&
    -        population != larger_set.population)
    +        population > larger_set.population)
           return false;
     
         uint32_t spi = 0;
    @@ -424,7 +426,7 @@ struct hb_bit_set_t
       private:
       bool allocate_compact_workspace (hb_vector_t& workspace)
       {
    -    if (unlikely (!workspace.resize (pages.length)))
    +    if (unlikely (!workspace.resize_exact (pages.length)))
         {
           successful = false;
           return false;
    @@ -465,12 +467,10 @@ struct hb_bit_set_t
       }
       public:
     
    -  template 
    -  void process (const Op& op, const hb_bit_set_t &other)
    +  void process_ (hb_bit_page_t::vector_t (*op) (const hb_bit_page_t::vector_t &, const hb_bit_page_t::vector_t &),
    +                 bool passthru_left, bool passthru_right,
    +                 const hb_bit_set_t &other)
       {
    -    const bool passthru_left = op (1, 0);
    -    const bool passthru_right = op (0, 1);
    -
         if (unlikely (!successful)) return;
     
         dirty ();
    @@ -542,21 +542,21 @@ struct hb_bit_set_t
         b = nb;
         for (; a && b; )
         {
    -      if (page_map[a - 1].major == other.page_map[b - 1].major)
    +      if (page_map.arrayZ[a - 1].major == other.page_map.arrayZ[b - 1].major)
           {
             a--;
             b--;
             count--;
    -        page_map[count] = page_map[a];
    +        page_map.arrayZ[count] = page_map.arrayZ[a];
             page_at (count).v = op (page_at (a).v, other.page_at (b).v);
           }
    -      else if (page_map[a - 1].major > other.page_map[b - 1].major)
    +      else if (page_map.arrayZ[a - 1].major > other.page_map.arrayZ[b - 1].major)
           {
             a--;
             if (passthru_left)
             {
               count--;
    -          page_map[count] = page_map[a];
    +          page_map.arrayZ[count] = page_map.arrayZ[a];
             }
           }
           else
    @@ -565,8 +565,8 @@ struct hb_bit_set_t
             if (passthru_right)
             {
               count--;
    -          page_map[count].major = other.page_map[b].major;
    -          page_map[count].index = next_page++;
    +          page_map.arrayZ[count].major = other.page_map.arrayZ[b].major;
    +          page_map.arrayZ[count].index = next_page++;
               page_at (count).v = other.page_at (b).v;
             }
           }
    @@ -576,20 +576,29 @@ struct hb_bit_set_t
           {
             a--;
             count--;
    -        page_map[count] = page_map [a];
    +        page_map.arrayZ[count] = page_map.arrayZ[a];
           }
         if (passthru_right)
           while (b)
           {
             b--;
             count--;
    -        page_map[count].major = other.page_map[b].major;
    -        page_map[count].index = next_page++;
    +        page_map.arrayZ[count].major = other.page_map.arrayZ[b].major;
    +        page_map.arrayZ[count].index = next_page++;
             page_at (count).v = other.page_at (b).v;
           }
         assert (!count);
         resize (newCount);
       }
    +  template 
    +  static hb_bit_page_t::vector_t
    +  op_ (const hb_bit_page_t::vector_t &a, const hb_bit_page_t::vector_t &b)
    +  { return Op{} (a, b); }
    +  template 
    +  void process (const Op& op, const hb_bit_set_t &other)
    +  {
    +    process_ (op_, op (1, 0), op (0, 1), other);
    +  }
     
       void union_ (const hb_bit_set_t &other) { process (hb_bitwise_or, other); }
       void intersect (const hb_bit_set_t &other) { process (hb_bitwise_and, other); }
    @@ -598,8 +607,6 @@ struct hb_bit_set_t
     
       bool next (hb_codepoint_t *codepoint) const
       {
    -    // TODO: this should be merged with prev() as both implementations
    -    //       are very similar.
         if (unlikely (*codepoint == INVALID)) {
           *codepoint = get_min ();
           return *codepoint != INVALID;
    @@ -607,7 +614,7 @@ struct hb_bit_set_t
     
         const auto* page_map_array = page_map.arrayZ;
         unsigned int major = get_major (*codepoint);
    -    unsigned int i = last_page_lookup.get_relaxed ();
    +    unsigned int i = last_page_lookup;
     
         if (unlikely (i >= page_map.length || page_map_array[i].major != major))
         {
    @@ -625,7 +632,7 @@ struct hb_bit_set_t
           if (pages_array[current.index].next (codepoint))
           {
             *codepoint += current.major * page_t::PAGE_BITS;
    -        last_page_lookup.set_relaxed (i);
    +        last_page_lookup = i;
             return true;
           }
           i++;
    @@ -633,16 +640,16 @@ struct hb_bit_set_t
     
         for (; i < page_map.length; i++)
         {
    -      const page_map_t ¤t = page_map.arrayZ[i];
    +      const page_map_t ¤t = page_map_array[i];
           hb_codepoint_t m = pages_array[current.index].get_min ();
           if (m != INVALID)
           {
             *codepoint = current.major * page_t::PAGE_BITS + m;
    -        last_page_lookup.set_relaxed (i);
    +        last_page_lookup = i;
             return true;
           }
         }
    -    last_page_lookup.set_relaxed (0);
    +    last_page_lookup = 0;
         *codepoint = INVALID;
         return false;
       }
    @@ -656,21 +663,21 @@ struct hb_bit_set_t
         page_map_t map = {get_major (*codepoint), 0};
         unsigned int i;
         page_map.bfind (map, &i, HB_NOT_FOUND_STORE_CLOSEST);
    -    if (i < page_map.length && page_map[i].major == map.major)
    +    if (i < page_map.length && page_map.arrayZ[i].major == map.major)
         {
    -      if (pages[page_map[i].index].previous (codepoint))
    +      if (pages[page_map.arrayZ[i].index].previous (codepoint))
           {
    -        *codepoint += page_map[i].major * page_t::PAGE_BITS;
    +        *codepoint += page_map.arrayZ[i].major * page_t::PAGE_BITS;
             return true;
           }
         }
         i--;
         for (; (int) i >= 0; i--)
         {
    -      hb_codepoint_t m = pages[page_map[i].index].get_max ();
    +      hb_codepoint_t m = pages.arrayZ[page_map.arrayZ[i].index].get_max ();
           if (m != INVALID)
           {
    -        *codepoint = page_map[i].major * page_t::PAGE_BITS + m;
    +        *codepoint = page_map.arrayZ[i].major * page_t::PAGE_BITS + m;
             return true;
           }
         }
    @@ -725,7 +732,7 @@ struct hb_bit_set_t
         {
           const auto* page_map_array = page_map.arrayZ;
           unsigned int major = get_major (codepoint);
    -      unsigned int i = last_page_lookup.get_relaxed ();
    +      unsigned int i = last_page_lookup;
           if (unlikely (i >= page_map.length || page_map_array[i].major != major))
           {
             page_map.bfind (major, &i, HB_NOT_FOUND_STORE_CLOSEST);
    @@ -766,7 +773,7 @@ struct hb_bit_set_t
         {
           const auto* page_map_array = page_map.arrayZ;
           unsigned int major = get_major (codepoint);
    -      unsigned int i = last_page_lookup.get_relaxed ();
    +      unsigned int i = last_page_lookup;
           if (unlikely (i >= page_map.length || page_map_array[i].major != major))
           {
             page_map.bfind(major, &i, HB_NOT_FOUND_STORE_CLOSEST);
    @@ -893,12 +900,12 @@ struct hb_bit_set_t
         /* The extra page_map length is necessary; can't just rely on vector here,
          * since the next check would be tricked because a null page also has
          * major==0, which we can't distinguish from an actualy major==0 page... */
    -    unsigned i = last_page_lookup.get_relaxed ();
    +    unsigned i = last_page_lookup;
         if (likely (i < page_map.length))
         {
           auto &cached_page = page_map.arrayZ[i];
           if (cached_page.major == major)
    -        return &pages[cached_page.index];
    +        return &pages.arrayZ[cached_page.index];
         }
     
         page_map_t map = {major, pages.length};
    @@ -910,15 +917,15 @@ struct hb_bit_set_t
           if (unlikely (!resize (pages.length + 1)))
             return nullptr;
     
    -      pages[map.index].init0 ();
    -      memmove (page_map + i + 1,
    -               page_map + i,
    +      pages.arrayZ[map.index].init0 ();
    +      memmove (page_map.arrayZ + i + 1,
    +               page_map.arrayZ + i,
                    (page_map.length - 1 - i) * page_map.item_size);
           page_map[i] = map;
         }
     
    -    last_page_lookup.set_relaxed (i);
    -    return &pages[page_map[i].index];
    +    last_page_lookup = i;
    +    return &pages.arrayZ[page_map.arrayZ[i].index];
       }
       const page_t *page_for (hb_codepoint_t g) const
       {
    @@ -927,23 +934,31 @@ struct hb_bit_set_t
         /* The extra page_map length is necessary; can't just rely on vector here,
          * since the next check would be tricked because a null page also has
          * major==0, which we can't distinguish from an actualy major==0 page... */
    -    unsigned i = last_page_lookup.get_relaxed ();
    +    unsigned i = last_page_lookup;
         if (likely (i < page_map.length))
         {
           auto &cached_page = page_map.arrayZ[i];
           if (cached_page.major == major)
    -        return &pages[cached_page.index];
    +        return &pages.arrayZ[cached_page.index];
         }
     
         page_map_t key = {major};
         if (!page_map.bfind (key, &i))
           return nullptr;
     
    -    last_page_lookup.set_relaxed (i);
    -    return &pages[page_map[i].index];
    +    last_page_lookup = i;
    +    return &pages.arrayZ[page_map[i].index];
    +  }
    +  page_t &page_at (unsigned int i)
    +  {
    +    assert (i < page_map.length);
    +    return pages.arrayZ[page_map.arrayZ[i].index];
    +  }
    +  const page_t &page_at (unsigned int i) const
    +  {
    +    assert (i < page_map.length);
    +    return pages.arrayZ[page_map.arrayZ[i].index];
       }
    -  page_t &page_at (unsigned int i) { return pages[page_map[i].index]; }
    -  const page_t &page_at (unsigned int i) const { return pages[page_map[i].index]; }
       unsigned int get_major (hb_codepoint_t g) const { return g >> page_t::PAGE_BITS_LOG_2; }
       unsigned int page_remainder (hb_codepoint_t g) const { return g & page_t::PAGE_BITMASK; }
       hb_codepoint_t major_start (unsigned int major) const { return major << page_t::PAGE_BITS_LOG_2; }
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-blob.cc b/src/java.desktop/share/native/libharfbuzz/hb-blob.cc
    index 1ee4195db7fd2..2a43afa1a435f 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-blob.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-blob.cc
    @@ -99,7 +99,7 @@ hb_blob_create (const char        *data,
      * is zero. This is in contrast to hb_blob_create(), which returns the singleton
      * empty blob (as returned by hb_blob_get_empty()) if @length is zero.
      *
    - * Return value: New blob, or %NULL if failed.  Destroy with hb_blob_destroy().
    + * Return value: New blob, or `NULL` if failed.  Destroy with hb_blob_destroy().
      *
      * Since: 2.8.2
      **/
    @@ -263,8 +263,6 @@ hb_blob_destroy (hb_blob_t *blob)
     {
       if (!hb_object_destroy (blob)) return;
     
    -  blob->fini_shallow ();
    -
       hb_free (blob);
     }
     
    @@ -278,7 +276,7 @@ hb_blob_destroy (hb_blob_t *blob)
      *
      * Attaches a user-data key/data pair to the specified blob.
      *
    - * Return value: %true if success, %false otherwise
    + * Return value: `true` if success, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -305,7 +303,7 @@ hb_blob_set_user_data (hb_blob_t          *blob,
      * Since: 0.9.2
      **/
     void *
    -hb_blob_get_user_data (hb_blob_t          *blob,
    +hb_blob_get_user_data (const hb_blob_t    *blob,
                            hb_user_data_key_t *key)
     {
       return hb_object_get_user_data (blob, key);
    @@ -335,7 +333,7 @@ hb_blob_make_immutable (hb_blob_t *blob)
      *
      * Tests whether a blob is immutable.
      *
    - * Return value: %true if @blob is immutable, %false otherwise
    + * Return value: `true` if @blob is immutable, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -394,7 +392,7 @@ hb_blob_get_data (hb_blob_t *blob, unsigned int *length)
      * fails.
      *
      * Returns: (transfer none) (array length=length): Writable blob data,
    - * or %NULL if failed.
    + * or `NULL` if failed.
      *
      * Since: 0.9.2
      **/
    @@ -497,7 +495,7 @@ hb_blob_t::try_make_writable ()
     
       DEBUG_MSG_FUNC (BLOB, this, "dupped successfully -> %p\n", this->data);
     
    -  memcpy (new_data, this->data, this->length);
    +  hb_memcpy (new_data, this->data, this->length);
       this->destroy_user_data ();
       this->mode = HB_MEMORY_MODE_WRITABLE;
       this->data = new_data;
    @@ -620,7 +618,7 @@ hb_blob_create_from_file (const char *file_name)
      * specified binary font file.
      *
      * Returns: An #hb_blob_t pointer with the content of the file,
    - * or %NULL if failed.
    + * or `NULL` if failed.
      *
      * Since: 2.8.2
      **/
    @@ -678,7 +676,7 @@ hb_blob_create_from_file_or_fail (const char *file_name)
       wchar_t * wchar_file_name = (wchar_t *) hb_malloc (sizeof (wchar_t) * size);
       if (unlikely (!wchar_file_name)) goto fail_without_close;
       mbstowcs (wchar_file_name, file_name, size);
    -#if !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
    +#if !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) && WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP)
       {
         CREATEFILE2_EXTENDED_PARAMETERS ceparams = { 0 };
         ceparams.dwSize = sizeof(CREATEFILE2_EXTENDED_PARAMETERS);
    @@ -699,7 +697,7 @@ hb_blob_create_from_file_or_fail (const char *file_name)
     
       if (unlikely (fd == INVALID_HANDLE_VALUE)) goto fail_without_close;
     
    -#if !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
    +#if !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) && WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP)
       {
         LARGE_INTEGER length;
         GetFileSizeEx (fd, &length);
    @@ -712,7 +710,7 @@ hb_blob_create_from_file_or_fail (const char *file_name)
     #endif
       if (unlikely (!file->mapping)) goto fail;
     
    -#if !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
    +#if !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) && WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP)
       file->contents = (char *) MapViewOfFileFromApp (file->mapping, FILE_MAP_READ, 0, 0);
     #else
       file->contents = (char *) MapViewOfFile (file->mapping, FILE_MAP_READ, 0, 0, 0);
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-blob.h b/src/java.desktop/share/native/libharfbuzz/hb-blob.h
    index 860b09bd08853..1761e6534cac8 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-blob.h
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-blob.h
    @@ -63,7 +63,7 @@ HB_BEGIN_DECLS
      *   HarfBuzz and doing that just once (no reuse!),
      *
      * - If the font is mmap()ed, it's okay to use
    - *   @HB_MEMORY_READONLY_MAY_MAKE_WRITABLE, however, using that mode
    + *   @HB_MEMORY_MODE_READONLY_MAY_MAKE_WRITABLE, however, using that mode
      *   correctly is very tricky.  Use @HB_MEMORY_MODE_READONLY instead.
      **/
     typedef enum {
    @@ -135,7 +135,7 @@ hb_blob_set_user_data (hb_blob_t          *blob,
     
     
     HB_EXTERN void *
    -hb_blob_get_user_data (hb_blob_t          *blob,
    +hb_blob_get_user_data (const hb_blob_t    *blob,
                            hb_user_data_key_t *key);
     
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-blob.hh b/src/java.desktop/share/native/libharfbuzz/hb-blob.hh
    index a3683a681e78b..b1b3b94d3ddc6 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-blob.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-blob.hh
    @@ -38,7 +38,7 @@
     
     struct hb_blob_t
     {
    -  void fini_shallow () { destroy_user_data (); }
    +  ~hb_blob_t () { destroy_user_data (); }
     
       void destroy_user_data ()
       {
    @@ -61,12 +61,12 @@ struct hb_blob_t
       public:
       hb_object_header_t header;
     
    -  const char *data;
    -  unsigned int length;
    -  hb_memory_mode_t mode;
    +  const char *data = nullptr;
    +  unsigned int length = 0;
    +  hb_memory_mode_t mode = (hb_memory_mode_t) 0;
     
    -  void *user_data;
    -  hb_destroy_func_t destroy;
    +  void *user_data = nullptr;
    +  hb_destroy_func_t destroy = nullptr;
     };
     
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-buffer-deserialize-json.hh b/src/java.desktop/share/native/libharfbuzz/hb-buffer-deserialize-json.hh
    index 970fae7dfa678..7b9fc557f737d 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-buffer-deserialize-json.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-buffer-deserialize-json.hh
    @@ -32,35 +32,38 @@
     #include "hb.hh"
     
     
    -#line 36 "hb-buffer-deserialize-json.hh"
    +#line 33 "hb-buffer-deserialize-json.hh"
     static const unsigned char _deserialize_json_trans_keys[] = {
             0u, 0u, 9u, 123u, 9u, 34u, 97u, 117u, 120u, 121u, 34u, 34u, 9u, 58u, 9u, 57u,
    -        48u, 57u, 9u, 125u, 9u, 125u, 9u, 125u, 34u, 34u, 9u, 58u, 9u, 57u, 48u, 57u,
    -        9u, 125u, 9u, 125u, 108u, 108u, 34u, 34u, 9u, 58u, 9u, 57u, 9u, 125u, 9u, 125u,
    -        120u, 121u, 34u, 34u, 9u, 58u, 9u, 57u, 48u, 57u, 9u, 125u, 9u, 125u, 34u, 34u,
    -        9u, 58u, 9u, 57u, 48u, 57u, 9u, 125u, 9u, 125u, 34u, 34u, 9u, 58u, 9u, 57u,
    -        34u, 92u, 9u, 125u, 34u, 92u, 9u, 125u, 9u, 125u, 34u, 34u, 9u, 58u, 9u, 57u,
    -        9u, 125u, 9u, 93u, 9u, 123u, 0u, 0u, 0
    +        48u, 57u, 9u, 125u, 9u, 125u, 9u, 93u, 9u, 125u, 34u, 34u, 9u, 58u, 9u, 57u,
    +        48u, 57u, 9u, 125u, 9u, 125u, 108u, 108u, 34u, 34u, 9u, 58u, 9u, 57u, 9u, 125u,
    +        9u, 125u, 120u, 121u, 34u, 34u, 9u, 58u, 9u, 57u, 48u, 57u, 9u, 125u, 9u, 125u,
    +        34u, 34u, 9u, 58u, 9u, 57u, 48u, 57u, 9u, 125u, 9u, 125u, 108u, 108u, 34u, 34u,
    +        9u, 58u, 9u, 57u, 9u, 125u, 9u, 125u, 34u, 34u, 9u, 58u, 9u, 57u, 34u, 92u,
    +        9u, 125u, 34u, 92u, 9u, 125u, 9u, 125u, 34u, 34u, 9u, 58u, 9u, 57u, 9u, 125u,
    +        9u, 123u, 0u, 0u, 0
     };
     
     static const char _deserialize_json_key_spans[] = {
             0, 115, 26, 21, 2, 1, 50, 49,
    -        10, 117, 117, 117, 1, 50, 49, 10,
    -        117, 117, 1, 1, 50, 49, 117, 117,
    -        2, 1, 50, 49, 10, 117, 117, 1,
    -        50, 49, 10, 117, 117, 1, 50, 49,
    -        59, 117, 59, 117, 117, 1, 50, 49,
    -        117, 85, 115, 0
    +        10, 117, 117, 85, 117, 1, 50, 49,
    +        10, 117, 117, 1, 1, 50, 49, 117,
    +        117, 2, 1, 50, 49, 10, 117, 117,
    +        1, 50, 49, 10, 117, 117, 1, 1,
    +        50, 49, 117, 117, 1, 50, 49, 59,
    +        117, 59, 117, 117, 1, 50, 49, 117,
    +        115, 0
     };
     
     static const short _deserialize_json_index_offsets[] = {
             0, 0, 116, 143, 165, 168, 170, 221,
    -        271, 282, 400, 518, 636, 638, 689, 739,
    -        750, 868, 986, 988, 990, 1041, 1091, 1209,
    -        1327, 1330, 1332, 1383, 1433, 1444, 1562, 1680,
    -        1682, 1733, 1783, 1794, 1912, 2030, 2032, 2083,
    -        2133, 2193, 2311, 2371, 2489, 2607, 2609, 2660,
    -        2710, 2828, 2914, 3030
    +        271, 282, 400, 518, 604, 722, 724, 775,
    +        825, 836, 954, 1072, 1074, 1076, 1127, 1177,
    +        1295, 1413, 1416, 1418, 1469, 1519, 1530, 1648,
    +        1766, 1768, 1819, 1869, 1880, 1998, 2116, 2118,
    +        2120, 2171, 2221, 2339, 2457, 2459, 2510, 2560,
    +        2620, 2738, 2798, 2916, 3034, 3036, 3087, 3137,
    +        3255, 3371
     };
     
     static const char _deserialize_json_indicies[] = {
    @@ -82,28 +85,28 @@ static const char _deserialize_json_indicies[] = {
             3, 3, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 3, 1, 4, 1,
    -        5, 1, 6, 7, 1, 1, 8, 1,
    +        5, 1, 6, 7, 1, 8, 9, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 9, 1, 10, 11,
    -        1, 12, 1, 12, 12, 12, 12, 12,
    +        1, 1, 1, 1, 10, 1, 11, 12,
    +        1, 13, 1, 13, 13, 13, 13, 13,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 12, 1, 1, 1, 1, 1,
    +        1, 1, 13, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 13, 1, 13, 13,
    -        13, 13, 13, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 14, 1, 14, 14,
    +        14, 14, 14, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 13, 1, 1,
    +        1, 1, 1, 1, 1, 14, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 14, 1, 1, 15, 16, 16,
    -        16, 16, 16, 16, 16, 16, 16, 1,
    -        17, 18, 18, 18, 18, 18, 18, 18,
    -        18, 18, 1, 19, 19, 19, 19, 19,
    +        1, 1, 15, 1, 1, 16, 17, 17,
    +        17, 17, 17, 17, 17, 17, 17, 1,
    +        18, 19, 19, 19, 19, 19, 19, 19,
    +        19, 19, 1, 20, 20, 20, 20, 20,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 19, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 20, 1,
    +        1, 1, 20, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 21, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    @@ -113,11 +116,11 @@ static const char _deserialize_json_indicies[] = {
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 21,
    -        1, 22, 22, 22, 22, 22, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 22,
    +        1, 23, 23, 23, 23, 23, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        22, 1, 1, 1, 1, 1, 1, 1,
    +        23, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 3, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    @@ -128,85 +131,94 @@ static const char _deserialize_json_indicies[] = {
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 23, 1, 19,
    -        19, 19, 19, 19, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 24, 1, 25,
    +        25, 25, 25, 25, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 19, 1,
    +        1, 1, 1, 1, 1, 1, 25, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 20, 1, 1, 1, 18, 18,
    -        18, 18, 18, 18, 18, 18, 18, 18,
    +        1, 1, 26, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 27, 1, 20, 20, 20,
    +        20, 20, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 20, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        21, 1, 1, 1, 19, 19, 19, 19,
    +        19, 19, 19, 19, 19, 19, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 21, 1, 24, 1, 24,
    -        24, 24, 24, 24, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 24, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        25, 1, 25, 25, 25, 25, 25, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 25, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 26, 1,
    -        1, 27, 28, 28, 28, 28, 28, 28,
    -        28, 28, 28, 1, 29, 30, 30, 30,
    -        30, 30, 30, 30, 30, 30, 1, 31,
    -        31, 31, 31, 31, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 31, 1,
    +        1, 22, 1, 28, 1, 28, 28, 28,
    +        28, 28, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 32, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 28, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 29, 1,
    +        29, 29, 29, 29, 29, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 29,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 30, 1, 1, 31,
    +        32, 32, 32, 32, 32, 32, 32, 32,
    +        32, 1, 33, 34, 34, 34, 34, 34,
    +        34, 34, 34, 34, 1, 35, 35, 35,
    +        35, 35, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 35, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        36, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 33, 1, 31, 31, 31,
    -        31, 31, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 31, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        32, 1, 1, 1, 30, 30, 30, 30,
    -        30, 30, 30, 30, 30, 30, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 37, 1, 35, 35, 35, 35, 35,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 35, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 36, 1,
    +        1, 1, 34, 34, 34, 34, 34, 34,
    +        34, 34, 34, 34, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 33, 1, 34, 1, 35, 1, 35,
    -        35, 35, 35, 35, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 35, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        36, 1, 36, 36, 36, 36, 36, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 37,
    +        1, 38, 1, 39, 1, 39, 39, 39,
    +        39, 39, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 36, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 39, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 37, 38, 38, 38, 38, 38, 38,
    -        38, 38, 38, 1, 39, 39, 39, 39,
    -        39, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 39, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 40, 1,
    +        40, 40, 40, 40, 40, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 40,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 41,
    +        42, 42, 42, 42, 42, 42, 42, 42,
    +        42, 1, 43, 43, 43, 43, 43, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 43, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 44, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    @@ -215,43 +227,42 @@ static const char _deserialize_json_indicies[] = {
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        41, 1, 39, 39, 39, 39, 39, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 45, 1,
    +        43, 43, 43, 43, 43, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 39, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 40, 1, 1,
    -        1, 42, 42, 42, 42, 42, 42, 42,
    -        42, 42, 42, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 43,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 44, 1, 1, 1, 46,
    +        46, 46, 46, 46, 46, 46, 46, 46,
    +        46, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 41, 1,
    -        43, 44, 1, 45, 1, 45, 45, 45,
    -        45, 45, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 45, 1, 1, 1,
    +        1, 1, 1, 1, 45, 1, 47, 48,
    +        1, 49, 1, 49, 49, 49, 49, 49,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 46, 1,
    -        46, 46, 46, 46, 46, 1, 1, 1,
    +        1, 1, 49, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 46,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 47, 1, 1, 48,
    -        49, 49, 49, 49, 49, 49, 49, 49,
    -        49, 1, 50, 51, 51, 51, 51, 51,
    -        51, 51, 51, 51, 1, 52, 52, 52,
    -        52, 52, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 50, 1, 50, 50,
    +        50, 50, 50, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 52, 1, 1, 1,
    +        1, 1, 1, 1, 1, 50, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        53, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 51, 1, 1, 52, 53, 53,
    +        53, 53, 53, 53, 53, 53, 53, 1,
    +        54, 55, 55, 55, 55, 55, 55, 55,
    +        55, 55, 1, 56, 56, 56, 56, 56,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 56, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 57, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    @@ -259,42 +270,43 @@ static const char _deserialize_json_indicies[] = {
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 54, 1, 52, 52, 52, 52, 52,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 52, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 53, 1,
    -        1, 1, 51, 51, 51, 51, 51, 51,
    -        51, 51, 51, 51, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 58,
    +        1, 56, 56, 56, 56, 56, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        56, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 57, 1, 1, 1,
    +        55, 55, 55, 55, 55, 55, 55, 55,
    +        55, 55, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 54,
    -        1, 55, 1, 55, 55, 55, 55, 55,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 55, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 58, 1, 59,
    +        1, 59, 59, 59, 59, 59, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 56, 1, 56, 56,
    -        56, 56, 56, 1, 1, 1, 1, 1,
    +        59, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 56, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 57, 1, 1, 58, 59, 59,
    -        59, 59, 59, 59, 59, 59, 59, 1,
    -        60, 61, 61, 61, 61, 61, 61, 61,
    -        61, 61, 1, 62, 62, 62, 62, 62,
    +        1, 1, 60, 1, 60, 60, 60, 60,
    +        60, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 60, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 62, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 63, 1,
    +        61, 1, 1, 62, 63, 63, 63, 63,
    +        63, 63, 63, 63, 63, 1, 64, 65,
    +        65, 65, 65, 65, 65, 65, 65, 65,
    +        1, 66, 66, 66, 66, 66, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        66, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 67, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    @@ -302,48 +314,42 @@ static const char _deserialize_json_indicies[] = {
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 64,
    -        1, 62, 62, 62, 62, 62, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        62, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 63, 1, 1, 1,
    -        61, 61, 61, 61, 61, 61, 61, 61,
    -        61, 61, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 68, 1, 66,
    +        66, 66, 66, 66, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 66, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 67, 1, 1, 1, 65, 65,
    +        65, 65, 65, 65, 65, 65, 65, 65,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 64, 1, 65,
    -        1, 65, 65, 65, 65, 65, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        65, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 68, 1, 69, 1, 70,
    +        1, 70, 70, 70, 70, 70, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        70, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 66, 1, 66, 66, 66, 66,
    -        66, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 66, 1, 67, 1, 1,
    +        1, 1, 71, 1, 71, 71, 71, 71,
    +        71, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 68, 69, 69, 69, 69,
    -        69, 69, 69, 69, 69, 1, 71, 70,
    -        70, 70, 70, 70, 70, 70, 70, 70,
    -        70, 70, 70, 70, 70, 70, 70, 70,
    -        70, 70, 70, 70, 70, 70, 70, 70,
    -        70, 70, 70, 70, 70, 70, 70, 70,
    -        70, 70, 70, 70, 70, 70, 70, 70,
    -        70, 70, 70, 70, 70, 70, 70, 70,
    -        70, 70, 70, 70, 70, 70, 70, 70,
    -        72, 70, 73, 73, 73, 73, 73, 1,
    +        1, 1, 1, 71, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 72, 73, 73, 73, 73,
    +        73, 73, 73, 73, 73, 1, 74, 74,
    +        74, 74, 74, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 73, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 74, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 75, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    @@ -352,86 +358,126 @@ static const char _deserialize_json_indicies[] = {
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 75, 1,
    -        70, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 76, 1, 74, 74, 74, 74,
    +        74, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 74, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 75,
    +        1, 1, 1, 77, 77, 77, 77, 77,
    +        77, 77, 77, 77, 77, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        76, 1, 78, 1, 78, 78, 78, 78,
    +        78, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 78, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 79, 1, 79,
    +        79, 79, 79, 79, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 79, 1,
    +        80, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 81, 82,
    +        82, 82, 82, 82, 82, 82, 82, 82,
    +        1, 84, 83, 83, 83, 83, 83, 83,
    +        83, 83, 83, 83, 83, 83, 83, 83,
    +        83, 83, 83, 83, 83, 83, 83, 83,
    +        83, 83, 83, 83, 83, 83, 83, 83,
    +        83, 83, 83, 83, 83, 83, 83, 83,
    +        83, 83, 83, 83, 83, 83, 83, 83,
    +        83, 83, 83, 83, 83, 83, 83, 83,
    +        83, 83, 83, 85, 83, 86, 86, 86,
    +        86, 86, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 86, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        87, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 70, 1, 76, 76, 76, 76,
    -        76, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 76, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 77,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 88, 1, 83, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 83, 1, 89,
    +        89, 89, 89, 89, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 89, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 90, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        78, 1, 76, 76, 76, 76, 76, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 76, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 77, 1, 1,
    -        1, 79, 79, 79, 79, 79, 79, 79,
    -        79, 79, 79, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 91, 1, 89, 89, 89,
    +        89, 89, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 78, 1,
    -        80, 1, 80, 80, 80, 80, 80, 1,
    +        1, 1, 1, 1, 89, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        90, 1, 1, 1, 92, 92, 92, 92,
    +        92, 92, 92, 92, 92, 92, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 80, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 81, 1, 81, 81, 81,
    -        81, 81, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 81, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 82, 83, 83, 83,
    -        83, 83, 83, 83, 83, 83, 1, 76,
    -        76, 76, 76, 76, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 76, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 77, 1, 1, 1, 84, 84,
    -        84, 84, 84, 84, 84, 84, 84, 84,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 91, 1, 93, 1, 93, 93, 93,
    +        93, 93, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 93, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 94, 1,
    +        94, 94, 94, 94, 94, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 94,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 95,
    +        96, 96, 96, 96, 96, 96, 96, 96,
    +        96, 1, 89, 89, 89, 89, 89, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 78, 1, 85, 85, 85,
    -        85, 85, 1, 1, 1, 1, 1, 1,
    +        1, 89, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 90, 1, 1,
    +        1, 97, 97, 97, 97, 97, 97, 97,
    +        97, 97, 97, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 85, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        86, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 87, 1, 0, 0, 0, 0, 0,
    +        1, 1, 1, 1, 1, 1, 91, 1,
    +        0, 0, 0, 0, 0, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 0,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 0, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    @@ -442,46 +488,49 @@ static const char _deserialize_json_indicies[] = {
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
             1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 2, 1, 1,
    -        0
    +        1, 1, 2, 1, 1, 0
     };
     
     static const char _deserialize_json_trans_targs[] = {
    -        1, 0, 2, 2, 3, 4, 18, 24,
    -        37, 45, 5, 12, 6, 7, 8, 9,
    -        11, 9, 11, 10, 2, 49, 10, 49,
    -        13, 14, 15, 16, 17, 16, 17, 10,
    -        2, 49, 19, 20, 21, 22, 23, 10,
    -        2, 49, 23, 25, 31, 26, 27, 28,
    -        29, 30, 29, 30, 10, 2, 49, 32,
    -        33, 34, 35, 36, 35, 36, 10, 2,
    -        49, 38, 39, 40, 43, 44, 40, 41,
    -        42, 10, 2, 49, 10, 2, 49, 44,
    -        46, 47, 43, 48, 48, 49, 50, 51
    +        1, 0, 2, 2, 3, 4, 19, 25,
    +        38, 44, 52, 5, 13, 6, 7, 8,
    +        9, 12, 9, 12, 10, 2, 11, 10,
    +        11, 11, 56, 57, 14, 15, 16, 17,
    +        18, 17, 18, 10, 2, 11, 20, 21,
    +        22, 23, 24, 10, 2, 11, 24, 26,
    +        32, 27, 28, 29, 30, 31, 30, 31,
    +        10, 2, 11, 33, 34, 35, 36, 37,
    +        36, 37, 10, 2, 11, 39, 40, 41,
    +        42, 43, 10, 2, 11, 43, 45, 46,
    +        47, 50, 51, 47, 48, 49, 10, 2,
    +        11, 10, 2, 11, 51, 53, 54, 50,
    +        55, 55
     };
     
     static const char _deserialize_json_trans_actions[] = {
             0, 0, 1, 0, 0, 0, 0, 0,
    -        0, 0, 0, 0, 0, 0, 2, 2,
    -        2, 0, 0, 3, 3, 4, 0, 5,
    -        0, 0, 2, 2, 2, 0, 0, 6,
    -        6, 7, 0, 0, 0, 2, 2, 8,
    -        8, 9, 0, 0, 0, 0, 0, 2,
    -        2, 2, 0, 0, 10, 10, 11, 0,
    -        0, 2, 2, 2, 0, 0, 12, 12,
    -        13, 0, 0, 2, 14, 14, 0, 15,
    -        0, 16, 16, 17, 18, 18, 19, 15,
    -        0, 0, 20, 20, 21, 0, 0, 0
    +        0, 0, 0, 0, 0, 0, 0, 2,
    +        2, 2, 0, 0, 3, 3, 4, 0,
    +        5, 0, 0, 0, 0, 0, 2, 2,
    +        2, 0, 0, 6, 6, 7, 0, 0,
    +        0, 2, 2, 8, 8, 9, 0, 0,
    +        0, 0, 0, 2, 2, 2, 0, 0,
    +        10, 10, 11, 0, 0, 2, 2, 2,
    +        0, 0, 12, 12, 13, 0, 0, 0,
    +        2, 2, 14, 14, 15, 0, 0, 0,
    +        2, 16, 16, 0, 17, 0, 18, 18,
    +        19, 20, 20, 21, 17, 0, 0, 22,
    +        22, 23
     };
     
     static const int deserialize_json_start = 1;
    -static const int deserialize_json_first_final = 49;
    +static const int deserialize_json_first_final = 56;
     static const int deserialize_json_error = 0;
     
     static const int deserialize_json_en_main = 1;
     
     
    -#line 108 "hb-buffer-deserialize-json.rl"
    +#line 111 "hb-buffer-deserialize-json.rl"
     
     
     static hb_bool_t
    @@ -499,21 +548,19 @@ _hb_buffer_deserialize_json (hb_buffer_t *buffer,
       while (p < pe && ISSPACE (*p))
         p++;
       if (p < pe && *p == (buffer->len ? ',' : '['))
    -  {
         *end_ptr = ++p;
    -  }
     
       const char *tok = nullptr;
       int cs;
       hb_glyph_info_t info = {0};
       hb_glyph_position_t pos = {0};
     
    -#line 512 "hb-buffer-deserialize-json.hh"
    +#line 552 "hb-buffer-deserialize-json.hh"
             {
             cs = deserialize_json_start;
             }
     
    -#line 517 "hb-buffer-deserialize-json.hh"
    +#line 555 "hb-buffer-deserialize-json.hh"
             {
             int _slen;
             int _trans;
    @@ -541,8 +588,8 @@ _resume:
             case 1:
     #line 38 "hb-buffer-deserialize-json.rl"
             {
    -        memset (&info, 0, sizeof (info));
    -        memset (&pos , 0, sizeof (pos ));
    +        hb_memset (&info, 0, sizeof (info));
    +        hb_memset (&pos , 0, sizeof (pos ));
     }
             break;
             case 5:
    @@ -561,25 +608,25 @@ _resume:
             tok = p;
     }
             break;
    -        case 15:
    +        case 17:
     #line 55 "hb-buffer-deserialize-json.rl"
             { if (unlikely (!buffer->ensure_glyphs ())) return false; }
             break;
    -        case 21:
    +        case 23:
     #line 56 "hb-buffer-deserialize-json.rl"
             { if (unlikely (!buffer->ensure_unicode ())) return false; }
             break;
    -        case 16:
    +        case 18:
     #line 58 "hb-buffer-deserialize-json.rl"
             {
             /* TODO Unescape \" and \\ if found. */
             if (!hb_font_glyph_from_string (font,
    -                                        tok, p - tok,
    +                                        tok+1, p - tok - 2, /* Skip "" */
                                             &info.codepoint))
               return false;
     }
             break;
    -        case 18:
    +        case 20:
     #line 66 "hb-buffer-deserialize-json.rl"
             { if (!parse_uint (tok, p, &info.codepoint)) return false; }
             break;
    @@ -604,6 +651,10 @@ _resume:
             { if (!parse_int  (tok, p, &pos.y_advance)) return false; }
             break;
             case 14:
    +#line 72 "hb-buffer-deserialize-json.rl"
    +        { if (!parse_uint (tok, p, &info.mask    )) return false; }
    +        break;
    +        case 16:
     #line 51 "hb-buffer-deserialize-json.rl"
             {
             tok = p;
    @@ -611,7 +662,7 @@ _resume:
     #line 55 "hb-buffer-deserialize-json.rl"
             { if (unlikely (!buffer->ensure_glyphs ())) return false; }
             break;
    -        case 20:
    +        case 22:
     #line 51 "hb-buffer-deserialize-json.rl"
             {
             tok = p;
    @@ -619,12 +670,12 @@ _resume:
     #line 56 "hb-buffer-deserialize-json.rl"
             { if (unlikely (!buffer->ensure_unicode ())) return false; }
             break;
    -        case 17:
    +        case 19:
     #line 58 "hb-buffer-deserialize-json.rl"
             {
             /* TODO Unescape \" and \\ if found. */
             if (!hb_font_glyph_from_string (font,
    -                                        tok, p - tok,
    +                                        tok+1, p - tok - 2, /* Skip "" */
                                             &info.codepoint))
               return false;
     }
    @@ -637,7 +688,7 @@ _resume:
             *end_ptr = p;
     }
             break;
    -        case 19:
    +        case 21:
     #line 66 "hb-buffer-deserialize-json.rl"
             { if (!parse_uint (tok, p, &info.codepoint)) return false; }
     #line 43 "hb-buffer-deserialize-json.rl"
    @@ -709,7 +760,19 @@ _resume:
             *end_ptr = p;
     }
             break;
    -#line 713 "hb-buffer-deserialize-json.hh"
    +        case 15:
    +#line 72 "hb-buffer-deserialize-json.rl"
    +        { if (!parse_uint (tok, p, &info.mask    )) return false; }
    +#line 43 "hb-buffer-deserialize-json.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +#line 733 "hb-buffer-deserialize-json.hh"
             }
     
     _again:
    @@ -721,7 +784,7 @@ _again:
             _out: {}
             }
     
    -#line 136 "hb-buffer-deserialize-json.rl"
    +#line 137 "hb-buffer-deserialize-json.rl"
     
     
       *end_ptr = p;
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-buffer-deserialize-text-glyphs.hh b/src/java.desktop/share/native/libharfbuzz/hb-buffer-deserialize-text-glyphs.hh
    new file mode 100644
    index 0000000000000..cf9c281e86ec5
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-buffer-deserialize-text-glyphs.hh
    @@ -0,0 +1,692 @@
    +
    +#line 1 "hb-buffer-deserialize-text-glyphs.rl"
    +/*
    + * Copyright © 2013  Google, Inc.
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + *
    + * Google Author(s): Behdad Esfahbod
    + */
    +
    +#ifndef HB_BUFFER_DESERIALIZE_TEXT_GLYPHS_HH
    +#define HB_BUFFER_DESERIALIZE_TEXT_GLYPHS_HH
    +
    +#include "hb.hh"
    +
    +
    +#line 33 "hb-buffer-deserialize-text-glyphs.hh"
    +static const unsigned char _deserialize_text_glyphs_trans_keys[] = {
    +        0u, 0u, 48u, 57u, 45u, 57u, 48u, 57u, 45u, 57u, 48u, 57u, 48u, 57u, 45u, 57u,
    +        48u, 57u, 44u, 44u, 45u, 57u, 48u, 57u, 44u, 57u, 43u, 124u, 9u, 124u, 9u, 124u,
    +        9u, 124u, 9u, 124u, 9u, 124u, 9u, 124u, 9u, 124u, 9u, 124u, 9u, 124u, 9u, 124u,
    +        9u, 124u, 9u, 124u, 9u, 124u, 0
    +};
    +
    +static const char _deserialize_text_glyphs_key_spans[] = {
    +        0, 10, 13, 10, 13, 10, 10, 13,
    +        10, 1, 13, 10, 14, 82, 116, 116,
    +        116, 116, 116, 116, 116, 116, 116, 116,
    +        116, 116, 116
    +};
    +
    +static const short _deserialize_text_glyphs_index_offsets[] = {
    +        0, 0, 11, 25, 36, 50, 61, 72,
    +        86, 97, 99, 113, 124, 139, 222, 339,
    +        456, 573, 690, 807, 924, 1041, 1158, 1275,
    +        1392, 1509, 1626
    +};
    +
    +static const char _deserialize_text_glyphs_indicies[] = {
    +        0, 2, 2, 2, 2, 2, 2,
    +        2, 2, 2, 1, 3, 1, 1, 4,
    +        5, 5, 5, 5, 5, 5, 5, 5,
    +        5, 1, 6, 7, 7, 7, 7, 7,
    +        7, 7, 7, 7, 1, 8, 1, 1,
    +        9, 10, 10, 10, 10, 10, 10, 10,
    +        10, 10, 1, 11, 12, 12, 12, 12,
    +        12, 12, 12, 12, 12, 1, 13, 14,
    +        14, 14, 14, 14, 14, 14, 14, 14,
    +        1, 15, 1, 1, 16, 17, 17, 17,
    +        17, 17, 17, 17, 17, 17, 1, 18,
    +        19, 19, 19, 19, 19, 19, 19, 19,
    +        19, 1, 20, 1, 21, 1, 1, 22,
    +        23, 23, 23, 23, 23, 23, 23, 23,
    +        23, 1, 24, 25, 25, 25, 25, 25,
    +        25, 25, 25, 25, 1, 20, 1, 1,
    +        1, 19, 19, 19, 19, 19, 19, 19,
    +        19, 19, 19, 1, 26, 26, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 26, 1,
    +        1, 26, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 26, 26, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 26, 1, 28,
    +        28, 28, 28, 28, 27, 27, 27, 27,
    +        27, 27, 27, 27, 27, 27, 27, 27,
    +        27, 27, 27, 27, 27, 27, 28, 27,
    +        27, 29, 27, 27, 27, 27, 27, 27,
    +        27, 30, 1, 27, 27, 27, 27, 27,
    +        27, 27, 27, 27, 27, 27, 27, 27,
    +        27, 27, 27, 31, 27, 27, 32, 27,
    +        27, 27, 27, 27, 27, 27, 27, 27,
    +        27, 27, 27, 27, 27, 27, 27, 27,
    +        27, 27, 27, 27, 27, 27, 27, 27,
    +        27, 27, 33, 1, 27, 27, 27, 27,
    +        27, 27, 27, 27, 27, 27, 27, 27,
    +        27, 27, 27, 27, 27, 27, 27, 27,
    +        27, 27, 27, 27, 27, 27, 27, 27,
    +        27, 27, 28, 27, 34, 34, 34, 34,
    +        34, 26, 26, 26, 26, 26, 26, 26,
    +        26, 26, 26, 26, 26, 26, 26, 26,
    +        26, 26, 26, 34, 26, 26, 35, 26,
    +        26, 26, 26, 26, 26, 26, 36, 1,
    +        26, 26, 26, 26, 26, 26, 26, 26,
    +        26, 26, 26, 26, 26, 26, 26, 26,
    +        37, 26, 26, 38, 26, 26, 26, 26,
    +        26, 26, 26, 26, 26, 26, 26, 26,
    +        26, 26, 26, 26, 26, 26, 26, 26,
    +        26, 26, 26, 26, 26, 26, 26, 39,
    +        1, 26, 26, 26, 26, 26, 26, 26,
    +        26, 26, 26, 26, 26, 26, 26, 26,
    +        26, 26, 26, 26, 26, 26, 26, 26,
    +        26, 26, 26, 26, 26, 26, 26, 40,
    +        26, 41, 41, 41, 41, 41, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        41, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 42, 1, 43, 43,
    +        43, 43, 43, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 43, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 44, 1, 41, 41, 41, 41, 41,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 41, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 45, 45, 45, 45, 45, 45,
    +        45, 45, 45, 45, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 42, 1,
    +        46, 46, 46, 46, 46, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 46,
    +        1, 1, 47, 1, 1, 1, 1, 1,
    +        1, 1, 1, 48, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 49, 1, 50, 50, 50,
    +        50, 50, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 50, 1, 1, 51,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        52, 1, 50, 50, 50, 50, 50, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 50, 1, 1, 51, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 12, 12, 12, 12, 12, 12, 12,
    +        12, 12, 12, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 52, 1, 46,
    +        46, 46, 46, 46, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 46, 1,
    +        1, 47, 1, 1, 1, 1, 1, 1,
    +        1, 1, 48, 1, 1, 1, 7, 7,
    +        7, 7, 7, 7, 7, 7, 7, 7,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 49, 1, 53, 53, 53, 53,
    +        53, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 53, 1, 1, 54, 1,
    +        1, 1, 1, 1, 1, 1, 55, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 56, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 57,
    +        1, 58, 58, 58, 58, 58, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        58, 1, 1, 59, 1, 1, 1, 1,
    +        1, 1, 1, 60, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 61, 1, 58, 58,
    +        58, 58, 58, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 58, 1, 1,
    +        59, 1, 1, 1, 1, 1, 1, 1,
    +        60, 1, 1, 1, 1, 25, 25, 25,
    +        25, 25, 25, 25, 25, 25, 25, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 61, 1, 53, 53, 53, 53, 53,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 53, 1, 1, 54, 1, 1,
    +        1, 1, 1, 1, 1, 55, 1, 1,
    +        1, 1, 62, 62, 62, 62, 62, 62,
    +        62, 62, 62, 62, 1, 1, 1, 1,
    +        1, 1, 56, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 57, 1,
    +        0
    +};
    +
    +static const char _deserialize_text_glyphs_trans_targs[] = {
    +        16, 0, 18, 3, 19, 22, 19, 22,
    +        5, 20, 21, 20, 21, 23, 26, 8,
    +        9, 12, 9, 12, 10, 11, 24, 25,
    +        24, 25, 15, 15, 14, 1, 2, 6,
    +        7, 13, 15, 1, 2, 6, 7, 13,
    +        14, 17, 14, 17, 14, 18, 17, 1,
    +        4, 14, 17, 1, 14, 17, 1, 2,
    +        7, 14, 17, 1, 2, 14, 26
    +};
    +
    +static const char _deserialize_text_glyphs_trans_actions[] = {
    +        1, 0, 1, 1, 1, 1, 0, 0,
    +        1, 1, 1, 0, 0, 1, 1, 1,
    +        1, 1, 0, 0, 2, 1, 1, 1,
    +        0, 0, 0, 4, 3, 5, 5, 5,
    +        5, 4, 6, 7, 7, 7, 7, 0,
    +        6, 8, 8, 0, 0, 0, 9, 10,
    +        10, 9, 11, 12, 11, 13, 14, 14,
    +        14, 13, 15, 16, 16, 15, 0
    +};
    +
    +static const char _deserialize_text_glyphs_eof_actions[] = {
    +        0, 0, 0, 0, 0, 0, 0, 0,
    +        0, 0, 0, 0, 0, 0, 3, 6,
    +        8, 0, 8, 9, 11, 11, 9, 13,
    +        15, 15, 13
    +};
    +
    +static const int deserialize_text_glyphs_start = 14;
    +static const int deserialize_text_glyphs_first_final = 14;
    +static const int deserialize_text_glyphs_error = 0;
    +
    +static const int deserialize_text_glyphs_en_main = 14;
    +
    +
    +#line 98 "hb-buffer-deserialize-text-glyphs.rl"
    +
    +
    +static hb_bool_t
    +_hb_buffer_deserialize_text_glyphs (hb_buffer_t *buffer,
    +                                    const char *buf,
    +                                    unsigned int buf_len,
    +                                    const char **end_ptr,
    +                                    hb_font_t *font)
    +{
    +  const char *p = buf, *pe = buf + buf_len, *eof = pe, *orig_pe = pe;
    +
    +  /* Ensure we have positions. */
    +  (void) hb_buffer_get_glyph_positions (buffer, nullptr);
    +
    +  while (p < pe && ISSPACE (*p))
    +    p++;
    +  if (p < pe && *p == (buffer->len ? '|' : '['))
    +    *end_ptr = ++p;
    +
    +  const char *end = strchr ((char *) p, ']');
    +  if (end)
    +    pe = eof = end;
    +  else
    +  {
    +    end = strrchr ((char *) p, '|');
    +    if (end)
    +      pe = eof = end;
    +    else
    +      pe = eof = p;
    +  }
    +
    +  const char *tok = nullptr;
    +  int cs;
    +  hb_glyph_info_t info = {0};
    +  hb_glyph_position_t pos = {0};
    +
    +#line 346 "hb-buffer-deserialize-text-glyphs.hh"
    +        {
    +        cs = deserialize_text_glyphs_start;
    +        }
    +
    +#line 349 "hb-buffer-deserialize-text-glyphs.hh"
    +        {
    +        int _slen;
    +        int _trans;
    +        const unsigned char *_keys;
    +        const char *_inds;
    +        if ( p == pe )
    +                goto _test_eof;
    +        if ( cs == 0 )
    +                goto _out;
    +_resume:
    +        _keys = _deserialize_text_glyphs_trans_keys + (cs<<1);
    +        _inds = _deserialize_text_glyphs_indicies + _deserialize_text_glyphs_index_offsets[cs];
    +
    +        _slen = _deserialize_text_glyphs_key_spans[cs];
    +        _trans = _inds[ _slen > 0 && _keys[0] <=(*p) &&
    +                (*p) <= _keys[1] ?
    +                (*p) - _keys[0] : _slen ];
    +
    +        cs = _deserialize_text_glyphs_trans_targs[_trans];
    +
    +        if ( _deserialize_text_glyphs_trans_actions[_trans] == 0 )
    +                goto _again;
    +
    +        switch ( _deserialize_text_glyphs_trans_actions[_trans] ) {
    +        case 1:
    +#line 51 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        tok = p;
    +}
    +        break;
    +        case 7:
    +#line 55 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        /* TODO Unescape delimiters. */
    +        if (!hb_font_glyph_from_string (font,
    +                                        tok, p - tok,
    +                                        &info.codepoint))
    +          return false;
    +}
    +        break;
    +        case 14:
    +#line 63 "hb-buffer-deserialize-text-glyphs.rl"
    +        { if (!parse_uint (tok, p, &info.cluster )) return false; }
    +        break;
    +        case 2:
    +#line 64 "hb-buffer-deserialize-text-glyphs.rl"
    +        { if (!parse_int  (tok, p, &pos.x_offset )) return false; }
    +        break;
    +        case 16:
    +#line 65 "hb-buffer-deserialize-text-glyphs.rl"
    +        { if (!parse_int  (tok, p, &pos.y_offset )) return false; }
    +        break;
    +        case 10:
    +#line 66 "hb-buffer-deserialize-text-glyphs.rl"
    +        { if (!parse_int  (tok, p, &pos.x_advance)) return false; }
    +        break;
    +        case 12:
    +#line 67 "hb-buffer-deserialize-text-glyphs.rl"
    +        { if (!parse_int  (tok, p, &pos.y_advance)) return false; }
    +        break;
    +        case 4:
    +#line 38 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        hb_memset (&info, 0, sizeof (info));
    +        hb_memset (&pos , 0, sizeof (pos ));
    +}
    +#line 51 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        tok = p;
    +}
    +        break;
    +        case 6:
    +#line 55 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        /* TODO Unescape delimiters. */
    +        if (!hb_font_glyph_from_string (font,
    +                                        tok, p - tok,
    +                                        &info.codepoint))
    +          return false;
    +}
    +#line 43 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +        case 13:
    +#line 63 "hb-buffer-deserialize-text-glyphs.rl"
    +        { if (!parse_uint (tok, p, &info.cluster )) return false; }
    +#line 43 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +        case 15:
    +#line 65 "hb-buffer-deserialize-text-glyphs.rl"
    +        { if (!parse_int  (tok, p, &pos.y_offset )) return false; }
    +#line 43 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +        case 9:
    +#line 66 "hb-buffer-deserialize-text-glyphs.rl"
    +        { if (!parse_int  (tok, p, &pos.x_advance)) return false; }
    +#line 43 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +        case 11:
    +#line 67 "hb-buffer-deserialize-text-glyphs.rl"
    +        { if (!parse_int  (tok, p, &pos.y_advance)) return false; }
    +#line 43 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +        case 8:
    +#line 68 "hb-buffer-deserialize-text-glyphs.rl"
    +        { if (!parse_uint (tok, p, &info.mask    )) return false; }
    +#line 43 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +        case 5:
    +#line 38 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        hb_memset (&info, 0, sizeof (info));
    +        hb_memset (&pos , 0, sizeof (pos ));
    +}
    +#line 51 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        tok = p;
    +}
    +#line 55 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        /* TODO Unescape delimiters. */
    +        if (!hb_font_glyph_from_string (font,
    +                                        tok, p - tok,
    +                                        &info.codepoint))
    +          return false;
    +}
    +        break;
    +        case 3:
    +#line 38 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        hb_memset (&info, 0, sizeof (info));
    +        hb_memset (&pos , 0, sizeof (pos ));
    +}
    +#line 51 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        tok = p;
    +}
    +#line 55 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        /* TODO Unescape delimiters. */
    +        if (!hb_font_glyph_from_string (font,
    +                                        tok, p - tok,
    +                                        &info.codepoint))
    +          return false;
    +}
    +#line 43 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +#line 516 "hb-buffer-deserialize-text-glyphs.hh"
    +        }
    +
    +_again:
    +        if ( cs == 0 )
    +                goto _out;
    +        if ( ++p != pe )
    +                goto _resume;
    +        _test_eof: {}
    +        if ( p == eof )
    +        {
    +        switch ( _deserialize_text_glyphs_eof_actions[cs] ) {
    +        case 6:
    +#line 55 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        /* TODO Unescape delimiters. */
    +        if (!hb_font_glyph_from_string (font,
    +                                        tok, p - tok,
    +                                        &info.codepoint))
    +          return false;
    +}
    +#line 43 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +        case 13:
    +#line 63 "hb-buffer-deserialize-text-glyphs.rl"
    +        { if (!parse_uint (tok, p, &info.cluster )) return false; }
    +#line 43 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +        case 15:
    +#line 65 "hb-buffer-deserialize-text-glyphs.rl"
    +        { if (!parse_int  (tok, p, &pos.y_offset )) return false; }
    +#line 43 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +        case 9:
    +#line 66 "hb-buffer-deserialize-text-glyphs.rl"
    +        { if (!parse_int  (tok, p, &pos.x_advance)) return false; }
    +#line 43 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +        case 11:
    +#line 67 "hb-buffer-deserialize-text-glyphs.rl"
    +        { if (!parse_int  (tok, p, &pos.y_advance)) return false; }
    +#line 43 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +        case 8:
    +#line 68 "hb-buffer-deserialize-text-glyphs.rl"
    +        { if (!parse_uint (tok, p, &info.mask    )) return false; }
    +#line 43 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +        case 3:
    +#line 38 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        hb_memset (&info, 0, sizeof (info));
    +        hb_memset (&pos , 0, sizeof (pos ));
    +}
    +#line 51 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        tok = p;
    +}
    +#line 55 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        /* TODO Unescape delimiters. */
    +        if (!hb_font_glyph_from_string (font,
    +                                        tok, p - tok,
    +                                        &info.codepoint))
    +          return false;
    +}
    +#line 43 "hb-buffer-deserialize-text-glyphs.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +#line 616 "hb-buffer-deserialize-text-glyphs.hh"
    +        }
    +        }
    +
    +        _out: {}
    +        }
    +
    +#line 136 "hb-buffer-deserialize-text-glyphs.rl"
    +
    +
    +  if (pe < orig_pe && *pe == ']')
    +  {
    +    pe++;
    +    if (p == pe)
    +      p++;
    +  }
    +
    +  *end_ptr = p;
    +
    +  return p == pe;
    +}
    +
    +#endif /* HB_BUFFER_DESERIALIZE_TEXT_GLYPHS_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-buffer-deserialize-text-unicode.hh b/src/java.desktop/share/native/libharfbuzz/hb-buffer-deserialize-text-unicode.hh
    new file mode 100644
    index 0000000000000..f0c9465453327
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-buffer-deserialize-text-unicode.hh
    @@ -0,0 +1,332 @@
    +
    +#line 1 "hb-buffer-deserialize-text-unicode.rl"
    +/*
    + * Copyright © 2013  Google, Inc.
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + *
    + * Google Author(s): Behdad Esfahbod
    + */
    +
    +#ifndef HB_BUFFER_DESERIALIZE_TEXT_UNICODE_HH
    +#define HB_BUFFER_DESERIALIZE_TEXT_UNICODE_HH
    +
    +#include "hb.hh"
    +
    +
    +#line 33 "hb-buffer-deserialize-text-unicode.hh"
    +static const unsigned char _deserialize_text_unicode_trans_keys[] = {
    +        0u, 0u, 9u, 117u, 43u, 102u, 48u, 102u, 48u, 57u, 9u, 124u, 9u, 124u, 9u, 124u,
    +        9u, 124u, 0
    +};
    +
    +static const char _deserialize_text_unicode_key_spans[] = {
    +        0, 109, 60, 55, 10, 116, 116, 116,
    +        116
    +};
    +
    +static const short _deserialize_text_unicode_index_offsets[] = {
    +        0, 0, 110, 171, 227, 238, 355, 472,
    +        589
    +};
    +
    +static const char _deserialize_text_unicode_indicies[] = {
    +        0, 0, 0, 0, 0, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        0, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 2, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 2, 1, 3,
    +        1, 1, 1, 1, 4, 4, 4, 4,
    +        4, 4, 4, 4, 4, 4, 1, 1,
    +        1, 1, 1, 1, 1, 4, 4, 4,
    +        4, 4, 4, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 4, 4, 4,
    +        4, 4, 4, 1, 4, 4, 4, 4,
    +        4, 4, 4, 4, 4, 4, 1, 1,
    +        1, 1, 1, 1, 1, 4, 4, 4,
    +        4, 4, 4, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 4, 4, 4,
    +        4, 4, 4, 1, 5, 6, 6, 6,
    +        6, 6, 6, 6, 6, 6, 1, 7,
    +        7, 7, 7, 7, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 7, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 8, 8,
    +        8, 8, 8, 8, 8, 8, 8, 8,
    +        1, 1, 1, 9, 1, 1, 1, 8,
    +        8, 8, 8, 8, 8, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 8,
    +        8, 8, 8, 8, 8, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 10, 1, 11, 11, 11, 11,
    +        11, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 11, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 0,
    +        1, 12, 12, 12, 12, 12, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        12, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 13, 1, 12, 12,
    +        12, 12, 12, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 12, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 14, 14, 14,
    +        14, 14, 14, 14, 14, 14, 14, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 1, 1, 1, 1, 1, 1, 1,
    +        1, 13, 1, 0
    +};
    +
    +static const char _deserialize_text_unicode_trans_targs[] = {
    +        1, 0, 2, 3, 5, 7, 8, 6,
    +        5, 4, 1, 6, 6, 1, 8
    +};
    +
    +static const char _deserialize_text_unicode_trans_actions[] = {
    +        0, 0, 1, 0, 2, 2, 2, 3,
    +        0, 4, 3, 0, 5, 5, 0
    +};
    +
    +static const char _deserialize_text_unicode_eof_actions[] = {
    +        0, 0, 0, 0, 0, 3, 0, 5,
    +        5
    +};
    +
    +static const int deserialize_text_unicode_start = 1;
    +static const int deserialize_text_unicode_first_final = 5;
    +static const int deserialize_text_unicode_error = 0;
    +
    +static const int deserialize_text_unicode_en_main = 1;
    +
    +
    +#line 79 "hb-buffer-deserialize-text-unicode.rl"
    +
    +
    +static hb_bool_t
    +_hb_buffer_deserialize_text_unicode (hb_buffer_t *buffer,
    +                                     const char *buf,
    +                                     unsigned int buf_len,
    +                                     const char **end_ptr,
    +                                     hb_font_t *font)
    +{
    +  const char *p = buf, *pe = buf + buf_len, *eof = pe, *orig_pe = pe;
    +
    +  while (p < pe && ISSPACE (*p))
    +    p++;
    +  if (p < pe && *p == (buffer->len ? '|' : '<'))
    +    *end_ptr = ++p;
    +
    +  const char *end = strchr ((char *) p, '>');
    +  if (end)
    +    pe = eof = end;
    +  else
    +  {
    +    end = strrchr ((char *) p, '|');
    +    if (end)
    +      pe = eof = end;
    +    else
    +      pe = eof = p;
    +  }
    +
    +
    +  const char *tok = nullptr;
    +  int cs;
    +  hb_glyph_info_t info = {0};
    +  const hb_glyph_position_t pos = {0};
    +
    +#line 194 "hb-buffer-deserialize-text-unicode.hh"
    +        {
    +        cs = deserialize_text_unicode_start;
    +        }
    +
    +#line 197 "hb-buffer-deserialize-text-unicode.hh"
    +        {
    +        int _slen;
    +        int _trans;
    +        const unsigned char *_keys;
    +        const char *_inds;
    +        if ( p == pe )
    +                goto _test_eof;
    +        if ( cs == 0 )
    +                goto _out;
    +_resume:
    +        _keys = _deserialize_text_unicode_trans_keys + (cs<<1);
    +        _inds = _deserialize_text_unicode_indicies + _deserialize_text_unicode_index_offsets[cs];
    +
    +        _slen = _deserialize_text_unicode_key_spans[cs];
    +        _trans = _inds[ _slen > 0 && _keys[0] <=(*p) &&
    +                (*p) <= _keys[1] ?
    +                (*p) - _keys[0] : _slen ];
    +
    +        cs = _deserialize_text_unicode_trans_targs[_trans];
    +
    +        if ( _deserialize_text_unicode_trans_actions[_trans] == 0 )
    +                goto _again;
    +
    +        switch ( _deserialize_text_unicode_trans_actions[_trans] ) {
    +        case 1:
    +#line 38 "hb-buffer-deserialize-text-unicode.rl"
    +        {
    +        hb_memset (&info, 0, sizeof (info));
    +}
    +        break;
    +        case 2:
    +#line 51 "hb-buffer-deserialize-text-unicode.rl"
    +        {
    +        tok = p;
    +}
    +        break;
    +        case 4:
    +#line 55 "hb-buffer-deserialize-text-unicode.rl"
    +        {if (!parse_hex (tok, p, &info.codepoint )) return false; }
    +        break;
    +        case 3:
    +#line 55 "hb-buffer-deserialize-text-unicode.rl"
    +        {if (!parse_hex (tok, p, &info.codepoint )) return false; }
    +#line 42 "hb-buffer-deserialize-text-unicode.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        if (buffer->have_positions)
    +          buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +        case 5:
    +#line 57 "hb-buffer-deserialize-text-unicode.rl"
    +        { if (!parse_uint (tok, p, &info.cluster )) return false; }
    +#line 42 "hb-buffer-deserialize-text-unicode.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        if (buffer->have_positions)
    +          buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +#line 256 "hb-buffer-deserialize-text-unicode.hh"
    +        }
    +
    +_again:
    +        if ( cs == 0 )
    +                goto _out;
    +        if ( ++p != pe )
    +                goto _resume;
    +        _test_eof: {}
    +        if ( p == eof )
    +        {
    +        switch ( _deserialize_text_unicode_eof_actions[cs] ) {
    +        case 3:
    +#line 55 "hb-buffer-deserialize-text-unicode.rl"
    +        {if (!parse_hex (tok, p, &info.codepoint )) return false; }
    +#line 42 "hb-buffer-deserialize-text-unicode.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        if (buffer->have_positions)
    +          buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +        case 5:
    +#line 57 "hb-buffer-deserialize-text-unicode.rl"
    +        { if (!parse_uint (tok, p, &info.cluster )) return false; }
    +#line 42 "hb-buffer-deserialize-text-unicode.rl"
    +        {
    +        buffer->add_info (info);
    +        if (unlikely (!buffer->successful))
    +          return false;
    +        if (buffer->have_positions)
    +          buffer->pos[buffer->len - 1] = pos;
    +        *end_ptr = p;
    +}
    +        break;
    +#line 289 "hb-buffer-deserialize-text-unicode.hh"
    +        }
    +        }
    +
    +        _out: {}
    +        }
    +
    +#line 115 "hb-buffer-deserialize-text-unicode.rl"
    +
    +
    +  if (pe < orig_pe && *pe == '>')
    +  {
    +    pe++;
    +    if (p == pe)
    +      p++;
    +  }
    +
    +  *end_ptr = p;
    +
    +  return p == pe;
    +}
    +
    +#endif /* HB_BUFFER_DESERIALIZE_TEXT_UNICODE_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-buffer-deserialize-text.hh b/src/java.desktop/share/native/libharfbuzz/hb-buffer-deserialize-text.hh
    deleted file mode 100644
    index d34d5a41a4a46..0000000000000
    --- a/src/java.desktop/share/native/libharfbuzz/hb-buffer-deserialize-text.hh
    +++ /dev/null
    @@ -1,853 +0,0 @@
    -
    -#line 1 "hb-buffer-deserialize-text.rl"
    -/*
    - * Copyright © 2013  Google, Inc.
    - *
    - *  This is part of HarfBuzz, a text shaping library.
    - *
    - * Permission is hereby granted, without written agreement and without
    - * license or royalty fees, to use, copy, modify, and distribute this
    - * software and its documentation for any purpose, provided that the
    - * above copyright notice and the following two paragraphs appear in
    - * all copies of this software.
    - *
    - * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    - * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    - * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    - * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    - * DAMAGE.
    - *
    - * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    - * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    - * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    - * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    - * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    - *
    - * Google Author(s): Behdad Esfahbod
    - */
    -
    -#ifndef HB_BUFFER_DESERIALIZE_TEXT_HH
    -#define HB_BUFFER_DESERIALIZE_TEXT_HH
    -
    -#include "hb.hh"
    -
    -
    -#line 36 "hb-buffer-deserialize-text.hh"
    -static const unsigned char _deserialize_text_trans_keys[] = {
    -        0u, 0u, 9u, 91u, 85u, 85u, 43u, 43u, 48u, 102u, 9u, 85u, 48u, 57u, 45u, 57u,
    -        48u, 57u, 48u, 57u, 45u, 57u, 48u, 57u, 44u, 44u, 45u, 57u, 48u, 57u, 44u, 57u,
    -        43u, 124u, 45u, 57u, 48u, 57u, 9u, 124u, 9u, 124u, 0u, 0u, 9u, 85u, 9u, 124u,
    -        9u, 124u, 9u, 124u, 9u, 124u, 9u, 124u, 9u, 124u, 9u, 124u, 9u, 124u, 9u, 124u,
    -        9u, 124u, 9u, 124u, 9u, 124u, 9u, 124u, 9u, 124u, 0
    -};
    -
    -static const char _deserialize_text_key_spans[] = {
    -        0, 83, 1, 1, 55, 77, 10, 13,
    -        10, 10, 13, 10, 1, 13, 10, 14,
    -        82, 13, 10, 116, 116, 0, 77, 116,
    -        116, 116, 116, 116, 116, 116, 116, 116,
    -        116, 116, 116, 116, 116
    -};
    -
    -static const short _deserialize_text_index_offsets[] = {
    -        0, 0, 84, 86, 88, 144, 222, 233,
    -        247, 258, 269, 283, 294, 296, 310, 321,
    -        336, 419, 433, 444, 561, 678, 679, 757,
    -        874, 991, 1108, 1225, 1342, 1459, 1576, 1693,
    -        1810, 1927, 2044, 2161, 2278
    -};
    -
    -static const char _deserialize_text_indicies[] = {
    -        0, 0, 0, 0, 0, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        0, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 2, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 3, 1, 4, 1, 5,
    -        1, 6, 6, 6, 6, 6, 6, 6,
    -        6, 6, 6, 1, 1, 1, 1, 1,
    -        1, 1, 6, 6, 6, 6, 6, 6,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 6, 6, 6, 6, 6, 6,
    -        1, 7, 7, 7, 7, 7, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        7, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 4, 1, 8,
    -        9, 9, 9, 9, 9, 9, 9, 9,
    -        9, 1, 10, 1, 1, 11, 12, 12,
    -        12, 12, 12, 12, 12, 12, 12, 1,
    -        13, 14, 14, 14, 14, 14, 14, 14,
    -        14, 14, 1, 15, 16, 16, 16, 16,
    -        16, 16, 16, 16, 16, 1, 17, 1,
    -        1, 18, 19, 19, 19, 19, 19, 19,
    -        19, 19, 19, 1, 20, 21, 21, 21,
    -        21, 21, 21, 21, 21, 21, 1, 22,
    -        1, 23, 1, 1, 24, 25, 25, 25,
    -        25, 25, 25, 25, 25, 25, 1, 26,
    -        27, 27, 27, 27, 27, 27, 27, 27,
    -        27, 1, 22, 1, 1, 1, 21, 21,
    -        21, 21, 21, 21, 21, 21, 21, 21,
    -        1, 28, 28, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 28, 1, 1, 28, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 28, 28, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 28, 1, 29, 1, 1, 30,
    -        31, 31, 31, 31, 31, 31, 31, 31,
    -        31, 1, 32, 33, 33, 33, 33, 33,
    -        33, 33, 33, 33, 1, 34, 34, 34,
    -        34, 34, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 34, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 35, 35, 35, 35,
    -        35, 35, 35, 35, 35, 35, 1, 1,
    -        1, 36, 37, 1, 1, 35, 35, 35,
    -        35, 35, 35, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 35, 35, 35,
    -        35, 35, 35, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        38, 1, 39, 39, 39, 39, 39, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 39, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 40,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 41, 1, 1,
    -        7, 7, 7, 7, 7, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 7,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 4, 1, 42, 42,
    -        42, 42, 42, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 42, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 43, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 44, 1, 42, 42, 42, 42, 42,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 42, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 45, 45, 45, 45, 45, 45,
    -        45, 45, 45, 45, 1, 1, 1, 1,
    -        43, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 44, 1,
    -        47, 47, 47, 47, 47, 46, 46, 46,
    -        46, 46, 46, 46, 46, 46, 46, 46,
    -        46, 46, 46, 46, 46, 46, 46, 47,
    -        46, 46, 46, 46, 46, 46, 46, 46,
    -        46, 46, 48, 1, 46, 46, 46, 46,
    -        46, 46, 46, 46, 46, 46, 46, 46,
    -        46, 46, 46, 46, 49, 46, 46, 50,
    -        46, 46, 46, 46, 46, 46, 46, 46,
    -        46, 46, 46, 46, 46, 46, 46, 46,
    -        46, 46, 46, 46, 46, 46, 46, 46,
    -        46, 46, 46, 51, 52, 46, 46, 46,
    -        46, 46, 46, 46, 46, 46, 46, 46,
    -        46, 46, 46, 46, 46, 46, 46, 46,
    -        46, 46, 46, 46, 46, 46, 46, 46,
    -        46, 46, 46, 53, 46, 54, 54, 54,
    -        54, 54, 28, 28, 28, 28, 28, 28,
    -        28, 28, 28, 28, 28, 28, 28, 28,
    -        28, 28, 28, 28, 54, 28, 28, 28,
    -        28, 28, 28, 28, 28, 28, 28, 55,
    -        1, 28, 28, 28, 28, 28, 28, 28,
    -        28, 28, 28, 28, 28, 28, 28, 28,
    -        28, 56, 28, 28, 57, 28, 28, 28,
    -        28, 28, 28, 28, 28, 28, 28, 28,
    -        28, 28, 28, 28, 28, 28, 28, 28,
    -        28, 28, 28, 28, 28, 28, 28, 28,
    -        58, 59, 28, 28, 28, 28, 28, 28,
    -        28, 28, 28, 28, 28, 28, 28, 28,
    -        28, 28, 28, 28, 28, 28, 28, 28,
    -        28, 28, 28, 28, 28, 28, 28, 28,
    -        60, 28, 61, 61, 61, 61, 61, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 61, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 62, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 63, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 64, 1, 65,
    -        65, 65, 65, 65, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 65, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 40, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 66, 1, 67, 67, 67, 67,
    -        67, 46, 46, 46, 46, 46, 46, 46,
    -        46, 46, 46, 46, 46, 46, 46, 46,
    -        46, 46, 46, 67, 46, 46, 46, 46,
    -        46, 46, 46, 46, 46, 46, 48, 1,
    -        46, 46, 46, 46, 46, 46, 46, 46,
    -        46, 46, 46, 46, 46, 46, 46, 46,
    -        49, 46, 46, 50, 46, 46, 46, 46,
    -        46, 46, 46, 46, 46, 46, 46, 46,
    -        46, 46, 46, 46, 46, 46, 46, 46,
    -        46, 46, 46, 46, 46, 46, 46, 51,
    -        52, 46, 46, 46, 46, 46, 46, 46,
    -        46, 46, 46, 46, 46, 46, 46, 46,
    -        46, 46, 46, 46, 46, 46, 46, 46,
    -        46, 46, 46, 46, 46, 46, 46, 53,
    -        46, 68, 68, 68, 68, 68, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        68, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 69, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        70, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 43, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 71, 1, 72, 72,
    -        72, 72, 72, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 72, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        73, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 74, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 75, 1, 72, 72, 72, 72, 72,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 72, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 73, 1, 1,
    -        1, 1, 27, 27, 27, 27, 27, 27,
    -        27, 27, 27, 27, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 74,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 75, 1,
    -        68, 68, 68, 68, 68, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 68,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 69, 1, 1, 1, 1, 76,
    -        76, 76, 76, 76, 76, 76, 76, 76,
    -        76, 1, 1, 1, 1, 1, 1, 70,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 43, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 71, 1, 77, 77, 77,
    -        77, 77, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 77, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 78, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        79, 1, 77, 77, 77, 77, 77, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 77, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 33, 33, 33, 33, 33, 33, 33,
    -        33, 33, 33, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 78, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 79, 1, 61,
    -        61, 61, 61, 61, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 61, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 62, 1, 1, 1, 14, 14,
    -        14, 14, 14, 14, 14, 14, 14, 14,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 63, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 1, 1, 1, 1, 1, 1,
    -        1, 1, 64, 1, 0
    -};
    -
    -static const char _deserialize_text_trans_targs[] = {
    -        1, 0, 2, 25, 3, 4, 19, 5,
    -        23, 24, 8, 27, 36, 27, 36, 30,
    -        33, 11, 12, 15, 12, 15, 13, 14,
    -        31, 32, 31, 32, 26, 18, 34, 35,
    -        34, 35, 20, 19, 6, 21, 22, 20,
    -        21, 22, 20, 21, 22, 24, 26, 26,
    -        7, 9, 10, 16, 21, 29, 26, 7,
    -        9, 10, 16, 21, 29, 28, 17, 21,
    -        29, 28, 29, 29, 28, 7, 10, 29,
    -        28, 7, 21, 29, 33, 28, 21, 29
    -};
    -
    -static const char _deserialize_text_trans_actions[] = {
    -        0, 0, 0, 0, 1, 0, 2, 0,
    -        2, 2, 3, 4, 4, 5, 5, 4,
    -        4, 3, 3, 3, 0, 0, 6, 3,
    -        4, 4, 5, 5, 5, 3, 4, 4,
    -        5, 5, 7, 8, 9, 7, 7, 0,
    -        0, 0, 10, 10, 10, 8, 12, 13,
    -        14, 14, 14, 15, 11, 11, 17, 18,
    -        18, 18, 0, 16, 16, 19, 20, 19,
    -        19, 0, 0, 13, 10, 21, 21, 10,
    -        22, 23, 22, 22, 5, 24, 24, 24
    -};
    -
    -static const char _deserialize_text_eof_actions[] = {
    -        0, 0, 0, 0, 0, 0, 0, 0,
    -        0, 0, 0, 0, 0, 0, 0, 0,
    -        0, 0, 0, 7, 0, 0, 0, 10,
    -        10, 11, 16, 19, 0, 11, 10, 22,
    -        22, 10, 24, 24, 19
    -};
    -
    -static const int deserialize_text_start = 1;
    -static const int deserialize_text_first_final = 19;
    -static const int deserialize_text_error = 0;
    -
    -static const int deserialize_text_en_main = 1;
    -
    -
    -#line 114 "hb-buffer-deserialize-text.rl"
    -
    -
    -static hb_bool_t
    -_hb_buffer_deserialize_text (hb_buffer_t *buffer,
    -                                    const char *buf,
    -                                    unsigned int buf_len,
    -                                    const char **end_ptr,
    -                                    hb_font_t *font)
    -{
    -  const char *p = buf, *pe = buf + buf_len;
    -
    -  /* Ensure we have positions. */
    -  (void) hb_buffer_get_glyph_positions (buffer, nullptr);
    -
    -  while (p < pe && ISSPACE (*p))
    -    p++;
    -
    -  const char *eof = pe, *tok = nullptr;
    -  int cs;
    -  hb_glyph_info_t info = {0};
    -  hb_glyph_position_t pos = {0};
    -
    -#line 428 "hb-buffer-deserialize-text.hh"
    -        {
    -        cs = deserialize_text_start;
    -        }
    -
    -#line 433 "hb-buffer-deserialize-text.hh"
    -        {
    -        int _slen;
    -        int _trans;
    -        const unsigned char *_keys;
    -        const char *_inds;
    -        if ( p == pe )
    -                goto _test_eof;
    -        if ( cs == 0 )
    -                goto _out;
    -_resume:
    -        _keys = _deserialize_text_trans_keys + (cs<<1);
    -        _inds = _deserialize_text_indicies + _deserialize_text_index_offsets[cs];
    -
    -        _slen = _deserialize_text_key_spans[cs];
    -        _trans = _inds[ _slen > 0 && _keys[0] <=(*p) &&
    -                (*p) <= _keys[1] ?
    -                (*p) - _keys[0] : _slen ];
    -
    -        cs = _deserialize_text_trans_targs[_trans];
    -
    -        if ( _deserialize_text_trans_actions[_trans] == 0 )
    -                goto _again;
    -
    -        switch ( _deserialize_text_trans_actions[_trans] ) {
    -        case 1:
    -#line 38 "hb-buffer-deserialize-text.rl"
    -        {
    -        memset (&info, 0, sizeof (info));
    -        memset (&pos , 0, sizeof (pos ));
    -}
    -        break;
    -        case 3:
    -#line 51 "hb-buffer-deserialize-text.rl"
    -        {
    -        tok = p;
    -}
    -        break;
    -        case 5:
    -#line 55 "hb-buffer-deserialize-text.rl"
    -        { if (unlikely (!buffer->ensure_glyphs ())) return false; }
    -        break;
    -        case 8:
    -#line 56 "hb-buffer-deserialize-text.rl"
    -        { if (unlikely (!buffer->ensure_unicode ())) return false; }
    -        break;
    -        case 18:
    -#line 58 "hb-buffer-deserialize-text.rl"
    -        {
    -        /* TODO Unescape delimiters. */
    -        if (!hb_font_glyph_from_string (font,
    -                                        tok, p - tok,
    -                                        &info.codepoint))
    -          return false;
    -}
    -        break;
    -        case 9:
    -#line 66 "hb-buffer-deserialize-text.rl"
    -        {if (!parse_hex (tok, p, &info.codepoint )) return false; }
    -        break;
    -        case 21:
    -#line 68 "hb-buffer-deserialize-text.rl"
    -        { if (!parse_uint (tok, p, &info.cluster )) return false; }
    -        break;
    -        case 6:
    -#line 69 "hb-buffer-deserialize-text.rl"
    -        { if (!parse_int  (tok, p, &pos.x_offset )) return false; }
    -        break;
    -        case 23:
    -#line 70 "hb-buffer-deserialize-text.rl"
    -        { if (!parse_int  (tok, p, &pos.y_offset )) return false; }
    -        break;
    -        case 20:
    -#line 71 "hb-buffer-deserialize-text.rl"
    -        { if (!parse_int  (tok, p, &pos.x_advance)) return false; }
    -        break;
    -        case 15:
    -#line 38 "hb-buffer-deserialize-text.rl"
    -        {
    -        memset (&info, 0, sizeof (info));
    -        memset (&pos , 0, sizeof (pos ));
    -}
    -#line 51 "hb-buffer-deserialize-text.rl"
    -        {
    -        tok = p;
    -}
    -        break;
    -        case 4:
    -#line 51 "hb-buffer-deserialize-text.rl"
    -        {
    -        tok = p;
    -}
    -#line 55 "hb-buffer-deserialize-text.rl"
    -        { if (unlikely (!buffer->ensure_glyphs ())) return false; }
    -        break;
    -        case 2:
    -#line 51 "hb-buffer-deserialize-text.rl"
    -        {
    -        tok = p;
    -}
    -#line 56 "hb-buffer-deserialize-text.rl"
    -        { if (unlikely (!buffer->ensure_unicode ())) return false; }
    -        break;
    -        case 16:
    -#line 58 "hb-buffer-deserialize-text.rl"
    -        {
    -        /* TODO Unescape delimiters. */
    -        if (!hb_font_glyph_from_string (font,
    -                                        tok, p - tok,
    -                                        &info.codepoint))
    -          return false;
    -}
    -#line 43 "hb-buffer-deserialize-text.rl"
    -        {
    -        buffer->add_info (info);
    -        if (unlikely (!buffer->successful))
    -          return false;
    -        buffer->pos[buffer->len - 1] = pos;
    -        *end_ptr = p;
    -}
    -        break;
    -        case 7:
    -#line 66 "hb-buffer-deserialize-text.rl"
    -        {if (!parse_hex (tok, p, &info.codepoint )) return false; }
    -#line 43 "hb-buffer-deserialize-text.rl"
    -        {
    -        buffer->add_info (info);
    -        if (unlikely (!buffer->successful))
    -          return false;
    -        buffer->pos[buffer->len - 1] = pos;
    -        *end_ptr = p;
    -}
    -        break;
    -        case 10:
    -#line 68 "hb-buffer-deserialize-text.rl"
    -        { if (!parse_uint (tok, p, &info.cluster )) return false; }
    -#line 43 "hb-buffer-deserialize-text.rl"
    -        {
    -        buffer->add_info (info);
    -        if (unlikely (!buffer->successful))
    -          return false;
    -        buffer->pos[buffer->len - 1] = pos;
    -        *end_ptr = p;
    -}
    -        break;
    -        case 22:
    -#line 70 "hb-buffer-deserialize-text.rl"
    -        { if (!parse_int  (tok, p, &pos.y_offset )) return false; }
    -#line 43 "hb-buffer-deserialize-text.rl"
    -        {
    -        buffer->add_info (info);
    -        if (unlikely (!buffer->successful))
    -          return false;
    -        buffer->pos[buffer->len - 1] = pos;
    -        *end_ptr = p;
    -}
    -        break;
    -        case 19:
    -#line 71 "hb-buffer-deserialize-text.rl"
    -        { if (!parse_int  (tok, p, &pos.x_advance)) return false; }
    -#line 43 "hb-buffer-deserialize-text.rl"
    -        {
    -        buffer->add_info (info);
    -        if (unlikely (!buffer->successful))
    -          return false;
    -        buffer->pos[buffer->len - 1] = pos;
    -        *end_ptr = p;
    -}
    -        break;
    -        case 24:
    -#line 72 "hb-buffer-deserialize-text.rl"
    -        { if (!parse_int  (tok, p, &pos.y_advance)) return false; }
    -#line 43 "hb-buffer-deserialize-text.rl"
    -        {
    -        buffer->add_info (info);
    -        if (unlikely (!buffer->successful))
    -          return false;
    -        buffer->pos[buffer->len - 1] = pos;
    -        *end_ptr = p;
    -}
    -        break;
    -        case 12:
    -#line 38 "hb-buffer-deserialize-text.rl"
    -        {
    -        memset (&info, 0, sizeof (info));
    -        memset (&pos , 0, sizeof (pos ));
    -}
    -#line 51 "hb-buffer-deserialize-text.rl"
    -        {
    -        tok = p;
    -}
    -#line 55 "hb-buffer-deserialize-text.rl"
    -        { if (unlikely (!buffer->ensure_glyphs ())) return false; }
    -        break;
    -        case 14:
    -#line 38 "hb-buffer-deserialize-text.rl"
    -        {
    -        memset (&info, 0, sizeof (info));
    -        memset (&pos , 0, sizeof (pos ));
    -}
    -#line 51 "hb-buffer-deserialize-text.rl"
    -        {
    -        tok = p;
    -}
    -#line 58 "hb-buffer-deserialize-text.rl"
    -        {
    -        /* TODO Unescape delimiters. */
    -        if (!hb_font_glyph_from_string (font,
    -                                        tok, p - tok,
    -                                        &info.codepoint))
    -          return false;
    -}
    -        break;
    -        case 17:
    -#line 58 "hb-buffer-deserialize-text.rl"
    -        {
    -        /* TODO Unescape delimiters. */
    -        if (!hb_font_glyph_from_string (font,
    -                                        tok, p - tok,
    -                                        &info.codepoint))
    -          return false;
    -}
    -#line 55 "hb-buffer-deserialize-text.rl"
    -        { if (unlikely (!buffer->ensure_glyphs ())) return false; }
    -#line 43 "hb-buffer-deserialize-text.rl"
    -        {
    -        buffer->add_info (info);
    -        if (unlikely (!buffer->successful))
    -          return false;
    -        buffer->pos[buffer->len - 1] = pos;
    -        *end_ptr = p;
    -}
    -        break;
    -        case 11:
    -#line 38 "hb-buffer-deserialize-text.rl"
    -        {
    -        memset (&info, 0, sizeof (info));
    -        memset (&pos , 0, sizeof (pos ));
    -}
    -#line 51 "hb-buffer-deserialize-text.rl"
    -        {
    -        tok = p;
    -}
    -#line 58 "hb-buffer-deserialize-text.rl"
    -        {
    -        /* TODO Unescape delimiters. */
    -        if (!hb_font_glyph_from_string (font,
    -                                        tok, p - tok,
    -                                        &info.codepoint))
    -          return false;
    -}
    -#line 43 "hb-buffer-deserialize-text.rl"
    -        {
    -        buffer->add_info (info);
    -        if (unlikely (!buffer->successful))
    -          return false;
    -        buffer->pos[buffer->len - 1] = pos;
    -        *end_ptr = p;
    -}
    -        break;
    -        case 13:
    -#line 38 "hb-buffer-deserialize-text.rl"
    -        {
    -        memset (&info, 0, sizeof (info));
    -        memset (&pos , 0, sizeof (pos ));
    -}
    -#line 51 "hb-buffer-deserialize-text.rl"
    -        {
    -        tok = p;
    -}
    -#line 58 "hb-buffer-deserialize-text.rl"
    -        {
    -        /* TODO Unescape delimiters. */
    -        if (!hb_font_glyph_from_string (font,
    -                                        tok, p - tok,
    -                                        &info.codepoint))
    -          return false;
    -}
    -#line 55 "hb-buffer-deserialize-text.rl"
    -        { if (unlikely (!buffer->ensure_glyphs ())) return false; }
    -#line 43 "hb-buffer-deserialize-text.rl"
    -        {
    -        buffer->add_info (info);
    -        if (unlikely (!buffer->successful))
    -          return false;
    -        buffer->pos[buffer->len - 1] = pos;
    -        *end_ptr = p;
    -}
    -        break;
    -#line 722 "hb-buffer-deserialize-text.hh"
    -        }
    -
    -_again:
    -        if ( cs == 0 )
    -                goto _out;
    -        if ( ++p != pe )
    -                goto _resume;
    -        _test_eof: {}
    -        if ( p == eof )
    -        {
    -        switch ( _deserialize_text_eof_actions[cs] ) {
    -        case 16:
    -#line 58 "hb-buffer-deserialize-text.rl"
    -        {
    -        /* TODO Unescape delimiters. */
    -        if (!hb_font_glyph_from_string (font,
    -                                        tok, p - tok,
    -                                        &info.codepoint))
    -          return false;
    -}
    -#line 43 "hb-buffer-deserialize-text.rl"
    -        {
    -        buffer->add_info (info);
    -        if (unlikely (!buffer->successful))
    -          return false;
    -        buffer->pos[buffer->len - 1] = pos;
    -        *end_ptr = p;
    -}
    -        break;
    -        case 7:
    -#line 66 "hb-buffer-deserialize-text.rl"
    -        {if (!parse_hex (tok, p, &info.codepoint )) return false; }
    -#line 43 "hb-buffer-deserialize-text.rl"
    -        {
    -        buffer->add_info (info);
    -        if (unlikely (!buffer->successful))
    -          return false;
    -        buffer->pos[buffer->len - 1] = pos;
    -        *end_ptr = p;
    -}
    -        break;
    -        case 10:
    -#line 68 "hb-buffer-deserialize-text.rl"
    -        { if (!parse_uint (tok, p, &info.cluster )) return false; }
    -#line 43 "hb-buffer-deserialize-text.rl"
    -        {
    -        buffer->add_info (info);
    -        if (unlikely (!buffer->successful))
    -          return false;
    -        buffer->pos[buffer->len - 1] = pos;
    -        *end_ptr = p;
    -}
    -        break;
    -        case 22:
    -#line 70 "hb-buffer-deserialize-text.rl"
    -        { if (!parse_int  (tok, p, &pos.y_offset )) return false; }
    -#line 43 "hb-buffer-deserialize-text.rl"
    -        {
    -        buffer->add_info (info);
    -        if (unlikely (!buffer->successful))
    -          return false;
    -        buffer->pos[buffer->len - 1] = pos;
    -        *end_ptr = p;
    -}
    -        break;
    -        case 19:
    -#line 71 "hb-buffer-deserialize-text.rl"
    -        { if (!parse_int  (tok, p, &pos.x_advance)) return false; }
    -#line 43 "hb-buffer-deserialize-text.rl"
    -        {
    -        buffer->add_info (info);
    -        if (unlikely (!buffer->successful))
    -          return false;
    -        buffer->pos[buffer->len - 1] = pos;
    -        *end_ptr = p;
    -}
    -        break;
    -        case 24:
    -#line 72 "hb-buffer-deserialize-text.rl"
    -        { if (!parse_int  (tok, p, &pos.y_advance)) return false; }
    -#line 43 "hb-buffer-deserialize-text.rl"
    -        {
    -        buffer->add_info (info);
    -        if (unlikely (!buffer->successful))
    -          return false;
    -        buffer->pos[buffer->len - 1] = pos;
    -        *end_ptr = p;
    -}
    -        break;
    -        case 11:
    -#line 38 "hb-buffer-deserialize-text.rl"
    -        {
    -        memset (&info, 0, sizeof (info));
    -        memset (&pos , 0, sizeof (pos ));
    -}
    -#line 51 "hb-buffer-deserialize-text.rl"
    -        {
    -        tok = p;
    -}
    -#line 58 "hb-buffer-deserialize-text.rl"
    -        {
    -        /* TODO Unescape delimiters. */
    -        if (!hb_font_glyph_from_string (font,
    -                                        tok, p - tok,
    -                                        &info.codepoint))
    -          return false;
    -}
    -#line 43 "hb-buffer-deserialize-text.rl"
    -        {
    -        buffer->add_info (info);
    -        if (unlikely (!buffer->successful))
    -          return false;
    -        buffer->pos[buffer->len - 1] = pos;
    -        *end_ptr = p;
    -}
    -        break;
    -#line 839 "hb-buffer-deserialize-text.hh"
    -        }
    -        }
    -
    -        _out: {}
    -        }
    -
    -#line 138 "hb-buffer-deserialize-text.rl"
    -
    -
    -  *end_ptr = p;
    -
    -  return p == pe && *(p-1) != ']';
    -}
    -
    -#endif /* HB_BUFFER_DESERIALIZE_TEXT_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-buffer-serialize.cc b/src/java.desktop/share/native/libharfbuzz/hb-buffer-serialize.cc
    index 4797d6a79fb4a..bb2cddcb65526 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-buffer-serialize.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-buffer-serialize.cc
    @@ -56,7 +56,7 @@ hb_buffer_serialize_list_formats ()
     /**
      * hb_buffer_serialize_format_from_string:
      * @str: (array length=len) (element-type uint8_t): a string to parse
    - * @len: length of @str, or -1 if string is %NULL terminated
    + * @len: length of @str, or -1 if string is `NULL` terminated
      *
      * Parses a string into an #hb_buffer_serialize_format_t. Does not check if
      * @str is a valid buffer serialization format, use
    @@ -78,11 +78,11 @@ hb_buffer_serialize_format_from_string (const char *str, int len)
      * hb_buffer_serialize_format_to_string:
      * @format: an #hb_buffer_serialize_format_t to convert.
      *
    - * Converts @format to the string corresponding it, or %NULL if it is not a valid
    + * Converts @format to the string corresponding it, or `NULL` if it is not a valid
      * #hb_buffer_serialize_format_t.
      *
      * Return value: (transfer none):
    - * A %NULL terminated string corresponding to @format. Should not be freed.
    + * A `NULL` terminated string corresponding to @format. Should not be freed.
      *
      * Since: 0.9.7
      **/
    @@ -183,7 +183,7 @@ _hb_buffer_serialize_glyphs_json (hb_buffer_t *buffer,
         unsigned int l = p - b;
         if (buf_size > l)
         {
    -      memcpy (buf, b, l);
    +      hb_memcpy (buf, b, l);
           buf += l;
           buf_size -= l;
           *buf_consumed += l;
    @@ -241,7 +241,7 @@ _hb_buffer_serialize_unicode_json (hb_buffer_t *buffer,
         unsigned int l = p - b;
         if (buf_size > l)
         {
    -      memcpy (buf, b, l);
    +      hb_memcpy (buf, b, l);
           buf += l;
           buf_size -= l;
           *buf_consumed += l;
    @@ -329,7 +329,7 @@ _hb_buffer_serialize_glyphs_text (hb_buffer_t *buffer,
         unsigned int l = p - b;
         if (buf_size > l)
         {
    -      memcpy (buf, b, l);
    +      hb_memcpy (buf, b, l);
           buf += l;
           buf_size -= l;
           *buf_consumed += l;
    @@ -381,7 +381,7 @@ _hb_buffer_serialize_unicode_text (hb_buffer_t *buffer,
         unsigned int l = p - b;
         if (buf_size > l)
         {
    -      memcpy (buf, b, l);
    +      hb_memcpy (buf, b, l);
           buf += l;
           buf_size -= l;
           *buf_consumed += l;
    @@ -400,9 +400,9 @@ _hb_buffer_serialize_unicode_text (hb_buffer_t *buffer,
      * @buf: (out) (array length=buf_size) (element-type uint8_t): output string to
      *       write serialized buffer into.
      * @buf_size: the size of @buf.
    - * @buf_consumed: (out) (optional): if not %NULL, will be set to the number of bytes written into @buf.
    + * @buf_consumed: (out) (optional): if not `NULL`, will be set to the number of bytes written into @buf.
      * @font: (nullable): the #hb_font_t used to shape this buffer, needed to
    - *        read glyph names and extents. If %NULL, an empty font will be used.
    + *        read glyph names and extents. If `NULL`, an empty font will be used.
      * @format: the #hb_buffer_serialize_format_t to use for formatting the output.
      * @flags: the #hb_buffer_serialize_flags_t that control what glyph properties
      *         to serialize.
    @@ -514,7 +514,7 @@ hb_buffer_serialize_glyphs (hb_buffer_t *buffer,
      * @buf: (out) (array length=buf_size) (element-type uint8_t): output string to
      *       write serialized buffer into.
      * @buf_size: the size of @buf.
    - * @buf_consumed: (out) (optional): if not %NULL, will be set to the number of bytes written into @buf.
    + * @buf_consumed: (out) (optional): if not `NULL`, will be set to the number of bytes written into @buf.
      * @format: the #hb_buffer_serialize_format_t to use for formatting the output.
      * @flags: the #hb_buffer_serialize_flags_t that control what glyph properties
      *         to serialize.
    @@ -637,9 +637,9 @@ _hb_buffer_serialize_invalid (hb_buffer_t *buffer,
      * @buf: (out) (array length=buf_size) (element-type uint8_t): output string to
      *       write serialized buffer into.
      * @buf_size: the size of @buf.
    - * @buf_consumed: (out) (optional): if not %NULL, will be set to the number of bytes written into @buf.
    + * @buf_consumed: (out) (optional): if not `NULL`, will be set to the number of bytes written into @buf.
      * @font: (nullable): the #hb_font_t used to shape this buffer, needed to
    - *        read glyph names and extents. If %NULL, an empty font will be used.
    + *        read glyph names and extents. If `NULL`, an empty font will be used.
      * @format: the #hb_buffer_serialize_format_t to use for formatting the output.
      * @flags: the #hb_buffer_serialize_flags_t that control what glyph properties
      *         to serialize.
    @@ -721,13 +721,14 @@ parse_hex (const char *pp, const char *end, uint32_t *pv)
     }
     
     #include "hb-buffer-deserialize-json.hh"
    -#include "hb-buffer-deserialize-text.hh"
    +#include "hb-buffer-deserialize-text-glyphs.hh"
    +#include "hb-buffer-deserialize-text-unicode.hh"
     
     /**
      * hb_buffer_deserialize_glyphs:
      * @buffer: an #hb_buffer_t buffer.
      * @buf: (array length=buf_len): string to deserialize
    - * @buf_len: the size of @buf, or -1 if it is %NULL-terminated
    + * @buf_len: the size of @buf, or -1 if it is `NULL`-terminated
      * @end_ptr: (out) (optional): output pointer to the character after last
      *                               consumed one.
      * @font: (nullable): font for getting glyph IDs
    @@ -736,7 +737,8 @@ parse_hex (const char *pp, const char *end, uint32_t *pv)
      * Deserializes glyphs @buffer from textual representation in the format
      * produced by hb_buffer_serialize_glyphs().
      *
    - * Return value: %true if @buf is not fully consumed, %false otherwise.
    + * Return value: `true` if parse was successful, `false` if an error
    + * occurred.
      *
      * Since: 0.9.7
      **/
    @@ -779,9 +781,9 @@ hb_buffer_deserialize_glyphs (hb_buffer_t *buffer,
       switch (format)
       {
         case HB_BUFFER_SERIALIZE_FORMAT_TEXT:
    -      return _hb_buffer_deserialize_text (buffer,
    -                                          buf, buf_len, end_ptr,
    -                                          font);
    +      return _hb_buffer_deserialize_text_glyphs (buffer,
    +                                                 buf, buf_len, end_ptr,
    +                                                 font);
     
         case HB_BUFFER_SERIALIZE_FORMAT_JSON:
           return _hb_buffer_deserialize_json (buffer,
    @@ -800,7 +802,7 @@ hb_buffer_deserialize_glyphs (hb_buffer_t *buffer,
      * hb_buffer_deserialize_unicode:
      * @buffer: an #hb_buffer_t buffer.
      * @buf: (array length=buf_len): string to deserialize
    - * @buf_len: the size of @buf, or -1 if it is %NULL-terminated
    + * @buf_len: the size of @buf, or -1 if it is `NULL`-terminated
      * @end_ptr: (out) (optional): output pointer to the character after last
      *                               consumed one.
      * @format: the #hb_buffer_serialize_format_t of the input @buf
    @@ -808,7 +810,8 @@ hb_buffer_deserialize_glyphs (hb_buffer_t *buffer,
      * Deserializes Unicode @buffer from textual representation in the format
      * produced by hb_buffer_serialize_unicode().
      *
    - * Return value: %true if @buf is not fully consumed, %false otherwise.
    + * Return value: `true` if parse was successful, `false` if an error
    + * occurred.
      *
      * Since: 2.7.3
      **/
    @@ -849,9 +852,9 @@ hb_buffer_deserialize_unicode (hb_buffer_t *buffer,
       switch (format)
       {
         case HB_BUFFER_SERIALIZE_FORMAT_TEXT:
    -      return _hb_buffer_deserialize_text (buffer,
    -                                          buf, buf_len, end_ptr,
    -                                          font);
    +      return _hb_buffer_deserialize_text_unicode (buffer,
    +                                                  buf, buf_len, end_ptr,
    +                                                  font);
     
         case HB_BUFFER_SERIALIZE_FORMAT_JSON:
           return _hb_buffer_deserialize_json (buffer,
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-buffer-verify.cc b/src/java.desktop/share/native/libharfbuzz/hb-buffer-verify.cc
    index 1b96fafb0d422..f2fef3137e529 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-buffer-verify.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-buffer-verify.cc
    @@ -150,7 +150,7 @@ buffer_verify_unsafe_to_break (hb_buffer_t  *buffer,
         assert (text_start < text_end);
     
         if (0)
    -      printf("start %d end %d text start %d end %d\n", start, end, text_start, text_end);
    +      printf("start %u end %u text start %u end %u\n", start, end, text_start, text_end);
     
         hb_buffer_clear_contents (fragment);
     
    @@ -186,7 +186,7 @@ buffer_verify_unsafe_to_break (hb_buffer_t  *buffer,
     
       bool ret = true;
       hb_buffer_diff_flags_t diff = hb_buffer_diff (reconstruction, buffer, (hb_codepoint_t) -1, 0);
    -  if (diff)
    +  if (diff & ~HB_BUFFER_DIFF_FLAG_GLYPH_FLAGS_MISMATCH)
       {
         buffer_verify_error (buffer, font, BUFFER_VERIFY_ERROR "unsafe-to-break test failed.");
         ret = false;
    @@ -292,7 +292,7 @@ buffer_verify_unsafe_to_concat (hb_buffer_t        *buffer,
           assert (text_start < text_end);
     
           if (0)
    -        printf("start %d end %d text start %d end %d\n", start, end, text_start, text_end);
    +        printf("start %u end %u text start %u end %u\n", start, end, text_start, text_end);
     
     #if 0
           hb_buffer_flags_t flags = hb_buffer_get_flags (fragment);
    @@ -313,7 +313,6 @@ buffer_verify_unsafe_to_concat (hb_buffer_t        *buffer,
     
       bool ret = true;
       hb_buffer_diff_flags_t diff;
    -
       /*
        * Shape the two fragment streams.
        */
    @@ -382,7 +381,7 @@ buffer_verify_unsafe_to_concat (hb_buffer_t        *buffer,
        * Diff results.
        */
       diff = hb_buffer_diff (reconstruction, buffer, (hb_codepoint_t) -1, 0);
    -  if (diff)
    +  if (diff & ~HB_BUFFER_DIFF_FLAG_GLYPH_FLAGS_MISMATCH)
       {
         buffer_verify_error (buffer, font, BUFFER_VERIFY_ERROR "unsafe-to-concat test failed.");
         ret = false;
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-buffer.cc b/src/java.desktop/share/native/libharfbuzz/hb-buffer.cc
    index e743e18125a24..69d8c961930a9 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-buffer.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-buffer.cc
    @@ -40,6 +40,11 @@
      * Buffers serve a dual role in HarfBuzz; before shaping, they hold
      * the input characters that are passed to hb_shape(), and after
      * shaping they hold the output glyphs.
    + *
    + * The input buffer is a sequence of Unicode codepoints, with
    + * associated attributes such as direction and script.  The output
    + * buffer is a sequence of glyphs, with associated attributes such
    + * as position and cluster.
      **/
     
     
    @@ -51,7 +56,7 @@
      * Checks the equality of two #hb_segment_properties_t's.
      *
      * Return value:
    - * %true if all properties of @a equal those of @b, %false otherwise.
    + * `true` if all properties of @a equal those of @b, `false` otherwise.
      *
      * Since: 0.9.7
      **/
    @@ -172,12 +177,13 @@ hb_buffer_t::enlarge (unsigned int size)
       while (size >= new_allocated)
         new_allocated += (new_allocated >> 1) + 32;
     
    -  static_assert ((sizeof (info[0]) == sizeof (pos[0])), "");
    -  if (unlikely (hb_unsigned_mul_overflows (new_allocated, sizeof (info[0]))))
    +  unsigned new_bytes;
    +  if (unlikely (hb_unsigned_mul_overflows (new_allocated, sizeof (info[0]), &new_bytes)))
         goto done;
     
    -  new_pos = (hb_glyph_position_t *) hb_realloc (pos, new_allocated * sizeof (pos[0]));
    -  new_info = (hb_glyph_info_t *) hb_realloc (info, new_allocated * sizeof (info[0]));
    +  static_assert (sizeof (info[0]) == sizeof (pos[0]), "");
    +  new_pos = (hb_glyph_position_t *) hb_realloc (pos, new_bytes);
    +  new_info = (hb_glyph_info_t *) hb_realloc (info, new_bytes);
     
     done:
       if (unlikely (!new_pos || !new_info))
    @@ -208,7 +214,7 @@ hb_buffer_t::make_room_for (unsigned int num_in,
         assert (have_output);
     
         out_info = (hb_glyph_info_t *) pos;
    -    memcpy (out_info, info, out_len * sizeof (out_info[0]));
    +    hb_memcpy (out_info, info, out_len * sizeof (out_info[0]));
       }
     
       return true;
    @@ -229,7 +235,7 @@ hb_buffer_t::shift_forward (unsigned int count)
          * Ideally, we should at least set Default_Ignorable bits on
          * these, as well as consistent cluster values.  But the former
          * is layering violation... */
    -    memset (info + len, 0, (idx + count - len) * sizeof (info[0]));
    +    hb_memset (info + len, 0, (idx + count - len) * sizeof (info[0]));
       }
       len += count;
       idx += count;
    @@ -298,8 +304,8 @@ hb_buffer_t::clear ()
       out_len = 0;
       out_info = info;
     
    -  memset (context, 0, sizeof context);
    -  memset (context_len, 0, sizeof context_len);
    +  hb_memset (context, 0, sizeof context);
    +  hb_memset (context_len, 0, sizeof context_len);
     
       deallocate_var_all ();
       serial = 0;
    @@ -313,15 +319,14 @@ hb_buffer_t::enter ()
       serial = 0;
       shaping_failed = false;
       scratch_flags = HB_BUFFER_SCRATCH_FLAG_DEFAULT;
    -  if (likely (!hb_unsigned_mul_overflows (len, HB_BUFFER_MAX_LEN_FACTOR)))
    +  unsigned mul;
    +  if (likely (!hb_unsigned_mul_overflows (len, HB_BUFFER_MAX_LEN_FACTOR, &mul)))
       {
    -    max_len = hb_max (len * HB_BUFFER_MAX_LEN_FACTOR,
    -                      (unsigned) HB_BUFFER_MAX_LEN_MIN);
    +    max_len = hb_max (mul, (unsigned) HB_BUFFER_MAX_LEN_MIN);
       }
    -  if (likely (!hb_unsigned_mul_overflows (len, HB_BUFFER_MAX_OPS_FACTOR)))
    +  if (likely (!hb_unsigned_mul_overflows (len, HB_BUFFER_MAX_OPS_FACTOR, &mul)))
       {
    -    max_ops = hb_max (len * HB_BUFFER_MAX_OPS_FACTOR,
    -                      (unsigned) HB_BUFFER_MAX_OPS_MIN);
    +    max_ops = hb_max (mul, (unsigned) HB_BUFFER_MAX_OPS_MIN);
       }
     }
     void
    @@ -345,7 +350,7 @@ hb_buffer_t::add (hb_codepoint_t  codepoint,
     
       glyph = &info[len];
     
    -  memset (glyph, 0, sizeof (*glyph));
    +  hb_memset (glyph, 0, sizeof (*glyph));
       glyph->codepoint = codepoint;
       glyph->mask = 0;
       glyph->cluster = cluster;
    @@ -387,9 +392,11 @@ hb_buffer_t::clear_positions ()
       hb_memset (pos, 0, sizeof (pos[0]) * len);
     }
     
    -void
    +bool
     hb_buffer_t::sync ()
     {
    +  bool ret = false;
    +
       assert (have_output);
     
       assert (idx <= len);
    @@ -403,12 +410,39 @@ hb_buffer_t::sync ()
         info = out_info;
       }
       len = out_len;
    +  ret = true;
     
     reset:
       have_output = false;
       out_len = 0;
       out_info = info;
       idx = 0;
    +
    +  return ret;
    +}
    +
    +int
    +hb_buffer_t::sync_so_far ()
    +{
    +  bool had_output = have_output;
    +  unsigned out_i = out_len;
    +  unsigned i = idx;
    +  unsigned old_idx = idx;
    +
    +  if (sync ())
    +    idx = out_i;
    +  else
    +    idx = i;
    +
    +  if (had_output)
    +  {
    +    have_output = true;
    +    out_len = idx;
    +  }
    +
    +  assert (idx <= len);
    +
    +  return idx - old_idx;
     }
     
     bool
    @@ -493,15 +527,17 @@ hb_buffer_t::merge_clusters_impl (unsigned int start,
         cluster = hb_min (cluster, info[i].cluster);
     
       /* Extend end */
    -  while (end < len && info[end - 1].cluster == info[end].cluster)
    -    end++;
    +  if (cluster != info[end - 1].cluster)
    +    while (end < len && info[end - 1].cluster == info[end].cluster)
    +      end++;
     
       /* Extend start */
    -  while (idx < start && info[start - 1].cluster == info[start].cluster)
    -    start--;
    +  if (cluster != info[start].cluster)
    +    while (idx < start && info[start - 1].cluster == info[start].cluster)
    +      start--;
     
       /* If we hit the start of buffer, continue in out-buffer. */
    -  if (idx == start)
    +  if (idx == start && info[start].cluster != cluster)
         for (unsigned int i = out_len; i && out_info[i - 1].cluster == info[start].cluster; i--)
           set_cluster (out_info[i - 1], cluster);
     
    @@ -576,6 +612,53 @@ hb_buffer_t::delete_glyph ()
       skip_glyph ();
     }
     
    +void
    +hb_buffer_t::delete_glyphs_inplace (bool (*filter) (const hb_glyph_info_t *info))
    +{
    +  /* Merge clusters and delete filtered glyphs.
    +   * NOTE! We can't use out-buffer as we have positioning data. */
    +  unsigned int j = 0;
    +  unsigned int count = len;
    +  for (unsigned int i = 0; i < count; i++)
    +  {
    +    if (filter (&info[i]))
    +    {
    +      /* Merge clusters.
    +       * Same logic as delete_glyph(), but for in-place removal. */
    +
    +      unsigned int cluster = info[i].cluster;
    +      if (i + 1 < count && cluster == info[i + 1].cluster)
    +        continue; /* Cluster survives; do nothing. */
    +
    +      if (j)
    +      {
    +        /* Merge cluster backward. */
    +        if (cluster < info[j - 1].cluster)
    +        {
    +          unsigned int mask = info[i].mask;
    +          unsigned int old_cluster = info[j - 1].cluster;
    +          for (unsigned k = j; k && info[k - 1].cluster == old_cluster; k--)
    +            set_cluster (info[k - 1], cluster, mask);
    +        }
    +        continue;
    +      }
    +
    +      if (i + 1 < count)
    +        merge_clusters (i, i + 2); /* Merge cluster forward. */
    +
    +      continue;
    +    }
    +
    +    if (j != i)
    +    {
    +      info[j] = info[i];
    +      pos[j] = pos[i];
    +    }
    +    j++;
    +  }
    +  len = j;
    +}
    +
     void
     hb_buffer_t::guess_segment_properties ()
     {
    @@ -643,9 +726,9 @@ DEFINE_NULL_INSTANCE (hb_buffer_t) =
      * Return value: (transfer full):
      * A newly allocated #hb_buffer_t with a reference count of 1. The initial
      * reference count should be released with hb_buffer_destroy() when you are done
    - * using the #hb_buffer_t. This function never returns %NULL. If memory cannot
    + * using the #hb_buffer_t. This function never returns `NULL`. If memory cannot
      * be allocated, a special #hb_buffer_t object will be returned on which
    - * hb_buffer_allocation_successful() returns %false.
    + * hb_buffer_allocation_successful() returns `false`.
      *
      * Since: 0.9.2
      **/
    @@ -775,7 +858,7 @@ hb_buffer_destroy (hb_buffer_t *buffer)
      *
      * Attaches a user-data key/data pair to the specified buffer.
      *
    - * Return value: %true if success, %false otherwise
    + * Return value: `true` if success, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -802,7 +885,7 @@ hb_buffer_set_user_data (hb_buffer_t        *buffer,
      * Since: 0.9.2
      **/
     void *
    -hb_buffer_get_user_data (hb_buffer_t        *buffer,
    +hb_buffer_get_user_data (const hb_buffer_t  *buffer,
                              hb_user_data_key_t *key)
     {
       return hb_object_get_user_data (buffer, key);
    @@ -817,6 +900,32 @@ hb_buffer_get_user_data (hb_buffer_t        *buffer,
      * Sets the type of @buffer contents. Buffers are either empty, contain
      * characters (before shaping), or contain glyphs (the result of shaping).
      *
    + * You rarely need to call this function, since a number of other
    + * functions transition the content type for you. Namely:
    + *
    + * - A newly created buffer starts with content type
    + *   %HB_BUFFER_CONTENT_TYPE_INVALID. Calling hb_buffer_reset(),
    + *   hb_buffer_clear_contents(), as well as calling hb_buffer_set_length()
    + *   with an argument of zero all set the buffer content type to invalid
    + *   as well.
    + *
    + * - Calling hb_buffer_add_utf8(), hb_buffer_add_utf16(),
    + *   hb_buffer_add_utf32(), hb_buffer_add_codepoints() and
    + *   hb_buffer_add_latin1() expect that buffer is either empty and
    + *   have a content type of invalid, or that buffer content type is
    + *   %HB_BUFFER_CONTENT_TYPE_UNICODE, and they also set the content
    + *   type to Unicode if they added anything to an empty buffer.
    + *
    + * - Finally hb_shape() and hb_shape_full() expect that the buffer
    + *   is either empty and have content type of invalid, or that buffer
    + *   content type is %HB_BUFFER_CONTENT_TYPE_UNICODE, and upon
    + *   success they set the buffer content type to
    + *   %HB_BUFFER_CONTENT_TYPE_GLYPHS.
    + *
    + * The above transitions are designed such that one can use a buffer
    + * in a loop of "reset : add-text : shape" without needing to ever
    + * modify the content type manually.
    + *
      * Since: 0.9.5
      **/
     void
    @@ -904,7 +1013,6 @@ hb_buffer_get_unicode_funcs (const hb_buffer_t *buffer)
     void
     hb_buffer_set_direction (hb_buffer_t    *buffer,
                              hb_direction_t  direction)
    -
     {
       if (unlikely (hb_object_is_immutable (buffer)))
         return;
    @@ -1278,7 +1386,7 @@ hb_buffer_clear_contents (hb_buffer_t *buffer)
      * Pre allocates memory for @buffer to fit at least @size number of items.
      *
      * Return value:
    - * %true if @buffer memory allocation succeeded, %false otherwise
    + * `true` if @buffer memory allocation succeeded, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -1295,7 +1403,7 @@ hb_buffer_pre_allocate (hb_buffer_t *buffer, unsigned int size)
      * Check if allocating memory for the buffer succeeded.
      *
      * Return value:
    - * %true if @buffer memory allocation succeeded, %false otherwise.
    + * `true` if @buffer memory allocation succeeded, `false` otherwise.
      *
      * Since: 0.9.2
      **/
    @@ -1340,7 +1448,7 @@ hb_buffer_add (hb_buffer_t    *buffer,
      * end.
      *
      * Return value:
    - * %true if @buffer memory allocation succeeded, %false otherwise.
    + * `true` if @buffer memory allocation succeeded, `false` otherwise.
      *
      * Since: 0.9.2
      **/
    @@ -1356,9 +1464,9 @@ hb_buffer_set_length (hb_buffer_t  *buffer,
     
       /* Wipe the new space */
       if (length > buffer->len) {
    -    memset (buffer->info + buffer->len, 0, sizeof (buffer->info[0]) * (length - buffer->len));
    +    hb_memset (buffer->info + buffer->len, 0, sizeof (buffer->info[0]) * (length - buffer->len));
         if (buffer->have_positions)
    -      memset (buffer->pos + buffer->len, 0, sizeof (buffer->pos[0]) * (length - buffer->len));
    +      hb_memset (buffer->pos + buffer->len, 0, sizeof (buffer->pos[0]) * (length - buffer->len));
       }
     
       buffer->len = length;
    @@ -1426,7 +1534,7 @@ hb_buffer_get_glyph_infos (hb_buffer_t  *buffer,
      * If buffer did not have positions before, the positions will be
      * initialized to zeros, unless this function is called from
      * within a buffer message callback (see hb_buffer_set_message_func()),
    - * in which case %NULL is returned.
    + * in which case `NULL` is returned.
      *
      * Return value: (transfer none) (array length=length):
      * The @buffer glyph position array.
    @@ -1461,7 +1569,7 @@ hb_buffer_get_glyph_positions (hb_buffer_t  *buffer,
      * and cleared of position data when hb_buffer_clear_contents() is called.
      *
      * Return value:
    - * %true if the @buffer has position array, %false otherwise.
    + * `true` if the @buffer has position array, `false` otherwise.
      *
      * Since: 2.7.3
      **/
    @@ -1645,10 +1753,10 @@ hb_buffer_add_utf (hb_buffer_t  *buffer,
      * @buffer: An #hb_buffer_t
      * @text: (array length=text_length) (element-type uint8_t): An array of UTF-8
      *               characters to append.
    - * @text_length: The length of the @text, or -1 if it is %NULL terminated.
    + * @text_length: The length of the @text, or -1 if it is `NULL` terminated.
      * @item_offset: The offset of the first character to add to the @buffer.
      * @item_length: The number of characters to add to the @buffer, or -1 for the
    - *               end of @text (assuming it is %NULL terminated).
    + *               end of @text (assuming it is `NULL` terminated).
      *
      * See hb_buffer_add_codepoints().
      *
    @@ -1671,10 +1779,10 @@ hb_buffer_add_utf8 (hb_buffer_t  *buffer,
      * hb_buffer_add_utf16:
      * @buffer: An #hb_buffer_t
      * @text: (array length=text_length): An array of UTF-16 characters to append
    - * @text_length: The length of the @text, or -1 if it is %NULL terminated
    + * @text_length: The length of the @text, or -1 if it is `NULL` terminated
      * @item_offset: The offset of the first character to add to the @buffer
      * @item_length: The number of characters to add to the @buffer, or -1 for the
    - *               end of @text (assuming it is %NULL terminated)
    + *               end of @text (assuming it is `NULL` terminated)
      *
      * See hb_buffer_add_codepoints().
      *
    @@ -1697,10 +1805,10 @@ hb_buffer_add_utf16 (hb_buffer_t    *buffer,
      * hb_buffer_add_utf32:
      * @buffer: An #hb_buffer_t
      * @text: (array length=text_length): An array of UTF-32 characters to append
    - * @text_length: The length of the @text, or -1 if it is %NULL terminated
    + * @text_length: The length of the @text, or -1 if it is `NULL` terminated
      * @item_offset: The offset of the first character to add to the @buffer
      * @item_length: The number of characters to add to the @buffer, or -1 for the
    - *               end of @text (assuming it is %NULL terminated)
    + *               end of @text (assuming it is `NULL` terminated)
      *
      * See hb_buffer_add_codepoints().
      *
    @@ -1724,10 +1832,10 @@ hb_buffer_add_utf32 (hb_buffer_t    *buffer,
      * @buffer: An #hb_buffer_t
      * @text: (array length=text_length) (element-type uint8_t): an array of UTF-8
      *               characters to append
    - * @text_length: the length of the @text, or -1 if it is %NULL terminated
    + * @text_length: the length of the @text, or -1 if it is `NULL` terminated
      * @item_offset: the offset of the first character to add to the @buffer
      * @item_length: the number of characters to add to the @buffer, or -1 for the
    - *               end of @text (assuming it is %NULL terminated)
    + *               end of @text (assuming it is `NULL` terminated)
      *
      * Similar to hb_buffer_add_codepoints(), but allows only access to first 256
      * Unicode code points that can fit in 8-bit strings.
    @@ -1750,10 +1858,10 @@ hb_buffer_add_latin1 (hb_buffer_t   *buffer,
      * hb_buffer_add_codepoints:
      * @buffer: a #hb_buffer_t to append characters to.
      * @text: (array length=text_length): an array of Unicode code points to append.
    - * @text_length: the length of the @text, or -1 if it is %NULL terminated.
    + * @text_length: the length of the @text, or -1 if it is `NULL` terminated.
      * @item_offset: the offset of the first code point to add to the @buffer.
      * @item_length: the number of code points to add to the @buffer, or -1 for the
    - *               end of @text (assuming it is %NULL terminated).
    + *               end of @text (assuming it is `NULL` terminated).
      *
      * Appends characters from @text array to @buffer. The @item_offset is the
      * position of the first character from @text that will be appended, and
    @@ -1766,7 +1874,9 @@ hb_buffer_add_latin1 (hb_buffer_t   *buffer,
      * marks at stat of run.
      *
      * This function does not check the validity of @text, it is up to the caller
    - * to ensure it contains a valid Unicode code points.
    + * to ensure it contains a valid Unicode scalar values.  In contrast,
    + * hb_buffer_add_utf32() can be used that takes similar input but performs
    + * sanity-check on the input.
      *
      * Since: 0.9.31
      **/
    @@ -1829,9 +1939,9 @@ hb_buffer_append (hb_buffer_t *buffer,
     
       hb_segment_properties_overlay (&buffer->props, &source->props);
     
    -  memcpy (buffer->info + orig_len, source->info + start, (end - start) * sizeof (buffer->info[0]));
    +  hb_memcpy (buffer->info + orig_len, source->info + start, (end - start) * sizeof (buffer->info[0]));
       if (buffer->have_positions)
    -    memcpy (buffer->pos + orig_len, source->pos + start, (end - start) * sizeof (buffer->pos[0]));
    +    hb_memcpy (buffer->pos + orig_len, source->pos + start, (end - start) * sizeof (buffer->pos[0]));
     
       if (source->content_type == HB_BUFFER_CONTENT_TYPE_UNICODE)
       {
    @@ -2019,7 +2129,7 @@ hb_buffer_diff (hb_buffer_t *buffer,
           result |= HB_BUFFER_DIFF_FLAG_CODEPOINT_MISMATCH;
         if (buf_info->cluster != ref_info->cluster)
           result |= HB_BUFFER_DIFF_FLAG_CLUSTER_MISMATCH;
    -    if ((buf_info->mask & ~ref_info->mask & HB_GLYPH_FLAG_DEFINED))
    +    if ((buf_info->mask ^ ref_info->mask) & HB_GLYPH_FLAG_DEFINED)
           result |= HB_BUFFER_DIFF_FLAG_GLYPH_FLAGS_MISMATCH;
         if (contains && ref_info->codepoint == dottedcircle_glyph)
           result |= HB_BUFFER_DIFF_FLAG_DOTTED_CIRCLE_PRESENT;
    @@ -2074,6 +2184,13 @@ hb_buffer_set_message_func (hb_buffer_t *buffer,
                                 hb_buffer_message_func_t func,
                                 void *user_data, hb_destroy_func_t destroy)
     {
    +  if (unlikely (hb_object_is_immutable (buffer)))
    +  {
    +    if (destroy)
    +      destroy (user_data);
    +    return;
    +  }
    +
       if (buffer->message_destroy)
         buffer->message_destroy (buffer->message_data);
     
    @@ -2090,8 +2207,16 @@ hb_buffer_set_message_func (hb_buffer_t *buffer,
     bool
     hb_buffer_t::message_impl (hb_font_t *font, const char *fmt, va_list ap)
     {
    +  assert (!have_output || (out_info == info && out_len == idx));
    +
    +  message_depth++;
    +
       char buf[100];
       vsnprintf (buf, sizeof (buf), fmt, ap);
    -  return (bool) this->message_func (this, font, buf, this->message_data);
    +  bool ret = (bool) this->message_func (this, font, buf, this->message_data);
    +
    +  message_depth--;
    +
    +  return ret;
     }
     #endif
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-buffer.h b/src/java.desktop/share/native/libharfbuzz/hb-buffer.h
    index d5c2b25f852c7..9b21ffb10fa15 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-buffer.h
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-buffer.h
    @@ -142,6 +142,15 @@ typedef struct hb_glyph_info_t {
      *                                 shaping, otherwise the buffer flag will not be
      *                                 reliably produced.
      *                                 Since: 4.0.0
    + * @HB_GLYPH_FLAG_SAFE_TO_INSERT_TATWEEL: In scripts that use elongation (Arabic,
    +                                   Mongolian, Syriac, etc.), this flag signifies
    +                                   that it is safe to insert a U+0640 TATWEEL
    +                                   character before this cluster for elongation.
    +                                   This flag does not determine the
    +                                   script-specific elongation places, but only
    +                                   when it is safe to do the elongation without
    +                                   interrupting text shaping.
    +                                   Since: 5.1.0
      * @HB_GLYPH_FLAG_DEFINED: All the currently defined flags.
      *
      * Flags for #hb_glyph_info_t.
    @@ -149,10 +158,11 @@ typedef struct hb_glyph_info_t {
      * Since: 1.5.0
      */
     typedef enum { /*< flags >*/
    -  HB_GLYPH_FLAG_UNSAFE_TO_BREAK         = 0x00000001,
    -  HB_GLYPH_FLAG_UNSAFE_TO_CONCAT        = 0x00000002,
    +  HB_GLYPH_FLAG_UNSAFE_TO_BREAK                 = 0x00000001,
    +  HB_GLYPH_FLAG_UNSAFE_TO_CONCAT                = 0x00000002,
    +  HB_GLYPH_FLAG_SAFE_TO_INSERT_TATWEEL          = 0x00000004,
     
    -  HB_GLYPH_FLAG_DEFINED                 = 0x00000003 /* OR of all defined flags */
    +  HB_GLYPH_FLAG_DEFINED                         = 0x00000007 /* OR of all defined flags */
     } hb_glyph_flags_t;
     
     HB_EXTERN hb_glyph_flags_t
    @@ -266,7 +276,7 @@ hb_buffer_set_user_data (hb_buffer_t        *buffer,
                              hb_bool_t           replace);
     
     HB_EXTERN void *
    -hb_buffer_get_user_data (hb_buffer_t        *buffer,
    +hb_buffer_get_user_data (const hb_buffer_t  *buffer,
                              hb_user_data_key_t *key);
     
     
    @@ -373,6 +383,10 @@ hb_buffer_guess_segment_properties (hb_buffer_t *buffer);
      *                      flag indicating that the @HB_GLYPH_FLAG_UNSAFE_TO_CONCAT
      *                      glyph-flag should be produced by the shaper. By default
      *                      it will not be produced since it incurs a cost. Since: 4.0.0
    + * @HB_BUFFER_FLAG_PRODUCE_SAFE_TO_INSERT_TATWEEL:
    + *                      flag indicating that the @HB_GLYPH_FLAG_SAFE_TO_INSERT_TATWEEL
    + *                      glyph-flag should be produced by the shaper. By default
    + *                      it will not be produced. Since: 5.1.0
      * @HB_BUFFER_FLAG_DEFINED: All currently defined flags: Since: 4.4.0
      *
      * Flags for #hb_buffer_t.
    @@ -388,8 +402,9 @@ typedef enum { /*< flags >*/
       HB_BUFFER_FLAG_DO_NOT_INSERT_DOTTED_CIRCLE    = 0x00000010u,
       HB_BUFFER_FLAG_VERIFY                         = 0x00000020u,
       HB_BUFFER_FLAG_PRODUCE_UNSAFE_TO_CONCAT       = 0x00000040u,
    +  HB_BUFFER_FLAG_PRODUCE_SAFE_TO_INSERT_TATWEEL = 0x00000080u,
     
    -  HB_BUFFER_FLAG_DEFINED                        = 0x0000007Fu
    +  HB_BUFFER_FLAG_DEFINED                        = 0x000000FFu
     } hb_buffer_flags_t;
     
     HB_EXTERN void
    @@ -748,23 +763,23 @@ hb_buffer_diff (hb_buffer_t *buffer,
     
     
     /*
    - * Debugging.
    + * Tracing.
      */
     
     /**
      * hb_buffer_message_func_t:
      * @buffer: An #hb_buffer_t to work upon
      * @font: The #hb_font_t the @buffer is shaped with
    - * @message: %NULL-terminated message passed to the function
    + * @message: `NULL`-terminated message passed to the function
      * @user_data: User data pointer passed by the caller
      *
      * A callback method for #hb_buffer_t. The method gets called with the
      * #hb_buffer_t it was set on, the #hb_font_t the buffer is shaped with and a
      * message describing what step of the shaping process will be performed.
    - * Returning %false from this method will skip this shaping step and move to
    + * Returning `false` from this method will skip this shaping step and move to
      * the next one.
      *
    - * Return value: %true to perform the shaping step, %false to skip it.
    + * Return value: `true` to perform the shaping step, `false` to skip it.
      *
      * Since: 1.1.3
      */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-buffer.hh b/src/java.desktop/share/native/libharfbuzz/hb-buffer.hh
    index 44f30d0a5dcb6..c1476364de571 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-buffer.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-buffer.hh
    @@ -32,28 +32,9 @@
     
     #include "hb.hh"
     #include "hb-unicode.hh"
    +#include "hb-set-digest.hh"
     
     
    -#ifndef HB_BUFFER_MAX_LEN_FACTOR
    -#define HB_BUFFER_MAX_LEN_FACTOR 64
    -#endif
    -#ifndef HB_BUFFER_MAX_LEN_MIN
    -#define HB_BUFFER_MAX_LEN_MIN 16384
    -#endif
    -#ifndef HB_BUFFER_MAX_LEN_DEFAULT
    -#define HB_BUFFER_MAX_LEN_DEFAULT 0x3FFFFFFF /* Shaping more than a billion chars? Let us know! */
    -#endif
    -
    -#ifndef HB_BUFFER_MAX_OPS_FACTOR
    -#define HB_BUFFER_MAX_OPS_FACTOR 1024
    -#endif
    -#ifndef HB_BUFFER_MAX_OPS_MIN
    -#define HB_BUFFER_MAX_OPS_MIN 16384
    -#endif
    -#ifndef HB_BUFFER_MAX_OPS_DEFAULT
    -#define HB_BUFFER_MAX_OPS_DEFAULT 0x1FFFFFFF /* Shaping more than a billion operations? Let us know! */
    -#endif
    -
     static_assert ((sizeof (hb_glyph_info_t) == 20), "");
     static_assert ((sizeof (hb_glyph_info_t) == sizeof (hb_glyph_position_t)), "");
     
    @@ -207,6 +188,14 @@ struct hb_buffer_t
       hb_glyph_info_t &prev ()      { return out_info[out_len ? out_len - 1 : 0]; }
       hb_glyph_info_t prev () const { return out_info[out_len ? out_len - 1 : 0]; }
     
    +  hb_set_digest_t digest () const
    +  {
    +    hb_set_digest_t d;
    +    d.init ();
    +    d.add_array (&info[0].codepoint, len, sizeof (info[0]));
    +    return d;
    +  }
    +
       HB_INTERNAL void similar (const hb_buffer_t &src);
       HB_INTERNAL void reset ();
       HB_INTERNAL void clear ();
    @@ -288,7 +277,8 @@ struct hb_buffer_t
     
       HB_INTERNAL void guess_segment_properties ();
     
    -  HB_INTERNAL void sync ();
    +  HB_INTERNAL bool sync ();
    +  HB_INTERNAL int sync_so_far ();
       HB_INTERNAL void clear_output ();
       HB_INTERNAL void clear_positions ();
     
    @@ -401,6 +391,8 @@ struct hb_buffer_t
       HB_INTERNAL void merge_out_clusters (unsigned int start, unsigned int end);
       /* Merge clusters for deleting current glyph, and skip it. */
       HB_INTERNAL void delete_glyph ();
    +  HB_INTERNAL void delete_glyphs_inplace (bool (*filter) (const hb_glyph_info_t *info));
    +
     
     
       /* Adds glyph flags in mask to infos with clusters between start and end.
    @@ -461,6 +453,17 @@ struct hb_buffer_t
                           start, end,
                           true);
       }
    +  void safe_to_insert_tatweel (unsigned int start = 0, unsigned int end = -1)
    +  {
    +    if ((flags & HB_BUFFER_FLAG_PRODUCE_SAFE_TO_INSERT_TATWEEL) == 0)
    +    {
    +      unsafe_to_break (start, end);
    +      return;
    +    }
    +    _set_glyph_flags (HB_GLYPH_FLAG_SAFE_TO_INSERT_TATWEEL,
    +                      start, end,
    +                      true);
    +  }
       void unsafe_to_concat (unsigned int start = 0, unsigned int end = -1)
       {
         if (likely ((flags & HB_BUFFER_FLAG_PRODUCE_UNSAFE_TO_CONCAT) == 0))
    @@ -555,15 +558,11 @@ struct hb_buffer_t
         if (likely (!messaging ()))
           return true;
     
    -    message_depth++;
    -
         va_list ap;
         va_start (ap, fmt);
         bool ret = message_impl (font, fmt, ap);
         va_end (ap);
     
    -    message_depth--;
    -
         return ret;
     #endif
       }
    @@ -582,21 +581,59 @@ struct hb_buffer_t
                               unsigned int cluster,
                               hb_mask_t mask)
       {
    -    for (unsigned int i = start; i < end; i++)
    -      if (cluster != infos[i].cluster)
    +    if (unlikely (start == end))
    +      return;
    +
    +    unsigned cluster_first = infos[start].cluster;
    +    unsigned cluster_last = infos[end - 1].cluster;
    +
    +    if (cluster_level == HB_BUFFER_CLUSTER_LEVEL_CHARACTERS ||
    +        (cluster != cluster_first && cluster != cluster_last))
    +    {
    +      for (unsigned int i = start; i < end; i++)
    +        if (cluster != infos[i].cluster)
    +        {
    +          scratch_flags |= HB_BUFFER_SCRATCH_FLAG_HAS_GLYPH_FLAGS;
    +          infos[i].mask |= mask;
    +        }
    +      return;
    +    }
    +
    +    /* Monotone clusters */
    +
    +    if (cluster == cluster_first)
    +    {
    +      for (unsigned int i = end; start < i && infos[i - 1].cluster != cluster_first; i--)
    +      {
    +        scratch_flags |= HB_BUFFER_SCRATCH_FLAG_HAS_GLYPH_FLAGS;
    +        infos[i - 1].mask |= mask;
    +      }
    +    }
    +    else /* cluster == cluster_last */
    +    {
    +      for (unsigned int i = start; i < end && infos[i].cluster != cluster_last; i++)
           {
             scratch_flags |= HB_BUFFER_SCRATCH_FLAG_HAS_GLYPH_FLAGS;
             infos[i].mask |= mask;
           }
    +    }
       }
    -  static unsigned
    +  unsigned
       _infos_find_min_cluster (const hb_glyph_info_t *infos,
                                unsigned start, unsigned end,
                                unsigned cluster = UINT_MAX)
       {
    -    for (unsigned int i = start; i < end; i++)
    -      cluster = hb_min (cluster, infos[i].cluster);
    -    return cluster;
    +    if (unlikely (start == end))
    +      return cluster;
    +
    +    if (cluster_level == HB_BUFFER_CLUSTER_LEVEL_CHARACTERS)
    +    {
    +      for (unsigned int i = start; i < end; i++)
    +        cluster = hb_min (cluster, infos[i].cluster);
    +      return cluster;
    +    }
    +
    +    return hb_min (cluster, hb_min (infos[start].cluster, infos[end - 1].cluster));
       }
     
       void clear_glyph_flags (hb_mask_t mask = 0)
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-cache.hh b/src/java.desktop/share/native/libharfbuzz/hb-cache.hh
    index de8a808ec4524..f40c8610dbb5a 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-cache.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-cache.hh
    @@ -30,29 +30,53 @@
     #include "hb.hh"
     
     
    -/* Implements a lockfree cache for int->int functions. */
    +/* Implements a lockfree cache for int->int functions.
    + *
    + * The cache is a fixed-size array of 16-bit or 32-bit integers.
    + * The key is split into two parts: the cache index and the rest.
    + *
    + * The cache index is used to index into the array.  The rest is used
    + * to store the key and the value.
    + *
    + * The value is stored in the least significant bits of the integer.
    + * The key is stored in the most significant bits of the integer.
    + * The key is shifted by cache_bits to the left to make room for the
    + * value.
    + */
     
    -template 
    +template 
     struct hb_cache_t
     {
    +  using item_t = typename std::conditional::type,
    +                                           typename std::conditional::type
    +                                          >::type;
    +
       static_assert ((key_bits >= cache_bits), "");
    -  static_assert ((key_bits + value_bits - cache_bits <= 8 * sizeof (hb_atomic_int_t)), "");
    -  static_assert (sizeof (hb_atomic_int_t) == sizeof (unsigned int), "");
    +  static_assert ((key_bits + value_bits <= cache_bits + 8 * sizeof (item_t)), "");
    +
    +  hb_cache_t () { init (); }
     
       void init () { clear (); }
    -  void fini () {}
     
       void clear ()
       {
         for (unsigned i = 0; i < ARRAY_LENGTH (values); i++)
    -      values[i].set_relaxed (-1);
    +      values[i] = -1;
       }
     
       bool get (unsigned int key, unsigned int *value) const
       {
         unsigned int k = key & ((1u<> value_bits) != (key >> cache_bits))
           return false;
         *value = v & ((1u<>cache_bits)< hb_cmap_cache_t;
    -typedef hb_cache_t<16, 24, 8> hb_advance_cache_t;
    -
     
     #endif /* HB_CACHE_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-cff-interp-common.hh b/src/java.desktop/share/native/libharfbuzz/hb-cff-interp-common.hh
    index 2c980c602799f..e92e8140fd0f8 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-cff-interp-common.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-cff-interp-common.hh
    @@ -284,65 +284,56 @@ struct UnsizedByteStr : UnsizedArrayOf 
     /* A byte string associated with the current offset and an error condition */
     struct byte_str_ref_t
     {
    -  byte_str_ref_t () { init (); }
    -
    -  void init ()
    -  {
    -    str = hb_ubytes_t ();
    -    offset = 0;
    -    error = false;
    -  }
    -
    -  void fini () {}
    +  byte_str_ref_t ()
    +    : str () {}
     
       byte_str_ref_t (const hb_ubytes_t &str_, unsigned int offset_ = 0)
    -    : str (str_), offset (offset_), error (false) {}
    +    : str (str_) { set_offset (offset_); }
     
       void reset (const hb_ubytes_t &str_, unsigned int offset_ = 0)
       {
         str = str_;
    -    offset = offset_;
    -    error = false;
    +    set_offset (offset_);
       }
     
       const unsigned char& operator [] (int i) {
    -    if (unlikely ((unsigned int) (offset + i) >= str.length))
    +    if (unlikely ((unsigned int) (get_offset () + i) >= str.length))
         {
           set_error ();
           return Null (unsigned char);
         }
    -    return str[offset + i];
    +    return str.arrayZ[get_offset () + i];
       }
     
    +  unsigned char head_unchecked () const { return str.arrayZ[get_offset ()]; }
    +
       /* Conversion to hb_ubytes_t */
    -  operator hb_ubytes_t () const { return str.sub_array (offset, str.length - offset); }
    +  operator hb_ubytes_t () const { return str.sub_array (get_offset ()); }
     
       hb_ubytes_t sub_array (unsigned int offset_, unsigned int len_) const
       { return str.sub_array (offset_, len_); }
     
       bool avail (unsigned int count=1) const
    -  { return (!in_error () && offset + count <= str.length); }
    +  { return get_offset () + count <= str.length; }
       void inc (unsigned int count=1)
       {
    -    if (likely (!in_error () && (offset <= str.length) && (offset + count <= str.length)))
    -    {
    -      offset += count;
    -    }
    -    else
    -    {
    -      offset = str.length;
    -      set_error ();
    -    }
    +    /* Automatically puts us in error if count is out-of-range. */
    +    set_offset (get_offset () + count);
       }
     
    -  void set_error ()      { error = true; }
    -  bool in_error () const { return error; }
    +  /* We (ab)use ubytes backwards_length as a cursor (called offset),
    +   * as well as to store error condition. */
     
    -  hb_ubytes_t       str;
    -  unsigned int  offset; /* beginning of the sub-string within str */
    +  unsigned get_offset () const { return str.backwards_length; }
    +  void set_offset (unsigned offset) { str.backwards_length = offset; }
    +
    +  void set_error ()      { str.backwards_length = str.length + 1; }
    +  bool in_error () const { return str.backwards_length > str.length; }
    +
    +  unsigned total_size () const { return str.length; }
     
       protected:
    -  bool    error;
    +  hb_ubytes_t       str;
     };
     
     using byte_str_array_t = hb_vector_t;
    @@ -491,8 +482,15 @@ struct arg_stack_t : cff_stack_t
     /* an operator prefixed by its operands in a byte string */
     struct op_str_t
     {
    -  hb_ubytes_t str;
    -  op_code_t  op;
    +  /* This used to have a hb_ubytes_t. Using a pointer and length
    +   * in a particular order, saves 8 bytes in this struct and more
    +   * in our parsed_cs_op_t subclass. */
    +
    +  const unsigned char *ptr = nullptr;
    +
    +  op_code_t  op = OpCode_Invalid;
    +
    +  uint8_t length = 0;
     };
     
     /* base of OP_SERIALIZER */
    @@ -503,9 +501,11 @@ struct op_serializer_t
       {
         TRACE_SERIALIZE (this);
     
    -    HBUINT8 *d = c->allocate_size (opstr.str.length);
    +    unsigned char *d = c->allocate_size (opstr.length);
         if (unlikely (!d)) return_trace (false);
    -    memcpy (d, &opstr.str[0], opstr.str.length);
    +    /* Faster than hb_memcpy for small strings. */
    +    for (unsigned i = 0; i < opstr.length; i++)
    +      d[i] = opstr.ptr[i];
         return_trace (true);
       }
     };
    @@ -522,23 +522,17 @@ struct parsed_values_t
     
       void alloc (unsigned n)
       {
    -    values.alloc (n);
    +    values.alloc (n, true);
       }
     
    -  void add_op (op_code_t op, const byte_str_ref_t& str_ref = byte_str_ref_t ())
    -  {
    -    VAL *val = values.push ();
    -    val->op = op;
    -    val->str = str_ref.str.sub_array (opStart, str_ref.offset - opStart);
    -    opStart = str_ref.offset;
    -  }
    -
    -  void add_op (op_code_t op, const byte_str_ref_t& str_ref, const VAL &v)
    +  void add_op (op_code_t op, const byte_str_ref_t& str_ref = byte_str_ref_t (), const VAL &v = VAL ())
       {
         VAL *val = values.push (v);
         val->op = op;
    -    val->str = str_ref.sub_array ( opStart, str_ref.offset - opStart);
    -    opStart = str_ref.offset;
    +    auto arr = str_ref.sub_array (opStart, str_ref.get_offset () - opStart);
    +    val->ptr = arr.arrayZ;
    +    val->length = arr.length;
    +    opStart = str_ref.get_offset ();
       }
     
       bool has_op (op_code_t op) const
    @@ -549,8 +543,7 @@ struct parsed_values_t
       }
     
       unsigned get_count () const { return values.length; }
    -  const VAL &get_value (unsigned int i)   const { return values[i]; }
    -  const VAL &operator [] (unsigned int i) const { return get_value (i); }
    +  const VAL &operator [] (unsigned int i) const { return values[i]; }
     
       unsigned int       opStart;
       hb_vector_t   values;
    @@ -565,23 +558,23 @@ struct interp_env_t
         str_ref.reset (str_);
       }
       bool in_error () const
    -  { return error || str_ref.in_error () || argStack.in_error (); }
    +  { return str_ref.in_error () || argStack.in_error (); }
     
    -  void set_error () { error = true; }
    +  void set_error () { str_ref.set_error (); }
     
       op_code_t fetch_op ()
       {
         op_code_t  op = OpCode_Invalid;
         if (unlikely (!str_ref.avail ()))
           return OpCode_Invalid;
    -    op = (op_code_t)(unsigned char)str_ref[0];
    +    op = (op_code_t) str_ref.head_unchecked ();
    +    str_ref.inc ();
         if (op == OpCode_escape) {
           if (unlikely (!str_ref.avail ()))
             return OpCode_Invalid;
    -      op = Make_OpCode_ESC(str_ref[1]);
    +      op = Make_OpCode_ESC (str_ref.head_unchecked ());
           str_ref.inc ();
         }
    -    str_ref.inc ();
         return op;
       }
     
    @@ -596,8 +589,6 @@ struct interp_env_t
                     str_ref;
       arg_stack_t
                     argStack;
    -  protected:
    -  bool          error = false;
     };
     
     using num_interp_env_t =  interp_env_t<>;
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-cff-interp-cs-common.hh b/src/java.desktop/share/native/libharfbuzz/hb-cff-interp-cs-common.hh
    index f4f413704952b..52b52c3f769d2 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-cff-interp-cs-common.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-cff-interp-cs-common.hh
    @@ -881,7 +881,13 @@ struct cs_interpreter_t : interpreter_t
       {
         SUPER::env.set_endchar (false);
     
    +    unsigned max_ops = HB_CFF_MAX_OPS;
         for (;;) {
    +      if (unlikely (!--max_ops))
    +      {
    +        SUPER::env.set_error ();
    +        break;
    +      }
           OPSET::process_op (SUPER::env.fetch_op (), SUPER::env, param);
           if (unlikely (SUPER::env.in_error ()))
             return false;
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-cff-interp-dict-common.hh b/src/java.desktop/share/native/libharfbuzz/hb-cff-interp-dict-common.hh
    index 041585a89a623..0c0cec9986a84 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-cff-interp-dict-common.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-cff-interp-dict-common.hh
    @@ -35,10 +35,8 @@ using namespace OT;
     /* an opstr and the parsed out dict value(s) */
     struct dict_val_t : op_str_t
     {
    -  void init () { single_val.set_int (0); }
    +  void init () {}
       void fini () {}
    -
    -  number_t            single_val;
     };
     
     typedef dict_val_t num_dict_val_t;
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-cff1-interp-cs.hh b/src/java.desktop/share/native/libharfbuzz/hb-cff1-interp-cs.hh
    index cfcf442ee7de2..fbb31c9ee1627 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-cff1-interp-cs.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-cff1-interp-cs.hh
    @@ -38,7 +38,8 @@ typedef biased_subrs_t   cff1_biased_subrs_t;
     struct cff1_cs_interp_env_t : cs_interp_env_t
     {
       template 
    -  cff1_cs_interp_env_t (const hb_ubytes_t &str, ACC &acc, unsigned int fd)
    +  cff1_cs_interp_env_t (const hb_ubytes_t &str, ACC &acc, unsigned int fd,
    +                        const int *coords_=nullptr, unsigned int num_coords_=0)
         : SUPER (str, acc.globalSubrs, acc.privateDicts[fd].localSubrs)
       {
         processed_width = false;
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-cff2-interp-cs.hh b/src/java.desktop/share/native/libharfbuzz/hb-cff2-interp-cs.hh
    index 791f3aab56a5c..c970633e06416 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-cff2-interp-cs.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-cff2-interp-cs.hh
    @@ -40,20 +40,22 @@ struct blend_arg_t : number_t
       void set_real (double v) { reset_blends (); number_t::set_real (v); }
     
       void set_blends (unsigned int numValues_, unsigned int valueIndex_,
    -                   unsigned int numBlends, hb_array_t blends_)
    +                   hb_array_t blends_)
       {
         numValues = numValues_;
         valueIndex = valueIndex_;
    -    deltas.resize (numBlends);
    +    unsigned numBlends = blends_.length;
    +    if (unlikely (!deltas.resize_exact (numBlends)))
    +      return;
         for (unsigned int i = 0; i < numBlends; i++)
    -      deltas[i] = blends_[i];
    +      deltas.arrayZ[i] = blends_.arrayZ[i];
       }
     
       bool blending () const { return deltas.length > 0; }
       void reset_blends ()
       {
         numValues = valueIndex = 0;
    -    deltas.resize (0);
    +    deltas.shrink (0);
       }
     
       unsigned int numValues;
    @@ -61,7 +63,6 @@ struct blend_arg_t : number_t
       hb_vector_t deltas;
     };
     
    -typedef interp_env_t BlendInterpEnv;
     typedef biased_subrs_t   cff2_biased_subrs_t;
     
     template 
    @@ -117,7 +118,7 @@ struct cff2_cs_interp_env_t : cs_interp_env_t
           region_count = varStore->varStore.get_region_index_count (get_ivs ());
           if (do_blend)
           {
    -        if (unlikely (!scalars.resize (region_count)))
    +        if (unlikely (!scalars.resize_exact (region_count)))
               SUPER::set_error ();
             else
               varStore->varStore.get_region_scalars (get_ivs (), coords, num_coords,
    @@ -154,13 +155,16 @@ struct cff2_cs_interp_env_t : cs_interp_env_t
         {
           if (likely (scalars.length == deltas.length))
           {
    -        for (unsigned int i = 0; i < scalars.length; i++)
    -          v += (double) scalars[i] * deltas[i].to_real ();
    +        unsigned count = scalars.length;
    +        for (unsigned i = 0; i < count; i++)
    +          v += (double) scalars.arrayZ[i] * deltas.arrayZ[i].to_real ();
           }
         }
         return v;
       }
     
    +  bool have_coords () const { return num_coords; }
    +
       protected:
       const int     *coords;
       unsigned int  num_coords;
    @@ -220,7 +224,10 @@ struct cff2_cs_opset_t : cs_opset_t, PAR
                                      const hb_array_t blends,
                                      unsigned n, unsigned i)
       {
    -    arg.set_blends (n, i, blends.length, blends);
    +    if (env.have_coords ())
    +      arg.set_int (round (arg.to_real () + env.blend_deltas (blends)));
    +    else
    +      arg.set_blends (n, i, blends);
       }
       template 
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-common.cc b/src/java.desktop/share/native/libharfbuzz/hb-common.cc
    index 310053e89dfed..8b94dcb366d9d 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-common.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-common.cc
    @@ -29,32 +29,6 @@
     #include "hb.hh"
     #include "hb-machinery.hh"
     
    -#if !defined(HB_NO_SETLOCALE) && (!defined(HAVE_NEWLOCALE) || !defined(HAVE_USELOCALE))
    -#define HB_NO_SETLOCALE 1
    -#endif
    -
    -#ifndef HB_NO_SETLOCALE
    -
    -#include 
    -#ifdef HAVE_XLOCALE_H
    -#include  // Needed on BSD/OS X for uselocale
    -#endif
    -
    -#ifdef WIN32
    -#define hb_locale_t _locale_t
    -#else
    -#define hb_locale_t locale_t
    -#endif
    -#define hb_setlocale setlocale
    -#define hb_uselocale uselocale
    -
    -#else
    -
    -#define hb_locale_t void *
    -#define hb_setlocale(Category, Locale) "C"
    -#define hb_uselocale(Locale) ((hb_locale_t) 0)
    -
    -#endif
     
     /**
      * SECTION:hb-common
    @@ -99,7 +73,7 @@ _hb_options_init ()
       }
     
       /* This is idempotent and threadsafe. */
    -  _hb_options.set_relaxed (u.i);
    +  _hb_options = u.i;
     }
     
     
    @@ -108,7 +82,7 @@ _hb_options_init ()
     /**
      * hb_tag_from_string:
      * @str: (array length=len) (element-type uint8_t): String to convert
    - * @len: Length of @str, or -1 if it is %NULL-terminated
    + * @len: Length of @str, or -1 if it is `NULL`-terminated
      *
      * Converts a string into an #hb_tag_t. Valid tags
      * are four characters. Shorter input strings will be
    @@ -170,7 +144,7 @@ static const char direction_strings[][4] = {
     /**
      * hb_direction_from_string:
      * @str: (array length=len) (element-type uint8_t): String to convert
    - * @len: Length of @str, or -1 if it is %NULL-terminated
    + * @len: Length of @str, or -1 if it is `NULL`-terminated
      *
      * Converts a string to an #hb_direction_t.
      *
    @@ -285,7 +259,7 @@ struct hb_language_item_t {
         lang = (hb_language_t) hb_malloc(len);
         if (likely (lang))
         {
    -      memcpy((unsigned char *) lang, s, len);
    +      hb_memcpy((unsigned char *) lang, s, len);
           for (unsigned char *p = (unsigned char *) lang; *p; p++)
             *p = canon_map[*p];
         }
    @@ -357,7 +331,7 @@ lang_find_or_insert (const char *key)
      * hb_language_from_string:
      * @str: (array length=len) (element-type uint8_t): a string representing
      *       a BCP 47 language tag
    - * @len: length of the @str, or -1 if it is %NULL-terminated.
    + * @len: length of the @str, or -1 if it is `NULL`-terminated.
      *
      * Converts @str representing a BCP 47 language tag to the corresponding
      * #hb_language_t.
    @@ -379,7 +353,7 @@ hb_language_from_string (const char *str, int len)
         /* NUL-terminate it. */
         char strbuf[64];
         len = hb_min (len, (int) sizeof (strbuf) - 1);
    -    memcpy (strbuf, str, len);
    +    hb_memcpy (strbuf, str, len);
         strbuf[len] = '\0';
         item = lang_find_or_insert (strbuf);
       }
    @@ -396,7 +370,7 @@ hb_language_from_string (const char *str, int len)
      * Converts an #hb_language_t to a string.
      *
      * Return value: (transfer none):
    - * A %NULL-terminated string representing the @language. Must not be freed by
    + * A `NULL`-terminated string representing the @language. Must not be freed by
      * the caller.
      *
      * Since: 0.9.2
    @@ -441,6 +415,38 @@ hb_language_get_default ()
       return language;
     }
     
    +/**
    + * hb_language_matches:
    + * @language: The #hb_language_t to work on
    + * @specific: Another #hb_language_t
    + *
    + * Check whether a second language tag is the same or a more
    + * specific version of the provided language tag.  For example,
    + * "fa_IR.utf8" is a more specific tag for "fa" or for "fa_IR".
    + *
    + * Return value: `true` if languages match, `false` otherwise.
    + *
    + * Since: 5.0.0
    + **/
    +hb_bool_t
    +hb_language_matches (hb_language_t language,
    +                     hb_language_t specific)
    +{
    +  if (language == specific) return true;
    +  if (!language || !specific) return false;
    +
    +  const char *l = language->s;
    +  const char *s = specific->s;
    +  unsigned ll = strlen (l);
    +  unsigned sl = strlen (s);
    +
    +  if (ll > sl)
    +    return false;
    +
    +  return strncmp (l, s, ll) == 0 &&
    +         (s[ll] == '\0' || s[ll] == '-');
    +}
    +
     
     /* hb_script_t */
     
    @@ -498,7 +504,7 @@ hb_script_from_iso15924_tag (hb_tag_t tag)
      * hb_script_from_string:
      * @str: (array length=len) (element-type uint8_t): a string representing an
      *       ISO 15924 tag.
    - * @len: length of the @str, or -1 if it is %NULL-terminated.
    + * @len: length of the @str, or -1 if it is `NULL`-terminated.
      *
      * Converts a string @str representing an ISO 15924 script tag to a
      * corresponding #hb_script_t. Shorthand for hb_tag_from_string() then
    @@ -626,6 +632,7 @@ hb_script_get_horizontal_direction (hb_script_t script)
         case HB_SCRIPT_OLD_HUNGARIAN:
         case HB_SCRIPT_OLD_ITALIC:
         case HB_SCRIPT_RUNIC:
    +    case HB_SCRIPT_TIFINAGH:
     
           return HB_DIRECTION_INVALID;
       }
    @@ -693,8 +700,8 @@ hb_version_string ()
      * Tests the library version against a minimum value,
      * as three integer components.
      *
    - * Return value: %true if the library is equal to or greater than
    - * the test value, %false otherwise
    + * Return value: `true` if the library is equal to or greater than
    + * the test value, `false` otherwise
      *
      * Since: 0.9.30
      **/
    @@ -881,7 +888,7 @@ parse_one_feature (const char **pp, const char *end, hb_feature_t *feature)
     /**
      * hb_feature_from_string:
      * @str: (array length=len) (element-type uint8_t): a string to parse
    - * @len: length of @str, or -1 if string is %NULL terminated
    + * @len: length of @str, or -1 if string is `NULL` terminated
      * @feature: (out): the #hb_feature_t to initialize with the parsed values
      *
      * Parses a string into a #hb_feature_t.
    @@ -923,7 +930,7 @@ parse_one_feature (const char **pp, const char *end, hb_feature_t *feature)
      * 
      *
      * Return value:
    - * %true if @str is successfully parsed, %false otherwise
    + * `true` if @str is successfully parsed, `false` otherwise
      *
      * Since: 0.9.5
      **/
    @@ -944,7 +951,7 @@ hb_feature_from_string (const char *str, int len,
       }
     
       if (feature)
    -    memset (feature, 0, sizeof (*feature));
    +    hb_memset (feature, 0, sizeof (*feature));
       return false;
     }
     
    @@ -954,7 +961,7 @@ hb_feature_from_string (const char *str, int len,
      * @buf: (array length=size) (out): output string
      * @size: the allocated size of @buf
      *
    - * Converts a #hb_feature_t into a %NULL-terminated string in the format
    + * Converts a #hb_feature_t into a `NULL`-terminated string in the format
      * understood by hb_feature_from_string(). The client in responsible for
      * allocating big enough size for @buf, 128 bytes is more than enough.
      *
    @@ -993,7 +1000,7 @@ hb_feature_to_string (hb_feature_t *feature,
       }
       assert (len < ARRAY_LENGTH (s));
       len = hb_min (len, size - 1);
    -  memcpy (buf, s, len);
    +  hb_memcpy (buf, s, len);
       buf[len] = '\0';
     }
     
    @@ -1022,7 +1029,7 @@ parse_one_variation (const char **pp, const char *end, hb_variation_t *variation
     /**
      * hb_variation_from_string:
      * @str: (array length=len) (element-type uint8_t): a string to parse
    - * @len: length of @str, or -1 if string is %NULL terminated
    + * @len: length of @str, or -1 if string is `NULL` terminated
      * @variation: (out): the #hb_variation_t to initialize with the parsed values
      *
      * Parses a string into a #hb_variation_t.
    @@ -1035,7 +1042,7 @@ parse_one_variation (const char **pp, const char *end, hb_variation_t *variation
      * number. For example `wght=500`, or `slnt=-7.5`.
      *
      * Return value:
    - * %true if @str is successfully parsed, %false otherwise
    + * `true` if @str is successfully parsed, `false` otherwise
      *
      * Since: 1.4.2
      */
    @@ -1056,7 +1063,7 @@ hb_variation_from_string (const char *str, int len,
       }
     
       if (variation)
    -    memset (variation, 0, sizeof (*variation));
    +    hb_memset (variation, 0, sizeof (*variation));
       return false;
     }
     
    @@ -1104,10 +1111,10 @@ get_C_locale ()
     /**
      * hb_variation_to_string:
      * @variation: an #hb_variation_t to convert
    - * @buf: (array length=size) (out): output string
    + * @buf: (array length=size) (out caller-allocates): output string
      * @size: the allocated size of @buf
      *
    - * Converts an #hb_variation_t into a %NULL-terminated string in the format
    + * Converts an #hb_variation_t into a `NULL`-terminated string in the format
      * understood by hb_variation_from_string(). The client in responsible for
      * allocating big enough size for @buf, 128 bytes is more than enough.
      *
    @@ -1134,7 +1141,7 @@ hb_variation_to_string (hb_variation_t *variation,
     
       assert (len < ARRAY_LENGTH (s));
       len = hb_min (len, size - 1);
    -  memcpy (buf, s, len);
    +  hb_memcpy (buf, s, len);
       buf[len] = '\0';
     }
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-common.h b/src/java.desktop/share/native/libharfbuzz/hb-common.h
    index 95daf0ed6a29c..ebdeadd1fd195 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-common.h
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-common.h
    @@ -326,6 +326,9 @@ hb_language_to_string (hb_language_t language);
     HB_EXTERN hb_language_t
     hb_language_get_default (void);
     
    +HB_EXTERN hb_bool_t
    +hb_language_matches (hb_language_t language,
    +                     hb_language_t specific);
     
     /**
      * hb_script_t:
    @@ -492,6 +495,8 @@ hb_language_get_default (void);
      * @HB_SCRIPT_TOTO: `Toto`, Since: 3.0.0
      * @HB_SCRIPT_VITHKUQI: `Vith`, Since: 3.0.0
      * @HB_SCRIPT_MATH: `Zmth`, Since: 3.4.0
    + * @HB_SCRIPT_KAWI: `Kawi`, Since: 5.2.0
    + * @HB_SCRIPT_NAG_MUNDARI: `Nagm`, Since: 5.2.0
      * @HB_SCRIPT_INVALID: No script set
      *
      * Data type for scripts. Each #hb_script_t's value is an #hb_tag_t corresponding
    @@ -713,6 +718,12 @@ typedef enum
        */
       HB_SCRIPT_MATH                        = HB_TAG ('Z','m','t','h'),
     
    +  /*
    +   * Since 5.2.0
    +   */
    +  HB_SCRIPT_KAWI                        = HB_TAG ('K','a','w','i'), /*15.0*/
    +  HB_SCRIPT_NAG_MUNDARI                 = HB_TAG ('N','a','g','m'), /*15.0*/
    +
       /* No script set. */
       HB_SCRIPT_INVALID                     = HB_TAG_NONE,
     
    @@ -886,6 +897,32 @@ HB_EXTERN uint8_t
     hb_color_get_blue (hb_color_t color);
     #define hb_color_get_blue(color)        (((color) >> 24) & 0xFF)
     
    +/**
    + * hb_glyph_extents_t:
    + * @x_bearing: Distance from the x-origin to the left extremum of the glyph.
    + * @y_bearing: Distance from the top extremum of the glyph to the y-origin.
    + * @width: Distance from the left extremum of the glyph to the right extremum.
    + * @height: Distance from the top extremum of the glyph to the bottom extremum.
    + *
    + * Glyph extent values, measured in font units.
    + *
    + * Note that @height is negative, in coordinate systems that grow up.
    + **/
    +typedef struct hb_glyph_extents_t {
    +  hb_position_t x_bearing;
    +  hb_position_t y_bearing;
    +  hb_position_t width;
    +  hb_position_t height;
    +} hb_glyph_extents_t;
    +
    +/**
    + * hb_font_t:
    + *
    + * Data type for holding fonts.
    + *
    + */
    +typedef struct hb_font_t hb_font_t;
    +
     HB_END_DECLS
     
     #endif /* HB_COMMON_H */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-config.hh b/src/java.desktop/share/native/libharfbuzz/hb-config.hh
    index 2578231d23889..52adaad4384c0 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-config.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-config.hh
    @@ -35,6 +35,11 @@
     #include "config.h"
     #endif
     
    +#ifndef HB_EXPERIMENTAL_API
    +#define HB_NO_BEYOND_64K
    +#define HB_NO_CUBIC_GLYF
    +#define HB_NO_VAR_COMPOSITES
    +#endif
     
     #ifdef HB_TINY
     #define HB_LEAN
    @@ -68,6 +73,7 @@
     #define HB_NO_LANGUAGE_PRIVATE_SUBTAG
     #define HB_NO_LAYOUT_FEATURE_PARAMS
     #define HB_NO_LAYOUT_COLLECT_GLYPHS
    +#define HB_NO_LAYOUT_RARELY_USED
     #define HB_NO_LAYOUT_UNUSED
     #define HB_NO_MATH
     #define HB_NO_META
    @@ -75,11 +81,13 @@
     #define HB_NO_MMAP
     #define HB_NO_NAME
     #define HB_NO_OPEN
    -#define HB_NO_SETLOCALE
     #define HB_NO_OT_FONT_GLYPH_NAMES
     #define HB_NO_OT_SHAPE_FRACTIONS
    +#define HB_NO_PAINT
    +#define HB_NO_SETLOCALE
     #define HB_NO_STYLE
     #define HB_NO_SUBSET_LAYOUT
    +#define HB_NO_VERTICAL
     #define HB_NO_VAR
     #endif
     
    @@ -98,12 +106,22 @@
     
     /* Closure of options. */
     
    +#ifdef HB_NO_BORING_EXPANSION
    +#define HB_NO_BEYOND_64K
    +#define HB_NO_AVAR2
    +#endif
    +
     #ifdef HB_DISABLE_DEPRECATED
     #define HB_IF_NOT_DEPRECATED(x)
     #else
     #define HB_IF_NOT_DEPRECATED(x) x
     #endif
     
    +#ifdef HB_NO_SHAPER
    +#define HB_NO_OT_SHAPE
    +#define HB_NO_AAT_SHAPE
    +#endif
    +
     #ifdef HB_NO_AAT
     #define HB_NO_OT_NAME_LANGUAGE_AAT
     #define HB_NO_AAT_SHAPE
    @@ -118,6 +136,10 @@
     #define HB_NO_SUBSET_CFF
     #endif
     
    +#ifdef HB_NO_DRAW
    +#define HB_NO_OUTLINE
    +#endif
    +
     #ifdef HB_NO_GETENV
     #define HB_NO_UNISCRIBE_BUG_COMPATIBLE
     #endif
    @@ -150,6 +172,7 @@
     #define HB_NO_OT_SHAPER_HEBREW_FALLBACK
     #define HB_NO_OT_SHAPER_THAI_FALLBACK
     #define HB_NO_OT_SHAPER_VOWEL_CONSTRAINTS
    +#define HB_NO_OT_SHAPER_MYANMAR_ZAWGYI
     #endif
     
     #ifdef NDEBUG
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-cplusplus.hh b/src/java.desktop/share/native/libharfbuzz/hb-cplusplus.hh
    index adfbc3c11bf7c..eac4ff03ed5f7 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-cplusplus.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-cplusplus.hh
    @@ -69,9 +69,9 @@ struct shared_ptr
       operator T * () const { return p; }
       T& operator * () const { return *get (); }
       T* operator -> () const { return get (); }
    -  operator bool () { return p; }
    -  bool operator == (const shared_ptr &o) { return p == o.p; }
    -  bool operator != (const shared_ptr &o) { return p != o.p; }
    +  operator bool () const { return p; }
    +  bool operator == (const shared_ptr &o) const { return p == o.p; }
    +  bool operator != (const shared_ptr &o) const { return p != o.p; }
     
       static T* get_empty() { return v::get_empty (); }
       T* reference() { return v::reference (p); }
    @@ -130,7 +130,7 @@ template 
     struct vtable_t
     {
    @@ -160,14 +160,43 @@ HB_DEFINE_VTABLE (map);
     HB_DEFINE_VTABLE (set);
     HB_DEFINE_VTABLE (shape_plan);
     HB_DEFINE_VTABLE (unicode_funcs);
    +HB_DEFINE_VTABLE (draw_funcs);
    +HB_DEFINE_VTABLE (paint_funcs);
     
     #undef HB_DEFINE_VTABLE
     
     
    +#ifdef HB_SUBSET_H
    +
    +#define HB_DEFINE_VTABLE(name) \
    +        template<> \
    +        struct vtable \
    +             : vtable_t {}
    +
    +
    +HB_DEFINE_VTABLE (subset_input);
    +HB_DEFINE_VTABLE (subset_plan);
    +
    +#undef HB_DEFINE_VTABLE
    +
    +#endif
    +
    +
     } // namespace hb
     
    +/* Workaround for GCC < 7, see:
    + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=56480
    + * https://stackoverflow.com/a/25594741 */
    +namespace std {
    +
    +
     template
    -struct std::hash>
    +struct hash>
     {
         std::size_t operator()(const hb::shared_ptr& v) const noexcept
         {
    @@ -177,7 +206,7 @@ struct std::hash>
     };
     
     template
    -struct std::hash>
    +struct hash>
     {
         std::size_t operator()(const hb::unique_ptr& v) const noexcept
         {
    @@ -187,6 +216,8 @@ struct std::hash>
     };
     
     
    +} // namespace std
    +
     #endif /* __cplusplus */
     
     #endif /* HB_CPLUSPLUS_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-debug.hh b/src/java.desktop/share/native/libharfbuzz/hb-debug.hh
    index 09c407e415d20..91a24a7152182 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-debug.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-debug.hh
    @@ -67,12 +67,12 @@ hb_options ()
     #endif
       /* Make a local copy, so we can access bitfield threadsafely. */
       hb_options_union_t u;
    -  u.i = _hb_options.get_relaxed ();
    +  u.i = _hb_options;
     
       if (unlikely (!u.i))
       {
         _hb_options_init ();
    -    u.i = _hb_options.get_relaxed ();
    +    u.i = _hb_options;
       }
     
       return u.opts;
    @@ -113,7 +113,7 @@ _hb_print_func (const char *func)
         const char *paren = strchr (func, '(');
         if (paren)
           func_len = paren - func;
    -    fprintf (stderr, "%.*s", func_len, func);
    +    fprintf (stderr, "%.*s", (int) func_len, func);
       }
     }
     
    @@ -142,9 +142,9 @@ _hb_debug_msg_va (const char *what,
       fprintf (stderr, "%-10s", what ? what : "");
     
       if (obj)
    -    fprintf (stderr, "(%*p) ", (unsigned int) (2 * sizeof (void *)), obj);
    +    fprintf (stderr, "(%*p) ", (int) (2 * sizeof (void *)), obj);
       else
    -    fprintf (stderr, " %*s  ", (unsigned int) (2 * sizeof (void *)), "");
    +    fprintf (stderr, " %*s  ", (int) (2 * sizeof (void *)), "");
     
       if (indented) {
     #define VBAR    "\342\224\202"  /* U+2502 BOX DRAWINGS LIGHT VERTICAL */
    @@ -306,7 +306,7 @@ struct hb_auto_trace_t
         }
     
         _hb_debug_msg (what, obj, func, true, plevel ? *plevel : 1, -1,
    -                              "return %s (line %d)",
    +                              "return %s (line %u)",
                                   hb_printer_t>().print (v), line);
         if (plevel) --*plevel;
         plevel = nullptr;
    @@ -373,6 +373,10 @@ struct hb_no_trace_t {
     #define HB_DEBUG_FT (HB_DEBUG+0)
     #endif
     
    +#ifndef HB_DEBUG_JUSTIFY
    +#define HB_DEBUG_JUSTIFY (HB_DEBUG+0)
    +#endif
    +
     #ifndef HB_DEBUG_OBJECT
     #define HB_DEBUG_OBJECT (HB_DEBUG+0)
     #endif
    @@ -396,7 +400,7 @@ struct hb_no_trace_t {
     #define TRACE_APPLY(this) \
             hb_auto_trace_t trace \
             (&c->debug_depth, c->get_name (), this, HB_FUNC, \
    -         "idx %d gid %u lookup %d", \
    +         "idx %u gid %u lookup %d", \
              c->buffer->idx, c->buffer->cur().codepoint, (int) c->lookup_index)
     #else
     #define TRACE_APPLY(this) hb_no_trace_t trace
    @@ -454,10 +458,15 @@ struct hb_no_trace_t {
     #define TRACE_DISPATCH(this, format) \
             hb_auto_trace_t trace \
             (&c->debug_depth, c->get_name (), this, HB_FUNC, \
    -         "format %d", (int) format)
    +         "format %u", (unsigned) format)
     #else
     #define TRACE_DISPATCH(this, format) hb_no_trace_t trace
     #endif
     
     
    +#ifndef HB_BUFFER_MESSAGE_MORE
    +#define HB_BUFFER_MESSAGE_MORE (HB_DEBUG+1)
    +#endif
    +
    +
     #endif /* HB_DEBUG_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-deprecated.h b/src/java.desktop/share/native/libharfbuzz/hb-deprecated.h
    index 55f1e1205dfd0..a9e63de853d33 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-deprecated.h
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-deprecated.h
    @@ -93,7 +93,7 @@ HB_BEGIN_DECLS
      * This method should retrieve the glyph ID for a specified Unicode code point
      * font, with an optional variation selector.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      * Deprecated: 1.2.3
      *
      **/
    @@ -102,11 +102,22 @@ typedef hb_bool_t (*hb_font_get_glyph_func_t) (hb_font_t *font, void *font_data,
                                                    hb_codepoint_t *glyph,
                                                    void *user_data);
     
    -HB_EXTERN HB_DEPRECATED_FOR(hb_font_funcs_set_nominal_glyph_func and hb_font_funcs_set_variation_glyph_func) void
    +HB_DEPRECATED_FOR (hb_font_funcs_set_nominal_glyph_func and hb_font_funcs_set_variation_glyph_func)
    +HB_EXTERN void
     hb_font_funcs_set_glyph_func (hb_font_funcs_t *ffuncs,
                                   hb_font_get_glyph_func_t func,
                                   void *user_data, hb_destroy_func_t destroy);
     
    +/* https://github.com/harfbuzz/harfbuzz/pull/4207 */
    +/**
    + * HB_UNICODE_COMBINING_CLASS_CCC133:
    + *
    + * [Tibetan]
    + *
    + * Deprecated: 7.2.0
    + **/
    +#define HB_UNICODE_COMBINING_CLASS_CCC133 133
    +
     /**
      * hb_unicode_eastasian_width_func_t:
      * @ufuncs: A Unicode-functions structure
    @@ -246,6 +257,7 @@ hb_font_get_glyph_v_kerning (hb_font_t *font,
     
     #endif
     
    +
     HB_END_DECLS
     
     #endif /* HB_DEPRECATED_H */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-draw.cc b/src/java.desktop/share/native/libharfbuzz/hb-draw.cc
    index 6a9ab01240918..f2821578b65f6 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-draw.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-draw.cc
    @@ -35,6 +35,8 @@
      * @include: hb.h
      *
      * Functions for drawing (extracting) glyph shapes.
    + *
    + * The #hb_draw_funcs_t struct can be used with hb_font_draw_glyph().
      **/
     
     static void
    @@ -80,6 +82,56 @@ hb_draw_close_path_nil (hb_draw_funcs_t *dfuncs HB_UNUSED, void *draw_data HB_UN
                             void *user_data HB_UNUSED) {}
     
     
    +static bool
    +_hb_draw_funcs_set_preamble (hb_draw_funcs_t    *dfuncs,
    +                             bool                func_is_null,
    +                             void              **user_data,
    +                             hb_destroy_func_t  *destroy)
    +{
    +  if (hb_object_is_immutable (dfuncs))
    +  {
    +    if (*destroy)
    +      (*destroy) (*user_data);
    +    return false;
    +  }
    +
    +  if (func_is_null)
    +  {
    +    if (*destroy)
    +      (*destroy) (*user_data);
    +    *destroy = nullptr;
    +    *user_data = nullptr;
    +  }
    +
    +  return true;
    +}
    +
    +static bool
    +_hb_draw_funcs_set_middle (hb_draw_funcs_t   *dfuncs,
    +                           void              *user_data,
    +                           hb_destroy_func_t  destroy)
    +{
    +  if (user_data && !dfuncs->user_data)
    +  {
    +    dfuncs->user_data = (decltype (dfuncs->user_data)) hb_calloc (1, sizeof (*dfuncs->user_data));
    +    if (unlikely (!dfuncs->user_data))
    +      goto fail;
    +  }
    +  if (destroy && !dfuncs->destroy)
    +  {
    +    dfuncs->destroy = (decltype (dfuncs->destroy)) hb_calloc (1, sizeof (*dfuncs->destroy));
    +    if (unlikely (!dfuncs->destroy))
    +      goto fail;
    +  }
    +
    +  return true;
    +
    +fail:
    +  if (destroy)
    +    (destroy) (user_data);
    +  return false;
    +}
    +
     #define HB_DRAW_FUNC_IMPLEMENT(name)                                            \
                                                                                     \
     void                                                                            \
    @@ -88,42 +140,24 @@ hb_draw_funcs_set_##name##_func (hb_draw_funcs_t         *dfuncs,
                                      void                    *user_data,            \
                                      hb_destroy_func_t        destroy)              \
     {                                                                               \
    -  if (hb_object_is_immutable (dfuncs))                                          \
    -    return;                                                                     \
    +  if (!_hb_draw_funcs_set_preamble (dfuncs, !func, &user_data, &destroy))\
    +      return;                                                            \
                                                                                     \
       if (dfuncs->destroy && dfuncs->destroy->name)                                 \
         dfuncs->destroy->name (!dfuncs->user_data ? nullptr : dfuncs->user_data->name); \
                                                                              \
    -  if (user_data && !dfuncs->user_data)                                   \
    -  {                                                                      \
    -    dfuncs->user_data = (decltype (dfuncs->user_data)) hb_calloc (1, sizeof (*dfuncs->user_data)); \
    -    if (unlikely (!dfuncs->user_data))                                   \
    -      goto fail;                                                         \
    -  }                                                                      \
    -  if (destroy && !dfuncs->destroy)                                       \
    -  {                                                                      \
    -    dfuncs->destroy = (decltype (dfuncs->destroy)) hb_calloc (1, sizeof (*dfuncs->destroy)); \
    -    if (unlikely (!dfuncs->destroy))                                     \
    -      goto fail;                                                         \
    -  }                                                                      \
    +  if (!_hb_draw_funcs_set_middle (dfuncs, user_data, destroy))           \
    +      return;                                                            \
                                                                             \
    -  if (func) {                                                           \
    +  if (func)                                                             \
         dfuncs->func.name = func;                                           \
    -    if (dfuncs->user_data)                                              \
    -      dfuncs->user_data->name = user_data;                              \
    -    if (dfuncs->destroy)                                                \
    -      dfuncs->destroy->name = destroy;                                  \
    -  } else {                                                              \
    +  else                                                                  \
         dfuncs->func.name = hb_draw_##name##_nil;                           \
    -    if (dfuncs->user_data)                                              \
    -      dfuncs->user_data->name = nullptr;                                \
    -    if (dfuncs->destroy)                                                \
    -      dfuncs->destroy->name = nullptr;                                  \
    -  }                                                                     \
    -                                                                         \
    -fail:                                                                    \
    -  if (destroy)                                                           \
    -    destroy (user_data);                                                 \
    +                                                                        \
    +  if (dfuncs->user_data)                                                \
    +    dfuncs->user_data->name = user_data;                                \
    +  if (dfuncs->destroy)                                                  \
    +    dfuncs->destroy->name = destroy;                                    \
     }
     
     HB_DRAW_FUNCS_IMPLEMENT_CALLBACKS
    @@ -137,7 +171,7 @@ HB_DRAW_FUNCS_IMPLEMENT_CALLBACKS
      * Return value: (transfer full):
      * A newly allocated #hb_draw_funcs_t with a reference count of 1. The initial
      * reference count should be released with hb_draw_funcs_destroy when you are
    - * done using the #hb_draw_funcs_t. This function never returns %NULL. If
    + * done using the #hb_draw_funcs_t. This function never returns `NULL`. If
      * memory cannot be allocated, a special singleton #hb_draw_funcs_t object will
      * be returned.
      *
    @@ -166,13 +200,29 @@ DEFINE_NULL_INSTANCE (hb_draw_funcs_t) =
       }
     };
     
    +/**
    + * hb_draw_funcs_get_empty:
    + *
    + * Fetches the singleton empty draw-functions structure.
    + *
    + * Return value: (transfer full): The empty draw-functions structure
    + *
    + * Since: 7.0.0
    + **/
    +hb_draw_funcs_t *
    +hb_draw_funcs_get_empty ()
    +{
    +  return const_cast (&Null (hb_draw_funcs_t));
    +}
     
     /**
      * hb_draw_funcs_reference: (skip)
      * @dfuncs: draw functions
      *
    - * Increases the reference count on @dfuncs by one. This prevents @buffer from
    - * being destroyed until a matching call to hb_draw_funcs_destroy() is made.
    + * Increases the reference count on @dfuncs by one.
    + *
    + * This prevents @dfuncs from being destroyed until a matching
    + * call to hb_draw_funcs_destroy() is made.
      *
      * Return value: (transfer full):
      * The referenced #hb_draw_funcs_t.
    @@ -208,9 +258,55 @@ hb_draw_funcs_destroy (hb_draw_funcs_t *dfuncs)
     #undef HB_DRAW_FUNC_IMPLEMENT
       }
     
    +  hb_free (dfuncs->destroy);
    +  hb_free (dfuncs->user_data);
    +
       hb_free (dfuncs);
     }
     
    +/**
    + * hb_draw_funcs_set_user_data: (skip)
    + * @dfuncs: The draw-functions structure
    + * @key: The user-data key
    + * @data: A pointer to the user data
    + * @destroy: (nullable): A callback to call when @data is not needed anymore
    + * @replace: Whether to replace an existing data with the same key
    + *
    + * Attaches a user-data key/data pair to the specified draw-functions structure.
    + *
    + * Return value: `true` if success, `false` otherwise
    + *
    + * Since: 7.0.0
    + **/
    +hb_bool_t
    +hb_draw_funcs_set_user_data (hb_draw_funcs_t *dfuncs,
    +                             hb_user_data_key_t *key,
    +                             void *              data,
    +                             hb_destroy_func_t   destroy,
    +                             hb_bool_t           replace)
    +{
    +  return hb_object_set_user_data (dfuncs, key, data, destroy, replace);
    +}
    +
    +/**
    + * hb_draw_funcs_get_user_data: (skip)
    + * @dfuncs: The draw-functions structure
    + * @key: The user-data key to query
    + *
    + * Fetches the user-data associated with the specified key,
    + * attached to the specified draw-functions structure.
    + *
    + * Return value: (transfer none): A pointer to the user data
    + *
    + * Since: 7.0.0
    + **/
    +void *
    +hb_draw_funcs_get_user_data (const hb_draw_funcs_t *dfuncs,
    +                             hb_user_data_key_t       *key)
    +{
    +  return hb_object_get_user_data (dfuncs, key);
    +}
    +
     /**
      * hb_draw_funcs_make_immutable:
      * @dfuncs: draw functions
    @@ -234,7 +330,7 @@ hb_draw_funcs_make_immutable (hb_draw_funcs_t *dfuncs)
      *
      * Checks whether @dfuncs is immutable.
      *
    - * Return value: %true if @dfuncs is immutable, %false otherwise
    + * Return value: `true` if @dfuncs is immutable, `false` otherwise
      *
      * Since: 4.0.0
      **/
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-draw.h b/src/java.desktop/share/native/libharfbuzz/hb-draw.h
    index 364db5f7f56be..eba791d10c08d 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-draw.h
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-draw.h
    @@ -92,11 +92,11 @@ typedef struct hb_draw_funcs_t hb_draw_funcs_t;
     /**
      * hb_draw_move_to_func_t:
      * @dfuncs: draw functions object
    - * @draw_data: The data accompanying the draw functions
    + * @draw_data: The data accompanying the draw functions in hb_font_draw_glyph()
      * @st: current draw state
      * @to_x: X component of target point
      * @to_y: Y component of target point
    - * @user_data: User data pointer passed by the caller
    + * @user_data: User data pointer passed to hb_draw_funcs_set_move_to_func()
      *
      * A virtual method for the #hb_draw_funcs_t to perform a "move-to" draw
      * operation.
    @@ -112,11 +112,11 @@ typedef void (*hb_draw_move_to_func_t) (hb_draw_funcs_t *dfuncs, void *draw_data
     /**
      * hb_draw_line_to_func_t:
      * @dfuncs: draw functions object
    - * @draw_data: The data accompanying the draw functions
    + * @draw_data: The data accompanying the draw functions in hb_font_draw_glyph()
      * @st: current draw state
      * @to_x: X component of target point
      * @to_y: Y component of target point
    - * @user_data: User data pointer passed by the caller
    + * @user_data: User data pointer passed to hb_draw_funcs_set_line_to_func()
      *
      * A virtual method for the #hb_draw_funcs_t to perform a "line-to" draw
      * operation.
    @@ -132,13 +132,13 @@ typedef void (*hb_draw_line_to_func_t) (hb_draw_funcs_t *dfuncs, void *draw_data
     /**
      * hb_draw_quadratic_to_func_t:
      * @dfuncs: draw functions object
    - * @draw_data: The data accompanying the draw functions
    + * @draw_data: The data accompanying the draw functions in hb_font_draw_glyph()
      * @st: current draw state
      * @control_x: X component of control point
      * @control_y: Y component of control point
      * @to_x: X component of target point
      * @to_y: Y component of target point
    - * @user_data: User data pointer passed by the caller
    + * @user_data: User data pointer passed to hb_draw_funcs_set_quadratic_to_func()
      *
      * A virtual method for the #hb_draw_funcs_t to perform a "quadratic-to" draw
      * operation.
    @@ -155,7 +155,7 @@ typedef void (*hb_draw_quadratic_to_func_t) (hb_draw_funcs_t *dfuncs, void *draw
     /**
      * hb_draw_cubic_to_func_t:
      * @dfuncs: draw functions object
    - * @draw_data: The data accompanying the draw functions
    + * @draw_data: The data accompanying the draw functions in hb_font_draw_glyph()
      * @st: current draw state
      * @control1_x: X component of first control point
      * @control1_y: Y component of first control point
    @@ -163,7 +163,7 @@ typedef void (*hb_draw_quadratic_to_func_t) (hb_draw_funcs_t *dfuncs, void *draw
      * @control2_y: Y component of second control point
      * @to_x: X component of target point
      * @to_y: Y component of target point
    - * @user_data: User data pointer passed by the caller
    + * @user_data: User data pointer passed to hb_draw_funcs_set_cubic_to_func()
      *
      * A virtual method for the #hb_draw_funcs_t to perform a "cubic-to" draw
      * operation.
    @@ -181,9 +181,9 @@ typedef void (*hb_draw_cubic_to_func_t) (hb_draw_funcs_t *dfuncs, void *draw_dat
     /**
      * hb_draw_close_path_func_t:
      * @dfuncs: draw functions object
    - * @draw_data: The data accompanying the draw functions
    + * @draw_data: The data accompanying the draw functions in hb_font_draw_glyph()
      * @st: current draw state
    - * @user_data: User data pointer passed by the caller
    + * @user_data: User data pointer passed to hb_draw_funcs_set_close_path_func()
      *
      * A virtual method for the #hb_draw_funcs_t to perform a "close-path" draw
      * operation.
    @@ -279,12 +279,27 @@ hb_draw_funcs_set_close_path_func (hb_draw_funcs_t           *dfuncs,
     HB_EXTERN hb_draw_funcs_t *
     hb_draw_funcs_create (void);
     
    +HB_EXTERN hb_draw_funcs_t *
    +hb_draw_funcs_get_empty (void);
    +
     HB_EXTERN hb_draw_funcs_t *
     hb_draw_funcs_reference (hb_draw_funcs_t *dfuncs);
     
     HB_EXTERN void
     hb_draw_funcs_destroy (hb_draw_funcs_t *dfuncs);
     
    +HB_EXTERN hb_bool_t
    +hb_draw_funcs_set_user_data (hb_draw_funcs_t *dfuncs,
    +                             hb_user_data_key_t *key,
    +                             void *              data,
    +                             hb_destroy_func_t   destroy,
    +                             hb_bool_t           replace);
    +
    +
    +HB_EXTERN void *
    +hb_draw_funcs_get_user_data (const hb_draw_funcs_t *dfuncs,
    +                             hb_user_data_key_t       *key);
    +
     HB_EXTERN void
     hb_draw_funcs_make_immutable (hb_draw_funcs_t *dfuncs);
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-face-builder.cc b/src/java.desktop/share/native/libharfbuzz/hb-face-builder.cc
    new file mode 100644
    index 0000000000000..ff723ac700040
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-face-builder.cc
    @@ -0,0 +1,246 @@
    +/*
    + * Copyright © 2009  Red Hat, Inc.
    + * Copyright © 2012  Google, Inc.
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + *
    + * Red Hat Author(s): Behdad Esfahbod
    + * Google Author(s): Behdad Esfahbod
    + */
    +
    +#include "hb.hh"
    +
    +#include "hb-face.hh"
    +
    +#include "hb-map.hh"
    +#include "hb-open-file.hh"
    +#include "hb-serialize.hh"
    +
    +
    +/*
    + * face-builder: A face that has add_table().
    + */
    +
    +struct face_table_info_t
    +{
    +  hb_blob_t* data;
    +  signed order;
    +};
    +
    +struct hb_face_builder_data_t
    +{
    +  hb_hashmap_t tables;
    +};
    +
    +static int compare_entries (const void* pa, const void* pb)
    +{
    +  const auto& a = * (const hb_pair_t *) pa;
    +  const auto& b = * (const hb_pair_t *) pb;
    +
    +  /* Order by blob size first (smallest to largest) and then table tag */
    +
    +  if (a.second.order != b.second.order)
    +    return a.second.order < b.second.order ? -1 : +1;
    +
    +  if (a.second.data->length != b.second.data->length)
    +    return a.second.data->length < b.second.data->length ? -1 : +1;
    +
    +  return a.first < b.first ? -1 : a.first == b.first ? 0 : +1;
    +}
    +
    +static hb_face_builder_data_t *
    +_hb_face_builder_data_create ()
    +{
    +  hb_face_builder_data_t *data = (hb_face_builder_data_t *) hb_calloc (1, sizeof (hb_face_builder_data_t));
    +  if (unlikely (!data))
    +    return nullptr;
    +
    +  data->tables.init ();
    +
    +  return data;
    +}
    +
    +static void
    +_hb_face_builder_data_destroy (void *user_data)
    +{
    +  hb_face_builder_data_t *data = (hb_face_builder_data_t *) user_data;
    +
    +  for (auto info : data->tables.values())
    +    hb_blob_destroy (info.data);
    +
    +  data->tables.fini ();
    +
    +  hb_free (data);
    +}
    +
    +static hb_blob_t *
    +_hb_face_builder_data_reference_blob (hb_face_builder_data_t *data)
    +{
    +
    +  unsigned int table_count = data->tables.get_population ();
    +  unsigned int face_length = table_count * 16 + 12;
    +
    +  for (auto info : data->tables.values())
    +    face_length += hb_ceil_to_4 (hb_blob_get_length (info.data));
    +
    +  char *buf = (char *) hb_malloc (face_length);
    +  if (unlikely (!buf))
    +    return nullptr;
    +
    +  hb_serialize_context_t c (buf, face_length);
    +  c.propagate_error (data->tables);
    +  OT::OpenTypeFontFile *f = c.start_serialize ();
    +
    +  bool is_cff = (data->tables.has (HB_TAG ('C','F','F',' '))
    +                 || data->tables.has (HB_TAG ('C','F','F','2')));
    +  hb_tag_t sfnt_tag = is_cff ? OT::OpenTypeFontFile::CFFTag : OT::OpenTypeFontFile::TrueTypeTag;
    +
    +  // Sort the tags so that produced face is deterministic.
    +  hb_vector_t> sorted_entries;
    +  data->tables.iter () | hb_sink (sorted_entries);
    +  if (unlikely (sorted_entries.in_error ()))
    +  {
    +    hb_free (buf);
    +    return nullptr;
    +  }
    +
    +  sorted_entries.qsort (compare_entries);
    +
    +  bool ret = f->serialize_single (&c,
    +                                  sfnt_tag,
    +                                  + sorted_entries.iter()
    +                                  | hb_map ([&] (hb_pair_t _) {
    +                                    return hb_pair_t (_.first, _.second.data);
    +                                  }));
    +
    +  c.end_serialize ();
    +
    +  if (unlikely (!ret))
    +  {
    +    hb_free (buf);
    +    return nullptr;
    +  }
    +
    +  return hb_blob_create (buf, face_length, HB_MEMORY_MODE_WRITABLE, buf, hb_free);
    +}
    +
    +static hb_blob_t *
    +_hb_face_builder_reference_table (hb_face_t *face HB_UNUSED, hb_tag_t tag, void *user_data)
    +{
    +  hb_face_builder_data_t *data = (hb_face_builder_data_t *) user_data;
    +
    +  if (!tag)
    +    return _hb_face_builder_data_reference_blob (data);
    +
    +  return hb_blob_reference (data->tables[tag].data);
    +}
    +
    +
    +/**
    + * hb_face_builder_create:
    + *
    + * Creates a #hb_face_t that can be used with hb_face_builder_add_table().
    + * After tables are added to the face, it can be compiled to a binary
    + * font file by calling hb_face_reference_blob().
    + *
    + * Return value: (transfer full): New face.
    + *
    + * Since: 1.9.0
    + **/
    +hb_face_t *
    +hb_face_builder_create ()
    +{
    +  hb_face_builder_data_t *data = _hb_face_builder_data_create ();
    +  if (unlikely (!data)) return hb_face_get_empty ();
    +
    +  return hb_face_create_for_tables (_hb_face_builder_reference_table,
    +                                    data,
    +                                    _hb_face_builder_data_destroy);
    +}
    +
    +/**
    + * hb_face_builder_add_table:
    + * @face: A face object created with hb_face_builder_create()
    + * @tag: The #hb_tag_t of the table to add
    + * @blob: The blob containing the table data to add
    + *
    + * Add table for @tag with data provided by @blob to the face.  @face must
    + * be created using hb_face_builder_create().
    + *
    + * Since: 1.9.0
    + **/
    +hb_bool_t
    +hb_face_builder_add_table (hb_face_t *face, hb_tag_t tag, hb_blob_t *blob)
    +{
    +  if (unlikely (face->destroy != (hb_destroy_func_t) _hb_face_builder_data_destroy))
    +    return false;
    +
    +  if (tag == HB_MAP_VALUE_INVALID)
    +    return false;
    +
    +  hb_face_builder_data_t *data = (hb_face_builder_data_t *) face->user_data;
    +
    +  hb_blob_t* previous = data->tables.get (tag).data;
    +  if (!data->tables.set (tag, face_table_info_t {hb_blob_reference (blob), -1}))
    +  {
    +    hb_blob_destroy (blob);
    +    return false;
    +  }
    +
    +  hb_blob_destroy (previous);
    +  return true;
    +}
    +
    +/**
    + * hb_face_builder_sort_tables:
    + * @face: A face object created with hb_face_builder_create()
    + * @tags: (array zero-terminated=1): ordered list of table tags terminated by
    + *   %HB_TAG_NONE
    + *
    + * Set the ordering of tables for serialization. Any tables not
    + * specified in the tags list will be ordered after the tables in
    + * tags, ordered by the default sort ordering.
    + *
    + * Since: 5.3.0
    + **/
    +void
    +hb_face_builder_sort_tables (hb_face_t *face,
    +                             const hb_tag_t  *tags)
    +{
    +  if (unlikely (face->destroy != (hb_destroy_func_t) _hb_face_builder_data_destroy))
    +    return;
    +
    +  hb_face_builder_data_t *data = (hb_face_builder_data_t *) face->user_data;
    +
    +  // Sort all unspecified tables after any specified tables.
    +  for (auto& info : data->tables.values_ref())
    +    info.order = (unsigned) -1;
    +
    +  signed order = 0;
    +  for (const hb_tag_t* tag = tags;
    +       *tag;
    +       tag++)
    +  {
    +    face_table_info_t* info;
    +    if (!data->tables.has (*tag, &info)) continue;
    +    info->order = order++;
    +  }
    +}
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-face.cc b/src/java.desktop/share/native/libharfbuzz/hb-face.cc
    index 43a32484701b3..bc19bb9d5c970 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-face.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-face.cc
    @@ -33,7 +33,6 @@
     #include "hb-open-file.hh"
     #include "hb-ot-face.hh"
     #include "hb-ot-cmap-table.hh"
    -#include "hb-map.hh"
     
     
     /**
    @@ -48,6 +47,12 @@
      * More precisely, a font face represents a single face in a binary font file.
      * Font faces are typically built from a binary blob and a face index.
      * Font faces are used to create fonts.
    + *
    + * A font face can be created from a binary blob using hb_face_create().
    + * The face index is used to select a face from a binary blob that contains
    + * multiple faces.  For example, a binary blob that contains both a regular
    + * and a bold face can be used to create two font faces, one for each face
    + * index.
      **/
     
     
    @@ -132,7 +137,7 @@ hb_face_create_for_tables (hb_reference_table_func_t  reference_table_func,
       face->user_data = user_data;
       face->destroy = destroy;
     
    -  face->num_glyphs.set_relaxed (-1);
    +  face->num_glyphs = -1;
     
       face->data.init0 (face);
       face->table.init0 (face);
    @@ -198,7 +203,7 @@ _hb_face_for_data_reference_table (hb_face_t *face HB_UNUSED, hb_tag_t tag, void
      * a face index into that blob.
      *
      * The face index is used for blobs of file formats such as TTC and
    - * and DFont that can contain more than one face.  Face indices within
    + * DFont that can contain more than one face.  Face indices within
      * such collections are zero-based.
      *
      * Note: If the blob font format is not a collection, @index
    @@ -288,6 +293,7 @@ hb_face_destroy (hb_face_t *face)
     {
       if (!hb_object_destroy (face)) return;
     
    +#ifndef HB_NO_SHAPER
       for (hb_face_t::plan_node_t *node = face->shape_plans; node; )
       {
         hb_face_t::plan_node_t *next = node->next;
    @@ -295,6 +301,7 @@ hb_face_destroy (hb_face_t *face)
         hb_free (node);
         node = next;
       }
    +#endif
     
       face->data.fini ();
       face->table.fini ();
    @@ -315,7 +322,7 @@ hb_face_destroy (hb_face_t *face)
      *
      * Attaches a user-data key/data pair to the given face object.
      *
    - * Return value: %true if success, %false otherwise
    + * Return value: `true` if success, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -342,7 +349,7 @@ hb_face_set_user_data (hb_face_t          *face,
      * Since: 0.9.2
      **/
     void *
    -hb_face_get_user_data (hb_face_t          *face,
    +hb_face_get_user_data (const hb_face_t    *face,
                            hb_user_data_key_t *key)
     {
       return hb_object_get_user_data (face, key);
    @@ -371,7 +378,7 @@ hb_face_make_immutable (hb_face_t *face)
      *
      * Tests whether the given face object is immutable.
      *
    - * Return value: %true is @face is immutable, %false otherwise
    + * Return value: `true` is @face is immutable, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -470,6 +477,8 @@ hb_face_get_index (const hb_face_t *face)
      *
      * Sets the units-per-em (upem) for a face object to the specified value.
      *
    + * This API is used in rare circumstances.
    + *
      * Since: 0.9.2
      **/
     void
    @@ -479,14 +488,17 @@ hb_face_set_upem (hb_face_t    *face,
       if (hb_object_is_immutable (face))
         return;
     
    -  face->upem.set_relaxed (upem);
    +  face->upem = upem;
     }
     
     /**
      * hb_face_get_upem:
      * @face: A face object
      *
    - * Fetches the units-per-em (upem) value of the specified face object.
    + * Fetches the units-per-em (UPEM) value of the specified face object.
    + *
    + * Typical UPEM values for fonts are 1000, or 2048, but any value
    + * in between 16 and 16,384 is allowed for OpenType fonts.
      *
      * Return value: The upem value of @face
      *
    @@ -505,6 +517,8 @@ hb_face_get_upem (const hb_face_t *face)
      *
      * Sets the glyph count for a face object to the specified value.
      *
    + * This API is used in rare circumstances.
    + *
      * Since: 0.9.7
      **/
     void
    @@ -514,7 +528,7 @@ hb_face_set_glyph_count (hb_face_t    *face,
       if (hb_object_is_immutable (face))
         return;
     
    -  face->num_glyphs.set_relaxed (glyph_count);
    +  face->num_glyphs = glyph_count;
     }
     
     /**
    @@ -579,7 +593,7 @@ hb_face_get_table_tags (const hb_face_t *face,
     /**
      * hb_face_collect_unicodes:
      * @face: A face object
    - * @out: The set to add Unicode characters to
    + * @out: (out): The set to add Unicode characters to
      *
      * Collects all of the Unicode characters covered by @face and adds
      * them to the #hb_set_t set @out.
    @@ -592,10 +606,31 @@ hb_face_collect_unicodes (hb_face_t *face,
     {
       face->table.cmap->collect_unicodes (out, face->get_num_glyphs ());
     }
    +/**
    + * hb_face_collect_nominal_glyph_mapping:
    + * @face: A face object
    + * @mapping: (out): The map to add Unicode-to-glyph mapping to
    + * @unicodes: (nullable) (out): The set to add Unicode characters to, or `NULL`
    + *
    + * Collects the mapping from Unicode characters to nominal glyphs of the @face,
    + * and optionally all of the Unicode characters covered by @face.
    + *
    + * Since: 7.0.0
    + */
    +void
    +hb_face_collect_nominal_glyph_mapping (hb_face_t *face,
    +                                       hb_map_t  *mapping,
    +                                       hb_set_t  *unicodes)
    +{
    +  hb_set_t stack_unicodes;
    +  if (!unicodes)
    +    unicodes = &stack_unicodes;
    +  face->table.cmap->collect_mapping (unicodes, mapping, face->get_num_glyphs ());
    +}
     /**
      * hb_face_collect_variation_selectors:
      * @face: A face object
    - * @out: The set to add Variation Selector characters to
    + * @out: (out): The set to add Variation Selector characters to
      *
      * Collects all Unicode "Variation Selector" characters covered by @face and adds
      * them to the #hb_set_t set @out.
    @@ -612,7 +647,7 @@ hb_face_collect_variation_selectors (hb_face_t *face,
      * hb_face_collect_variation_unicodes:
      * @face: A face object
      * @variation_selector: The Variation Selector to query
    - * @out: The set to add Unicode characters to
    + * @out: (out): The set to add Unicode characters to
      *
      * Collects all Unicode characters for @variation_selector covered by @face and adds
      * them to the #hb_set_t set @out.
    @@ -627,163 +662,3 @@ hb_face_collect_variation_unicodes (hb_face_t *face,
       face->table.cmap->collect_variation_unicodes (variation_selector, out);
     }
     #endif
    -
    -
    -/*
    - * face-builder: A face that has add_table().
    - */
    -
    -struct hb_face_builder_data_t
    -{
    -  hb_hashmap_t tables;
    -};
    -
    -static int compare_entries (const void* pa, const void* pb)
    -{
    -  const auto& a = * (const hb_pair_t *) pa;
    -  const auto& b = * (const hb_pair_t *) pb;
    -
    -  /* Order by blob size first (smallest to largest) and then table tag */
    -
    -  if (a.second->length != b.second->length)
    -    return a.second->length < b.second->length ? -1 : +1;
    -
    -  return a.first < b.first ? -1 : a.first == b.first ? 0 : +1;
    -}
    -
    -static hb_face_builder_data_t *
    -_hb_face_builder_data_create ()
    -{
    -  hb_face_builder_data_t *data = (hb_face_builder_data_t *) hb_calloc (1, sizeof (hb_face_builder_data_t));
    -  if (unlikely (!data))
    -    return nullptr;
    -
    -  data->tables.init ();
    -
    -  return data;
    -}
    -
    -static void
    -_hb_face_builder_data_destroy (void *user_data)
    -{
    -  hb_face_builder_data_t *data = (hb_face_builder_data_t *) user_data;
    -
    -  for (hb_blob_t* b : data->tables.values())
    -    hb_blob_destroy (b);
    -
    -  data->tables.fini ();
    -
    -  hb_free (data);
    -}
    -
    -static hb_blob_t *
    -_hb_face_builder_data_reference_blob (hb_face_builder_data_t *data)
    -{
    -
    -  unsigned int table_count = data->tables.get_population ();
    -  unsigned int face_length = table_count * 16 + 12;
    -
    -  for (hb_blob_t* b : data->tables.values())
    -    face_length += hb_ceil_to_4 (hb_blob_get_length (b));
    -
    -  char *buf = (char *) hb_malloc (face_length);
    -  if (unlikely (!buf))
    -    return nullptr;
    -
    -  hb_serialize_context_t c (buf, face_length);
    -  c.propagate_error (data->tables);
    -  OT::OpenTypeFontFile *f = c.start_serialize ();
    -
    -  bool is_cff = (data->tables.has (HB_TAG ('C','F','F',' '))
    -                 || data->tables.has (HB_TAG ('C','F','F','2')));
    -  hb_tag_t sfnt_tag = is_cff ? OT::OpenTypeFontFile::CFFTag : OT::OpenTypeFontFile::TrueTypeTag;
    -
    -  // Sort the tags so that produced face is deterministic.
    -  hb_vector_t> sorted_entries;
    -  data->tables.iter () | hb_sink (sorted_entries);
    -  if (unlikely (sorted_entries.in_error ()))
    -  {
    -    hb_free (buf);
    -    return nullptr;
    -  }
    -
    -  sorted_entries.qsort (compare_entries);
    -  bool ret = f->serialize_single (&c, sfnt_tag, + sorted_entries.iter());
    -
    -  c.end_serialize ();
    -
    -  if (unlikely (!ret))
    -  {
    -    hb_free (buf);
    -    return nullptr;
    -  }
    -
    -  return hb_blob_create (buf, face_length, HB_MEMORY_MODE_WRITABLE, buf, hb_free);
    -}
    -
    -static hb_blob_t *
    -_hb_face_builder_reference_table (hb_face_t *face HB_UNUSED, hb_tag_t tag, void *user_data)
    -{
    -  hb_face_builder_data_t *data = (hb_face_builder_data_t *) user_data;
    -
    -  if (!tag)
    -    return _hb_face_builder_data_reference_blob (data);
    -
    -  return hb_blob_reference (data->tables[tag]);
    -}
    -
    -
    -/**
    - * hb_face_builder_create:
    - *
    - * Creates a #hb_face_t that can be used with hb_face_builder_add_table().
    - * After tables are added to the face, it can be compiled to a binary
    - * font file by calling hb_face_reference_blob().
    - *
    - * Return value: (transfer full): New face.
    - *
    - * Since: 1.9.0
    - **/
    -hb_face_t *
    -hb_face_builder_create ()
    -{
    -  hb_face_builder_data_t *data = _hb_face_builder_data_create ();
    -  if (unlikely (!data)) return hb_face_get_empty ();
    -
    -  return hb_face_create_for_tables (_hb_face_builder_reference_table,
    -                                    data,
    -                                    _hb_face_builder_data_destroy);
    -}
    -
    -/**
    - * hb_face_builder_add_table:
    - * @face: A face object created with hb_face_builder_create()
    - * @tag: The #hb_tag_t of the table to add
    - * @blob: The blob containing the table data to add
    - *
    - * Add table for @tag with data provided by @blob to the face.  @face must
    - * be created using hb_face_builder_create().
    - *
    - * Since: 1.9.0
    - **/
    -hb_bool_t
    -hb_face_builder_add_table (hb_face_t *face, hb_tag_t tag, hb_blob_t *blob)
    -{
    -  if (tag == HB_MAP_VALUE_INVALID)
    -    return false;
    -
    -  if (unlikely (face->destroy != (hb_destroy_func_t) _hb_face_builder_data_destroy))
    -    return false;
    -
    -  hb_face_builder_data_t *data = (hb_face_builder_data_t *) face->user_data;
    -
    -  hb_blob_t* previous = data->tables.get (tag);
    -  if (!data->tables.set (tag, hb_blob_reference (blob)))
    -  {
    -    hb_blob_destroy (blob);
    -    return false;
    -  }
    -
    -  hb_blob_destroy (previous);
    -  return true;
    -}
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-face.h b/src/java.desktop/share/native/libharfbuzz/hb-face.h
    index e74db2933b9a1..2e5fb1509272b 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-face.h
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-face.h
    @@ -33,6 +33,7 @@
     
     #include "hb-common.h"
     #include "hb-blob.h"
    +#include "hb-map.h"
     #include "hb-set.h"
     
     HB_BEGIN_DECLS
    @@ -96,7 +97,7 @@ hb_face_set_user_data (hb_face_t          *face,
                            hb_bool_t           replace);
     
     HB_EXTERN void *
    -hb_face_get_user_data (hb_face_t          *face,
    +hb_face_get_user_data (const hb_face_t    *face,
                            hb_user_data_key_t *key);
     
     HB_EXTERN void
    @@ -149,6 +150,11 @@ HB_EXTERN void
     hb_face_collect_unicodes (hb_face_t *face,
                               hb_set_t  *out);
     
    +HB_EXTERN void
    +hb_face_collect_nominal_glyph_mapping (hb_face_t *face,
    +                                       hb_map_t  *mapping,
    +                                       hb_set_t  *unicodes);
    +
     HB_EXTERN void
     hb_face_collect_variation_selectors (hb_face_t *face,
                                          hb_set_t  *out);
    @@ -171,6 +177,10 @@ hb_face_builder_add_table (hb_face_t *face,
                                hb_tag_t   tag,
                                hb_blob_t *blob);
     
    +HB_EXTERN void
    +hb_face_builder_sort_tables (hb_face_t *face,
    +                             const hb_tag_t  *tags);
    +
     
     HB_END_DECLS
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-face.hh b/src/java.desktop/share/native/libharfbuzz/hb-face.hh
    index b4a74e9580f8b..4c6b252e88eaf 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-face.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-face.hh
    @@ -65,7 +65,9 @@ struct hb_face_t
         hb_shape_plan_t *shape_plan;
         plan_node_t *next;
       };
    +#ifndef HB_NO_SHAPER
       hb_atomic_ptr_t shape_plans;
    +#endif
     
       hb_blob_t *reference_table (hb_tag_t tag) const
       {
    @@ -74,7 +76,7 @@ struct hb_face_t
         if (unlikely (!reference_table_func))
           return hb_blob_get_empty ();
     
    -    blob = reference_table_func (/*XXX*/const_cast (this), tag, user_data);
    +    blob = reference_table_func (/*Oh, well.*/const_cast (this), tag, user_data);
         if (unlikely (!blob))
           return hb_blob_get_empty ();
     
    @@ -83,7 +85,7 @@ struct hb_face_t
     
       unsigned int get_upem () const
       {
    -    unsigned int ret = upem.get_relaxed ();
    +    unsigned int ret = upem;
         if (unlikely (!ret))
         {
           return load_upem ();
    @@ -93,7 +95,7 @@ struct hb_face_t
     
       unsigned int get_num_glyphs () const
       {
    -    unsigned int ret = num_glyphs.get_relaxed ();
    +    unsigned int ret = num_glyphs;
         if (unlikely (ret == UINT_MAX))
           return load_num_glyphs ();
         return ret;
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-fallback-shape.cc b/src/java.desktop/share/native/libharfbuzz/hb-fallback-shape.cc
    index cd1eaf5e21fd8..e37f90111ebd2 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-fallback-shape.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-fallback-shape.cc
    @@ -75,16 +75,6 @@ _hb_fallback_shape (hb_shape_plan_t    *shape_plan HB_UNUSED,
                         const hb_feature_t *features HB_UNUSED,
                         unsigned int        num_features HB_UNUSED)
     {
    -  /* TODO
    -   *
    -   * - Apply fallback kern.
    -   * - Handle Variation Selectors?
    -   * - Apply normalization?
    -   *
    -   * This will make the fallback shaper into a dumb "TrueType"
    -   * shaper which many people unfortunately still request.
    -   */
    -
       hb_codepoint_t space;
       bool has_space = (bool) font->get_nominal_glyph (' ', &space);
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-font.cc b/src/java.desktop/share/native/libharfbuzz/hb-font.cc
    index b36b53613f801..5cfed3b04903a 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-font.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-font.cc
    @@ -30,6 +30,7 @@
     
     #include "hb-font.hh"
     #include "hb-draw.hh"
    +#include "hb-paint.hh"
     #include "hb-machinery.hh"
     
     #include "hb-ot.h"
    @@ -58,6 +59,11 @@
      *
      * HarfBuzz provides a built-in set of lightweight default
      * functions for each method in #hb_font_funcs_t.
    + *
    + * The default font functions are implemented in terms of the
    + * #hb_font_funcs_t methods of the parent font object.  This allows
    + * client programs to override only the methods they need to, and
    + * otherwise inherit the parent font's implementation, if any.
      **/
     
     
    @@ -71,7 +77,7 @@ hb_font_get_font_h_extents_nil (hb_font_t         *font HB_UNUSED,
                                     hb_font_extents_t *extents,
                                     void              *user_data HB_UNUSED)
     {
    -  memset (extents, 0, sizeof (*extents));
    +  hb_memset (extents, 0, sizeof (*extents));
       return false;
     }
     
    @@ -96,7 +102,7 @@ hb_font_get_font_v_extents_nil (hb_font_t         *font HB_UNUSED,
                                     hb_font_extents_t *extents,
                                     void              *user_data HB_UNUSED)
     {
    -  memset (extents, 0, sizeof (*extents));
    +  hb_memset (extents, 0, sizeof (*extents));
       return false;
     }
     
    @@ -409,7 +415,7 @@ hb_font_get_glyph_extents_nil (hb_font_t          *font HB_UNUSED,
                                    hb_glyph_extents_t *extents,
                                    void               *user_data HB_UNUSED)
     {
    -  memset (extents, 0, sizeof (*extents));
    +  hb_memset (extents, 0, sizeof (*extents));
       return false;
     }
     
    @@ -503,22 +509,34 @@ hb_font_get_glyph_from_name_default (hb_font_t      *font,
     }
     
     static void
    -hb_font_get_glyph_shape_nil (hb_font_t       *font HB_UNUSED,
    -                             void            *font_data HB_UNUSED,
    -                             hb_codepoint_t   glyph,
    -                             hb_draw_funcs_t *draw_funcs,
    -                             void            *draw_data,
    -                             void            *user_data HB_UNUSED)
    +hb_font_draw_glyph_nil (hb_font_t       *font HB_UNUSED,
    +                        void            *font_data HB_UNUSED,
    +                        hb_codepoint_t   glyph,
    +                        hb_draw_funcs_t *draw_funcs,
    +                        void            *draw_data,
    +                        void            *user_data HB_UNUSED)
     {
     }
     
    +static void
    +hb_font_paint_glyph_nil (hb_font_t *font HB_UNUSED,
    +                         void *font_data HB_UNUSED,
    +                         hb_codepoint_t glyph HB_UNUSED,
    +                         hb_paint_funcs_t *paint_funcs HB_UNUSED,
    +                         void *paint_data HB_UNUSED,
    +                         unsigned int palette HB_UNUSED,
    +                         hb_color_t foreground HB_UNUSED,
    +                         void *user_data HB_UNUSED)
    +{
    +}
     
    -typedef struct hb_font_get_glyph_shape_default_adaptor_t {
    +typedef struct hb_font_draw_glyph_default_adaptor_t {
       hb_draw_funcs_t *draw_funcs;
       void            *draw_data;
       float            x_scale;
       float            y_scale;
    -} hb_font_get_glyph_shape_default_adaptor_t;
    +  float            slant;
    +} hb_font_draw_glyph_default_adaptor_t;
     
     static void
     hb_draw_move_to_default (hb_draw_funcs_t *dfuncs HB_UNUSED,
    @@ -527,12 +545,13 @@ hb_draw_move_to_default (hb_draw_funcs_t *dfuncs HB_UNUSED,
                              float to_x, float to_y,
                              void *user_data HB_UNUSED)
     {
    -  hb_font_get_glyph_shape_default_adaptor_t *adaptor = (hb_font_get_glyph_shape_default_adaptor_t *) draw_data;
    +  hb_font_draw_glyph_default_adaptor_t *adaptor = (hb_font_draw_glyph_default_adaptor_t *) draw_data;
       float x_scale = adaptor->x_scale;
       float y_scale = adaptor->y_scale;
    +  float slant   = adaptor->slant;
     
       adaptor->draw_funcs->emit_move_to (adaptor->draw_data, *st,
    -                                     x_scale * to_x, y_scale * to_y);
    +                                     x_scale * to_x + slant * to_y, y_scale * to_y);
     }
     
     static void
    @@ -541,15 +560,16 @@ hb_draw_line_to_default (hb_draw_funcs_t *dfuncs HB_UNUSED, void *draw_data,
                              float to_x, float to_y,
                              void *user_data HB_UNUSED)
     {
    -  hb_font_get_glyph_shape_default_adaptor_t *adaptor = (hb_font_get_glyph_shape_default_adaptor_t *) draw_data;
    +  hb_font_draw_glyph_default_adaptor_t *adaptor = (hb_font_draw_glyph_default_adaptor_t *) draw_data;
       float x_scale = adaptor->x_scale;
       float y_scale = adaptor->y_scale;
    +  float slant   = adaptor->slant;
     
    -  st->current_x *= x_scale;
    -  st->current_y *= y_scale;
    +  st->current_x = st->current_x * x_scale + st->current_y * slant;
    +  st->current_y = st->current_y * y_scale;
     
       adaptor->draw_funcs->emit_line_to (adaptor->draw_data, *st,
    -                                     x_scale * to_x, y_scale * to_y);
    +                                     x_scale * to_x + slant * to_y, y_scale * to_y);
     }
     
     static void
    @@ -559,16 +579,17 @@ hb_draw_quadratic_to_default (hb_draw_funcs_t *dfuncs HB_UNUSED, void *draw_data
                                   float to_x, float to_y,
                                   void *user_data HB_UNUSED)
     {
    -  hb_font_get_glyph_shape_default_adaptor_t *adaptor = (hb_font_get_glyph_shape_default_adaptor_t *) draw_data;
    +  hb_font_draw_glyph_default_adaptor_t *adaptor = (hb_font_draw_glyph_default_adaptor_t *) draw_data;
       float x_scale = adaptor->x_scale;
       float y_scale = adaptor->y_scale;
    +  float slant   = adaptor->slant;
     
    -  st->current_x *= x_scale;
    -  st->current_y *= y_scale;
    +  st->current_x = st->current_x * x_scale + st->current_y * slant;
    +  st->current_y = st->current_y * y_scale;
     
       adaptor->draw_funcs->emit_quadratic_to (adaptor->draw_data, *st,
    -                                          x_scale * control_x, y_scale * control_y,
    -                                          x_scale * to_x, y_scale * to_y);
    +                                          x_scale * control_x + slant * control_y, y_scale * control_y,
    +                                          x_scale * to_x + slant * to_y, y_scale * to_y);
     }
     
     static void
    @@ -579,17 +600,18 @@ hb_draw_cubic_to_default (hb_draw_funcs_t *dfuncs HB_UNUSED, void *draw_data,
                               float to_x, float to_y,
                               void *user_data HB_UNUSED)
     {
    -  hb_font_get_glyph_shape_default_adaptor_t *adaptor = (hb_font_get_glyph_shape_default_adaptor_t *) draw_data;
    +  hb_font_draw_glyph_default_adaptor_t *adaptor = (hb_font_draw_glyph_default_adaptor_t *) draw_data;
       float x_scale = adaptor->x_scale;
       float y_scale = adaptor->y_scale;
    +  float slant   = adaptor->slant;
     
    -  st->current_x *= x_scale;
    -  st->current_y *= y_scale;
    +  st->current_x = st->current_x * x_scale + st->current_y * slant;
    +  st->current_y = st->current_y * y_scale;
     
       adaptor->draw_funcs->emit_cubic_to (adaptor->draw_data, *st,
    -                                      x_scale * control1_x, y_scale * control1_y,
    -                                      x_scale * control2_x, y_scale * control2_y,
    -                                      x_scale * to_x, y_scale * to_y);
    +                                      x_scale * control1_x + slant * control1_y, y_scale * control1_y,
    +                                      x_scale * control2_x + slant * control2_y, y_scale * control2_y,
    +                                      x_scale * to_x + slant * to_y, y_scale * to_y);
     }
     
     static void
    @@ -597,7 +619,7 @@ hb_draw_close_path_default (hb_draw_funcs_t *dfuncs HB_UNUSED, void *draw_data,
                                 hb_draw_state_t *st,
                                 void *user_data HB_UNUSED)
     {
    -  hb_font_get_glyph_shape_default_adaptor_t *adaptor = (hb_font_get_glyph_shape_default_adaptor_t *) draw_data;
    +  hb_font_draw_glyph_default_adaptor_t *adaptor = (hb_font_draw_glyph_default_adaptor_t *) draw_data;
     
       adaptor->draw_funcs->emit_close_path (adaptor->draw_data, *st);
     }
    @@ -613,25 +635,50 @@ static const hb_draw_funcs_t _hb_draw_funcs_default = {
     };
     
     static void
    -hb_font_get_glyph_shape_default (hb_font_t       *font,
    +hb_font_draw_glyph_default (hb_font_t       *font,
                                      void            *font_data HB_UNUSED,
                                      hb_codepoint_t   glyph,
                                      hb_draw_funcs_t *draw_funcs,
                                      void            *draw_data,
                                      void            *user_data HB_UNUSED)
     {
    -  hb_font_get_glyph_shape_default_adaptor_t adaptor = {
    +  hb_font_draw_glyph_default_adaptor_t adaptor = {
         draw_funcs,
         draw_data,
    -    (float) font->x_scale / (float) font->parent->x_scale,
    -    (float) font->y_scale / (float) font->parent->y_scale
    +    font->parent->x_scale ? (float) font->x_scale / (float) font->parent->x_scale : 0.f,
    +    font->parent->y_scale ? (float) font->y_scale / (float) font->parent->y_scale : 0.f,
    +    font->parent->y_scale ? (font->slant - font->parent->slant) *
    +                            (float) font->x_scale / (float) font->parent->y_scale : 0.f
       };
     
    -  font->parent->get_glyph_shape (glyph,
    +  font->parent->draw_glyph (glyph,
                                      const_cast (&_hb_draw_funcs_default),
                                      &adaptor);
     }
     
    +static void
    +hb_font_paint_glyph_default (hb_font_t *font,
    +                             void *font_data,
    +                             hb_codepoint_t glyph,
    +                             hb_paint_funcs_t *paint_funcs,
    +                             void *paint_data,
    +                             unsigned int palette,
    +                             hb_color_t foreground,
    +                             void *user_data)
    +{
    +  paint_funcs->push_transform (paint_data,
    +    font->parent->x_scale ? (float) font->x_scale / (float) font->parent->x_scale : 0.f,
    +    font->parent->y_scale ? (font->slant - font->parent->slant) *
    +                            (float) font->x_scale / (float) font->parent->y_scale : 0.f,
    +    0.f,
    +    font->parent->y_scale ? (float) font->y_scale / (float) font->parent->y_scale : 0.f,
    +    0.f, 0.f);
    +
    +  font->parent->paint_glyph (glyph, paint_funcs, paint_data, palette, foreground);
    +
    +  paint_funcs->pop_transform (paint_data);
    +}
    +
     DEFINE_NULL_INSTANCE (hb_font_funcs_t) =
     {
       HB_OBJECT_HEADER_STATIC,
    @@ -640,7 +687,7 @@ DEFINE_NULL_INSTANCE (hb_font_funcs_t) =
       nullptr,
       {
         {
    -#define HB_FONT_FUNC_IMPLEMENT(name) hb_font_get_##name##_nil,
    +#define HB_FONT_FUNC_IMPLEMENT(get_,name) hb_font_##get_##name##_nil,
           HB_FONT_FUNCS_IMPLEMENT_CALLBACKS
     #undef HB_FONT_FUNC_IMPLEMENT
         }
    @@ -654,7 +701,7 @@ static const hb_font_funcs_t _hb_font_funcs_default = {
       nullptr,
       {
         {
    -#define HB_FONT_FUNC_IMPLEMENT(name) hb_font_get_##name##_default,
    +#define HB_FONT_FUNC_IMPLEMENT(get_,name) hb_font_##get_##name##_default,
           HB_FONT_FUNCS_IMPLEMENT_CALLBACKS
     #undef HB_FONT_FUNC_IMPLEMENT
         }
    @@ -732,7 +779,7 @@ hb_font_funcs_destroy (hb_font_funcs_t *ffuncs)
     
       if (ffuncs->destroy)
       {
    -#define HB_FONT_FUNC_IMPLEMENT(name) if (ffuncs->destroy->name) \
    +#define HB_FONT_FUNC_IMPLEMENT(get_,name) if (ffuncs->destroy->name) \
         ffuncs->destroy->name (!ffuncs->user_data ? nullptr : ffuncs->user_data->name);
         HB_FONT_FUNCS_IMPLEMENT_CALLBACKS
     #undef HB_FONT_FUNC_IMPLEMENT
    @@ -754,7 +801,7 @@ hb_font_funcs_destroy (hb_font_funcs_t *ffuncs)
      *
      * Attaches a user-data key/data pair to the specified font-functions structure.
      *
    - * Return value: %true if success, %false otherwise
    + * Return value: `true` if success, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -781,8 +828,8 @@ hb_font_funcs_set_user_data (hb_font_funcs_t    *ffuncs,
      * Since: 0.9.2
      **/
     void *
    -hb_font_funcs_get_user_data (hb_font_funcs_t    *ffuncs,
    -                             hb_user_data_key_t *key)
    +hb_font_funcs_get_user_data (const hb_font_funcs_t *ffuncs,
    +                             hb_user_data_key_t    *key)
     {
       return hb_object_get_user_data (ffuncs, key);
     }
    @@ -811,7 +858,7 @@ hb_font_funcs_make_immutable (hb_font_funcs_t *ffuncs)
      *
      * Tests whether a font-functions structure is immutable.
      *
    - * Return value: %true if @ffuncs is immutable, %false otherwise
    + * Return value: `true` if @ffuncs is immutable, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -822,59 +869,82 @@ hb_font_funcs_is_immutable (hb_font_funcs_t *ffuncs)
     }
     
     
    -#define HB_FONT_FUNC_IMPLEMENT(name) \
    +static bool
    +_hb_font_funcs_set_preamble (hb_font_funcs_t    *ffuncs,
    +                             bool                func_is_null,
    +                             void              **user_data,
    +                             hb_destroy_func_t  *destroy)
    +{
    +  if (hb_object_is_immutable (ffuncs))
    +  {
    +    if (*destroy)
    +      (*destroy) (*user_data);
    +    return false;
    +  }
    +
    +  if (func_is_null)
    +  {
    +    if (*destroy)
    +      (*destroy) (*user_data);
    +    *destroy = nullptr;
    +    *user_data = nullptr;
    +  }
    +
    +  return true;
    +}
    +
    +static bool
    +_hb_font_funcs_set_middle (hb_font_funcs_t   *ffuncs,
    +                           void              *user_data,
    +                           hb_destroy_func_t  destroy)
    +{
    +  if (user_data && !ffuncs->user_data)
    +  {
    +    ffuncs->user_data = (decltype (ffuncs->user_data)) hb_calloc (1, sizeof (*ffuncs->user_data));
    +    if (unlikely (!ffuncs->user_data))
    +      goto fail;
    +  }
    +  if (destroy && !ffuncs->destroy)
    +  {
    +    ffuncs->destroy = (decltype (ffuncs->destroy)) hb_calloc (1, sizeof (*ffuncs->destroy));
    +    if (unlikely (!ffuncs->destroy))
    +      goto fail;
    +  }
    +
    +  return true;
    +
    +fail:
    +  if (destroy)
    +    (destroy) (user_data);
    +  return false;
    +}
    +
    +#define HB_FONT_FUNC_IMPLEMENT(get_,name) \
                                                                              \
     void                                                                     \
     hb_font_funcs_set_##name##_func (hb_font_funcs_t             *ffuncs,    \
    -                                 hb_font_get_##name##_func_t  func,      \
    +                                 hb_font_##get_##name##_func_t func,     \
                                      void                        *user_data, \
                                      hb_destroy_func_t            destroy)   \
     {                                                                        \
    -  if (hb_object_is_immutable (ffuncs))                                   \
    -    goto fail;                                                           \
    -                                                                         \
    -  if (!func)                                                             \
    -  {                                                                      \
    -    if (destroy)                                                         \
    -      destroy (user_data);                                               \
    -    destroy = nullptr;                                                   \
    -    user_data = nullptr;                                                 \
    -  }                                                                      \
    +  if (!_hb_font_funcs_set_preamble (ffuncs, !func, &user_data, &destroy))\
    +      return;                                                            \
                                                                              \
       if (ffuncs->destroy && ffuncs->destroy->name)                          \
         ffuncs->destroy->name (!ffuncs->user_data ? nullptr : ffuncs->user_data->name); \
                                                                              \
    -  if (user_data && !ffuncs->user_data)                                   \
    -  {                                                                      \
    -    ffuncs->user_data = (decltype (ffuncs->user_data)) hb_calloc (1, sizeof (*ffuncs->user_data)); \
    -    if (unlikely (!ffuncs->user_data))                                   \
    -      goto fail;                                                         \
    -  }                                                                      \
    -  if (destroy && !ffuncs->destroy)                                       \
    -  {                                                                      \
    -    ffuncs->destroy = (decltype (ffuncs->destroy)) hb_calloc (1, sizeof (*ffuncs->destroy)); \
    -    if (unlikely (!ffuncs->destroy))                                     \
    -      goto fail;                                                         \
    -  }                                                                      \
    +  if (!_hb_font_funcs_set_middle (ffuncs, user_data, destroy))           \
    +      return;                                                            \
                                                                              \
    -  if (func) {                                                            \
    +  if (func)                                                              \
         ffuncs->get.f.name = func;                                           \
    -    if (ffuncs->user_data)                                               \
    -      ffuncs->user_data->name = user_data;                               \
    -    if (ffuncs->destroy)                                                 \
    -      ffuncs->destroy->name = destroy;                                   \
    -  } else {                                                               \
    -    ffuncs->get.f.name = hb_font_get_##name##_default;                   \
    -    if (ffuncs->user_data)                                               \
    -      ffuncs->user_data->name = nullptr;                                 \
    -    if (ffuncs->destroy)                                                 \
    -      ffuncs->destroy->name = nullptr;                                   \
    -  }                                                                      \
    -  return;                                                                \
    +  else                                                                   \
    +    ffuncs->get.f.name = hb_font_##get_##name##_default;                   \
                                                                              \
    -fail:                                                                    \
    -  if (destroy)                                                           \
    -    destroy (user_data);                                                 \
    +  if (ffuncs->user_data)                                                 \
    +    ffuncs->user_data->name = user_data;                                 \
    +  if (ffuncs->destroy)                                                   \
    +    ffuncs->destroy->name = destroy;                                     \
     }
     
     HB_FONT_FUNCS_IMPLEMENT_CALLBACKS
    @@ -903,7 +973,7 @@ hb_font_t::has_func (unsigned int i)
      * Fetches the extents for a specified font, for horizontal
      * text segments.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 1.1.3
      **/
    @@ -922,7 +992,7 @@ hb_font_get_h_extents (hb_font_t         *font,
      * Fetches the extents for a specified font, for vertical
      * text segments.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 1.1.3
      **/
    @@ -946,7 +1016,7 @@ hb_font_get_v_extents (hb_font_t         *font,
      * If @variation_selector is 0, calls hb_font_get_nominal_glyph();
      * otherwise calls hb_font_get_variation_glyph().
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -974,7 +1044,7 @@ hb_font_get_glyph (hb_font_t      *font,
      * for code points modified by variation selectors. For variation-selector
      * support, user hb_font_get_variation_glyph() or use hb_font_get_glyph().
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 1.2.3
      **/
    @@ -996,7 +1066,8 @@ hb_font_get_nominal_glyph (hb_font_t      *font,
      * @glyph_stride: The stride between successive glyph IDs
      *
      * Fetches the nominal glyph IDs for a sequence of Unicode code points. Glyph
    - * IDs must be returned in a #hb_codepoint_t output parameter.
    + * IDs must be returned in a #hb_codepoint_t output parameter. Stopes at the
    + * first unsupported glyph ID.
      *
      * Return value: the number of code points processed
      *
    @@ -1026,7 +1097,7 @@ hb_font_get_nominal_glyphs (hb_font_t *font,
      * by the specified variation-selector code point, in the specified
      * font.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 1.2.3
      **/
    @@ -1136,7 +1207,7 @@ hb_font_get_glyph_v_advances (hb_font_t*            font,
      * Fetches the (X,Y) coordinates of the origin for a glyph ID
      * in the specified font, for horizontal text segments.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -1159,7 +1230,7 @@ hb_font_get_glyph_h_origin (hb_font_t      *font,
      * Fetches the (X,Y) coordinates of the origin for a glyph ID
      * in the specified font, for vertical text segments.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -1232,7 +1303,7 @@ hb_font_get_glyph_v_kerning (hb_font_t      *font,
      * Fetches the #hb_glyph_extents_t data for a glyph ID
      * in the specified font.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -1255,7 +1326,7 @@ hb_font_get_glyph_extents (hb_font_t          *font,
      * Fetches the (x,y) coordinates of a specified contour-point index
      * in the specified glyph, within the specified font.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -1278,7 +1349,10 @@ hb_font_get_glyph_contour_point (hb_font_t      *font,
      *
      * Fetches the glyph-name string for a glyph ID in the specified @font.
      *
    - * Return value: %true if data found, %false otherwise
    + * According to the OpenType specification, glyph names are limited to 63
    + * characters and can only contain (a subset of) ASCII.
    + *
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -1302,7 +1376,7 @@ hb_font_get_glyph_name (hb_font_t      *font,
      *
      * Note: @len == -1 means the name string is null-terminated.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -1318,22 +1392,76 @@ hb_font_get_glyph_from_name (hb_font_t      *font,
     /**
      * hb_font_get_glyph_shape:
      * @font: #hb_font_t to work upon
    - * @glyph: : The glyph ID
    + * @glyph: The glyph ID
      * @dfuncs: #hb_draw_funcs_t to draw to
      * @draw_data: User data to pass to draw callbacks
      *
      * Fetches the glyph shape that corresponds to a glyph in the specified @font.
    - * The shape is returned by way of calls to the callsbacks of the @dfuncs
    + * The shape is returned by way of calls to the callbacks of the @dfuncs
      * objects, with @draw_data passed to them.
      *
      * Since: 4.0.0
    - **/
    + * Deprecated: 7.0.0: Use hb_font_draw_glyph() instead
    + */
     void
     hb_font_get_glyph_shape (hb_font_t *font,
                              hb_codepoint_t glyph,
                              hb_draw_funcs_t *dfuncs, void *draw_data)
     {
    -  font->get_glyph_shape (glyph, dfuncs, draw_data);
    +  hb_font_draw_glyph (font, glyph, dfuncs, draw_data);
    +}
    +
    +/**
    + * hb_font_draw_glyph:
    + * @font: #hb_font_t to work upon
    + * @glyph: The glyph ID
    + * @dfuncs: #hb_draw_funcs_t to draw to
    + * @draw_data: User data to pass to draw callbacks
    + *
    + * Draws the outline that corresponds to a glyph in the specified @font.
    + *
    + * The outline is returned by way of calls to the callbacks of the @dfuncs
    + * objects, with @draw_data passed to them.
    + *
    + * Since: 7.0.0
    + **/
    +void
    +hb_font_draw_glyph (hb_font_t *font,
    +                         hb_codepoint_t glyph,
    +                         hb_draw_funcs_t *dfuncs, void *draw_data)
    +{
    +  font->draw_glyph (glyph, dfuncs, draw_data);
    +}
    +
    +/**
    + * hb_font_paint_glyph:
    + * @font: #hb_font_t to work upon
    + * @glyph: The glyph ID
    + * @pfuncs: #hb_paint_funcs_t to paint with
    + * @paint_data: User data to pass to paint callbacks
    + * @palette_index: The index of the font's color palette to use
    + * @foreground: The foreground color, unpremultipled
    + *
    + * Paints the glyph.
    + *
    + * The painting instructions are returned by way of calls to
    + * the callbacks of the @funcs object, with @paint_data passed
    + * to them.
    + *
    + * If the font has color palettes (see hb_ot_color_has_palettes()),
    + * then @palette_index selects the palette to use. If the font only
    + * has one palette, this will be 0.
    + *
    + * Since: 7.0.0
    + */
    +void
    +hb_font_paint_glyph (hb_font_t *font,
    +                     hb_codepoint_t glyph,
    +                     hb_paint_funcs_t *pfuncs, void *paint_data,
    +                     unsigned int palette_index,
    +                     hb_color_t foreground)
    +{
    +  font->paint_glyph (glyph, pfuncs, paint_data, palette_index, foreground);
     }
     
     /* A bit higher-level, and with fallback */
    @@ -1537,7 +1665,7 @@ hb_font_get_glyph_kerning_for_direction (hb_font_t      *font,
      * Calls the appropriate direction-specific variant (horizontal
      * or vertical) depending on the value of @direction.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -1566,7 +1694,7 @@ hb_font_get_glyph_extents_for_origin (hb_font_t          *font,
      * Calls the appropriate direction-specific variant (horizontal
      * or vertical) depending on the value of @direction.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -1594,6 +1722,9 @@ hb_font_get_glyph_contour_point_for_origin (hb_font_t      *font,
      * If the glyph ID has no name in @font, a string of the form `gidDDD` is
      * generated, with `DDD` being the glyph ID.
      *
    + * According to the OpenType specification, glyph names are limited to 63
    + * characters and can only contain (a subset of) ASCII.
    + *
      * Since: 0.9.2
      **/
     void
    @@ -1617,7 +1748,7 @@ hb_font_glyph_to_string (hb_font_t      *font,
      *
      * Note: @len == -1 means the string is null-terminated.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -1647,8 +1778,13 @@ DEFINE_NULL_INSTANCE (hb_font_t) =
     
       1000, /* x_scale */
       1000, /* y_scale */
    -  0., /* slant */
    -  0., /* slant_xy; */
    +  0.f, /* x_embolden */
    +  0.f, /* y_embolden */
    +  true, /* embolden_in_place */
    +  0, /* x_strength */
    +  0, /* y_strength */
    +  0.f, /* slant */
    +  0.f, /* slant_xy; */
       1.f, /* x_multf */
       1.f, /* y_multf */
       1<<16, /* x_mult */
    @@ -1658,6 +1794,7 @@ DEFINE_NULL_INSTANCE (hb_font_t) =
       0, /* y_ppem */
       0, /* ptem */
     
    +  HB_FONT_NO_VAR_NAMED_INSTANCE, /* instance_index */
       0, /* num_coords */
       nullptr, /* coords */
       nullptr, /* design_coords */
    @@ -1675,6 +1812,7 @@ _hb_font_create (hb_face_t *face)
     
       if (unlikely (!face))
         face = hb_face_get_empty ();
    +
       if (!(font = hb_object_create ()))
         return hb_font_get_empty ();
     
    @@ -1684,8 +1822,10 @@ _hb_font_create (hb_face_t *face)
       font->klass = hb_font_funcs_get_empty ();
       font->data.init0 (font);
       font->x_scale = font->y_scale = face->get_upem ();
    +  font->embolden_in_place = true;
       font->x_multf = font->y_multf = 1.f;
       font->x_mult = font->y_mult = 1 << 16;
    +  font->instance_index = HB_FONT_NO_VAR_NAMED_INSTANCE;
     
       return font;
     }
    @@ -1767,6 +1907,9 @@ hb_font_create_sub_font (hb_font_t *parent)
     
       font->x_scale = parent->x_scale;
       font->y_scale = parent->y_scale;
    +  font->x_embolden = parent->x_embolden;
    +  font->y_embolden = parent->y_embolden;
    +  font->embolden_in_place = parent->embolden_in_place;
       font->slant = parent->slant;
       font->x_ppem = parent->x_ppem;
       font->y_ppem = parent->y_ppem;
    @@ -1779,8 +1922,8 @@ hb_font_create_sub_font (hb_font_t *parent)
         float *design_coords = (float *) hb_calloc (num_coords, sizeof (parent->design_coords[0]));
         if (likely (coords && design_coords))
         {
    -      memcpy (coords, parent->coords, num_coords * sizeof (parent->coords[0]));
    -      memcpy (design_coords, parent->design_coords, num_coords * sizeof (parent->design_coords[0]));
    +      hb_memcpy (coords, parent->coords, num_coords * sizeof (parent->coords[0]));
    +      hb_memcpy (design_coords, parent->design_coords, num_coords * sizeof (parent->design_coords[0]));
           _hb_font_adopt_var_coords (font, coords, design_coords, num_coords);
         }
         else
    @@ -1866,7 +2009,7 @@ hb_font_destroy (hb_font_t *font)
      *
      * Attaches a user-data key/data pair to the specified font object.
      *
    - * Return value: %true if success, %false otherwise
    + * Return value: `true` if success, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -1896,7 +2039,7 @@ hb_font_set_user_data (hb_font_t          *font,
      * Since: 0.9.2
      **/
     void *
    -hb_font_get_user_data (hb_font_t          *font,
    +hb_font_get_user_data (const hb_font_t    *font,
                            hb_user_data_key_t *key)
     {
       return hb_object_get_user_data (font, key);
    @@ -1928,7 +2071,7 @@ hb_font_make_immutable (hb_font_t *font)
      *
      * Tests whether a font object is immutable.
      *
    - * Return value: %true if @font is immutable, %false otherwise
    + * Return value: `true` if @font is immutable, `false` otherwise
      *
      * Since: 0.9.2
      **/
    @@ -1948,7 +2091,7 @@ hb_font_is_immutable (hb_font_t *font)
      *
      * Return value: serial number
      *
    - * Since: 4.4.0.
    + * Since: 4.4.0
      **/
     unsigned int
     hb_font_get_serial (hb_font_t *font)
    @@ -1964,7 +2107,7 @@ hb_font_get_serial (hb_font_t *font)
      * This has the effect of increasing the serial as returned
      * by hb_font_get_serial(), which invalidates internal caches.
      *
    - * Since: 4.4.0.
    + * Since: 4.4.0
      **/
     void
     hb_font_changed (hb_font_t *font)
    @@ -2156,6 +2299,31 @@ hb_font_set_funcs_data (hb_font_t         *font,
      *
      * Sets the horizontal and vertical scale of a font.
      *
    + * The font scale is a number related to, but not the same as,
    + * font size. Typically the client establishes a scale factor
    + * to be used between the two. For example, 64, or 256, which
    + * would be the fractional-precision part of the font scale.
    + * This is necessary because #hb_position_t values are integer
    + * types and you need to leave room for fractional values
    + * in there.
    + *
    + * For example, to set the font size to 20, with 64
    + * levels of fractional precision you would call
    + * `hb_font_set_scale(font, 20 * 64, 20 * 64)`.
    + *
    + * In the example above, even what font size 20 means is up to
    + * you. It might be 20 pixels, or 20 points, or 20 millimeters.
    + * HarfBuzz does not care about that.  You can set the point
    + * size of the font using hb_font_set_ptem(), and the pixel
    + * size using hb_font_set_ppem().
    + *
    + * The choice of scale is yours but needs to be consistent between
    + * what you set here, and what you expect out of #hb_position_t
    + * as well has draw / paint API output values.
    + *
    + * Fonts default to a scale equal to the UPEM value of their face.
    + * A font with this setting is sometimes called an "unscaled" font.
    + *
      * Since: 0.9.2
      **/
     void
    @@ -2201,7 +2369,11 @@ hb_font_get_scale (hb_font_t *font,
      * @x_ppem: Horizontal ppem value to assign
      * @y_ppem: Vertical ppem value to assign
      *
    - * Sets the horizontal and vertical pixels-per-em (ppem) of a font.
    + * Sets the horizontal and vertical pixels-per-em (PPEM) of a font.
    + *
    + * These values are used for pixel-size-specific adjustment to
    + * shaping and draw results, though for the most part they are
    + * unused and can be left unset.
      *
      * Since: 0.9.2
      **/
    @@ -2285,6 +2457,76 @@ hb_font_get_ptem (hb_font_t *font)
       return font->ptem;
     }
     
    +/**
    + * hb_font_set_synthetic_bold:
    + * @font: #hb_font_t to work upon
    + * @x_embolden: the amount to embolden horizontally
    + * @y_embolden: the amount to embolden vertically
    + * @in_place: whether to embolden glyphs in-place
    + *
    + * Sets the "synthetic boldness" of a font.
    + *
    + * Positive values for @x_embolden / @y_embolden make a font
    + * bolder, negative values thinner. Typical values are in the
    + * 0.01 to 0.05 range. The default value is zero.
    + *
    + * Synthetic boldness is applied by offsetting the contour
    + * points of the glyph shape.
    + *
    + * Synthetic boldness is applied when rendering a glyph via
    + * hb_font_draw_glyph().
    + *
    + * If @in_place is `false`, then glyph advance-widths are also
    + * adjusted, otherwise they are not.  The in-place mode is
    + * useful for simulating [font grading](https://fonts.google.com/knowledge/glossary/grade).
    + *
    + *
    + * Since: 7.0.0
    + **/
    +void
    +hb_font_set_synthetic_bold (hb_font_t *font,
    +                            float x_embolden,
    +                            float y_embolden,
    +                            hb_bool_t in_place)
    +{
    +  if (hb_object_is_immutable (font))
    +    return;
    +
    +  if (font->x_embolden == x_embolden &&
    +      font->y_embolden == y_embolden &&
    +      font->embolden_in_place == (bool) in_place)
    +    return;
    +
    +  font->serial++;
    +
    +  font->x_embolden = x_embolden;
    +  font->y_embolden = y_embolden;
    +  font->embolden_in_place = in_place;
    +  font->mults_changed ();
    +}
    +
    +/**
    + * hb_font_get_synthetic_bold:
    + * @font: #hb_font_t to work upon
    + * @x_embolden: (out): return location for horizontal value
    + * @y_embolden: (out): return location for vertical value
    + * @in_place: (out): return location for in-place value
    + *
    + * Fetches the "synthetic boldness" parameters of a font.
    + *
    + * Since: 7.0.0
    + **/
    +void
    +hb_font_get_synthetic_bold (hb_font_t *font,
    +                            float *x_embolden,
    +                            float *y_embolden,
    +                            hb_bool_t *in_place)
    +{
    +  if (x_embolden) *x_embolden = font->x_embolden;
    +  if (y_embolden) *y_embolden = font->y_embolden;
    +  if (in_place) *in_place = font->embolden_in_place;
    +}
    +
     /**
      * hb_font_set_synthetic_slant:
      * @font: #hb_font_t to work upon
    @@ -2297,9 +2539,8 @@ hb_font_get_ptem (hb_font_t *font)
      * HarfBuzz needs to know this value to adjust shaping results,
      * metrics, and style values to match the slanted rendering.
      *
    - * Note: The glyph shape fetched via the
    - * hb_font_get_glyph_shape() is slanted to reflect this value
    - * as well.
    + * Note: The glyph shape fetched via the hb_font_draw_glyph()
    + * function is slanted to reflect this value as well.
      *
      * Note: The slant value is a ratio.  For example, a
      * 20% slant would be represented as a 0.2 value.
    @@ -2366,7 +2607,7 @@ hb_font_set_variations (hb_font_t            *font,
     
       font->serial_coords = ++font->serial;
     
    -  if (!variations_length)
    +  if (!variations_length && font->instance_index == HB_FONT_NO_VAR_NAMED_INSTANCE)
       {
         hb_font_set_var_coords_normalized (font, nullptr, 0);
         return;
    @@ -2386,20 +2627,104 @@ hb_font_set_variations (hb_font_t            *font,
         return;
       }
     
    +  /* Initialize design coords. */
    +  for (unsigned int i = 0; i < coords_length; i++)
    +    design_coords[i] = axes[i].get_default ();
    +  if (font->instance_index != HB_FONT_NO_VAR_NAMED_INSTANCE)
    +  {
    +    unsigned count = coords_length;
    +    /* This may fail if index is out-of-range;
    +     * That's why we initialize design_coords from fvar above
    +     * unconditionally. */
    +    hb_ot_var_named_instance_get_design_coords (font->face, font->instance_index,
    +                                                &count, design_coords);
    +  }
    +
       for (unsigned int i = 0; i < variations_length; i++)
       {
         const auto tag = variations[i].tag;
         const auto v = variations[i].value;
         for (unsigned axis_index = 0; axis_index < coords_length; axis_index++)
           if (axes[axis_index].axisTag == tag)
    -      {
             design_coords[axis_index] = v;
    -        normalized[axis_index] = fvar.normalize_axis_value (axis_index, v);
    -      }
       }
       font->face->table.avar->map_coords (normalized, coords_length);
     
    +  hb_ot_var_normalize_coords (font->face, coords_length, design_coords, normalized);
    +  _hb_font_adopt_var_coords (font, normalized, design_coords, coords_length);
    +}
    +
    +/**
    + * hb_font_set_variation:
    + * @font: #hb_font_t to work upon
    + * @tag: The #hb_tag_t tag of the variation-axis name
    + * @value: The value of the variation axis
    + *
    + * Change the value of one variation axis on the font.
    + *
    + * Note: This function is expensive to be called repeatedly.
    + *   If you want to set multiple variation axes at the same time,
    + *   use hb_font_set_variations() instead.
    + *
    + * Since: 7.1.0
    + */
    +void
    +hb_font_set_variation (hb_font_t *font,
    +                       hb_tag_t tag,
    +                       float    value)
    +{
    +  if (hb_object_is_immutable (font))
    +    return;
    +
    +  font->serial_coords = ++font->serial;
    +
    +  // TODO Share some of this code with set_variations()
    +
    +  const OT::fvar &fvar = *font->face->table.fvar;
    +  auto axes = fvar.get_axes ();
    +  const unsigned coords_length = axes.length;
    +
    +  int *normalized = coords_length ? (int *) hb_calloc (coords_length, sizeof (int)) : nullptr;
    +  float *design_coords = coords_length ? (float *) hb_calloc (coords_length, sizeof (float)) : nullptr;
    +
    +  if (unlikely (coords_length && !(normalized && design_coords)))
    +  {
    +    hb_free (normalized);
    +    hb_free (design_coords);
    +    return;
    +  }
    +
    +  /* Initialize design coords. */
    +  if (font->design_coords)
    +  {
    +    assert (coords_length == font->num_coords);
    +    for (unsigned int i = 0; i < coords_length; i++)
    +      design_coords[i] = font->design_coords[i];
    +  }
    +  else
    +  {
    +    for (unsigned int i = 0; i < coords_length; i++)
    +      design_coords[i] = axes[i].get_default ();
    +    if (font->instance_index != HB_FONT_NO_VAR_NAMED_INSTANCE)
    +    {
    +      unsigned count = coords_length;
    +      /* This may fail if index is out-of-range;
    +       * That's why we initialize design_coords from fvar above
    +       * unconditionally. */
    +      hb_ot_var_named_instance_get_design_coords (font->face, font->instance_index,
    +                                                  &count, design_coords);
    +    }
    +  }
    +
    +  for (unsigned axis_index = 0; axis_index < coords_length; axis_index++)
    +    if (axes[axis_index].axisTag == tag)
    +      design_coords[axis_index] = value;
    +
    +  font->face->table.avar->map_coords (normalized, coords_length);
    +
    +  hb_ot_var_normalize_coords (font->face, coords_length, design_coords, normalized);
       _hb_font_adopt_var_coords (font, normalized, design_coords, coords_length);
    +
     }
     
     /**
    @@ -2438,7 +2763,7 @@ hb_font_set_var_coords_design (hb_font_t    *font,
       }
     
       if (coords_length)
    -    memcpy (design_coords, coords, coords_length * sizeof (font->design_coords[0]));
    +    hb_memcpy (design_coords, coords, coords_length * sizeof (font->design_coords[0]));
     
       hb_ot_var_normalize_coords (font->face, coords_length, coords, normalized);
       _hb_font_adopt_var_coords (font, normalized, design_coords, coords_length);
    @@ -2449,28 +2774,40 @@ hb_font_set_var_coords_design (hb_font_t    *font,
      * @font: a font.
      * @instance_index: named instance index.
      *
    - * Sets design coords of a font from a named instance index.
    + * Sets design coords of a font from a named-instance index.
      *
      * Since: 2.6.0
      */
     void
     hb_font_set_var_named_instance (hb_font_t *font,
    -                                unsigned instance_index)
    +                                unsigned int instance_index)
     {
       if (hb_object_is_immutable (font))
         return;
     
    -  font->serial_coords = ++font->serial;
    +  if (font->instance_index == instance_index)
    +    return;
     
    -  unsigned int coords_length = hb_ot_var_named_instance_get_design_coords (font->face, instance_index, nullptr, nullptr);
    +  font->serial_coords = ++font->serial;
     
    -  float *coords = coords_length ? (float *) hb_calloc (coords_length, sizeof (float)) : nullptr;
    -  if (unlikely (coords_length && !coords))
    -    return;
    +  font->instance_index = instance_index;
    +  hb_font_set_variations (font, nullptr, 0);
    +}
     
    -  hb_ot_var_named_instance_get_design_coords (font->face, instance_index, &coords_length, coords);
    -  hb_font_set_var_coords_design (font, coords, coords_length);
    -  hb_free (coords);
    +/**
    + * hb_font_get_var_named_instance:
    + * @font: a font.
    + *
    + * Returns the currently-set named-instance index of the font.
    + *
    + * Return value: Named-instance index or %HB_FONT_NO_VAR_NAMED_INSTANCE.
    + *
    + * Since: 7.0.0
    + **/
    +unsigned int
    +hb_font_get_var_named_instance (hb_font_t *font)
    +{
    +  return font->instance_index;
     }
     
     /**
    @@ -2514,8 +2851,8 @@ hb_font_set_var_coords_normalized (hb_font_t    *font,
     
       if (coords_length)
       {
    -    memcpy (copy, coords, coords_length * sizeof (coords[0]));
    -    memcpy (unmapped, coords, coords_length * sizeof (coords[0]));
    +    hb_memcpy (copy, coords, coords_length * sizeof (coords[0]));
    +    hb_memcpy (unmapped, coords, coords_length * sizeof (coords[0]));
       }
     
       /* Best effort design coords simulation */
    @@ -2719,3 +3056,13 @@ hb_font_funcs_set_glyph_func (hb_font_funcs_t          *ffuncs,
                                               trampoline_destroy);
     }
     #endif
    +
    +
    +void
    +hb_font_funcs_set_glyph_shape_func (hb_font_funcs_t               *ffuncs,
    +                                   hb_font_get_glyph_shape_func_t  func,
    +                                   void                           *user_data,
    +                                   hb_destroy_func_t               destroy /* May be NULL. */)
    +{
    +  hb_font_funcs_set_draw_glyph_func (ffuncs, func, user_data, destroy);
    +}
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-font.h b/src/java.desktop/share/native/libharfbuzz/hb-font.h
    index a6aeeef47705b..23301c13fc85c 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-font.h
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-font.h
    @@ -34,18 +34,10 @@
     #include "hb-common.h"
     #include "hb-face.h"
     #include "hb-draw.h"
    +#include "hb-paint.h"
     
     HB_BEGIN_DECLS
     
    -/**
    - * hb_font_t:
    - *
    - * Data type for holding fonts.
    - *
    - */
    -typedef struct hb_font_t hb_font_t;
    -
    -
     /*
      * hb_font_funcs_t
      */
    @@ -86,8 +78,8 @@ hb_font_funcs_set_user_data (hb_font_funcs_t    *ffuncs,
     
     
     HB_EXTERN void *
    -hb_font_funcs_get_user_data (hb_font_funcs_t    *ffuncs,
    -                             hb_user_data_key_t *key);
    +hb_font_funcs_get_user_data (const hb_font_funcs_t *ffuncs,
    +                             hb_user_data_key_t    *key);
     
     
     HB_EXTERN void
    @@ -97,7 +89,7 @@ HB_EXTERN hb_bool_t
     hb_font_funcs_is_immutable (hb_font_funcs_t *ffuncs);
     
     
    -/* font and glyph extents */
    +/* font extents */
     
     /**
      * hb_font_extents_t:
    @@ -126,24 +118,6 @@ typedef struct hb_font_extents_t {
       hb_position_t reserved1;
     } hb_font_extents_t;
     
    -/**
    - * hb_glyph_extents_t:
    - * @x_bearing: Distance from the x-origin to the left extremum of the glyph.
    - * @y_bearing: Distance from the top extremum of the glyph to the y-origin.
    - * @width: Distance from the left extremum of the glyph to the right extremum.
    - * @height: Distance from the top extremum of the glyph to the bottom extremum.
    - *
    - * Glyph extent values, measured in font units.
    - *
    - * Note that @height is negative, in coordinate systems that grow up.
    - **/
    -typedef struct hb_glyph_extents_t {
    -  hb_position_t x_bearing;
    -  hb_position_t y_bearing;
    -  hb_position_t width;
    -  hb_position_t height;
    -} hb_glyph_extents_t;
    -
     /* func types */
     
     /**
    @@ -198,7 +172,7 @@ typedef hb_font_get_font_extents_func_t hb_font_get_font_v_extents_func_t;
      * This method should retrieve the nominal glyph ID for a specified Unicode code
      * point. Glyph IDs must be returned in a #hb_codepoint_t output parameter.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      **/
     typedef hb_bool_t (*hb_font_get_nominal_glyph_func_t) (hb_font_t *font, void *font_data,
    @@ -221,7 +195,7 @@ typedef hb_bool_t (*hb_font_get_nominal_glyph_func_t) (hb_font_t *font, void *fo
      * followed by a specified Variation Selector code point. Glyph IDs must be
      * returned in a #hb_codepoint_t output parameter.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      **/
     typedef hb_bool_t (*hb_font_get_variation_glyph_func_t) (hb_font_t *font, void *font_data,
    @@ -362,7 +336,7 @@ typedef hb_font_get_glyph_advances_func_t hb_font_get_glyph_v_advances_func_t;
      * origin for a glyph. Each coordinate must be returned in an #hb_position_t
      * output parameter.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      **/
     typedef hb_bool_t (*hb_font_get_glyph_origin_func_t) (hb_font_t *font, void *font_data,
    @@ -434,7 +408,7 @@ typedef hb_font_get_glyph_kerning_func_t hb_font_get_glyph_h_kerning_func_t;
      * This method should retrieve the extents for a specified glyph. Extents must be
      * returned in an #hb_glyph_extents output parameter.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      **/
     typedef hb_bool_t (*hb_font_get_glyph_extents_func_t) (hb_font_t *font, void *font_data,
    @@ -458,7 +432,7 @@ typedef hb_bool_t (*hb_font_get_glyph_extents_func_t) (hb_font_t *font, void *fo
      * specified contour point in a glyph. Each coordinate must be returned as
      * an #hb_position_t output parameter.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      **/
     typedef hb_bool_t (*hb_font_get_glyph_contour_point_func_t) (hb_font_t *font, void *font_data,
    @@ -481,7 +455,7 @@ typedef hb_bool_t (*hb_font_get_glyph_contour_point_func_t) (hb_font_t *font, vo
      * This method should retrieve the glyph name that corresponds to a
      * glyph ID. The name should be returned in a string output parameter.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      **/
     typedef hb_bool_t (*hb_font_get_glyph_name_func_t) (hb_font_t *font, void *font_data,
    @@ -503,7 +477,7 @@ typedef hb_bool_t (*hb_font_get_glyph_name_func_t) (hb_font_t *font, void *font_
      * This method should retrieve the glyph ID that corresponds to a glyph-name
      * string.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      **/
     typedef hb_bool_t (*hb_font_get_glyph_from_name_func_t) (hb_font_t *font, void *font_data,
    @@ -523,13 +497,53 @@ typedef hb_bool_t (*hb_font_get_glyph_from_name_func_t) (hb_font_t *font, void *
      * A virtual method for the #hb_font_funcs_t of an #hb_font_t object.
      *
      * Since: 4.0.0
    - *
    + * Deprecated: 7.0.0: Use #hb_font_draw_glyph_func_t instead
      **/
     typedef void (*hb_font_get_glyph_shape_func_t) (hb_font_t *font, void *font_data,
                                                     hb_codepoint_t glyph,
                                                     hb_draw_funcs_t *draw_funcs, void *draw_data,
                                                     void *user_data);
     
    +/**
    + * hb_font_draw_glyph_func_t:
    + * @font: #hb_font_t to work upon
    + * @font_data: @font user data pointer
    + * @glyph: The glyph ID to query
    + * @draw_funcs: The draw functions to send the shape data to
    + * @draw_data: The data accompanying the draw functions
    + * @user_data: User data pointer passed by the caller
    + *
    + * A virtual method for the #hb_font_funcs_t of an #hb_font_t object.
    + *
    + * Since: 7.0.0
    + *
    + **/
    +typedef void (*hb_font_draw_glyph_func_t) (hb_font_t *font, void *font_data,
    +                                           hb_codepoint_t glyph,
    +                                           hb_draw_funcs_t *draw_funcs, void *draw_data,
    +                                           void *user_data);
    +
    +/**
    + * hb_font_paint_glyph_func_t:
    + * @font: #hb_font_t to work upon
    + * @font_data: @font user data pointer
    + * @glyph: The glyph ID to query
    + * @paint_funcs: The paint functions to use
    + * @paint_data: The data accompanying the paint functions
    + * @palette_index: The color palette to use
    + * @foreground: The foreground color
    + * @user_data: User data pointer passed by the caller
    + *
    + * A virtual method for the #hb_font_funcs_t of an #hb_font_t object.
    + *
    + * Since: 7.0.0
    + */
    +typedef void (*hb_font_paint_glyph_func_t) (hb_font_t *font, void *font_data,
    +                                            hb_codepoint_t glyph,
    +                                            hb_paint_funcs_t *paint_funcs, void *paint_data,
    +                                            unsigned int palette_index,
    +                                            hb_color_t foreground,
    +                                            void *user_data);
     
     /* func setters */
     
    @@ -796,15 +810,50 @@ hb_font_funcs_set_glyph_from_name_func (hb_font_funcs_t *ffuncs,
      * @user_data: Data to pass to @func
      * @destroy: (nullable): The function to call when @user_data is not needed anymore
      *
    - * Sets the implementation function for #hb_font_get_glyph_shape_func_t.
    + * Sets the implementation function for #hb_font_get_glyph_shape_func_t,
    + * which is the same as #hb_font_draw_glyph_func_t.
      *
      * Since: 4.0.0
    + * Deprecated: 7.0.0: Use hb_font_funcs_set_draw_glyph_func() instead
      **/
     HB_EXTERN void
     hb_font_funcs_set_glyph_shape_func (hb_font_funcs_t *ffuncs,
                                         hb_font_get_glyph_shape_func_t func,
                                         void *user_data, hb_destroy_func_t destroy);
     
    +/**
    + * hb_font_funcs_set_draw_glyph_func:
    + * @ffuncs: A font-function structure
    + * @func: (closure user_data) (destroy destroy) (scope notified): The callback function to assign
    + * @user_data: Data to pass to @func
    + * @destroy: (nullable): The function to call when @user_data is not needed anymore
    + *
    + * Sets the implementation function for #hb_font_draw_glyph_func_t,
    + * which is the same as #hb_font_get_glyph_shape_func_t.
    + *
    + * Since: 7.0.0
    + **/
    +HB_EXTERN void
    +hb_font_funcs_set_draw_glyph_func (hb_font_funcs_t *ffuncs,
    +                                   hb_font_draw_glyph_func_t func,
    +                                   void *user_data, hb_destroy_func_t destroy);
    +
    +/**
    + * hb_font_funcs_set_paint_glyph_func:
    + * @ffuncs: A font-function structure
    + * @func: (closure user_data) (destroy destroy) (scope notified): The callback function to assign
    + * @user_data: Data to pass to @func
    + * @destroy: (nullable): The function to call when @user_data is no longer needed
    + *
    + * Sets the implementation function for #hb_font_paint_glyph_func_t.
    + *
    + * Since: 7.0.0
    + */
    +HB_EXTERN void
    +hb_font_funcs_set_paint_glyph_func (hb_font_funcs_t *ffuncs,
    +                                    hb_font_paint_glyph_func_t func,
    +                                    void *user_data, hb_destroy_func_t destroy);
    +
     /* func dispatch */
     
     HB_EXTERN hb_bool_t
    @@ -890,6 +939,17 @@ hb_font_get_glyph_shape (hb_font_t *font,
                              hb_codepoint_t glyph,
                              hb_draw_funcs_t *dfuncs, void *draw_data);
     
    +HB_EXTERN void
    +hb_font_draw_glyph (hb_font_t *font,
    +                    hb_codepoint_t glyph,
    +                    hb_draw_funcs_t *dfuncs, void *draw_data);
    +
    +HB_EXTERN void
    +hb_font_paint_glyph (hb_font_t *font,
    +                     hb_codepoint_t glyph,
    +                     hb_paint_funcs_t *pfuncs, void *paint_data,
    +                     unsigned int palette_index,
    +                     hb_color_t foreground);
     
     /* high-level funcs, with fallback */
     
    @@ -993,7 +1053,7 @@ hb_font_set_user_data (hb_font_t          *font,
     
     
     HB_EXTERN void *
    -hb_font_get_user_data (hb_font_t          *font,
    +hb_font_get_user_data (const hb_font_t    *font,
                            hb_user_data_key_t *key);
     
     HB_EXTERN void
    @@ -1069,6 +1129,16 @@ hb_font_set_ptem (hb_font_t *font, float ptem);
     HB_EXTERN float
     hb_font_get_ptem (hb_font_t *font);
     
    +HB_EXTERN void
    +hb_font_set_synthetic_bold (hb_font_t *font,
    +                            float x_embolden, float y_embolden,
    +                            hb_bool_t in_place);
    +
    +HB_EXTERN void
    +hb_font_get_synthetic_bold (hb_font_t *font,
    +                            float *x_embolden, float *y_embolden,
    +                            hb_bool_t *in_place);
    +
     HB_EXTERN void
     hb_font_set_synthetic_slant (hb_font_t *font, float slant);
     
    @@ -1080,6 +1150,11 @@ hb_font_set_variations (hb_font_t *font,
                             const hb_variation_t *variations,
                             unsigned int variations_length);
     
    +HB_EXTERN void
    +hb_font_set_variation (hb_font_t *font,
    +                       hb_tag_t tag,
    +                       float    value);
    +
     HB_EXTERN void
     hb_font_set_var_coords_design (hb_font_t *font,
                                    const float *coords,
    @@ -1098,10 +1173,23 @@ HB_EXTERN const int *
     hb_font_get_var_coords_normalized (hb_font_t *font,
                                        unsigned int *length);
     
    +/**
    + * HB_FONT_NO_VAR_NAMED_INSTANCE:
    + *
    + * Constant signifying that a font does not have any
    + * named-instance index set.  This is the default of
    + * a font.
    + *
    + * Since: 7.0.0
    + */
    +#define HB_FONT_NO_VAR_NAMED_INSTANCE 0xFFFFFFFF
    +
     HB_EXTERN void
     hb_font_set_var_named_instance (hb_font_t *font,
    -                                unsigned instance_index);
    +                                unsigned int instance_index);
     
    +HB_EXTERN unsigned int
    +hb_font_get_var_named_instance (hb_font_t *font);
     
     HB_END_DECLS
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-font.hh b/src/java.desktop/share/native/libharfbuzz/hb-font.hh
    index 3af9d74012a7e..c3198830cd33b 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-font.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-font.hh
    @@ -40,24 +40,25 @@
      */
     
     #define HB_FONT_FUNCS_IMPLEMENT_CALLBACKS \
    -  HB_FONT_FUNC_IMPLEMENT (font_h_extents) \
    -  HB_FONT_FUNC_IMPLEMENT (font_v_extents) \
    -  HB_FONT_FUNC_IMPLEMENT (nominal_glyph) \
    -  HB_FONT_FUNC_IMPLEMENT (nominal_glyphs) \
    -  HB_FONT_FUNC_IMPLEMENT (variation_glyph) \
    -  HB_FONT_FUNC_IMPLEMENT (glyph_h_advance) \
    -  HB_FONT_FUNC_IMPLEMENT (glyph_v_advance) \
    -  HB_FONT_FUNC_IMPLEMENT (glyph_h_advances) \
    -  HB_FONT_FUNC_IMPLEMENT (glyph_v_advances) \
    -  HB_FONT_FUNC_IMPLEMENT (glyph_h_origin) \
    -  HB_FONT_FUNC_IMPLEMENT (glyph_v_origin) \
    -  HB_FONT_FUNC_IMPLEMENT (glyph_h_kerning) \
    -  HB_IF_NOT_DEPRECATED (HB_FONT_FUNC_IMPLEMENT (glyph_v_kerning)) \
    -  HB_FONT_FUNC_IMPLEMENT (glyph_extents) \
    -  HB_FONT_FUNC_IMPLEMENT (glyph_contour_point) \
    -  HB_FONT_FUNC_IMPLEMENT (glyph_name) \
    -  HB_FONT_FUNC_IMPLEMENT (glyph_from_name) \
    -  HB_FONT_FUNC_IMPLEMENT (glyph_shape) \
    +  HB_FONT_FUNC_IMPLEMENT (get_,font_h_extents) \
    +  HB_FONT_FUNC_IMPLEMENT (get_,font_v_extents) \
    +  HB_FONT_FUNC_IMPLEMENT (get_,nominal_glyph) \
    +  HB_FONT_FUNC_IMPLEMENT (get_,nominal_glyphs) \
    +  HB_FONT_FUNC_IMPLEMENT (get_,variation_glyph) \
    +  HB_FONT_FUNC_IMPLEMENT (get_,glyph_h_advance) \
    +  HB_FONT_FUNC_IMPLEMENT (get_,glyph_v_advance) \
    +  HB_FONT_FUNC_IMPLEMENT (get_,glyph_h_advances) \
    +  HB_FONT_FUNC_IMPLEMENT (get_,glyph_v_advances) \
    +  HB_FONT_FUNC_IMPLEMENT (get_,glyph_h_origin) \
    +  HB_FONT_FUNC_IMPLEMENT (get_,glyph_v_origin) \
    +  HB_FONT_FUNC_IMPLEMENT (get_,glyph_h_kerning) \
    +  HB_IF_NOT_DEPRECATED (HB_FONT_FUNC_IMPLEMENT (get_,glyph_v_kerning)) \
    +  HB_FONT_FUNC_IMPLEMENT (get_,glyph_extents) \
    +  HB_FONT_FUNC_IMPLEMENT (get_,glyph_contour_point) \
    +  HB_FONT_FUNC_IMPLEMENT (get_,glyph_name) \
    +  HB_FONT_FUNC_IMPLEMENT (get_,glyph_from_name) \
    +  HB_FONT_FUNC_IMPLEMENT (,draw_glyph) \
    +  HB_FONT_FUNC_IMPLEMENT (,paint_glyph) \
       /* ^--- Add new callbacks here */
     
     struct hb_font_funcs_t
    @@ -65,13 +66,13 @@ struct hb_font_funcs_t
       hb_object_header_t header;
     
       struct {
    -#define HB_FONT_FUNC_IMPLEMENT(name) void *name;
    +#define HB_FONT_FUNC_IMPLEMENT(get_,name) void *name;
         HB_FONT_FUNCS_IMPLEMENT_CALLBACKS
     #undef HB_FONT_FUNC_IMPLEMENT
       } *user_data;
     
       struct {
    -#define HB_FONT_FUNC_IMPLEMENT(name) hb_destroy_func_t name;
    +#define HB_FONT_FUNC_IMPLEMENT(get_,name) hb_destroy_func_t name;
         HB_FONT_FUNCS_IMPLEMENT_CALLBACKS
     #undef HB_FONT_FUNC_IMPLEMENT
       } *destroy;
    @@ -79,12 +80,12 @@ struct hb_font_funcs_t
       /* Don't access these directly.  Call font->get_*() instead. */
       union get_t {
         struct get_funcs_t {
    -#define HB_FONT_FUNC_IMPLEMENT(name) hb_font_get_##name##_func_t name;
    +#define HB_FONT_FUNC_IMPLEMENT(get_,name) hb_font_##get_##name##_func_t name;
           HB_FONT_FUNCS_IMPLEMENT_CALLBACKS
     #undef HB_FONT_FUNC_IMPLEMENT
         } f;
         void (*array[0
    -#define HB_FONT_FUNC_IMPLEMENT(name) +1
    +#define HB_FONT_FUNC_IMPLEMENT(get_,name) +1
           HB_FONT_FUNCS_IMPLEMENT_CALLBACKS
     #undef HB_FONT_FUNC_IMPLEMENT
                     ]) ();
    @@ -112,8 +113,16 @@ struct hb_font_t
     
       int32_t x_scale;
       int32_t y_scale;
    +
    +  float x_embolden;
    +  float y_embolden;
    +  bool embolden_in_place;
    +  int32_t x_strength; /* x_embolden, in scaled units. */
    +  int32_t y_strength; /* y_embolden, in scaled units. */
    +
       float slant;
       float slant_xy;
    +
       float x_multf;
       float y_multf;
       int64_t x_mult;
    @@ -125,6 +134,7 @@ struct hb_font_t
       float ptem;
     
       /* Font variation coordinates. */
    +  unsigned int instance_index;
       unsigned int num_coords;
       int *coords;
       float *design_coords;
    @@ -179,6 +189,42 @@ struct hb_font_t
         *y = parent_scale_y_position (*y);
       }
     
    +  void scale_glyph_extents (hb_glyph_extents_t *extents)
    +  {
    +    float x1 = em_fscale_x (extents->x_bearing);
    +    float y1 = em_fscale_y (extents->y_bearing);
    +    float x2 = em_fscale_x (extents->x_bearing + extents->width);
    +    float y2 = em_fscale_y (extents->y_bearing + extents->height);
    +
    +    /* Apply slant. */
    +    if (slant_xy)
    +    {
    +      x1 += hb_min (y1 * slant_xy, y2 * slant_xy);
    +      x2 += hb_max (y1 * slant_xy, y2 * slant_xy);
    +    }
    +
    +    extents->x_bearing = floorf (x1);
    +    extents->y_bearing = floorf (y1);
    +    extents->width = ceilf (x2) - extents->x_bearing;
    +    extents->height = ceilf (y2) - extents->y_bearing;
    +
    +    if (x_strength || y_strength)
    +    {
    +      /* Y */
    +      int y_shift = y_strength;
    +      if (y_scale < 0) y_shift = -y_shift;
    +      extents->y_bearing += y_shift;
    +      extents->height -= y_shift;
    +
    +      /* X */
    +      int x_shift = x_strength;
    +      if (x_scale < 0) x_shift = -x_shift;
    +      if (embolden_in_place)
    +        extents->x_bearing -= x_shift / 2;
    +      extents->width += x_shift;
    +    }
    +  }
    +
     
       /* Public getters */
     
    @@ -186,7 +232,7 @@ struct hb_font_t
       HB_INTERNAL bool has_func_set (unsigned int i);
     
       /* has_* ... */
    -#define HB_FONT_FUNC_IMPLEMENT(name) \
    +#define HB_FONT_FUNC_IMPLEMENT(get_,name) \
       bool \
       has_##name##_func () \
       { \
    @@ -206,14 +252,14 @@ struct hb_font_t
     
       hb_bool_t get_font_h_extents (hb_font_extents_t *extents)
       {
    -    memset (extents, 0, sizeof (*extents));
    +    hb_memset (extents, 0, sizeof (*extents));
         return klass->get.f.font_h_extents (this, user_data,
                                             extents,
                                             !klass->user_data ? nullptr : klass->user_data->font_h_extents);
       }
       hb_bool_t get_font_v_extents (hb_font_extents_t *extents)
       {
    -    memset (extents, 0, sizeof (*extents));
    +    hb_memset (extents, 0, sizeof (*extents));
         return klass->get.f.font_v_extents (this, user_data,
                                             extents,
                                             !klass->user_data ? nullptr : klass->user_data->font_v_extents);
    @@ -342,7 +388,7 @@ struct hb_font_t
       hb_bool_t get_glyph_extents (hb_codepoint_t glyph,
                                    hb_glyph_extents_t *extents)
       {
    -    memset (extents, 0, sizeof (*extents));
    +    hb_memset (extents, 0, sizeof (*extents));
         return klass->get.f.glyph_extents (this, user_data,
                                            glyph,
                                            extents,
    @@ -380,15 +426,26 @@ struct hb_font_t
                                              !klass->user_data ? nullptr : klass->user_data->glyph_from_name);
       }
     
    -  void get_glyph_shape (hb_codepoint_t glyph,
    -                        hb_draw_funcs_t *draw_funcs, void *draw_data)
    +  void draw_glyph (hb_codepoint_t glyph,
    +                   hb_draw_funcs_t *draw_funcs, void *draw_data)
       {
    -    klass->get.f.glyph_shape (this, user_data,
    -                              glyph,
    -                              draw_funcs, draw_data,
    -                              !klass->user_data ? nullptr : klass->user_data->glyph_shape);
    +    klass->get.f.draw_glyph (this, user_data,
    +                             glyph,
    +                             draw_funcs, draw_data,
    +                             !klass->user_data ? nullptr : klass->user_data->draw_glyph);
       }
     
    +  void paint_glyph (hb_codepoint_t glyph,
    +                    hb_paint_funcs_t *paint_funcs, void *paint_data,
    +                    unsigned int palette,
    +                    hb_color_t foreground)
    +  {
    +    klass->get.f.paint_glyph (this, user_data,
    +                              glyph,
    +                              paint_funcs, paint_data,
    +                              palette, foreground,
    +                              !klass->user_data ? nullptr : klass->user_data->paint_glyph);
    +  }
     
       /* A bit higher-level, and with fallback */
     
    @@ -632,12 +689,17 @@ struct hb_font_t
       void mults_changed ()
       {
         float upem = face->get_upem ();
    +
         x_multf = x_scale / upem;
         y_multf = y_scale / upem;
         bool x_neg = x_scale < 0;
         x_mult = (x_neg ? -((int64_t) -x_scale << 16) : ((int64_t) x_scale << 16)) / upem;
         bool y_neg = y_scale < 0;
         y_mult = (y_neg ? -((int64_t) -y_scale << 16) : ((int64_t) y_scale << 16)) / upem;
    +
    +    x_strength = fabsf (roundf (x_scale * x_embolden));
    +    y_strength = fabsf (roundf (y_scale * y_embolden));
    +
         slant_xy = y_scale ? slant * x_scale / y_scale : 0.f;
     
         data.fini ();
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ft.cc b/src/java.desktop/share/native/libharfbuzz/hb-ft.cc
    index 291d1e8d70c5f..9b1fa8d53f268 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ft.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-ft.cc
    @@ -33,17 +33,22 @@
     
     #include "hb-ft.h"
     
    +#include "hb-cache.hh"
     #include "hb-draw.hh"
     #include "hb-font.hh"
     #include "hb-machinery.hh"
    -#include "hb-cache.hh"
     #include "hb-ot-os2-table.hh"
     #include "hb-ot-shaper-arabic-pua.hh"
    +#include "hb-paint.hh"
     
     #include FT_ADVANCES_H
     #include FT_MULTIPLE_MASTERS_H
     #include FT_OUTLINE_H
     #include FT_TRUETYPE_TABLES_H
    +#include FT_SYNTHESIS_H
    +#if (FREETYPE_MAJOR*10000 + FREETYPE_MINOR*100 + FREETYPE_PATCH) >= 21300
    +#include FT_COLOR_H
    +#endif
     
     
     /**
    @@ -80,16 +85,19 @@
      */
     
     
    +using hb_ft_advance_cache_t = hb_cache_t<16, 24, 8, false>;
    +
     struct hb_ft_font_t
     {
       int load_flags;
       bool symbol; /* Whether selected cmap is symbol cmap. */
       bool unref; /* Whether to destroy ft_face when done. */
    +  bool transform; /* Whether to apply FT_Face's transform. */
     
    -  mutable hb_mutex_t lock;
    +  mutable hb_mutex_t lock; /* Protects members below. */
       FT_Face ft_face;
       mutable unsigned cached_serial;
    -  mutable hb_advance_cache_t advance_cache;
    +  mutable hb_ft_advance_cache_t advance_cache;
     };
     
     static hb_ft_font_t *
    @@ -122,8 +130,6 @@ _hb_ft_font_destroy (void *data)
     {
       hb_ft_font_t *ft_font = (hb_ft_font_t *) data;
     
    -  ft_font->advance_cache.fini ();
    -
       if (ft_font->unref)
         _hb_ft_face_destroy (ft_font->ft_face);
     
    @@ -136,32 +142,59 @@ _hb_ft_font_destroy (void *data)
     /* hb_font changed, update FT_Face. */
     static void _hb_ft_hb_font_changed (hb_font_t *font, FT_Face ft_face)
     {
    +  hb_ft_font_t *ft_font = (hb_ft_font_t *) font->user_data;
    +
    +  float x_mult = 1.f, y_mult = 1.f;
    +
    +  if (font->x_scale < 0) x_mult = -x_mult;
    +  if (font->y_scale < 0) y_mult = -y_mult;
     
    -  FT_Set_Char_Size (ft_face,
    -                    abs (font->x_scale), abs (font->y_scale),
    -                    0, 0);
    +  if (FT_Set_Char_Size (ft_face,
    +                        abs (font->x_scale), abs (font->y_scale),
    +                        0, 0
     #if 0
                         font->x_ppem * 72 * 64 / font->x_scale,
    -                    font->y_ppem * 72 * 64 / font->y_scale);
    +                    font->y_ppem * 72 * 64 / font->y_scale
     #endif
    -  if (font->x_scale < 0 || font->y_scale < 0)
    +     ) && ft_face->num_fixed_sizes)
       {
    -    FT_Matrix matrix = { font->x_scale < 0 ? -1 : +1, 0,
    -                          0, font->y_scale < 0 ? -1 : +1};
    +#ifdef HAVE_FT_GET_TRANSFORM
    +    /* Bitmap font, eg. bitmap color emoji. */
    +    /* Pick largest size? */
    +    int x_scale  = ft_face->available_sizes[ft_face->num_fixed_sizes - 1].x_ppem;
    +    int y_scale = ft_face->available_sizes[ft_face->num_fixed_sizes - 1].y_ppem;
    +    FT_Set_Char_Size (ft_face,
    +                      x_scale, y_scale,
    +                      0, 0);
    +
    +    /* This contains the sign that was previously in x_mult/y_mult. */
    +    x_mult = (float) font->x_scale / x_scale;
    +    y_mult = (float) font->y_scale / y_scale;
    +#endif
    +  }
    +  else
    +  { /* Shrug */ }
    +
    +
    +  if (x_mult != 1.f || y_mult != 1.f)
    +  {
    +    FT_Matrix matrix = { (int) roundf (x_mult * (1<<16)), 0,
    +                          0, (int) roundf (y_mult * (1<<16))};
         FT_Set_Transform (ft_face, &matrix, nullptr);
    +    ft_font->transform = true;
       }
     
     #if defined(HAVE_FT_GET_VAR_BLEND_COORDINATES) && !defined(HB_NO_VAR)
       unsigned int num_coords;
    -  const int *coords = hb_font_get_var_coords_normalized (font, &num_coords);
    +  const float *coords = hb_font_get_var_coords_design (font, &num_coords);
       if (num_coords)
       {
         FT_Fixed *ft_coords = (FT_Fixed *) hb_calloc (num_coords, sizeof (FT_Fixed));
         if (ft_coords)
         {
           for (unsigned int i = 0; i < num_coords; i++)
    -        ft_coords[i] = coords[i] * 4;
    -      FT_Set_Var_Blend_Coordinates (ft_face, num_coords, ft_coords);
    +          ft_coords[i] = coords[i] * 65536.f;
    +      FT_Set_Var_Design_Coordinates (ft_face, num_coords, ft_coords);
           hb_free (ft_coords);
         }
       }
    @@ -194,6 +227,9 @@ _hb_ft_hb_font_check_changed (hb_font_t *font,
      * For more information, see
      * https://www.freetype.org/freetype2/docs/reference/ft2-base_interface.html#ft_load_xxx
      *
    + * This function works with #hb_font_t objects created by
    + * hb_ft_font_create() or hb_ft_font_create_referenced().
    + *
      * Since: 1.0.5
      **/
     void
    @@ -219,7 +255,10 @@ hb_ft_font_set_load_flags (hb_font_t *font, int load_flags)
      * For more information, see
      * https://www.freetype.org/freetype2/docs/reference/ft2-base_interface.html#ft_load_xxx
      *
    - * Return value: FT_Load_Glyph flags found
    + * This function works with #hb_font_t objects created by
    + * hb_ft_font_create() or hb_ft_font_create_referenced().
    + *
    + * Return value: FT_Load_Glyph flags found, or 0
      *
      * Since: 1.0.5
      **/
    @@ -241,7 +280,10 @@ hb_ft_font_get_load_flags (hb_font_t *font)
      * Fetches the FT_Face associated with the specified #hb_font_t
      * font object.
      *
    - * Return value: (nullable): the FT_Face found or %NULL
    + * This function works with #hb_font_t objects created by
    + * hb_ft_font_create() or hb_ft_font_create_referenced().
    + *
    + * Return value: (nullable): the FT_Face found or `NULL`
      *
      * Since: 0.9.2
      **/
    @@ -260,10 +302,15 @@ hb_ft_font_get_face (hb_font_t *font)
      * hb_ft_font_lock_face: (skip)
      * @font: #hb_font_t to work upon
      *
    - * Gets the FT_Face associated with @font, This face will be kept around until
    - * you call hb_ft_font_unlock_face().
    + * Gets the FT_Face associated with @font.
    + *
    + * This face will be kept around and access to the FT_Face object
    + * from other HarfBuzz API wil be blocked until you call hb_ft_font_unlock_face().
      *
    - * Return value: (nullable): the FT_Face associated with @font or %NULL
    + * This function works with #hb_font_t objects created by
    + * hb_ft_font_create() or hb_ft_font_create_referenced().
    + *
    + * Return value: (nullable) (transfer none): the FT_Face associated with @font or `NULL`
      * Since: 2.6.5
      **/
     FT_Face
    @@ -280,7 +327,7 @@ hb_ft_font_lock_face (hb_font_t *font)
     }
     
     /**
    - * hb_ft_font_unlock_face:
    + * hb_ft_font_unlock_face: (skip)
      * @font: #hb_font_t to work upon
      *
      * Releases an FT_Face previously obtained with hb_ft_font_lock_face().
    @@ -401,10 +448,24 @@ hb_ft_get_glyph_h_advances (hb_font_t* font, void* font_data,
                                 void *user_data HB_UNUSED)
     {
       const hb_ft_font_t *ft_font = (const hb_ft_font_t *) font_data;
    +  hb_position_t *orig_first_advance = first_advance;
       hb_lock_t lock (ft_font->lock);
       FT_Face ft_face = ft_font->ft_face;
       int load_flags = ft_font->load_flags;
    -  int mult = font->x_scale < 0 ? -1 : +1;
    +  float x_mult;
    +#ifdef HAVE_FT_GET_TRANSFORM
    +  if (ft_font->transform)
    +  {
    +    FT_Matrix matrix;
    +    FT_Get_Transform (ft_face, &matrix, nullptr);
    +    x_mult = sqrtf ((float)matrix.xx * matrix.xx + (float)matrix.xy * matrix.xy) / 65536.f;
    +    x_mult *= font->x_scale < 0 ? -1 : +1;
    +  }
    +  else
    +#endif
    +  {
    +    x_mult = font->x_scale < 0 ? -1 : +1;
    +  }
     
       for (unsigned int i = 0; i < count; i++)
       {
    @@ -417,13 +478,29 @@ hb_ft_get_glyph_h_advances (hb_font_t* font, void* font_data,
         else
         {
           FT_Get_Advance (ft_face, glyph, load_flags, &v);
    +      /* Work around bug that FreeType seems to return negative advance
    +       * for variable-set fonts if x_scale is negative! */
    +      v = abs (v);
    +      v = (int) (v * x_mult + (1<<9)) >> 10;
           ft_font->advance_cache.set (glyph, v);
         }
     
    -    *first_advance = (v * mult + (1<<9)) >> 10;
    +    *first_advance = v;
         first_glyph = &StructAtOffsetUnaligned (first_glyph, glyph_stride);
         first_advance = &StructAtOffsetUnaligned (first_advance, advance_stride);
       }
    +
    +  if (font->x_strength && !font->embolden_in_place)
    +  {
    +    /* Emboldening. */
    +    hb_position_t x_strength = font->x_scale >= 0 ? font->x_strength : -font->x_strength;
    +    first_advance = orig_first_advance;
    +    for (unsigned int i = 0; i < count; i++)
    +    {
    +      *first_advance += *first_advance ? x_strength : 0;
    +      first_advance = &StructAtOffsetUnaligned (first_advance, advance_stride);
    +    }
    +  }
     }
     
     #ifndef HB_NO_VERTICAL
    @@ -436,17 +513,31 @@ hb_ft_get_glyph_v_advance (hb_font_t *font,
       const hb_ft_font_t *ft_font = (const hb_ft_font_t *) font_data;
       hb_lock_t lock (ft_font->lock);
       FT_Fixed v;
    +  float y_mult;
    +#ifdef HAVE_FT_GET_TRANSFORM
    +  if (ft_font->transform)
    +  {
    +    FT_Matrix matrix;
    +    FT_Get_Transform (ft_font->ft_face, &matrix, nullptr);
    +    y_mult = sqrtf ((float)matrix.yx * matrix.yx + (float)matrix.yy * matrix.yy) / 65536.f;
    +    y_mult *= font->y_scale < 0 ? -1 : +1;
    +  }
    +  else
    +#endif
    +  {
    +    y_mult = font->y_scale < 0 ? -1 : +1;
    +  }
     
       if (unlikely (FT_Get_Advance (ft_font->ft_face, glyph, ft_font->load_flags | FT_LOAD_VERTICAL_LAYOUT, &v)))
         return 0;
     
    -  if (font->y_scale < 0)
    -    v = -v;
    +  v = (int) (y_mult * v);
     
       /* Note: FreeType's vertical metrics grows downward while other FreeType coordinates
        * have a Y growing upward.  Hence the extra negation. */
     
    -  return (-v + (1<<9)) >> 10;
    +  hb_position_t y_strength = font->y_scale >= 0 ? font->y_strength : -font->y_strength;
    +  return ((-v + (1<<9)) >> 10) + (font->embolden_in_place ? 0 : y_strength);
     }
     #endif
     
    @@ -462,6 +553,23 @@ hb_ft_get_glyph_v_origin (hb_font_t *font,
       const hb_ft_font_t *ft_font = (const hb_ft_font_t *) font_data;
       hb_lock_t lock (ft_font->lock);
       FT_Face ft_face = ft_font->ft_face;
    +  float x_mult, y_mult;
    +#ifdef HAVE_FT_GET_TRANSFORM
    +  if (ft_font->transform)
    +  {
    +    FT_Matrix matrix;
    +    FT_Get_Transform (ft_face, &matrix, nullptr);
    +    x_mult = sqrtf ((float)matrix.xx * matrix.xx + (float)matrix.xy * matrix.xy) / 65536.f;
    +    x_mult *= font->x_scale < 0 ? -1 : +1;
    +    y_mult = sqrtf ((float)matrix.yx * matrix.yx + (float)matrix.yy * matrix.yy) / 65536.f;
    +    y_mult *= font->y_scale < 0 ? -1 : +1;
    +  }
    +  else
    +#endif
    +  {
    +    x_mult = font->x_scale < 0 ? -1 : +1;
    +    y_mult = font->y_scale < 0 ? -1 : +1;
    +  }
     
       if (unlikely (FT_Load_Glyph (ft_face, glyph, ft_font->load_flags)))
         return false;
    @@ -471,10 +579,8 @@ hb_ft_get_glyph_v_origin (hb_font_t *font,
       *x = ft_face->glyph->metrics.horiBearingX -   ft_face->glyph->metrics.vertBearingX;
       *y = ft_face->glyph->metrics.horiBearingY - (-ft_face->glyph->metrics.vertBearingY);
     
    -  if (font->x_scale < 0)
    -    *x = -*x;
    -  if (font->y_scale < 0)
    -    *y = -*y;
    +  *x = (hb_position_t) (x_mult * *x);
    +  *y = (hb_position_t) (y_mult * *y);
     
       return true;
     }
    @@ -510,24 +616,63 @@ hb_ft_get_glyph_extents (hb_font_t *font,
       const hb_ft_font_t *ft_font = (const hb_ft_font_t *) font_data;
       hb_lock_t lock (ft_font->lock);
       FT_Face ft_face = ft_font->ft_face;
    +  float x_mult, y_mult;
    +  float slant_xy = font->slant_xy;
    +#ifdef HAVE_FT_GET_TRANSFORM
    +  if (ft_font->transform)
    +  {
    +    FT_Matrix matrix;
    +    FT_Get_Transform (ft_face, &matrix, nullptr);
    +    x_mult = sqrtf ((float)matrix.xx * matrix.xx + (float)matrix.xy * matrix.xy) / 65536.f;
    +    x_mult *= font->x_scale < 0 ? -1 : +1;
    +    y_mult = sqrtf ((float)matrix.yx * matrix.yx + (float)matrix.yy * matrix.yy) / 65536.f;
    +    y_mult *= font->y_scale < 0 ? -1 : +1;
    +  }
    +  else
    +#endif
    +  {
    +    x_mult = font->x_scale < 0 ? -1 : +1;
    +    y_mult = font->y_scale < 0 ? -1 : +1;
    +  }
     
       if (unlikely (FT_Load_Glyph (ft_face, glyph, ft_font->load_flags)))
         return false;
     
    -  extents->x_bearing = ft_face->glyph->metrics.horiBearingX;
    -  extents->y_bearing = ft_face->glyph->metrics.horiBearingY;
    -  extents->width = ft_face->glyph->metrics.width;
    -  extents->height = -ft_face->glyph->metrics.height;
    -  if (font->x_scale < 0)
    +  /* Copied from hb_font_t::scale_glyph_extents. */
    +
    +  float x1 = x_mult * ft_face->glyph->metrics.horiBearingX;
    +  float y1 = y_mult * ft_face->glyph->metrics.horiBearingY;
    +  float x2 = x1 + x_mult *  ft_face->glyph->metrics.width;
    +  float y2 = y1 + y_mult * -ft_face->glyph->metrics.height;
    +
    +  /* Apply slant. */
    +  if (slant_xy)
       {
    -    extents->x_bearing = -extents->x_bearing;
    -    extents->width = -extents->width;
    +    x1 += hb_min (y1 * slant_xy, y2 * slant_xy);
    +    x2 += hb_max (y1 * slant_xy, y2 * slant_xy);
       }
    -  if (font->y_scale < 0)
    +
    +  extents->x_bearing = floorf (x1);
    +  extents->y_bearing = floorf (y1);
    +  extents->width = ceilf (x2) - extents->x_bearing;
    +  extents->height = ceilf (y2) - extents->y_bearing;
    +
    +  if (font->x_strength || font->y_strength)
       {
    -    extents->y_bearing = -extents->y_bearing;
    -    extents->height = -extents->height;
    +    /* Y */
    +    int y_shift = font->y_strength;
    +    if (font->y_scale < 0) y_shift = -y_shift;
    +    extents->y_bearing += y_shift;
    +    extents->height -= y_shift;
    +
    +    /* X */
    +    int x_shift = font->x_strength;
    +    if (font->x_scale < 0) x_shift = -x_shift;
    +    if (font->embolden_in_place)
    +      extents->x_bearing -= x_shift / 2;
    +    extents->width += x_shift;
       }
    +
       return true;
     }
     
    @@ -620,16 +765,39 @@ hb_ft_get_font_h_extents (hb_font_t *font HB_UNUSED,
       const hb_ft_font_t *ft_font = (const hb_ft_font_t *) font_data;
       hb_lock_t lock (ft_font->lock);
       FT_Face ft_face = ft_font->ft_face;
    +  float y_mult;
    +#ifdef HAVE_FT_GET_TRANSFORM
    +  if (ft_font->transform)
    +  {
    +    FT_Matrix matrix;
    +    FT_Get_Transform (ft_face, &matrix, nullptr);
    +    y_mult = sqrtf ((float)matrix.yx * matrix.yx + (float)matrix.yy * matrix.yy) / 65536.f;
    +    y_mult *= font->y_scale < 0 ? -1 : +1;
    +  }
    +  else
    +#endif
    +  {
    +    y_mult = font->y_scale < 0 ? -1 : +1;
    +  }
     
    -  metrics->ascender = FT_MulFix(ft_face->ascender, ft_face->size->metrics.y_scale);
    -  metrics->descender = FT_MulFix(ft_face->descender, ft_face->size->metrics.y_scale);
    -  metrics->line_gap = FT_MulFix( ft_face->height, ft_face->size->metrics.y_scale ) - (metrics->ascender - metrics->descender);
    -  if (font->y_scale < 0)
    +  if (ft_face->units_per_EM != 0)
       {
    -    metrics->ascender = -metrics->ascender;
    -    metrics->descender = -metrics->descender;
    -    metrics->line_gap = -metrics->line_gap;
    +    metrics->ascender = FT_MulFix(ft_face->ascender, ft_face->size->metrics.y_scale);
    +    metrics->descender = FT_MulFix(ft_face->descender, ft_face->size->metrics.y_scale);
    +    metrics->line_gap = FT_MulFix( ft_face->height, ft_face->size->metrics.y_scale ) - (metrics->ascender - metrics->descender);
       }
    +  else
    +  {
    +    /* Bitmap-only font, eg. color bitmap font. */
    +    metrics->ascender = ft_face->size->metrics.ascender;
    +    metrics->descender = ft_face->size->metrics.descender;
    +    metrics->line_gap = ft_face->size->metrics.height - (metrics->ascender - metrics->descender);
    +  }
    +
    +  metrics->ascender  = (hb_position_t) (y_mult * (metrics->ascender + font->y_strength));
    +  metrics->descender = (hb_position_t) (y_mult * metrics->descender);
    +  metrics->line_gap  = (hb_position_t) (y_mult * metrics->line_gap);
    +
       return true;
     }
     
    @@ -637,16 +805,18 @@ hb_ft_get_font_h_extents (hb_font_t *font HB_UNUSED,
     
     static int
     _hb_ft_move_to (const FT_Vector *to,
    -                hb_draw_session_t *drawing)
    +                void *arg)
     {
    +  hb_draw_session_t *drawing = (hb_draw_session_t *) arg;
       drawing->move_to (to->x, to->y);
       return FT_Err_Ok;
     }
     
     static int
     _hb_ft_line_to (const FT_Vector *to,
    -                hb_draw_session_t *drawing)
    +                void *arg)
     {
    +  hb_draw_session_t *drawing = (hb_draw_session_t *) arg;
       drawing->line_to (to->x, to->y);
       return FT_Err_Ok;
     }
    @@ -654,8 +824,9 @@ _hb_ft_line_to (const FT_Vector *to,
     static int
     _hb_ft_conic_to (const FT_Vector *control,
                      const FT_Vector *to,
    -                 hb_draw_session_t *drawing)
    +                 void *arg)
     {
    +  hb_draw_session_t *drawing = (hb_draw_session_t *) arg;
       drawing->quadratic_to (control->x, control->y,
                              to->x, to->y);
       return FT_Err_Ok;
    @@ -665,8 +836,9 @@ static int
     _hb_ft_cubic_to (const FT_Vector *control1,
                      const FT_Vector *control2,
                      const FT_Vector *to,
    -                 hb_draw_session_t *drawing)
    +                 void *arg)
     {
    +  hb_draw_session_t *drawing = (hb_draw_session_t *) arg;
       drawing->cubic_to (control1->x, control1->y,
                          control2->x, control2->y,
                          to->x, to->y);
    @@ -674,18 +846,16 @@ _hb_ft_cubic_to (const FT_Vector *control1,
     }
     
     static void
    -hb_ft_get_glyph_shape (hb_font_t *font HB_UNUSED,
    -                       void *font_data,
    -                       hb_codepoint_t glyph,
    -                       hb_draw_funcs_t *draw_funcs, void *draw_data,
    -                       void *user_data HB_UNUSED)
    +hb_ft_draw_glyph (hb_font_t *font,
    +                  void *font_data,
    +                  hb_codepoint_t glyph,
    +                  hb_draw_funcs_t *draw_funcs, void *draw_data,
    +                  void *user_data HB_UNUSED)
     {
       const hb_ft_font_t *ft_font = (const hb_ft_font_t *) font_data;
       hb_lock_t lock (ft_font->lock);
       FT_Face ft_face = ft_font->ft_face;
     
    -  _hb_ft_hb_font_check_changed (font, ft_font);
    -
       if (unlikely (FT_Load_Glyph (ft_face, glyph,
                                    FT_LOAD_NO_BITMAP | ft_font->load_flags)))
         return;
    @@ -694,22 +864,139 @@ hb_ft_get_glyph_shape (hb_font_t *font HB_UNUSED,
         return;
     
       const FT_Outline_Funcs outline_funcs = {
    -    (FT_Outline_MoveToFunc) _hb_ft_move_to,
    -    (FT_Outline_LineToFunc) _hb_ft_line_to,
    -    (FT_Outline_ConicToFunc) _hb_ft_conic_to,
    -    (FT_Outline_CubicToFunc) _hb_ft_cubic_to,
    +    _hb_ft_move_to,
    +    _hb_ft_line_to,
    +    _hb_ft_conic_to,
    +    _hb_ft_cubic_to,
         0, /* shift */
         0, /* delta */
       };
     
       hb_draw_session_t draw_session (draw_funcs, draw_data, font->slant_xy);
     
    +  /* Embolden */
    +  if (font->x_strength || font->y_strength)
    +  {
    +    FT_Outline_EmboldenXY (&ft_face->glyph->outline, font->x_strength, font->y_strength);
    +
    +    int x_shift = 0;
    +    int y_shift = 0;
    +    if (font->embolden_in_place)
    +    {
    +      /* Undo the FreeType shift. */
    +      x_shift = -font->x_strength / 2;
    +      y_shift = 0;
    +      if (font->y_scale < 0) y_shift = -font->y_strength;
    +    }
    +    else
    +    {
    +      /* FreeType applied things in the wrong direction for negative scale; fix up. */
    +      if (font->x_scale < 0) x_shift = -font->x_strength;
    +      if (font->y_scale < 0) y_shift = -font->y_strength;
    +    }
    +    if (x_shift || y_shift)
    +    {
    +      auto &outline = ft_face->glyph->outline;
    +      for (auto &point : hb_iter (outline.points, outline.contours[outline.n_contours - 1] + 1))
    +      {
    +        point.x += x_shift;
    +        point.y += y_shift;
    +      }
    +    }
    +  }
    +
    +
       FT_Outline_Decompose (&ft_face->glyph->outline,
                             &outline_funcs,
                             &draw_session);
     }
     #endif
     
    +#ifndef HB_NO_PAINT
    +#if (FREETYPE_MAJOR*10000 + FREETYPE_MINOR*100 + FREETYPE_PATCH) >= 21300
    +
    +#include "hb-ft-colr.hh"
    +
    +static void
    +hb_ft_paint_glyph (hb_font_t *font,
    +                   void *font_data,
    +                   hb_codepoint_t gid,
    +                   hb_paint_funcs_t *paint_funcs, void *paint_data,
    +                   unsigned int palette_index,
    +                   hb_color_t foreground,
    +                   void *user_data)
    +{
    +  const hb_ft_font_t *ft_font = (const hb_ft_font_t *) font_data;
    +  hb_lock_t lock (ft_font->lock);
    +  FT_Face ft_face = ft_font->ft_face;
    +
    +  /* We release the lock before calling into glyph callbacks, such that
    +   * eg. draw API can call back into the face.*/
    +
    +  if (unlikely (FT_Load_Glyph (ft_face, gid,
    +                               ft_font->load_flags | FT_LOAD_COLOR)))
    +    return;
    +
    +  if (ft_face->glyph->format == FT_GLYPH_FORMAT_OUTLINE)
    +  {
    +    if (hb_ft_paint_glyph_colr (font, font_data, gid,
    +                                paint_funcs, paint_data,
    +                                palette_index, foreground,
    +                                user_data))
    +      return;
    +
    +    /* Simple outline. */
    +    ft_font->lock.unlock ();
    +    paint_funcs->push_clip_glyph (paint_data, gid, font);
    +    ft_font->lock.lock ();
    +    paint_funcs->color (paint_data, true, foreground);
    +    paint_funcs->pop_clip (paint_data);
    +
    +    return;
    +  }
    +
    +  auto *glyph = ft_face->glyph;
    +  if (glyph->format == FT_GLYPH_FORMAT_BITMAP)
    +  {
    +    auto &bitmap = glyph->bitmap;
    +    if (bitmap.pixel_mode == FT_PIXEL_MODE_BGRA)
    +    {
    +      if (bitmap.pitch != (signed) bitmap.width * 4)
    +        return;
    +
    +      ft_font->lock.unlock ();
    +
    +      hb_blob_t *blob = hb_blob_create ((const char *) bitmap.buffer,
    +                                        bitmap.pitch * bitmap.rows,
    +                                        HB_MEMORY_MODE_DUPLICATE,
    +                                        nullptr, nullptr);
    +
    +      hb_glyph_extents_t extents;
    +      if (!hb_font_get_glyph_extents (font, gid, &extents))
    +        goto out;
    +
    +      if (!paint_funcs->image (paint_data,
    +                               blob,
    +                               bitmap.width,
    +                               bitmap.rows,
    +                               HB_PAINT_IMAGE_FORMAT_BGRA,
    +                               font->slant_xy,
    +                               &extents))
    +      {
    +        /* TODO Try a forced outline load and paint? */
    +      }
    +
    +    out:
    +      hb_blob_destroy (blob);
    +      ft_font->lock.lock ();
    +    }
    +
    +    return;
    +  }
    +}
    +#endif
    +#endif
    +
     
     static inline void free_static_ft_funcs ();
     
    @@ -743,7 +1030,13 @@ static struct hb_ft_font_funcs_lazy_loader_t : hb_font_funcs_lazy_loader_t= 21300
    +    hb_font_funcs_set_paint_glyph_func (funcs, hb_ft_paint_glyph, nullptr, nullptr);
    +#endif
     #endif
     
         hb_font_funcs_make_immutable (funcs);
    @@ -818,6 +1111,10 @@ _hb_ft_reference_table (hb_face_t *face HB_UNUSED, hb_tag_t tag, void *user_data
      *
      * Creates an #hb_face_t face object from the specified FT_Face.
      *
    + * Note that this is using the FT_Face object just to get at the underlying
    + * font data, and fonts created from the returned #hb_face_t will use the native
    + * HarfBuzz font implementation, unless you call hb_ft_font_set_funcs() on them.
    + *
      * This variant of the function does not provide any life-cycle management.
      *
      * Most client programs should use hb_ft_face_create_referenced()
    @@ -862,6 +1159,10 @@ hb_ft_face_create (FT_Face           ft_face,
      *
      * Creates an #hb_face_t face object from the specified FT_Face.
      *
    + * Note that this is using the FT_Face object just to get at the underlying
    + * font data, and fonts created from the returned #hb_face_t will use the native
    + * HarfBuzz font implementation, unless you call hb_ft_font_set_funcs() on them.
    + *
      * This is the preferred variant of the hb_ft_face_create*
      * function family, because it calls FT_Reference_Face() on @ft_face,
      * ensuring that @ft_face remains alive as long as the resulting
    @@ -882,8 +1183,9 @@ hb_ft_face_create_referenced (FT_Face ft_face)
     }
     
     static void
    -hb_ft_face_finalize (FT_Face ft_face)
    +hb_ft_face_finalize (void *arg)
     {
    +  FT_Face ft_face = (FT_Face) arg;
       hb_face_destroy ((hb_face_t *) ft_face->generic.data);
     }
     
    @@ -893,6 +1195,10 @@ hb_ft_face_finalize (FT_Face ft_face)
      *
      * Creates an #hb_face_t face object from the specified FT_Face.
      *
    + * Note that this is using the FT_Face object just to get at the underlying
    + * font data, and fonts created from the returned #hb_face_t will use the native
    + * HarfBuzz font implementation, unless you call hb_ft_font_set_funcs() on them.
    + *
      * This variant of the function caches the newly created #hb_face_t
      * face object, using the @generic pointer of @ft_face. Subsequent function
      * calls that are passed the same @ft_face parameter will have the same
    @@ -915,7 +1221,7 @@ hb_ft_face_create_cached (FT_Face ft_face)
           ft_face->generic.finalizer (ft_face);
     
         ft_face->generic.data = hb_ft_face_create (ft_face, nullptr);
    -    ft_face->generic.finalizer = (FT_Generic_Finalizer) hb_ft_face_finalize;
    +    ft_face->generic.finalizer = hb_ft_face_finalize;
       }
     
       return hb_face_reference ((hb_face_t *) ft_face->generic.data);
    @@ -1028,6 +1334,9 @@ hb_ft_font_changed (hb_font_t *font)
     #endif
       }
     #endif
    +
    +  ft_font->advance_cache.clear ();
    +  ft_font->cached_serial = font->serial;
     }
     
     /**
    @@ -1121,8 +1430,9 @@ get_ft_library ()
     }
     
     static void
    -_release_blob (FT_Face ft_face)
    +_release_blob (void *arg)
     {
    +  FT_Face ft_face = (FT_Face) arg;
       hb_blob_destroy ((hb_blob_t *) ft_face->generic.data);
     }
     
    @@ -1139,10 +1449,14 @@ _release_blob (FT_Face ft_face)
      * created with hb_face_create(), and therefore was not
      * initially configured to use FreeType font functions.
      *
    - * An #hb_face_t face object created with hb_ft_face_create()
    + * An #hb_font_t object created with hb_ft_font_create()
      * is preconfigured for FreeType font functions and does not
      * require this function to be used.
      *
    + * Note that if you modify the underlying #hb_font_t after
    + * calling this function, you need to call hb_ft_hb_font_changed()
    + * to update the underlying FT_Face.
    + *
      * Note: Internally, this function creates an FT_Face.
     * 
      *
    @@ -1173,14 +1487,14 @@ hb_ft_font_set_funcs (hb_font_t *font)
       if (FT_Select_Charmap (ft_face, FT_ENCODING_MS_SYMBOL))
         FT_Select_Charmap (ft_face, FT_ENCODING_UNICODE);
     
    -  _hb_ft_hb_font_changed (font, ft_face);
     
       ft_face->generic.data = blob;
    -  ft_face->generic.finalizer = (FT_Generic_Finalizer) _release_blob;
    +  ft_face->generic.finalizer = _release_blob;
     
       _hb_ft_font_set_funcs (font, ft_face, true);
       hb_ft_font_set_load_flags (font, FT_LOAD_DEFAULT | FT_LOAD_NO_HINTING);
    -}
     
    +  _hb_ft_hb_font_changed (font, ft_face);
    +}
     
     #endif
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-iter.hh b/src/java.desktop/share/native/libharfbuzz/hb-iter.hh
    index ff7a4c75895e1..bcd4eb8ebc4f2 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-iter.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-iter.hh
    @@ -73,8 +73,10 @@ struct hb_iter_t
       /* Operators. */
       iter_t iter () const { return *thiz(); }
       iter_t operator + () const { return *thiz(); }
    -  iter_t begin () const { return *thiz(); }
    -  iter_t end () const { return thiz()->__end__ (); }
    +  iter_t _begin () const { return *thiz(); }
    +  iter_t begin () const { return _begin (); }
    +  iter_t _end () const { return thiz()->__end__ (); }
    +  iter_t end () const { return _end (); }
       explicit operator bool () const { return thiz()->__more__ (); }
       unsigned len () const { return thiz()->__len__ (); }
       /* The following can only be enabled if item_t is reference type.  Otherwise
    @@ -118,7 +120,9 @@ struct hb_iter_t
     
     #define HB_ITER_USING(Name) \
       using item_t = typename Name::item_t; \
    +  using Name::_begin; \
       using Name::begin; \
    +  using Name::_end; \
       using Name::end; \
       using Name::get_item_size; \
       using Name::is_iterator; \
    @@ -168,10 +172,16 @@ struct
     HB_FUNCOBJ (hb_iter);
     struct
     {
    -  template  unsigned
    -  operator () (T&& c) const
    -  { return c.len (); }
    +  template  auto
    +  impl (T&& c, hb_priority<1>) const HB_RETURN (unsigned, c.len ())
    +
    +  template  auto
    +  impl (T&& c, hb_priority<0>) const HB_RETURN (unsigned, c.len)
    +
    +  public:
     
    +  template  auto
    +  operator () (T&& c) const HB_RETURN (unsigned, impl (std::forward (c), hb_prioritize))
     }
     HB_FUNCOBJ (hb_len);
     
    @@ -253,6 +263,8 @@ struct hb_is_iterator_of
     };
     #define hb_is_iterator_of(Iter, Item) hb_is_iterator_of::value
     #define hb_is_iterator(Iter) hb_is_iterator_of (Iter, typename Iter::item_t)
    +#define hb_is_sorted_iterator_of(Iter, Item) (hb_is_iterator_of::value && Iter::is_sorted_iterator)
    +#define hb_is_sorted_iterator(Iter) hb_is_sorted_iterator_of (Iter, typename Iter::item_t)
     
     /* hb_is_iterable() */
     
    @@ -375,7 +387,7 @@ struct hb_map_iter_t :
       void __forward__ (unsigned n) { it += n; }
       void __prev__ () { --it; }
       void __rewind__ (unsigned n) { it -= n; }
    -  hb_map_iter_t __end__ () const { return hb_map_iter_t (it.end (), f); }
    +  hb_map_iter_t __end__ () const { return hb_map_iter_t (it._end (), f); }
       bool operator != (const hb_map_iter_t& o) const
       { return it != o.it; }
     
    @@ -438,7 +450,7 @@ struct hb_filter_iter_t :
       bool __more__ () const { return bool (it); }
       void __next__ () { do ++it; while (it && !hb_has (p.get (), hb_get (f.get (), *it))); }
       void __prev__ () { do --it; while (it && !hb_has (p.get (), hb_get (f.get (), *it))); }
    -  hb_filter_iter_t __end__ () const { return hb_filter_iter_t (it.end (), p, f); }
    +  hb_filter_iter_t __end__ () const { return hb_filter_iter_t (it._end (), p, f); }
       bool operator != (const hb_filter_iter_t& o) const
       { return it != o.it; }
     
    @@ -551,7 +563,7 @@ struct hb_zip_iter_t :
       void __forward__ (unsigned n) { a += n; b += n; }
       void __prev__ () { --a; --b; }
       void __rewind__ (unsigned n) { a -= n; b -= n; }
    -  hb_zip_iter_t __end__ () const { return hb_zip_iter_t (a.end (), b.end ()); }
    +  hb_zip_iter_t __end__ () const { return hb_zip_iter_t (a._end (), b._end ()); }
       /* Note, we should stop if ANY of the iters reaches end.  As such two compare
        * unequal if both items are unequal, NOT if either is unequal. */
       bool operator != (const hb_zip_iter_t& o) const
    @@ -635,7 +647,7 @@ struct hb_concat_iter_t :
         }
       }
     
    -  hb_concat_iter_t __end__ () const { return hb_concat_iter_t (a.end (), b.end ()); }
    +  hb_concat_iter_t __end__ () const { return hb_concat_iter_t (a._end (), b._end ()); }
       bool operator != (const hb_concat_iter_t& o) const
       {
         return a != o.a
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-limits.hh b/src/java.desktop/share/native/libharfbuzz/hb-limits.hh
    new file mode 100644
    index 0000000000000..0f60e9e21013b
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-limits.hh
    @@ -0,0 +1,109 @@
    +/*
    + * Copyright © 2022  Behdad Esfahbod
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + */
    +
    +#ifndef HB_LIMITS_HH
    +#define HB_LIMITS_HH
    +
    +#include "hb.hh"
    +
    +
    +#ifndef HB_BUFFER_MAX_LEN_FACTOR
    +#define HB_BUFFER_MAX_LEN_FACTOR 64
    +#endif
    +#ifndef HB_BUFFER_MAX_LEN_MIN
    +#define HB_BUFFER_MAX_LEN_MIN 16384
    +#endif
    +#ifndef HB_BUFFER_MAX_LEN_DEFAULT
    +#define HB_BUFFER_MAX_LEN_DEFAULT 0x3FFFFFFF /* Shaping more than a billion chars? Let us know! */
    +#endif
    +
    +#ifndef HB_BUFFER_MAX_OPS_FACTOR
    +#define HB_BUFFER_MAX_OPS_FACTOR 1024
    +#endif
    +#ifndef HB_BUFFER_MAX_OPS_MIN
    +#define HB_BUFFER_MAX_OPS_MIN 16384
    +#endif
    +#ifndef HB_BUFFER_MAX_OPS_DEFAULT
    +#define HB_BUFFER_MAX_OPS_DEFAULT 0x1FFFFFFF /* Shaping more than a billion operations? Let us know! */
    +#endif
    +
    +
    +#ifndef HB_MAX_NESTING_LEVEL
    +#define HB_MAX_NESTING_LEVEL 64
    +#endif
    +
    +
    +#ifndef HB_MAX_CONTEXT_LENGTH
    +#define HB_MAX_CONTEXT_LENGTH 64
    +#endif
    +
    +#ifndef HB_CLOSURE_MAX_STAGES
    +/*
    + * The maximum number of times a lookup can be applied during shaping.
    + * Used to limit the number of iterations of the closure algorithm.
    + * This must be larger than the number of times add_gsub_pause() is
    + * called in a collect_features call of any shaper.
    + */
    +#define HB_CLOSURE_MAX_STAGES 12
    +#endif
    +
    +#ifndef HB_MAX_SCRIPTS
    +#define HB_MAX_SCRIPTS 500
    +#endif
    +
    +#ifndef HB_MAX_LANGSYS
    +#define HB_MAX_LANGSYS 2000
    +#endif
    +
    +#ifndef HB_MAX_LANGSYS_FEATURE_COUNT
    +#define HB_MAX_LANGSYS_FEATURE_COUNT 50000
    +#endif
    +
    +#ifndef HB_MAX_FEATURE_INDICES
    +#define HB_MAX_FEATURE_INDICES 1500
    +#endif
    +
    +#ifndef HB_MAX_LOOKUP_VISIT_COUNT
    +#define HB_MAX_LOOKUP_VISIT_COUNT 35000
    +#endif
    +
    +
    +#ifndef HB_GLYF_MAX_POINTS
    +#define HB_GLYF_MAX_POINTS 20000
    +#endif
    +
    +#ifndef HB_GLYF_MAX_EDGE_COUNT
    +#define HB_GLYF_MAX_EDGE_COUNT 1024
    +#endif
    +
    +#ifndef HB_CFF_MAX_OPS
    +#define HB_CFF_MAX_OPS 10000
    +#endif
    +
    +#ifndef HB_COLRV1_MAX_EDGE_COUNT
    +#define HB_COLRV1_MAX_EDGE_COUNT 1024
    +#endif
    +
    +
    +#endif /* HB_LIMITS_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-machinery.hh b/src/java.desktop/share/native/libharfbuzz/hb-machinery.hh
    index 5938a25504ad5..3a048fc124265 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-machinery.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-machinery.hh
    @@ -34,7 +34,6 @@
     
     #include "hb-dispatch.hh"
     #include "hb-sanitize.hh"
    -#include "hb-serialize.hh"
     
     
     /*
    @@ -136,6 +135,13 @@ static inline Type& StructAfter(TObject &X)
     
     /*
      * Lazy loaders.
    + *
    + * The lazy-loaders are thread-safe pointer-like objects that create their
    + * instead on-demand.  They also support access to a "data" object that is
    + * necessary for creating their instance.  The data object, if specified,
    + * is accessed via pointer math, located at a location before the position
    + * of the loader itself.  This avoids having to store a pointer to data
    + * for every lazy-loader.  Multiple lazy-loaders can access the same data.
      */
     
     template 
    @@ -176,12 +182,12 @@ struct hb_lazy_loader_t : hb_data_wrapper_t
     
       void init0 () {} /* Init, when memory is already set to 0. No-op for us. */
       void init ()  { instance.set_relaxed (nullptr); }
    -  void fini ()  { do_destroy (instance.get ()); init (); }
    +  void fini ()  { do_destroy (instance.get_acquire ()); init (); }
     
       void free_instance ()
       {
       retry:
    -    Stored *p = instance.get ();
    +    Stored *p = instance.get_acquire ();
         if (unlikely (p && !cmpexch (p, nullptr)))
           goto retry;
         do_destroy (p);
    @@ -203,7 +209,7 @@ struct hb_lazy_loader_t : hb_data_wrapper_t
       Stored * get_stored () const
       {
       retry:
    -    Stored *p = this->instance.get ();
    +    Stored *p = this->instance.get_acquire ();
         if (unlikely (!p))
         {
           if (unlikely (this->is_inert ()))
    @@ -228,7 +234,8 @@ struct hb_lazy_loader_t : hb_data_wrapper_t
     
       bool cmpexch (Stored *current, Stored *value) const
       {
    -    /* This *must* be called when there are no other threads accessing. */
    +    /* This function can only be safely called directly if no
    +     * other thread is accessing. */
         return this->instance.cmpexch (current, value);
       }
     
    @@ -261,7 +268,7 @@ struct hb_lazy_loader_t : hb_data_wrapper_t
         hb_free (p);
       }
     
    -//  private:
    +  private:
       /* Must only have one pointer. */
       hb_atomic_ptr_t instance;
     };
    @@ -283,7 +290,7 @@ struct hb_table_lazy_loader_t : hb_lazy_loader_t (face);
       }
       static void destroy (hb_blob_t *p) { hb_blob_destroy (p); }
    @@ -297,22 +304,22 @@ struct hb_table_lazy_loader_t : hb_lazy_loader_tget_stored (); }
     };
     
    -template 
    -struct hb_font_funcs_lazy_loader_t : hb_lazy_loader_t
    -{
    -  static void destroy (hb_font_funcs_t *p)
    -  { hb_font_funcs_destroy (p); }
    -  static const hb_font_funcs_t *get_null ()
    -  { return hb_font_funcs_get_empty (); }
    -};
    -template 
    -struct hb_unicode_funcs_lazy_loader_t : hb_lazy_loader_t
    -{
    -  static void destroy (hb_unicode_funcs_t *p)
    -  { hb_unicode_funcs_destroy (p); }
    -  static const hb_unicode_funcs_t *get_null ()
    -  { return hb_unicode_funcs_get_empty (); }
    -};
    +#define HB_DEFINE_TYPE_FUNCS_LAZY_LOADER_T(Type) \
    +  template  \
    +  struct hb_##Type##_funcs_lazy_loader_t : hb_lazy_loader_t \
    +  { \
    +    static void destroy (hb_##Type##_funcs_t *p) \
    +    { hb_##Type##_funcs_destroy (p); } \
    +    static const hb_##Type##_funcs_t *get_null () \
    +    { return hb_##Type##_funcs_get_empty (); } \
    +  }
    +
    +HB_DEFINE_TYPE_FUNCS_LAZY_LOADER_T (font);
    +HB_DEFINE_TYPE_FUNCS_LAZY_LOADER_T (unicode);
    +HB_DEFINE_TYPE_FUNCS_LAZY_LOADER_T (draw);
    +HB_DEFINE_TYPE_FUNCS_LAZY_LOADER_T (paint);
    +
    +#undef HB_DEFINE_TYPE_FUNCS_LAZY_LOADER_T
     
     
     #endif /* HB_MACHINERY_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-map.cc b/src/java.desktop/share/native/libharfbuzz/hb-map.cc
    index af449bea41102..48913a6a10df1 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-map.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-map.cc
    @@ -56,8 +56,6 @@ hb_map_create ()
       if (!(map = hb_object_create ()))
         return hb_map_get_empty ();
     
    -  map->init_shallow ();
    -
       return map;
     }
     
    @@ -107,8 +105,6 @@ hb_map_destroy (hb_map_t *map)
     {
       if (!hb_object_destroy (map)) return;
     
    -  map->fini_shallow ();
    -
       hb_free (map);
     }
     
    @@ -122,7 +118,7 @@ hb_map_destroy (hb_map_t *map)
      *
      * Attaches a user-data key/data pair to the specified map.
      *
    - * Return value: %true if success, %false otherwise
    + * Return value: `true` if success, `false` otherwise
      *
      * Since: 1.7.7
      **/
    @@ -149,7 +145,7 @@ hb_map_set_user_data (hb_map_t           *map,
      * Since: 1.7.7
      **/
     void *
    -hb_map_get_user_data (hb_map_t           *map,
    +hb_map_get_user_data (const hb_map_t     *map,
                           hb_user_data_key_t *key)
     {
       return hb_object_get_user_data (map, key);
    @@ -162,7 +158,7 @@ hb_map_get_user_data (hb_map_t           *map,
      *
      * Tests whether memory allocation for a set was successful.
      *
    - * Return value: %true if allocation succeeded, %false otherwise
    + * Return value: `true` if allocation succeeded, `false` otherwise
      *
      * Since: 1.7.7
      **/
    @@ -178,7 +174,7 @@ hb_map_allocation_successful (const hb_map_t  *map)
      *
      * Allocate a copy of @map.
      *
    - * Return value: Newly-allocated map.
    + * Return value: (transfer full): Newly-allocated map.
      *
      * Since: 4.4.0
      **/
    @@ -186,9 +182,10 @@ hb_map_t *
     hb_map_copy (const hb_map_t *map)
     {
       hb_map_t *copy = hb_map_create ();
    -  if (unlikely (!copy)) return nullptr;
    -  copy->resize (map->population);
    -  hb_copy (*map, *copy);
    +  if (unlikely (copy->in_error ()))
    +    return hb_map_get_empty ();
    +
    +  *copy = *map;
       return copy;
     }
     
    @@ -251,7 +248,7 @@ hb_map_del (hb_map_t       *map,
      *
      * Tests whether @key is an element of @map.
      *
    - * Return value: %true if @key is found in @map, %false otherwise
    + * Return value: `true` if @key is found in @map, `false` otherwise
      *
      * Since: 1.7.7
      **/
    @@ -283,7 +280,7 @@ hb_map_clear (hb_map_t *map)
      *
      * Tests whether @map is empty (contains no elements).
      *
    - * Return value: %true if @map is empty
    + * Return value: `true` if @map is empty
      *
      * Since: 1.7.7
      **/
    @@ -317,7 +314,7 @@ hb_map_get_population (const hb_map_t *map)
      * Tests whether @map and @other are equal (contain the same
      * elements).
      *
    - * Return value: %true if the two maps are equal, %false otherwise.
    + * Return value: `true` if the two maps are equal, `false` otherwise.
      *
      * Since: 4.3.0
      **/
    @@ -339,9 +336,84 @@ hb_map_is_equal (const hb_map_t *map,
      *
      * Since: 4.4.0
      **/
    -HB_EXTERN unsigned int
    +unsigned int
     hb_map_hash (const hb_map_t *map)
     {
       return map->hash ();
     }
     
    +/**
    + * hb_map_update:
    + * @map: A map
    + * @other: Another map
    + *
    + * Add the contents of @other to @map.
    + *
    + * Since: 7.0.0
    + **/
    +HB_EXTERN void
    +hb_map_update (hb_map_t *map,
    +               const hb_map_t *other)
    +{
    +  map->update (*other);
    +}
    +
    +/**
    + * hb_map_next:
    + * @map: A map
    + * @idx: (inout): Iterator internal state
    + * @key: (out): Key retrieved
    + * @value: (out): Value retrieved
    + *
    + * Fetches the next key/value paire in @map.
    + *
    + * Set @idx to -1 to get started.
    + *
    + * If the map is modified during iteration, the behavior is undefined.
    + *
    + * The order in which the key/values are returned is undefined.
    + *
    + * Return value: `true` if there was a next value, `false` otherwise
    + *
    + * Since: 7.0.0
    + **/
    +hb_bool_t
    +hb_map_next (const hb_map_t *map,
    +             int *idx,
    +             hb_codepoint_t *key,
    +             hb_codepoint_t *value)
    +{
    +  return map->next (idx, key, value);
    +}
    +
    +/**
    + * hb_map_keys:
    + * @map: A map
    + * @keys: A set
    + *
    + * Add the keys of @map to @keys.
    + *
    + * Since: 7.0.0
    + **/
    +void
    +hb_map_keys (const hb_map_t *map,
    +             hb_set_t *keys)
    +{
    +  hb_copy (map->keys() , *keys);
    +}
    +
    +/**
    + * hb_map_values:
    + * @map: A map
    + * @values: A set
    + *
    + * Add the values of @map to @values.
    + *
    + * Since: 7.0.0
    + **/
    +void
    +hb_map_values (const hb_map_t *map,
    +               hb_set_t *values)
    +{
    +  hb_copy (map->values() , *values);
    +}
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-map.h b/src/java.desktop/share/native/libharfbuzz/hb-map.h
    index 27c19c2b5899f..12d8970f48f28 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-map.h
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-map.h
    @@ -32,6 +32,7 @@
     #define HB_MAP_H
     
     #include "hb-common.h"
    +#include "hb-set.h"
     
     HB_BEGIN_DECLS
     
    @@ -74,7 +75,7 @@ hb_map_set_user_data (hb_map_t           *map,
                           hb_bool_t           replace);
     
     HB_EXTERN void *
    -hb_map_get_user_data (hb_map_t           *map,
    +hb_map_get_user_data (const hb_map_t     *map,
                           hb_user_data_key_t *key);
     
     
    @@ -118,6 +119,24 @@ HB_EXTERN hb_bool_t
     hb_map_has (const hb_map_t *map,
                 hb_codepoint_t  key);
     
    +HB_EXTERN void
    +hb_map_update (hb_map_t *map,
    +               const hb_map_t *other);
    +
    +/* Pass -1 in for idx to get started. */
    +HB_EXTERN hb_bool_t
    +hb_map_next (const hb_map_t *map,
    +             int *idx,
    +             hb_codepoint_t *key,
    +             hb_codepoint_t *value);
    +
    +HB_EXTERN void
    +hb_map_keys (const hb_map_t *map,
    +             hb_set_t *keys);
    +
    +HB_EXTERN void
    +hb_map_values (const hb_map_t *map,
    +               hb_set_t *values);
     
     HB_END_DECLS
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-map.hh b/src/java.desktop/share/native/libharfbuzz/hb-map.hh
    index ff5a476d7a281..3b24dc9455ac4 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-map.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-map.hh
    @@ -29,6 +29,8 @@
     
     #include "hb.hh"
     
    +#include "hb-set.hh"
    +
     
     /*
      * hb_hashmap_t
    @@ -43,9 +45,9 @@ struct hb_hashmap_t
       hb_hashmap_t ()  { init (); }
       ~hb_hashmap_t () { fini (); }
     
    -  hb_hashmap_t (const hb_hashmap_t& o) : hb_hashmap_t () { resize (population); hb_copy (o, *this); }
    +  hb_hashmap_t (const hb_hashmap_t& o) : hb_hashmap_t () { resize (o.population); hb_copy (o, *this); }
       hb_hashmap_t (hb_hashmap_t&& o) : hb_hashmap_t () { hb_swap (*this, o); }
    -  hb_hashmap_t& operator= (const hb_hashmap_t& o)  { resize (population); hb_copy (o, *this); return *this; }
    +  hb_hashmap_t& operator= (const hb_hashmap_t& o)  { reset (); resize (o.population); hb_copy (o, *this); return *this; }
       hb_hashmap_t& operator= (hb_hashmap_t&& o)  { hb_swap (*this, o); return *this; }
     
       hb_hashmap_t (std::initializer_list> lst) : hb_hashmap_t ()
    @@ -71,6 +73,11 @@ struct hb_hashmap_t
         uint32_t is_tombstone_ : 1;
         V value;
     
    +    item_t () : key (),
    +                hash (0),
    +                is_used_ (false), is_tombstone_ (false),
    +                value () {}
    +
         bool is_used () const { return is_used_; }
         void set_used (bool is_used) { is_used_ = is_used; }
         bool is_tombstone () const { return is_tombstone_; }
    @@ -88,17 +95,8 @@ struct hb_hashmap_t
           return minus_1;
         };
     
    -    void clear ()
    -    {
    -      new (std::addressof (key)) K ();
    -      new (std::addressof (value)) V ();
    -      hash = 0;
    -      is_used_ = false;
    -      is_tombstone_ = false;
    -    }
    -
    -    bool operator == (const K &o) { return hb_deref (key) == hb_deref (o); }
    -    bool operator == (const item_t &o) { return *this == o.key; }
    +    bool operator == (const K &o) const { return hb_deref (key) == hb_deref (o); }
    +    bool operator == (const item_t &o) const { return *this == o.key; }
         hb_pair_t get_pair() const { return hb_pair_t (key, value); }
         hb_pair_t get_pair_ref() const { return hb_pair_t (key, value); }
     
    @@ -107,8 +105,8 @@ struct hb_hashmap_t
       };
     
       hb_object_header_t header;
    -  bool successful; /* Allocations successful */
    -  unsigned int population; /* Not including tombstones. */
    +  unsigned int successful : 1; /* Allocations successful */
    +  unsigned int population : 31; /* Not including tombstones. */
       unsigned int occupancy; /* Including tombstones. */
       unsigned int mask;
       unsigned int prime;
    @@ -118,27 +116,29 @@ struct hb_hashmap_t
       {
         if (unlikely (!a.successful || !b.successful))
           return;
    -    hb_swap (a.population, b.population);
    +    unsigned tmp = a.population;
    +    a.population = b.population;
    +    b.population = tmp;
    +    //hb_swap (a.population, b.population);
         hb_swap (a.occupancy, b.occupancy);
         hb_swap (a.mask, b.mask);
         hb_swap (a.prime, b.prime);
         hb_swap (a.items, b.items);
       }
    -  void init_shallow ()
    +  void init ()
       {
    +    hb_object_init (this);
    +
         successful = true;
         population = occupancy = 0;
         mask = 0;
         prime = 0;
         items = nullptr;
       }
    -  void init ()
    -  {
    -    hb_object_init (this);
    -    init_shallow ();
    -  }
    -  void fini_shallow ()
    +  void fini ()
       {
    +    hb_object_fini (this);
    +
         if (likely (items)) {
           unsigned size = mask + 1;
           for (unsigned i = 0; i < size; i++)
    @@ -148,11 +148,6 @@ struct hb_hashmap_t
         }
         population = occupancy = 0;
       }
    -  void fini ()
    -  {
    -    hb_object_fini (this);
    -    fini_shallow ();
    -  }
     
       void reset ()
       {
    @@ -166,7 +161,9 @@ struct hb_hashmap_t
       {
         if (unlikely (!successful)) return false;
     
    -    unsigned int power = hb_bit_storage (hb_max (population, new_population) * 2 + 8);
    +    if (new_population != 0 && (new_population + new_population / 2) < mask) return true;
    +
    +    unsigned int power = hb_bit_storage (hb_max ((unsigned) population, new_population) * 2 + 8);
         unsigned int new_size = 1u << power;
         item_t *new_items = (item_t *) hb_malloc ((size_t) new_size * sizeof (item_t));
         if (unlikely (!new_items))
    @@ -175,9 +172,9 @@ struct hb_hashmap_t
           return false;
         }
         for (auto &_ : hb_iter (new_items, new_size))
    -      _.clear ();
    +      new (&_) item_t ();
     
    -    unsigned int old_size = mask + 1;
    +    unsigned int old_size = size ();
         item_t *old_items = items;
     
         /* Switch to new, empty, array. */
    @@ -187,67 +184,102 @@ struct hb_hashmap_t
         items = new_items;
     
         /* Insert back old items. */
    -    if (old_items)
    -      for (unsigned int i = 0; i < old_size; i++)
    +    for (unsigned int i = 0; i < old_size; i++)
    +    {
    +      if (old_items[i].is_real ())
           {
    -        if (old_items[i].is_real ())
    -        {
    -          set_with_hash (old_items[i].key,
    -                         old_items[i].hash,
    -                         std::move (old_items[i].value));
    -        }
    -        old_items[i].~item_t ();
    +        set_with_hash (std::move (old_items[i].key),
    +                       old_items[i].hash,
    +                       std::move (old_items[i].value));
           }
    +      old_items[i].~item_t ();
    +    }
     
         hb_free (old_items);
     
         return true;
       }
     
    +  template 
    +  bool set_with_hash (KK&& key, uint32_t hash, VV&& value, bool is_delete=false)
    +  {
    +    if (unlikely (!successful)) return false;
    +    if (unlikely ((occupancy + occupancy / 2) >= mask && !resize ())) return false;
    +    item_t &item = item_for_hash (key, hash);
    +
    +    if (is_delete && !(item == key))
    +      return true; /* Trying to delete non-existent key. */
    +
    +    if (item.is_used ())
    +    {
    +      occupancy--;
    +      if (!item.is_tombstone ())
    +        population--;
    +    }
    +
    +    item.key = std::forward (key);
    +    item.value = std::forward (value);
    +    item.hash = hash;
    +    item.set_used (true);
    +    item.set_tombstone (is_delete);
    +
    +    occupancy++;
    +    if (!is_delete)
    +      population++;
    +
    +    return true;
    +  }
    +
    +  template 
    +  bool set (const K &key, VV&& value) { return set_with_hash (key, hb_hash (key), std::forward (value)); }
       template 
    -  bool set (K key, VV&& value) { return set_with_hash (key, hb_hash (key), std::forward (value)); }
    +  bool set (K &&key, VV&& value) { return set_with_hash (std::move (key), hb_hash (key), std::forward (value)); }
     
    -  const V& get (K key) const
    +  const V& get_with_hash (const K &key, uint32_t hash) const
       {
         if (unlikely (!items)) return item_t::default_value ();
    -    unsigned int i = bucket_for (key);
    -    return items[i].is_real () && items[i] == key ? items[i].value : item_t::default_value ();
    +    auto &item = item_for_hash (key, hash);
    +    return item.is_real () && item == key ? item.value : item_t::default_value ();
    +  }
    +  const V& get (const K &key) const
    +  {
    +    if (unlikely (!items)) return item_t::default_value ();
    +    return get_with_hash (key, hb_hash (key));
       }
     
    -  void del (K key) { set_with_hash (key, hb_hash (key), item_t::default_value (), true); }
    +  void del (const K &key) { set_with_hash (key, hb_hash (key), item_t::default_value (), true); }
     
       /* Has interface. */
    -  typedef const V& value_t;
    -  value_t operator [] (K k) const { return get (k); }
    -  bool has (K key, const V **vp = nullptr) const
    +  const V& operator [] (K k) const { return get (k); }
    +  template 
    +  bool has (K key, VV **vp = nullptr) const
       {
         if (unlikely (!items))
    -    {
    -      if (vp) *vp = &item_t::default_value ();
           return false;
    -    }
    -    unsigned int i = bucket_for (key);
    -    if (items[i].is_real () && items[i] == key)
    +    auto &item = item_for_hash (key, hb_hash (key));
    +    if (item.is_real () && item == key)
         {
    -      if (vp) *vp = &items[i].value;
    +      if (vp) *vp = std::addressof (item.value);
           return true;
         }
         else
    -    {
    -      if (vp) *vp = &item_t::default_value ();
           return false;
    -    }
       }
       /* Projection. */
       V operator () (K k) const { return get (k); }
     
    +  unsigned size () const { return mask ? mask + 1 : 0; }
    +
       void clear ()
       {
         if (unlikely (!successful)) return;
     
    -    if (items)
    -      for (auto &_ : hb_iter (items, mask + 1))
    -        _.clear ();
    +    for (auto &_ : hb_iter (items, size ()))
    +    {
    +      /* Reconstruct items. */
    +      _.~item_t ();
    +      new (&_) item_t ();
    +    }
     
         population = occupancy = 0;
       }
    @@ -257,11 +289,10 @@ struct hb_hashmap_t
     
       uint32_t hash () const
       {
    -    uint32_t h = 0;
    -    for (const auto &item : + hb_array (items, mask ? mask + 1 : 0)
    -                            | hb_filter (&item_t::is_real))
    -      h ^= item.total_hash ();
    -    return h;
    +    return
    +    + iter_items ()
    +    | hb_reduce ([] (uint32_t h, const item_t &_) { return h ^ _.total_hash (); }, (uint32_t) 0u)
    +    ;
       }
     
       bool is_equal (const hb_hashmap_t &other) const
    @@ -269,7 +300,7 @@ struct hb_hashmap_t
         if (population != other.population) return false;
     
         for (auto pair : iter ())
    -      if (get (pair.first) != pair.second)
    +      if (other.get (pair.first) != pair.second)
             return false;
     
         return true;
    @@ -279,78 +310,90 @@ struct hb_hashmap_t
     
       unsigned int get_population () const { return population; }
     
    +  void update (const hb_hashmap_t &other)
    +  {
    +    if (unlikely (!successful)) return;
    +
    +    hb_copy (other, *this);
    +  }
    +
       /*
        * Iterator
        */
    -  auto iter () const HB_AUTO_RETURN
    +
    +  auto iter_items () const HB_AUTO_RETURN
       (
    -    + hb_array (items, mask ? mask + 1 : 0)
    +    + hb_iter (items, size ())
         | hb_filter (&item_t::is_real)
    -    | hb_map (&item_t::get_pair)
       )
       auto iter_ref () const HB_AUTO_RETURN
       (
    -    + hb_array (items, mask ? mask + 1 : 0)
    -    | hb_filter (&item_t::is_real)
    +    + iter_items ()
         | hb_map (&item_t::get_pair_ref)
       )
    +  auto iter () const HB_AUTO_RETURN
    +  (
    +    + iter_items ()
    +    | hb_map (&item_t::get_pair)
    +  )
    +  auto keys_ref () const HB_AUTO_RETURN
    +  (
    +    + iter_items ()
    +    | hb_map (&item_t::key)
    +  )
       auto keys () const HB_AUTO_RETURN
       (
    -    + hb_array (items, mask ? mask + 1 : 0)
    -    | hb_filter (&item_t::is_real)
    +    + iter_items ()
         | hb_map (&item_t::key)
         | hb_map (hb_ridentity)
       )
    +  auto values_ref () const HB_AUTO_RETURN
    +  (
    +    + iter_items ()
    +    | hb_map (&item_t::value)
    +  )
       auto values () const HB_AUTO_RETURN
       (
    -    + hb_array (items, mask ? mask + 1 : 0)
    -    | hb_filter (&item_t::is_real)
    +    + iter_items ()
         | hb_map (&item_t::value)
         | hb_map (hb_ridentity)
       )
     
    -  /* Sink interface. */
    -  hb_hashmap_t& operator << (const hb_pair_t& v)
    -  { set (v.first, v.second); return *this; }
    -
    -  protected:
    -
    -  template 
    -  bool set_with_hash (K key, uint32_t hash, VV&& value, bool is_delete=false)
    +  /* C iterator. */
    +  bool next (int *idx,
    +             K *key,
    +             V *value) const
       {
    -    if (unlikely (!successful)) return false;
    -    if (unlikely ((occupancy + occupancy / 2) >= mask && !resize ())) return false;
    -    unsigned int i = bucket_for_hash (key, hash);
    +    unsigned i = (unsigned) (*idx + 1);
     
    -    if (is_delete && items[i].key != key)
    -      return true; /* Trying to delete non-existent key. */
    +    unsigned count = size ();
    +    while (i < count && !items[i].is_real ())
    +      i++;
     
    -    if (items[i].is_used ())
    +    if (i >= count)
         {
    -      occupancy--;
    -      if (!items[i].is_tombstone ())
    -        population--;
    +      *idx = -1;
    +      return false;
         }
     
    -    items[i].key = key;
    -    items[i].value = std::forward (value);
    -    items[i].hash = hash;
    -    items[i].set_used (true);
    -    items[i].set_tombstone (is_delete);
    -
    -    occupancy++;
    -    if (!is_delete)
    -      population++;
    +    *key = items[i].key;
    +    *value = items[i].value;
     
    +    *idx = (signed) i;
         return true;
       }
     
    -  unsigned int bucket_for (const K &key) const
    -  {
    -    return bucket_for_hash (key, hb_hash (key));
    -  }
    -
    -  unsigned int bucket_for_hash (const K &key, uint32_t hash) const
    +  /* Sink interface. */
    +  hb_hashmap_t& operator << (const hb_pair_t& v)
    +  { set (v.first, v.second); return *this; }
    +  hb_hashmap_t& operator << (const hb_pair_t& v)
    +  { set (v.first, std::move (v.second)); return *this; }
    +  hb_hashmap_t& operator << (const hb_pair_t& v)
    +  { set (std::move (v.first), v.second); return *this; }
    +  hb_hashmap_t& operator << (const hb_pair_t& v)
    +  { set (std::move (v.first), std::move (v.second)); return *this; }
    +
    +  item_t& item_for_hash (const K &key, uint32_t hash) const
       {
         hash &= 0x3FFFFFFF; // We only store lower 30bit of hash
         unsigned int i = hash % prime;
    @@ -359,12 +402,12 @@ struct hb_hashmap_t
         while (items[i].is_used ())
         {
           if (items[i].hash == hash && items[i] == key)
    -        return i;
    +        return items[i];
           if (tombstone == (unsigned) -1 && items[i].is_tombstone ())
             tombstone = i;
           i = (i + ++step) & mask;
         }
    -    return tombstone == (unsigned) -1 ? i : tombstone;
    +    return items[tombstone == (unsigned) -1 ? i : tombstone];
       }
     
       static unsigned int prime_for (unsigned int shift)
    @@ -443,4 +486,5 @@ struct hb_map_t : hb_hashmap_t auto _hb_try_add_pointer (hb_priority<1>) -> hb_type_ident
     template  using hb_add_pointer = decltype (_hb_try_add_pointer (hb_prioritize));
     
     
    -/* TODO Add feature-parity to std::decay. */
    -template  using hb_decay = hb_remove_const>;
    +template  using hb_decay = typename std::decay::type;
     
     #define hb_is_convertible(From,To) std::is_convertible::value
     
    @@ -133,6 +132,18 @@ struct
     
       template  constexpr auto
       operator () (T *v) const HB_AUTO_RETURN (*v)
    +
    +  template  constexpr auto
    +  operator () (const hb::shared_ptr& v) const HB_AUTO_RETURN (*v)
    +
    +  template  constexpr auto
    +  operator () (hb::shared_ptr& v) const HB_AUTO_RETURN (*v)
    +
    +  template  constexpr auto
    +  operator () (const hb::unique_ptr& v) const HB_AUTO_RETURN (*v)
    +
    +  template  constexpr auto
    +  operator () (hb::unique_ptr& v) const HB_AUTO_RETURN (*v)
     }
     HB_FUNCOBJ (hb_deref);
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-multimap.hh b/src/java.desktop/share/native/libharfbuzz/hb-multimap.hh
    new file mode 100644
    index 0000000000000..b4a8cc62a3e3c
    --- /dev/null
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-multimap.hh
    @@ -0,0 +1,92 @@
    +/*
    + * Copyright © 2022  Behdad Esfahbod
    + *
    + *  This is part of HarfBuzz, a text shaping library.
    + *
    + * Permission is hereby granted, without written agreement and without
    + * license or royalty fees, to use, copy, modify, and distribute this
    + * software and its documentation for any purpose, provided that the
    + * above copyright notice and the following two paragraphs appear in
    + * all copies of this software.
    + *
    + * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
    + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    + * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
    + * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
    + * DAMAGE.
    + *
    + * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
    + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
    + * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
    + * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
    + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
    + */
    +
    +#ifndef HB_MULTIMAP_HH
    +#define HB_MULTIMAP_HH
    +
    +#include "hb.hh"
    +#include "hb-map.hh"
    +#include "hb-vector.hh"
    +
    +
    +/*
    + * hb_multimap_t
    + */
    +
    +struct hb_multimap_t
    +{
    +  void add (hb_codepoint_t k, hb_codepoint_t v)
    +  {
    +    hb_codepoint_t *i;
    +    if (multiples_indices.has (k, &i))
    +    {
    +      multiples_values[*i].push (v);
    +      return;
    +    }
    +
    +    hb_codepoint_t *old_v;
    +    if (singulars.has (k, &old_v))
    +    {
    +      hb_codepoint_t old = *old_v;
    +      singulars.del (k);
    +
    +      multiples_indices.set (k, multiples_values.length);
    +      auto *vec = multiples_values.push ();
    +
    +      vec->push (old);
    +      vec->push (v);
    +
    +      return;
    +    }
    +
    +    singulars.set (k, v);
    +  }
    +
    +  hb_array_t get (hb_codepoint_t k) const
    +  {
    +    const hb_codepoint_t *v;
    +    if (singulars.has (k, &v))
    +      return hb_array (v, 1);
    +
    +    hb_codepoint_t *i;
    +    if (multiples_indices.has (k, &i))
    +      return multiples_values[*i].as_array ();
    +
    +    return hb_array_t ();
    +  }
    +
    +  bool in_error () const
    +  {
    +    return singulars.in_error () || multiples_indices.in_error () || multiples_values.in_error ();
    +  }
    +
    +  protected:
    +  hb_map_t singulars;
    +  hb_map_t multiples_indices;
    +  hb_vector_t> multiples_values;
    +};
    +
    +
    +
    +#endif /* HB_MULTIMAP_HH */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-mutex.hh b/src/java.desktop/share/native/libharfbuzz/hb-mutex.hh
    index 8b62afaf94ef5..72012bd2e641e 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-mutex.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-mutex.hh
    @@ -60,7 +60,7 @@ typedef pthread_mutex_t hb_mutex_impl_t;
     #elif !defined(HB_NO_MT) && !defined(HB_MUTEX_IMPL_STD_MUTEX) && defined(_WIN32)
     
     typedef CRITICAL_SECTION hb_mutex_impl_t;
    -#if !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
    +#if !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) && WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP)
     #define hb_mutex_impl_init(M)   InitializeCriticalSectionEx (M, 0, 0)
     #else
     #define hb_mutex_impl_init(M)   InitializeCriticalSection (M)
    @@ -97,6 +97,9 @@ struct hb_mutex_t
       /* Create space for, but do not initialize m. */
       alignas(hb_mutex_impl_t) char m[sizeof (hb_mutex_impl_t)];
     
    +  hb_mutex_t () { init (); }
    +  ~hb_mutex_t () { fini (); }
    +
     #pragma GCC diagnostic push
     #pragma GCC diagnostic ignored "-Wcast-align"
       void init   () { hb_mutex_impl_init   ((hb_mutex_impl_t *) m); }
    @@ -108,10 +111,11 @@ struct hb_mutex_t
     
     struct hb_lock_t
     {
    -  hb_lock_t (hb_mutex_t &mutex_) : mutex (mutex_) { mutex.lock (); }
    -  ~hb_lock_t () { mutex.unlock (); }
    +  hb_lock_t (hb_mutex_t &mutex_) : mutex (&mutex_) { mutex->lock (); }
    +  hb_lock_t (hb_mutex_t *mutex_) : mutex (mutex_) { if (mutex) mutex->lock (); }
    +  ~hb_lock_t () { if (mutex) mutex->unlock (); }
       private:
    -  hb_mutex_t &mutex;
    +  hb_mutex_t *mutex;
     };
     
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-null.hh b/src/java.desktop/share/native/libharfbuzz/hb-null.hh
    index 4403867f5c35d..8a9ebbfc2d758 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-null.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-null.hh
    @@ -39,6 +39,24 @@
     
     #define HB_NULL_POOL_SIZE 448
     
    +template 
    +struct _hb_has_min_size : hb_false_type {};
    +template 
    +struct _hb_has_min_size>
    +        : hb_true_type {};
    +template 
    +using hb_has_min_size = _hb_has_min_size;
    +#define hb_has_min_size(T) hb_has_min_size::value
    +
    +template 
    +struct _hb_has_null_size : hb_false_type {};
    +template 
    +struct _hb_has_null_size>
    +        : hb_true_type {};
    +template 
    +using hb_has_null_size = _hb_has_null_size;
    +#define hb_has_null_size(T) hb_has_null_size::value
    +
     /* Use SFINAE to sniff whether T has min_size; in which case return the larger
      * of sizeof(T) and T::null_size, otherwise return sizeof(T).
      *
    @@ -117,8 +135,19 @@ struct NullHelper
             }; \
             namespace Namespace { \
             static_assert (true, "") /* Require semicolon after. */
    +#define DECLARE_NULL_NAMESPACE_BYTES_TEMPLATE1(Namespace, Type, Size) \
    +        } /* Close namespace. */ \
    +        extern HB_INTERNAL const unsigned char _hb_Null_##Namespace##_##Type[Size]; \
    +        template  \
    +        struct Null> { \
    +          static Namespace::Type const & get_null () { \
    +            return *reinterpret_cast *> (_hb_Null_##Namespace##_##Type); \
    +          } \
    +        }; \
    +        namespace Namespace { \
    +        static_assert (true, "") /* Require semicolon after. */
     #define DEFINE_NULL_NAMESPACE_BYTES(Namespace, Type) \
    -        const unsigned char _hb_Null_##Namespace##_##Type[hb_null_size (Namespace::Type)]
    +        const unsigned char _hb_Null_##Namespace##_##Type[sizeof (_hb_Null_##Namespace##_##Type)]
     
     /* Specializations for arbitrary-content Null objects expressed as struct initializer. */
     #define DECLARE_NULL_INSTANCE(Type) \
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-number-parser.hh b/src/java.desktop/share/native/libharfbuzz/hb-number-parser.hh
    index 9d2867e483550..07f3ebe0c7dff 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-number-parser.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-number-parser.hh
    @@ -31,7 +31,7 @@
     #include "hb.hh"
     
     
    -#line 35 "hb-number-parser.hh"
    +#line 32 "hb-number-parser.hh"
     static const unsigned char _double_parser_trans_keys[] = {
             0u, 0u, 43u, 57u, 46u, 57u, 48u, 57u, 43u, 57u, 48u, 57u, 48u, 101u, 48u, 57u,
             46u, 101u, 0
    @@ -135,12 +135,12 @@ strtod_rl (const char *p, const char **end_ptr /* IN/OUT */)
     
       int cs;
     
    -#line 139 "hb-number-parser.hh"
    +#line 132 "hb-number-parser.hh"
             {
             cs = double_parser_start;
             }
     
    -#line 144 "hb-number-parser.hh"
    +#line 135 "hb-number-parser.hh"
             {
             int _slen;
             int _trans;
    @@ -198,7 +198,7 @@ _resume:
               exp_overflow = true;
     }
             break;
    -#line 202 "hb-number-parser.hh"
    +#line 187 "hb-number-parser.hh"
             }
     
     _again:
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-number.cc b/src/java.desktop/share/native/libharfbuzz/hb-number.cc
    index f4ce693d8ed88..37bc8b873997d 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-number.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-number.cc
    @@ -24,7 +24,6 @@
      */
     
     #include "hb.hh"
    -#include "hb-machinery.hh"
     #include "hb-number.hh"
     #include "hb-number-parser.hh"
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-object.hh b/src/java.desktop/share/native/libharfbuzz/hb-object.hh
    index 436c37aa1b752..ef675f835f119 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-object.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-object.hh
    @@ -69,7 +69,7 @@ struct hb_lockable_set_t
           item = items.push (v);
           l.unlock ();
         }
    -    return item;
    +    return items.in_error () ? nullptr : item;
       }
     
       template 
    @@ -80,7 +80,7 @@ struct hb_lockable_set_t
         if (item)
         {
           item_t old = *item;
    -      *item = items[items.length - 1];
    +      *item = std::move (items.tail ());
           items.pop ();
           l.unlock ();
           old.fini ();
    @@ -123,7 +123,7 @@ struct hb_lockable_set_t
         l.lock ();
         while (items.length)
         {
    -      item_t old = items[items.length - 1];
    +      item_t old = items.tail ();
           items.pop ();
           l.unlock ();
           old.fini ();
    @@ -144,14 +144,14 @@ struct hb_reference_count_t
     {
       mutable hb_atomic_int_t ref_count;
     
    -  void init (int v = 1) { ref_count.set_relaxed (v); }
    -  int get_relaxed () const { return ref_count.get_relaxed (); }
    +  void init (int v = 1) { ref_count = v; }
    +  int get_relaxed () const { return ref_count; }
       int inc () const { return ref_count.inc (); }
       int dec () const { return ref_count.dec (); }
    -  void fini () { ref_count.set_relaxed (-0x0000DEAD); }
    +  void fini () { ref_count = -0x0000DEAD; }
     
    -  bool is_inert () const { return !ref_count.get_relaxed (); }
    -  bool is_valid () const { return ref_count.get_relaxed () > 0; }
    +  bool is_inert () const { return !ref_count; }
    +  bool is_valid () const { return ref_count > 0; }
     };
     
     
    @@ -175,14 +175,34 @@ struct hb_user_data_array_t
     
       void init () { lock.init (); items.init (); }
     
    -  HB_INTERNAL bool set (hb_user_data_key_t *key,
    -                        void *              data,
    -                        hb_destroy_func_t   destroy,
    -                        hb_bool_t           replace);
    +  void fini () { items.fini (lock); lock.fini (); }
    +
    +  bool set (hb_user_data_key_t *key,
    +            void *              data,
    +            hb_destroy_func_t   destroy,
    +            hb_bool_t           replace)
    +  {
    +    if (!key)
    +      return false;
     
    -  HB_INTERNAL void *get (hb_user_data_key_t *key);
    +    if (replace) {
    +      if (!data && !destroy) {
    +        items.remove (key, lock);
    +        return true;
    +      }
    +    }
    +    hb_user_data_item_t item = {key, data, destroy};
    +    bool ret = !!items.replace_or_insert (item, lock, (bool) replace);
     
    -  void fini () { items.fini (lock); lock.fini (); }
    +    return ret;
    +  }
    +
    +  void *get (hb_user_data_key_t *key)
    +  {
    +    hb_user_data_item_t item = {nullptr, nullptr, nullptr};
    +
    +    return items.find (key, &item, lock) ? item.data : nullptr;
    +  }
     };
     
     
    @@ -214,23 +234,26 @@ static inline void hb_object_trace (const Type *obj, const char *function)
                  obj ? obj->header.ref_count.get_relaxed () : 0);
     }
     
    -template 
    -static inline Type *hb_object_create ()
    +template 
    +static inline Type *hb_object_create (Ts... ds)
     {
       Type *obj = (Type *) hb_calloc (1, sizeof (Type));
     
       if (unlikely (!obj))
         return obj;
     
    +  new (obj) Type (std::forward (ds)...);
    +
       hb_object_init (obj);
       hb_object_trace (obj, HB_FUNC);
    +
       return obj;
     }
     template 
     static inline void hb_object_init (Type *obj)
     {
       obj->header.ref_count.init ();
    -  obj->header.writable.set_relaxed (true);
    +  obj->header.writable = true;
       obj->header.user_data.init ();
     }
     template 
    @@ -241,12 +264,12 @@ static inline bool hb_object_is_valid (const Type *obj)
     template 
     static inline bool hb_object_is_immutable (const Type *obj)
     {
    -  return !obj->header.writable.get_relaxed ();
    +  return !obj->header.writable;
     }
     template 
     static inline void hb_object_make_immutable (const Type *obj)
     {
    -  obj->header.writable.set_relaxed (false);
    +  obj->header.writable = false;
     }
     template 
     static inline Type *hb_object_reference (Type *obj)
    @@ -269,18 +292,22 @@ static inline bool hb_object_destroy (Type *obj)
         return false;
     
       hb_object_fini (obj);
    +
    +  if (!std::is_trivially_destructible::value)
    +    obj->~Type ();
    +
       return true;
     }
     template 
     static inline void hb_object_fini (Type *obj)
     {
       obj->header.ref_count.fini (); /* Do this before user_data */
    -  hb_user_data_array_t *user_data = obj->header.user_data.get ();
    +  hb_user_data_array_t *user_data = obj->header.user_data.get_acquire ();
       if (user_data)
       {
         user_data->fini ();
         hb_free (user_data);
    -    user_data = nullptr;
    +    obj->header.user_data.set_relaxed (nullptr);
       }
     }
     template 
    @@ -295,7 +322,7 @@ static inline bool hb_object_set_user_data (Type               *obj,
       assert (hb_object_is_valid (obj));
     
     retry:
    -  hb_user_data_array_t *user_data = obj->header.user_data.get ();
    +  hb_user_data_array_t *user_data = obj->header.user_data.get_acquire ();
       if (unlikely (!user_data))
       {
         user_data = (hb_user_data_array_t *) hb_calloc (sizeof (hb_user_data_array_t), 1);
    @@ -320,7 +347,7 @@ static inline void *hb_object_get_user_data (Type               *obj,
       if (unlikely (!obj || obj->header.is_inert ()))
         return nullptr;
       assert (hb_object_is_valid (obj));
    -  hb_user_data_array_t *user_data = obj->header.user_data.get ();
    +  hb_user_data_array_t *user_data = obj->header.user_data.get_acquire ();
       if (!user_data)
         return nullptr;
       return user_data->get (key);
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-open-file.hh b/src/java.desktop/share/native/libharfbuzz/hb-open-file.hh
    index 1dff7c414cc53..c02eb41d1001e 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-open-file.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-open-file.hh
    @@ -90,7 +90,7 @@ typedef struct OpenTypeOffsetTable
       {
         if (table_count)
         {
    -      + tables.sub_array (start_offset, table_count)
    +      + tables.as_array ().sub_array (start_offset, table_count)
           | hb_map (&TableRecord::tag)
           | hb_sink (hb_array (table_tags, *table_count))
           ;
    @@ -158,7 +158,7 @@ typedef struct OpenTypeOffsetTable
             return_trace (false);
     
           if (likely (len))
    -        memcpy (start, blob->data, len);
    +        hb_memcpy (start, blob->data, len);
     
           /* 4-byte alignment. */
           c->align (4);
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-open-type.hh b/src/java.desktop/share/native/libharfbuzz/hb-open-type.hh
    index a4dfb3615e3cd..4639d80babc86 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-open-type.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-open-type.hh
    @@ -105,7 +105,7 @@ struct IntType
       bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
    -    return_trace (likely (c->check_struct (this)));
    +    return_trace (c->check_struct (this));
       }
       protected:
       BEInt v;
    @@ -141,27 +141,29 @@ typedef HBINT32 FWORD32;
     /* 16-bit unsigned integer (HBUINT16) that describes a quantity in FUnits. */
     typedef HBUINT16 UFWORD;
     
    -/* 16-bit signed fixed number with the low 14 bits of fraction (2.14). */
    -struct F2DOT14 : HBINT16
    +template 
    +struct HBFixed : Type
     {
    -  F2DOT14& operator = (uint16_t i ) { HBINT16::operator= (i); return *this; }
    -  // 16384 means 1<<14
    -  float to_float () const  { return ((int32_t) v) / 16384.f; }
    -  void set_float (float f) { v = roundf (f * 16384.f); }
    +  static constexpr float shift = (float) (1 << fraction_bits);
    +  static_assert (Type::static_size * 8 > fraction_bits, "");
    +
    +  operator signed () const = delete;
    +  operator unsigned () const = delete;
    +  typename Type::type to_int () const { return Type::v; }
    +  void set_int (typename Type::type i ) { Type::v = i; }
    +  float to_float (float offset = 0) const  { return ((int32_t) Type::v + offset) / shift; }
    +  void set_float (float f) { Type::v = roundf (f * shift); }
       public:
    -  DEFINE_SIZE_STATIC (2);
    +  DEFINE_SIZE_STATIC (Type::static_size);
     };
     
    +/* 16-bit signed fixed number with the low 14 bits of fraction (2.14). */
    +using F2DOT14 = HBFixed;
    +using F4DOT12 = HBFixed;
    +using F6DOT10 = HBFixed;
    +
     /* 32-bit signed fixed-point number (16.16). */
    -struct HBFixed : HBINT32
    -{
    -  HBFixed& operator = (uint32_t i) { HBINT32::operator= (i); return *this; }
    -  // 65536 means 1<<16
    -  float to_float () const  { return ((int32_t) v) / 65536.f; }
    -  void set_float (float f) { v = roundf (f * 65536.f); }
    -  public:
    -  DEFINE_SIZE_STATIC (4);
    -};
    +using F16DOT16 = HBFixed;
     
     /* Date represented in number of seconds since 12:00 midnight, January 1,
      * 1904. The value is represented as a signed 64-bit integer. */
    @@ -170,7 +172,7 @@ struct LONGDATETIME
       bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
    -    return_trace (likely (c->check_struct (this)));
    +    return_trace (c->check_struct (this));
       }
       protected:
       HBINT32 major;
    @@ -196,6 +198,10 @@ struct HBGlyphID16 : HBUINT16
     {
       HBGlyphID16& operator = (uint16_t i) { HBUINT16::operator= (i); return *this; }
     };
    +struct HBGlyphID24 : HBUINT24
    +{
    +  HBGlyphID24& operator = (uint32_t i) { HBUINT24::operator= (i); return *this; }
    +};
     
     /* Script/language-system/feature index */
     struct Index : HBUINT16 {
    @@ -208,6 +214,12 @@ typedef Index NameID;
     
     struct VarIdx : HBUINT32 {
       static constexpr unsigned NO_VARIATION = 0xFFFFFFFFu;
    +  static_assert (NO_VARIATION == HB_OT_LAYOUT_NO_VARIATIONS_INDEX, "");
    +  static uint32_t add (uint32_t i, unsigned short v)
    +  {
    +    if (i == NO_VARIATION) return i;
    +    return i + v;
    +  }
       VarIdx& operator = (uint32_t i) { HBUINT32::operator= (i); return *this; }
     };
     DECLARE_NULL_NAMESPACE_BYTES (OT, VarIdx);
    @@ -300,6 +312,10 @@ struct _hb_has_null
     template 
     struct OffsetTo : Offset
     {
    +  // Make sure Type is not unbounded; works only for types that are fully defined at OffsetTo time.
    +  static_assert (has_null == false ||
    +                 (hb_has_null_size (Type) || !hb_has_min_size (Type)), "");
    +
       HB_DELETE_COPY_ASSIGN (OffsetTo);
       OffsetTo () = default;
     
    @@ -450,14 +466,16 @@ struct UnsizedArrayOf
       {
         unsigned int i = (unsigned int) i_;
         const Type *p = &arrayZ[i];
    -    if (unlikely (p < arrayZ)) return Null (Type); /* Overflowed. */
    +    if (unlikely ((const void *) p < (const void *) arrayZ)) return Null (Type); /* Overflowed. */
    +    _hb_compiler_memory_r_barrier ();
         return *p;
       }
       Type& operator [] (int i_)
       {
         unsigned int i = (unsigned int) i_;
         Type *p = &arrayZ[i];
    -    if (unlikely (p < arrayZ)) return Crap (Type); /* Overflowed. */
    +    if (unlikely ((const void *) p < (const void *) arrayZ)) return Crap (Type); /* Overflowed. */
    +    _hb_compiler_memory_r_barrier ();
         return *p;
       }
     
    @@ -486,10 +504,10 @@ struct UnsizedArrayOf
       void qsort (unsigned int len, unsigned int start = 0, unsigned int end = (unsigned int) -1)
       { as_array (len).qsort (start, end); }
     
    -  bool serialize (hb_serialize_context_t *c, unsigned int items_len)
    +  bool serialize (hb_serialize_context_t *c, unsigned int items_len, bool clear = true)
       {
         TRACE_SERIALIZE (this);
    -    if (unlikely (!c->extend (this, items_len))) return_trace (false);
    +    if (unlikely (!c->extend_size (this, get_size (items_len), clear))) return_trace (false);
         return_trace (true);
       }
       template  *p = &this->arrayZ[i];
    -    if (unlikely (p < this->arrayZ)) return Null (Type); /* Overflowed. */
    +    if (unlikely ((const void *) p < (const void *) this->arrayZ)) return Null (Type); /* Overflowed. */
    +    _hb_compiler_memory_r_barrier ();
         return this+*p;
       }
       Type& operator [] (int i_)
       {
         unsigned int i = (unsigned int) i_;
         const OffsetTo *p = &this->arrayZ[i];
    -    if (unlikely (p < this->arrayZ)) return Crap (Type); /* Overflowed. */
    +    if (unlikely ((const void *) p < (const void *) this->arrayZ)) return Crap (Type); /* Overflowed. */
    +    _hb_compiler_memory_r_barrier ();
         return this+*p;
       }
     
    @@ -608,12 +628,14 @@ struct ArrayOf
       {
         unsigned int i = (unsigned int) i_;
         if (unlikely (i >= len)) return Null (Type);
    +    _hb_compiler_memory_r_barrier ();
         return arrayZ[i];
       }
       Type& operator [] (int i_)
       {
         unsigned int i = (unsigned int) i_;
         if (unlikely (i >= len)) return Crap (Type);
    +    _hb_compiler_memory_r_barrier ();
         return arrayZ[i];
       }
     
    @@ -635,14 +657,9 @@ struct ArrayOf
       operator   iter_t () const { return   iter (); }
       operator writer_t ()       { return writer (); }
     
    -  hb_array_t sub_array (unsigned int start_offset, unsigned int count) const
    -  { return as_array ().sub_array (start_offset, count); }
    -  hb_array_t sub_array (unsigned int start_offset, unsigned int *count = nullptr /* IN/OUT */) const
    -  { return as_array ().sub_array (start_offset, count); }
    -  hb_array_t sub_array (unsigned int start_offset, unsigned int count)
    -  { return as_array ().sub_array (start_offset, count); }
    -  hb_array_t sub_array (unsigned int start_offset, unsigned int *count = nullptr /* IN/OUT */)
    -  { return as_array ().sub_array (start_offset, count); }
    +  /* Faster range-based for loop. */
    +  const Type *begin () const { return arrayZ; }
    +  const Type *end () const { return arrayZ + len; }
     
       template 
       Type &lsearch (const T &x, Type ¬_found = Crap (Type))
    @@ -656,15 +673,15 @@ struct ArrayOf
                   unsigned int to_store = (unsigned int) -1) const
       { return as_array ().lfind (x, i, not_found, to_store); }
     
    -  void qsort (unsigned int start = 0, unsigned int end = (unsigned int) -1)
    -  { as_array ().qsort (start, end); }
    +  void qsort ()
    +  { as_array ().qsort (); }
     
    -  HB_NODISCARD bool serialize (hb_serialize_context_t *c, unsigned items_len)
    +  HB_NODISCARD bool serialize (hb_serialize_context_t *c, unsigned items_len, bool clear = true)
       {
         TRACE_SERIALIZE (this);
         if (unlikely (!c->extend_min (this))) return_trace (false);
         c->check_assign (len, items_len, HB_SERIALIZE_ERROR_ARRAY_OVERFLOW);
    -    if (unlikely (!c->extend (this))) return_trace (false);
    +    if (unlikely (!c->extend_size (this, get_size (), clear))) return_trace (false);
         return_trace (true);
       }
       template  using Array16Of = ArrayOf;
    +template  using Array24Of = ArrayOf;
     template  using Array32Of = ArrayOf;
     using PString = ArrayOf;
     
    @@ -738,26 +756,28 @@ template  using Array16OfOffset32To = ArrayOf using Array32OfOffset32To = ArrayOf, HBUINT32>;
     
     /* Array of offsets relative to the beginning of the array itself. */
    -template 
    -struct List16OfOffset16To : Array16OfOffset16To
    +template 
    +struct List16OfOffsetTo : ArrayOf, HBUINT16>
     {
       const Type& operator [] (int i_) const
       {
         unsigned int i = (unsigned int) i_;
         if (unlikely (i >= this->len)) return Null (Type);
    +    _hb_compiler_memory_r_barrier ();
         return this+this->arrayZ[i];
       }
       const Type& operator [] (int i_)
       {
         unsigned int i = (unsigned int) i_;
         if (unlikely (i >= this->len)) return Crap (Type);
    +    _hb_compiler_memory_r_barrier ();
         return this+this->arrayZ[i];
       }
     
       bool subset (hb_subset_context_t *c) const
       {
         TRACE_SUBSET (this);
    -    struct List16OfOffset16To *out = c->serializer->embed (*this);
    +    struct List16OfOffsetTo *out = c->serializer->embed (*this);
         if (unlikely (!out)) return_trace (false);
         unsigned int count = this->len;
         for (unsigned int i = 0; i < count; i++)
    @@ -769,10 +789,13 @@ struct List16OfOffset16To : Array16OfOffset16To
       bool sanitize (hb_sanitize_context_t *c, Ts&&... ds) const
       {
         TRACE_SANITIZE (this);
    -    return_trace (Array16OfOffset16To::sanitize (c, this, std::forward (ds)...));
    +    return_trace ((Array16Of>::sanitize (c, this, std::forward (ds)...)));
       }
     };
     
    +template 
    +using List16OfOffset16To = List16OfOffsetTo;
    +
     /* An array starting at second element. */
     template 
     struct HeadlessArrayOf
    @@ -785,12 +808,14 @@ struct HeadlessArrayOf
       {
         unsigned int i = (unsigned int) i_;
         if (unlikely (i >= lenP1 || !i)) return Null (Type);
    +    _hb_compiler_memory_r_barrier ();
         return arrayZ[i-1];
       }
       Type& operator [] (int i_)
       {
         unsigned int i = (unsigned int) i_;
         if (unlikely (i >= lenP1 || !i)) return Crap (Type);
    +    _hb_compiler_memory_r_barrier ();
         return arrayZ[i-1];
       }
       unsigned int get_size () const
    @@ -809,21 +834,25 @@ struct HeadlessArrayOf
       operator   iter_t () const { return   iter (); }
       operator writer_t ()       { return writer (); }
     
    -  bool serialize (hb_serialize_context_t *c, unsigned int items_len)
    +  /* Faster range-based for loop. */
    +  const Type *begin () const { return arrayZ; }
    +  const Type *end () const { return arrayZ + get_length (); }
    +
    +  HB_NODISCARD bool serialize (hb_serialize_context_t *c, unsigned int items_len, bool clear = true)
       {
         TRACE_SERIALIZE (this);
         if (unlikely (!c->extend_min (this))) return_trace (false);
         c->check_assign (lenP1, items_len + 1, HB_SERIALIZE_ERROR_ARRAY_OVERFLOW);
    -    if (unlikely (!c->extend (this))) return_trace (false);
    +    if (unlikely (!c->extend_size (this, get_size (), clear))) return_trace (false);
         return_trace (true);
       }
       template 
    -  bool serialize (hb_serialize_context_t *c, Iterator items)
    +  HB_NODISCARD bool serialize (hb_serialize_context_t *c, Iterator items)
       {
         TRACE_SERIALIZE (this);
    -    unsigned count = items.len ();
    -    if (unlikely (!serialize (c, count))) return_trace (false);
    +    unsigned count = hb_len (items);
    +    if (unlikely (!serialize (c, count, false))) return_trace (false);
         /* TODO Umm. Just exhaust the iterator instead?  Being extra
          * cautious right now.. */
         for (unsigned i = 0; i < count; i++, ++items)
    @@ -869,12 +898,14 @@ struct ArrayOfM1
       {
         unsigned int i = (unsigned int) i_;
         if (unlikely (i > lenM1)) return Null (Type);
    +    _hb_compiler_memory_r_barrier ();
         return arrayZ[i];
       }
       Type& operator [] (int i_)
       {
         unsigned int i = (unsigned int) i_;
         if (unlikely (i > lenM1)) return Crap (Type);
    +    _hb_compiler_memory_r_barrier ();
         return arrayZ[i];
       }
       unsigned int get_size () const
    @@ -923,14 +954,9 @@ struct SortedArrayOf : ArrayOf
       operator   iter_t () const { return   iter (); }
       operator writer_t ()       { return writer (); }
     
    -  hb_sorted_array_t sub_array (unsigned int start_offset, unsigned int count) const
    -  { return as_array ().sub_array (start_offset, count); }
    -  hb_sorted_array_t sub_array (unsigned int start_offset, unsigned int *count = nullptr /* IN/OUT */) const
    -  { return as_array ().sub_array (start_offset, count); }
    -  hb_sorted_array_t sub_array (unsigned int start_offset, unsigned int count)
    -  { return as_array ().sub_array (start_offset, count); }
    -  hb_sorted_array_t sub_array (unsigned int start_offset, unsigned int *count = nullptr /* IN/OUT */)
    -  { return as_array ().sub_array (start_offset, count); }
    +  /* Faster range-based for loop. */
    +  const Type *begin () const { return this->arrayZ; }
    +  const Type *end () const { return this->arrayZ + this->len; }
     
       bool serialize (hb_serialize_context_t *c, unsigned int items_len)
       {
    @@ -961,6 +987,7 @@ struct SortedArrayOf : ArrayOf
     };
     
     template  using SortedArray16Of = SortedArrayOf;
    +template  using SortedArray24Of = SortedArrayOf;
     template  using SortedArray32Of = SortedArrayOf;
     
     /*
    @@ -1053,12 +1080,14 @@ struct VarSizedBinSearchArrayOf
       {
         unsigned int i = (unsigned int) i_;
         if (unlikely (i >= get_length ())) return Null (Type);
    +    _hb_compiler_memory_r_barrier ();
         return StructAtOffset (&bytesZ, i * header.unitSize);
       }
       Type& operator [] (int i_)
       {
         unsigned int i = (unsigned int) i_;
         if (unlikely (i >= get_length ())) return Crap (Type);
    +    _hb_compiler_memory_r_barrier ();
         return StructAtOffset (&bytesZ, i * header.unitSize);
       }
       unsigned int get_length () const
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-cff-common.hh b/src/java.desktop/share/native/libharfbuzz/hb-ot-cff-common.hh
    index 54c7ecdf294ed..d31a328e8adf7 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-cff-common.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-ot-cff-common.hh
    @@ -66,95 +66,25 @@ struct CFFIndex
       {
         TRACE_SERIALIZE (this);
         unsigned int size = get_size ();
    -    CFFIndex *out = c->allocate_size (size);
    +    CFFIndex *out = c->allocate_size (size, false);
         if (likely (out))
    -      memcpy (out, this, size);
    +      hb_memcpy (out, this, size);
         return_trace (out);
       }
     
    +  template 
       bool serialize (hb_serialize_context_t *c,
    -                  unsigned int offSize_,
    -                  const byte_str_array_t &byteArray)
    +                  const Iterable &iterable)
       {
         TRACE_SERIALIZE (this);
    -
    -    if (byteArray.length == 0)
    -    {
    -      COUNT *dest = c->allocate_min ();
    -      if (unlikely (!dest)) return_trace (false);
    -      *dest = 0;
    -      return_trace (true);
    -    }
    -
    -    /* serialize CFFIndex header */
    -    if (unlikely (!c->extend_min (this))) return_trace (false);
    -    this->count = byteArray.length;
    -    this->offSize = offSize_;
    -    if (unlikely (!c->allocate_size (offSize_ * (byteArray.length + 1))))
    -      return_trace (false);
    -
    -    /* serialize indices */
    -    unsigned int  offset = 1;
    -    unsigned int  i = 0;
    -    for (; i < byteArray.length; i++)
    -    {
    -      set_offset_at (i, offset);
    -      offset += byteArray[i].get_size ();
    -    }
    -    set_offset_at (i, offset);
    -
    -    /* serialize data */
    -    for (unsigned int i = 0; i < byteArray.length; i++)
    -    {
    -      const hb_ubytes_t &bs = byteArray[i];
    -      unsigned char *dest = c->allocate_size (bs.length);
    -      if (unlikely (!dest)) return_trace (false);
    -      memcpy (dest, &bs[0], bs.length);
    -    }
    -
    -    return_trace (true);
    -  }
    -
    -  bool serialize (hb_serialize_context_t *c,
    -                  unsigned int offSize_,
    -                  const str_buff_vec_t &buffArray)
    -  {
    -    byte_str_array_t  byteArray;
    -    byteArray.init ();
    -    byteArray.resize (buffArray.length);
    -    for (unsigned int i = 0; i < byteArray.length; i++)
    -      byteArray[i] = hb_ubytes_t (buffArray[i].arrayZ, buffArray[i].length);
    -    bool result = this->serialize (c, offSize_, byteArray);
    -    byteArray.fini ();
    -    return result;
    -  }
    -
    -  template 
    -  bool serialize (hb_serialize_context_t *c,
    -                  Iterator it)
    -  {
    -    TRACE_SERIALIZE (this);
    -    serialize_header(c, + it | hb_map ([] (const hb_ubytes_t &_) { return _.length; }));
    +    auto it = hb_iter (iterable);
    +    serialize_header(c, + it | hb_map (hb_iter) | hb_map (hb_len));
         for (const auto &_ : +it)
    -      _.copy (c);
    +      hb_iter (_).copy (c);
         return_trace (true);
       }
     
    -  bool serialize (hb_serialize_context_t *c,
    -                  const byte_str_array_t &byteArray)
    -  { return serialize (c, + hb_iter (byteArray)); }
    -
    -  bool serialize (hb_serialize_context_t *c,
    -                  const str_buff_vec_t &buffArray)
    -  {
    -    auto it =
    -    + hb_iter (buffArray)
    -    | hb_map ([] (const str_buff_t &_) { return hb_ubytes_t (_.arrayZ, _.length); })
    -    ;
    -    return serialize (c, it);
    -  }
    -
       template 
       bool serialize_header (hb_serialize_context_t *c,
    @@ -171,7 +101,7 @@ struct CFFIndex
         if (!this->count) return_trace (true);
         if (unlikely (!c->extend (this->offSize))) return_trace (false);
         this->offSize = off_size;
    -    if (unlikely (!c->allocate_size (off_size * (this->count + 1))))
    +    if (unlikely (!c->allocate_size (off_size * (this->count + 1), false)))
           return_trace (false);
     
         /* serialize indices */
    @@ -179,14 +109,27 @@ struct CFFIndex
         unsigned int i = 0;
         for (unsigned _ : +it)
         {
    -      CFFIndex::set_offset_at (i++, offset);
    +      set_offset_at (i++, offset);
           offset += _;
         }
    -    CFFIndex::set_offset_at (i, offset);
    +    set_offset_at (i, offset);
     
         return_trace (true);
       }
     
    +  template 
    +  static unsigned total_size (const Iterable &iterable)
    +  {
    +    auto it = + hb_iter (iterable) | hb_map (hb_iter) | hb_map (hb_len);
    +    if (!it) return 0;
    +
    +    unsigned total = + it | hb_reduce (hb_add, 0);
    +    unsigned off_size = (hb_bit_storage (total + 1) + 7) / 8;
    +
    +    return min_size + HBUINT8::static_size + (hb_len (it) + 1) * off_size + total;
    +  }
    +
       void set_offset_at (unsigned int index, unsigned int offset)
       {
         assert (index <= count);
    @@ -207,10 +150,14 @@ struct CFFIndex
     
         unsigned int size = offSize;
         const HBUINT8 *p = offsets + size * index;
    -    unsigned int offset = 0;
    -    for (; size; size--)
    -      offset = (offset << 8) + *p++;
    -    return offset;
    +    switch (size)
    +    {
    +      case 1: return * (HBUINT8  *) p;
    +      case 2: return * (HBUINT16 *) p;
    +      case 3: return * (HBUINT24 *) p;
    +      case 4: return * (HBUINT32 *) p;
    +      default: return 0;
    +    }
       }
     
       unsigned int length_at (unsigned int index) const
    @@ -229,6 +176,7 @@ struct CFFIndex
       hb_ubytes_t operator [] (unsigned int index) const
       {
         if (unlikely (index >= count)) return hb_ubytes_t ();
    +    _hb_compiler_memory_r_barrier ();
         unsigned length = length_at (index);
         if (unlikely (!length)) return hb_ubytes_t ();
         return hb_ubytes_t (data_base () + offset_at (index) - 1, length);
    @@ -280,7 +228,7 @@ struct CFFIndexOf : CFFIndex
         if (unlikely (!c->extend_min (this))) return_trace (false);
         this->count = dataArrayLen;
         this->offSize = offSize_;
    -    if (unlikely (!c->allocate_size (offSize_ * (dataArrayLen + 1))))
    +    if (unlikely (!c->allocate_size (offSize_ * (dataArrayLen + 1), false)))
           return_trace (false);
     
         /* serialize indices */
    @@ -288,10 +236,10 @@ struct CFFIndexOf : CFFIndex
         unsigned int  i = 0;
         for (; i < dataArrayLen; i++)
         {
    -      CFFIndex::set_offset_at (i, offset);
    +      this->set_offset_at (i, offset);
           offset += dataSizeArray[i];
         }
    -    CFFIndex::set_offset_at (i, offset);
    +    this->set_offset_at (i, offset);
     
         /* serialize data */
         for (unsigned int i = 0; i < dataArrayLen; i++)
    @@ -324,13 +272,12 @@ struct Dict : UnsizedByteStr
       template 
       static bool serialize_int_op (hb_serialize_context_t *c, op_code_t op, V value, op_code_t intOp)
       {
    -    // XXX: not sure why but LLVM fails to compile the following 'unlikely' macro invocation
    -    if (/*unlikely*/ (!serialize_int (c, intOp, value)))
    +    if (unlikely ((!serialize_int (c, intOp, value))))
           return false;
     
         TRACE_SERIALIZE (this);
         /* serialize the opcode */
    -    HBUINT8 *p = c->allocate_size (OpCode_Size (op));
    +    HBUINT8 *p = c->allocate_size (OpCode_Size (op), false);
         if (unlikely (!p)) return_trace (false);
         if (Is_OpCode_ESC (op))
         {
    @@ -415,9 +362,8 @@ struct FDSelect0 {
         TRACE_SANITIZE (this);
         if (unlikely (!(c->check_struct (this))))
           return_trace (false);
    -    for (unsigned int i = 0; i < c->get_num_glyphs (); i++)
    -      if (unlikely (!fds[i].sanitize (c)))
    -        return_trace (false);
    +    if (unlikely (!c->check_array (fds, c->get_num_glyphs ())))
    +      return_trace (false);
     
         return_trace (true);
       }
    @@ -471,14 +417,20 @@ struct FDSelect3_4
         return_trace (true);
       }
     
    -  hb_codepoint_t get_fd (hb_codepoint_t glyph) const
    +  static int _cmp_range (const void *_key, const void *_item)
       {
    -    unsigned int i;
    -    for (i = 1; i < nRanges (); i++)
    -      if (glyph < ranges[i].first)
    -        break;
    +    hb_codepoint_t glyph = * (hb_codepoint_t *) _key;
    +    FDSelect3_4_Range *range = (FDSelect3_4_Range *) _item;
     
    -    return (hb_codepoint_t) ranges[i - 1].fd;
    +    if (glyph < range[0].first) return -1;
    +    if (glyph < range[1].first) return 0;
    +    return +1;
    +  }
    +
    +  hb_codepoint_t get_fd (hb_codepoint_t glyph) const
    +  {
    +    auto *range = hb_bsearch (glyph, &ranges[0], nRanges () - 1, sizeof (ranges[0]), _cmp_range);
    +    return range ? range->fd : ranges[nRanges () - 1].fd;
       }
     
       GID_TYPE        &nRanges ()       { return ranges.len; }
    @@ -501,9 +453,9 @@ struct FDSelect
       {
         TRACE_SERIALIZE (this);
         unsigned int size = src.get_size (num_glyphs);
    -    FDSelect *dest = c->allocate_size (size);
    +    FDSelect *dest = c->allocate_size (size, false);
         if (unlikely (!dest)) return_trace (false);
    -    memcpy (dest, &src, size);
    +    hb_memcpy (dest, &src, size);
         return_trace (true);
       }
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-cff1-table.cc b/src/java.desktop/share/native/libharfbuzz/hb-ot-cff1-table.cc
    index 9173b9d86007b..505af15e5c572 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-cff1-table.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-ot-cff1-table.cc
    @@ -422,8 +422,8 @@ bool OT::cff1::accelerator_t::get_extents (hb_font_t *font, hb_codepoint_t glyph
       }
       else
       {
    -    extents->x_bearing = font->em_scalef_x (bounds.min.x.to_real ());
    -    extents->width = font->em_scalef_x (bounds.max.x.to_real ()) - extents->x_bearing;
    +    extents->x_bearing = roundf (bounds.min.x.to_real ());
    +    extents->width = roundf (bounds.max.x.to_real () - extents->x_bearing);
       }
       if (bounds.min.y >= bounds.max.y)
       {
    @@ -432,10 +432,12 @@ bool OT::cff1::accelerator_t::get_extents (hb_font_t *font, hb_codepoint_t glyph
       }
       else
       {
    -    extents->y_bearing = font->em_scalef_y (bounds.max.y.to_real ());
    -    extents->height = font->em_scalef_y (bounds.min.y.to_real ()) - extents->y_bearing;
    +    extents->y_bearing = roundf (bounds.max.y.to_real ());
    +    extents->height = roundf (bounds.min.y.to_real () - extents->y_bearing);
       }
     
    +  font->scale_glyph_extents (extents);
    +
       return true;
     }
     
    @@ -551,6 +553,15 @@ bool _get_path (const OT::cff1::accelerator_t *cff, hb_font_t *font, hb_codepoin
       return true;
     }
     
    +bool OT::cff1::accelerator_t::paint_glyph (hb_font_t *font, hb_codepoint_t glyph, hb_paint_funcs_t *funcs, void *data, hb_color_t foreground) const
    +{
    +  funcs->push_clip_glyph (data, glyph, font);
    +  funcs->color (data, true, foreground);
    +  funcs->pop_clip (data);
    +
    +  return true;
    +}
    +
     bool OT::cff1::accelerator_t::get_path (hb_font_t *font, hb_codepoint_t glyph, hb_draw_session_t &draw_session) const
     {
     #ifdef HB_NO_OT_FONT_CFF
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-cff1-table.hh b/src/java.desktop/share/native/libharfbuzz/hb-ot-cff1-table.hh
    index b773956662f4f..60bc308f90cc2 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-cff1-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-ot-cff1-table.hh
    @@ -30,6 +30,7 @@
     #include "hb-ot-cff-common.hh"
     #include "hb-subset-cff1.hh"
     #include "hb-draw.hh"
    +#include "hb-paint.hh"
     
     #define HB_STRING_ARRAY_NAME cff1_std_strings
     #define HB_STRING_ARRAY_LIST "hb-ot-cff1-std-str.hh"
    @@ -175,7 +176,7 @@ struct Encoding
         unsigned int size = src.get_size ();
         Encoding *dest = c->allocate_size (size);
         if (unlikely (!dest)) return_trace (false);
    -    memcpy (dest, &src, size);
    +    hb_memcpy (dest, &src, size);
         return_trace (true);
       }
     
    @@ -471,7 +472,7 @@ struct Charset
         unsigned int size = src.get_size (num_glyphs);
         Charset *dest = c->allocate_size (size);
         if (unlikely (!dest)) return_trace (false);
    -    memcpy (dest, &src, size);
    +    hb_memcpy (dest, &src, size);
         return_trace (true);
       }
     
    @@ -617,7 +618,6 @@ struct CFF1StringIndex : CFF1Index
         }
     
         byte_str_array_t bytesArray;
    -    bytesArray.init ();
         if (!bytesArray.resize (sidmap.get_population ()))
           return_trace (false);
         for (unsigned int i = 0; i < strings.count; i++)
    @@ -628,7 +628,6 @@ struct CFF1StringIndex : CFF1Index
         }
     
         bool result = CFF1Index::serialize (c, bytesArray);
    -    bytesArray.fini ();
         return_trace (result);
       }
     };
    @@ -813,7 +812,7 @@ struct cff1_top_dict_opset_t : top_dict_opset_t
             break;
     
           default:
    -        env.last_offset = env.str_ref.offset;
    +        env.last_offset = env.str_ref.get_offset ();
             top_dict_opset_t::process_op (op, env, dictval);
             /* Record this operand below if stack is empty, otherwise done */
             if (!env.argStack.is_empty ()) return;
    @@ -903,8 +902,6 @@ struct cff1_private_dict_opset_t : dict_opset_t
           case OpCode_FamilyOtherBlues:
           case OpCode_StemSnapH:
           case OpCode_StemSnapV:
    -        env.clear_args ();
    -        break;
           case OpCode_StdHW:
           case OpCode_StdVW:
           case OpCode_BlueScale:
    @@ -916,7 +913,6 @@ struct cff1_private_dict_opset_t : dict_opset_t
           case OpCode_initialRandomSeed:
           case OpCode_defaultWidthX:
           case OpCode_nominalWidthX:
    -        val.single_val = env.argStack.pop_num ();
             env.clear_args ();
             break;
           case OpCode_Subrs:
    @@ -1295,10 +1291,10 @@ struct cff1
         }
     
         protected:
    -    hb_blob_t              *blob = nullptr;
         hb_sanitize_context_t   sc;
     
         public:
    +    hb_blob_t               *blob = nullptr;
         const Encoding          *encoding = nullptr;
         const Charset           *charset = nullptr;
         const CFF1NameIndex     *nameIndex = nullptr;
    @@ -1345,6 +1341,7 @@ struct cff1
         bool get_glyph_name (hb_codepoint_t glyph,
                              char *buf, unsigned int buf_len) const
         {
    +      if (unlikely (glyph >= num_glyphs)) return false;
           if (unlikely (!is_valid ())) return false;
           if (is_CID()) return false;
           if (unlikely (!buf_len)) return true;
    @@ -1379,7 +1376,7 @@ struct cff1
           if (unlikely (!len)) return false;
     
         retry:
    -      hb_sorted_vector_t *names = glyph_names.get ();
    +      hb_sorted_vector_t *names = glyph_names.get_acquire ();
           if (unlikely (!names))
           {
             names = (hb_sorted_vector_t *) hb_calloc (sizeof (hb_sorted_vector_t), 1);
    @@ -1428,6 +1425,7 @@ struct cff1
         }
     
         HB_INTERNAL bool get_extents (hb_font_t *font, hb_codepoint_t glyph, hb_glyph_extents_t *extents) const;
    +    HB_INTERNAL bool paint_glyph (hb_font_t *font, hb_codepoint_t glyph, hb_paint_funcs_t *funcs, void *data, hb_color_t foreground) const;
         HB_INTERNAL bool get_seac_components (hb_codepoint_t glyph, hb_codepoint_t *base, hb_codepoint_t *accent) const;
         HB_INTERNAL bool get_path (hb_font_t *font, hb_codepoint_t glyph, hb_draw_session_t &draw_session) const;
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-cff2-table.cc b/src/java.desktop/share/native/libharfbuzz/hb-ot-cff2-table.cc
    index 1e757ebfba9c9..ed430ee59d56b 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-cff2-table.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-ot-cff2-table.cc
    @@ -124,8 +124,8 @@ bool OT::cff2::accelerator_t::get_extents (hb_font_t *font,
       }
       else
       {
    -    extents->x_bearing = font->em_scalef_x (param.min_x.to_real ());
    -    extents->width = font->em_scalef_x (param.max_x.to_real ()) - extents->x_bearing;
    +    extents->x_bearing = roundf (param.min_x.to_real ());
    +    extents->width = roundf (param.max_x.to_real () - extents->x_bearing);
       }
       if (param.min_y >= param.max_y)
       {
    @@ -134,10 +134,21 @@ bool OT::cff2::accelerator_t::get_extents (hb_font_t *font,
       }
       else
       {
    -    extents->y_bearing = font->em_scalef_y (param.max_y.to_real ());
    -    extents->height = font->em_scalef_y (param.min_y.to_real ()) - extents->y_bearing;
    +    extents->y_bearing = roundf (param.max_y.to_real ());
    +    extents->height = roundf (param.min_y.to_real () - extents->y_bearing);
       }
     
    +  font->scale_glyph_extents (extents);
    +
    +  return true;
    +}
    +
    +bool OT::cff2::accelerator_t::paint_glyph (hb_font_t *font, hb_codepoint_t glyph, hb_paint_funcs_t *funcs, void *data, hb_color_t foreground) const
    +{
    +  funcs->push_clip_glyph (data, glyph, font);
    +  funcs->color (data, true, foreground);
    +  funcs->pop_clip (data);
    +
       return true;
     }
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-cff2-table.hh b/src/java.desktop/share/native/libharfbuzz/hb-ot-cff2-table.hh
    index 7e1a3d0dba1ab..bfbc26b96ec43 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-cff2-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-ot-cff2-table.hh
    @@ -30,6 +30,7 @@
     #include "hb-ot-cff-common.hh"
     #include "hb-subset-cff2.hh"
     #include "hb-draw.hh"
    +#include "hb-paint.hh"
     
     namespace CFF {
     
    @@ -56,7 +57,7 @@ struct CFF2FDSelect
         unsigned int size = src.get_size (num_glyphs);
         CFF2FDSelect *dest = c->allocate_size (size);
         if (unlikely (!dest)) return_trace (false);
    -    memcpy (dest, &src, size);
    +    hb_memcpy (dest, &src, size);
         return_trace (true);
       }
     
    @@ -124,7 +125,7 @@ struct CFF2VariationStore
         unsigned int size_ = varStore->get_size ();
         CFF2VariationStore *dest = c->allocate_size (size_);
         if (unlikely (!dest)) return_trace (false);
    -    memcpy (dest, varStore, size_);
    +    hb_memcpy (dest, varStore, size_);
         return_trace (true);
       }
     
    @@ -282,9 +283,6 @@ struct cff2_private_dict_opset_t : dict_opset_t
           case OpCode_BlueFuzz:
           case OpCode_ExpansionFactor:
           case OpCode_LanguageGroup:
    -        val.single_val = env.argStack.pop_num ();
    -        env.clear_args ();
    -        break;
           case OpCode_BlueValues:
           case OpCode_OtherBlues:
           case OpCode_FamilyBlues:
    @@ -483,13 +481,18 @@ struct cff2
           blob = nullptr;
         }
     
    +    hb_map_t *create_glyph_to_sid_map () const
    +    {
    +      return nullptr;
    +    }
    +
         bool is_valid () const { return blob; }
     
         protected:
    -    hb_blob_t                   *blob = nullptr;
         hb_sanitize_context_t       sc;
     
         public:
    +    hb_blob_t                   *blob = nullptr;
         cff2_top_dict_values_t      topDict;
         const CFF2Subrs             *globalSubrs = nullptr;
         const CFF2VariationStore    *varStore = nullptr;
    @@ -511,6 +514,7 @@ struct cff2
         HB_INTERNAL bool get_extents (hb_font_t *font,
                                       hb_codepoint_t glyph,
                                       hb_glyph_extents_t *extents) const;
    +    HB_INTERNAL bool paint_glyph (hb_font_t *font, hb_codepoint_t glyph, hb_paint_funcs_t *funcs, void *data, hb_color_t foreground) const;
         HB_INTERNAL bool get_path (hb_font_t *font, hb_codepoint_t glyph, hb_draw_session_t &draw_session) const;
       };
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-cmap-table.hh b/src/java.desktop/share/native/libharfbuzz/hb-ot-cmap-table.hh
    index 4b6e542fb8726..eb1dd2bc0d184 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-cmap-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-ot-cmap-table.hh
    @@ -31,6 +31,7 @@
     #include "hb-ot-shaper-arabic-pua.hh"
     #include "hb-open-type.hh"
     #include "hb-set.hh"
    +#include "hb-cache.hh"
     
     /*
      * cmap -- Character to Glyph Index Mapping
    @@ -403,7 +404,7 @@ struct CmapSubtableFormat4
                      unsigned distance) const
             {
               if (k > last) return +1;
    -          if (k < (&last)[distance]) return -1;
    +          if (k < (&last)[distance]/*first*/) return -1;
               return 0;
             }
             HBUINT16 last;
    @@ -412,7 +413,7 @@ struct CmapSubtableFormat4
           const HBUINT16 *found = hb_bsearch (codepoint,
                                               this->endCount,
                                               this->segCount,
    -                                          2,
    +                                          sizeof (CustomRange),
                                               _hb_cmp_method,
                                               this->segCount + 1);
           if (unlikely (!found))
    @@ -909,7 +910,7 @@ struct DefaultUVS : SortedArray32Of
           hb_codepoint_t first = arrayZ[i].startUnicodeValue;
           hb_codepoint_t last = hb_min ((hb_codepoint_t) (first + arrayZ[i].additionalCount),
                                         (hb_codepoint_t) HB_UNICODE_MAX);
    -      out->add_range (first, hb_min (last, 0x10FFFFu));
    +      out->add_range (first, last);
         }
       }
     
    @@ -925,37 +926,75 @@ struct DefaultUVS : SortedArray32Of
         if (unlikely (!c->copy (len))) return nullptr;
         unsigned init_len = c->length ();
     
    -    hb_codepoint_t lastCode = HB_MAP_VALUE_INVALID;
    -    int count = -1;
    -
    -    for (const UnicodeValueRange& _ : as_array ())
    +    if (this->len > unicodes->get_population () * hb_bit_storage ((unsigned) this->len))
         {
    -      for (const unsigned addcnt : hb_range ((unsigned) _.additionalCount + 1))
    +      hb_codepoint_t start = HB_SET_VALUE_INVALID;
    +      hb_codepoint_t end = HB_SET_VALUE_INVALID;
    +
    +      for (hb_codepoint_t u = HB_SET_VALUE_INVALID;
    +           unicodes->next (&u);)
           {
    -        unsigned curEntry = (unsigned) _.startUnicodeValue + addcnt;
    -        if (!unicodes->has (curEntry)) continue;
    -        count += 1;
    -        if (lastCode == HB_MAP_VALUE_INVALID)
    -          lastCode = curEntry;
    -        else if (lastCode + count != curEntry)
    +        if (!as_array ().bsearch (u))
    +          continue;
    +        if (start == HB_SET_VALUE_INVALID)
    +        {
    +          start = u;
    +          end = start - 1;
    +        }
    +        if (end + 1 != u || end - start == 255)
             {
               UnicodeValueRange rec;
    -          rec.startUnicodeValue = lastCode;
    -          rec.additionalCount = count - 1;
    +          rec.startUnicodeValue = start;
    +          rec.additionalCount = end - start;
               c->copy (rec);
    -
    -          lastCode = curEntry;
    -          count = 0;
    +          start = u;
             }
    +        end = u;
    +      }
    +      if (start != HB_SET_VALUE_INVALID)
    +      {
    +        UnicodeValueRange rec;
    +        rec.startUnicodeValue = start;
    +        rec.additionalCount = end - start;
    +        c->copy (rec);
           }
    -    }
     
    -    if (lastCode != HB_MAP_VALUE_INVALID)
    +    }
    +    else
         {
    -      UnicodeValueRange rec;
    -      rec.startUnicodeValue = lastCode;
    -      rec.additionalCount = count;
    -      c->copy (rec);
    +      hb_codepoint_t lastCode = HB_SET_VALUE_INVALID;
    +      int count = -1;
    +
    +      for (const UnicodeValueRange& _ : *this)
    +      {
    +        hb_codepoint_t curEntry = (hb_codepoint_t) (_.startUnicodeValue - 1);
    +        hb_codepoint_t end = curEntry + _.additionalCount + 2;
    +
    +        for (; unicodes->next (&curEntry) && curEntry < end;)
    +        {
    +          count += 1;
    +          if (lastCode == HB_SET_VALUE_INVALID)
    +            lastCode = curEntry;
    +          else if (lastCode + count != curEntry)
    +          {
    +            UnicodeValueRange rec;
    +            rec.startUnicodeValue = lastCode;
    +            rec.additionalCount = count - 1;
    +            c->copy (rec);
    +
    +            lastCode = curEntry;
    +            count = 0;
    +          }
    +        }
    +      }
    +
    +      if (lastCode != HB_MAP_VALUE_INVALID)
    +      {
    +        UnicodeValueRange rec;
    +        rec.startUnicodeValue = lastCode;
    +        rec.additionalCount = count;
    +        c->copy (rec);
    +      }
         }
     
         if (c->length () - init_len == 0)
    @@ -1376,7 +1415,7 @@ struct CmapSubtable
         switch (format) {
         case  4: return u.format4.serialize (c, it);
         case 12: return u.format12.serialize (c, it);
    -    case 14: return u.format14.serialize (c, plan->unicodes, plan->glyphs_requested, plan->glyph_map, base);
    +    case 14: return u.format14.serialize (c, &plan->unicodes, &plan->glyphs_requested, plan->glyph_map, base);
         default: return;
         }
       }
    @@ -1474,19 +1513,67 @@ struct EncodingRecord
       DEFINE_SIZE_STATIC (8);
     };
     
    +struct cmap;
    +
     struct SubtableUnicodesCache {
     
      private:
    -  const void* base;
    -  hb_hashmap_t> cached_unicodes;
    +  hb_blob_ptr_t base_blob;
    +  const char* base;
    +  hb_hashmap_t> cached_unicodes;
     
      public:
    +
    +  static SubtableUnicodesCache* create (hb_blob_ptr_t source_table)
    +  {
    +    SubtableUnicodesCache* cache =
    +        (SubtableUnicodesCache*) hb_malloc (sizeof(SubtableUnicodesCache));
    +    new (cache) SubtableUnicodesCache (source_table);
    +    return cache;
    +  }
    +
    +  static void destroy (void* value) {
    +    if (!value) return;
    +
    +    SubtableUnicodesCache* cache = (SubtableUnicodesCache*) value;
    +    cache->~SubtableUnicodesCache ();
    +    hb_free (cache);
    +  }
    +
       SubtableUnicodesCache(const void* cmap_base)
    -      : base(cmap_base), cached_unicodes() {}
    +      : base_blob(),
    +        base ((const char*) cmap_base),
    +        cached_unicodes ()
    +  {}
    +
    +  SubtableUnicodesCache(hb_blob_ptr_t base_blob_)
    +      : base_blob(base_blob_),
    +        base ((const char *) base_blob.get()),
    +        cached_unicodes ()
    +  {}
    +
    +  ~SubtableUnicodesCache()
    +  {
    +    base_blob.destroy ();
    +  }
     
    -  hb_set_t* set_for (const EncodingRecord* record)
    +  bool same_base(const void* other) const
       {
    -    if (!cached_unicodes.has ((intptr_t) record))
    +    return other == (const void*) base;
    +  }
    +
    +  const hb_set_t* set_for (const EncodingRecord* record,
    +                           SubtableUnicodesCache& mutable_cache) const
    +  {
    +    if (cached_unicodes.has ((unsigned) ((const char *) record - base)))
    +      return cached_unicodes.get ((unsigned) ((const char *) record - base));
    +
    +    return mutable_cache.set_for (record);
    +  }
    +
    +  const hb_set_t* set_for (const EncodingRecord* record)
    +  {
    +    if (!cached_unicodes.has ((unsigned) ((const char *) record - base)))
         {
           hb_set_t *s = hb_set_create ();
           if (unlikely (s->in_error ()))
    @@ -1494,12 +1581,12 @@ struct SubtableUnicodesCache {
     
           (base+record->subtable).collect_unicodes (s);
     
    -      if (unlikely (!cached_unicodes.set ((intptr_t) record, hb::unique_ptr {s})))
    +      if (unlikely (!cached_unicodes.set ((unsigned) ((const char *) record - base), hb::unique_ptr {s})))
             return hb_set_get_empty ();
     
           return s;
         }
    -    return cached_unicodes.get ((intptr_t) record);
    +    return cached_unicodes.get ((unsigned) ((const char *) record - base));
       }
     
     };
    @@ -1523,13 +1610,30 @@ struct cmap
     {
       static constexpr hb_tag_t tableTag = HB_OT_TAG_cmap;
     
    +
    +  static SubtableUnicodesCache* create_filled_cache(hb_blob_ptr_t source_table) {
    +    const cmap* cmap = source_table.get();
    +    auto it =
    +    + hb_iter (cmap->encodingRecord)
    +    | hb_filter ([&](const EncodingRecord& _) {
    +      return cmap::filter_encoding_records_for_subset (cmap, _);
    +    })
    +    ;
    +
    +    SubtableUnicodesCache* cache = SubtableUnicodesCache::create(source_table);
    +    for (const EncodingRecord& _ : it)
    +      cache->set_for(&_); // populate the cache for this encoding record.
    +
    +    return cache;
    +  }
    +
       template
       bool serialize (hb_serialize_context_t *c,
                       Iterator it,
                       EncodingRecIter encodingrec_iter,
                       const void *base,
    -                  const hb_subset_plan_t *plan,
    +                  hb_subset_plan_t *plan,
                       bool drop_format_4 = false)
       {
         if (unlikely (!c->extend_min ((*this))))  return false;
    @@ -1538,7 +1642,14 @@ struct cmap
         unsigned format4objidx = 0, format12objidx = 0, format14objidx = 0;
         auto snap = c->snapshot ();
     
    -    SubtableUnicodesCache unicodes_cache (base);
    +    SubtableUnicodesCache local_unicodes_cache (base);
    +    const SubtableUnicodesCache* unicodes_cache = &local_unicodes_cache;
    +
    +    if (plan->accelerator &&
    +        plan->accelerator->cmap_cache &&
    +        plan->accelerator->cmap_cache->same_base (base))
    +      unicodes_cache = plan->accelerator->cmap_cache;
    +
         for (const EncodingRecord& _ : encodingrec_iter)
         {
           if (c->in_error ())
    @@ -1547,7 +1658,7 @@ struct cmap
           unsigned format = (base+_.subtable).u.format;
           if (format != 4 && format != 12 && format != 14) continue;
     
    -      hb_set_t* unicodes_set = unicodes_cache.set_for (&_);
    +      const hb_set_t* unicodes_set = unicodes_cache->set_for (&_, local_unicodes_cache);
     
           if (!drop_format_4 && format == 4)
           {
    @@ -1566,7 +1677,13 @@ struct cmap
     
           else if (format == 12)
           {
    -        if (_can_drop (_, *unicodes_set, base, unicodes_cache, + it | hb_map (hb_first), encodingrec_iter)) continue;
    +        if (_can_drop (_,
    +                       *unicodes_set,
    +                       base,
    +                       *unicodes_cache,
    +                       local_unicodes_cache,
    +                       + it | hb_map (hb_first), encodingrec_iter))
    +          continue;
             c->copy (_, + it | hb_filter (*unicodes_set, hb_first), 12u, base, plan, &format12objidx);
           }
           else if (format == 14) c->copy (_, it, 14u, base, plan, &format14objidx);
    @@ -1585,7 +1702,8 @@ struct cmap
       bool _can_drop (const EncodingRecord& cmap12,
                       const hb_set_t& cmap12_unicodes,
                       const void* base,
    -                  SubtableUnicodesCache& unicodes_cache,
    +                  const SubtableUnicodesCache& unicodes_cache,
    +                  SubtableUnicodesCache& local_unicodes_cache,
                       Iterator subset_unicodes,
                       EncodingRecordIterator encoding_records)
       {
    @@ -1616,7 +1734,7 @@ struct cmap
               || (base+_.subtable).get_language() != target_language)
             continue;
     
    -      hb_set_t* sibling_unicodes = unicodes_cache.set_for (&_);
    +      const hb_set_t* sibling_unicodes = unicodes_cache.set_for (&_, local_unicodes_cache);
     
           auto cmap12 = + subset_unicodes | hb_filter (cmap12_unicodes);
           auto sibling = + subset_unicodes | hb_filter (*sibling_unicodes);
    @@ -1653,17 +1771,9 @@ struct cmap
     
         auto encodingrec_iter =
         + hb_iter (encodingRecord)
    -    | hb_filter ([&] (const EncodingRecord& _)
    -                {
    -                  if ((_.platformID == 0 && _.encodingID == 3) ||
    -                      (_.platformID == 0 && _.encodingID == 4) ||
    -                      (_.platformID == 3 && _.encodingID == 1) ||
    -                      (_.platformID == 3 && _.encodingID == 10) ||
    -                      (this + _.subtable).u.format == 14)
    -                    return true;
    -
    -                  return false;
    -                })
    +    | hb_filter ([&](const EncodingRecord& _) {
    +      return cmap::filter_encoding_records_for_subset (this, _);
    +    })
         ;
     
         if (unlikely (!encodingrec_iter.len ())) return_trace (false);
    @@ -1692,7 +1802,11 @@ struct cmap
                      { return (_.second != HB_MAP_VALUE_INVALID); })
         ;
     
    -    return_trace (cmap_prime->serialize (c->serializer, it, encodingrec_iter, this, c->plan));
    +    return_trace (cmap_prime->serialize (c->serializer,
    +                                         it,
    +                                         encodingrec_iter,
    +                                         this,
    +                                         c->plan));
       }
     
       const CmapSubtable *find_best_subtable (bool *symbol = nullptr) const
    @@ -1728,6 +1842,8 @@ struct cmap
     
       struct accelerator_t
       {
    +    using cache_t = hb_cache_t<21, 16, 8, true>;
    +
         accelerator_t (hb_face_t *face)
         {
           this->table = hb_sanitize_context_t ().reference_table (face);
    @@ -1782,26 +1898,43 @@ struct cmap
         }
         ~accelerator_t () { this->table.destroy (); }
     
    +    inline bool _cached_get (hb_codepoint_t unicode,
    +                             hb_codepoint_t *glyph,
    +                             cache_t *cache) const
    +    {
    +      unsigned v;
    +      if (cache && cache->get (unicode, &v))
    +      {
    +        *glyph = v;
    +        return true;
    +      }
    +      bool ret  = this->get_glyph_funcZ (this->get_glyph_data, unicode, glyph);
    +
    +      if (cache && ret)
    +        cache->set (unicode, *glyph);
    +      return ret;
    +    }
    +
         bool get_nominal_glyph (hb_codepoint_t  unicode,
    -                            hb_codepoint_t *glyph) const
    +                            hb_codepoint_t *glyph,
    +                            cache_t *cache = nullptr) const
         {
    -      if (unlikely (!this->get_glyph_funcZ)) return false;
    -      return this->get_glyph_funcZ (this->get_glyph_data, unicode, glyph);
    +      if (unlikely (!this->get_glyph_funcZ)) return 0;
    +      return _cached_get (unicode, glyph, cache);
         }
    +
         unsigned int get_nominal_glyphs (unsigned int count,
                                          const hb_codepoint_t *first_unicode,
                                          unsigned int unicode_stride,
                                          hb_codepoint_t *first_glyph,
    -                                     unsigned int glyph_stride) const
    +                                     unsigned int glyph_stride,
    +                                     cache_t *cache = nullptr) const
         {
           if (unlikely (!this->get_glyph_funcZ)) return 0;
     
    -      hb_cmap_get_glyph_func_t get_glyph_funcZ = this->get_glyph_funcZ;
    -      const void *get_glyph_data = this->get_glyph_data;
    -
           unsigned int done;
           for (done = 0;
    -           done < count && get_glyph_funcZ (get_glyph_data, *first_unicode, first_glyph);
    +           done < count && _cached_get (*first_unicode, first_glyph, cache);
                done++)
           {
             first_unicode = &StructAtOffsetUnaligned (first_unicode, unicode_stride);
    @@ -1812,7 +1945,8 @@ struct cmap
     
         bool get_variation_glyph (hb_codepoint_t  unicode,
                                   hb_codepoint_t  variation_selector,
    -                              hb_codepoint_t *glyph) const
    +                              hb_codepoint_t *glyph,
    +                              cache_t *cache = nullptr) const
         {
           switch (this->subtable_uvs->get_glyph_variant (unicode,
                                                          variation_selector,
    @@ -1823,7 +1957,7 @@ struct cmap
             case GLYPH_VARIANT_USE_DEFAULT: break;
           }
     
    -      return get_nominal_glyph (unicode, glyph);
    +      return get_nominal_glyph (unicode, glyph, cache);
         }
     
         void collect_unicodes (hb_set_t *out, unsigned int num_glyphs) const
    @@ -1928,6 +2062,19 @@ struct cmap
                       encodingRecord.sanitize (c, this));
       }
     
    + private:
    +
    +  static bool filter_encoding_records_for_subset(const cmap* cmap,
    +                                                 const EncodingRecord& _)
    +  {
    +    return
    +        (_.platformID == 0 && _.encodingID == 3) ||
    +        (_.platformID == 0 && _.encodingID == 4) ||
    +        (_.platformID == 3 && _.encodingID == 1) ||
    +        (_.platformID == 3 && _.encodingID == 10) ||
    +        (cmap + _.subtable).u.format == 14;
    +  }
    +
       protected:
       HBUINT16      version;        /* Table version number (0). */
       SortedArray16Of
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-color.cc b/src/java.desktop/share/native/libharfbuzz/hb-ot-color.cc
    index 5b45dbcd9fc91..ef46fe5429a57 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-color.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-ot-color.cc
    @@ -31,11 +31,11 @@
     
     #include "hb-ot.h"
     
    -#include "hb-ot-color-cbdt-table.hh"
    -#include "hb-ot-color-colr-table.hh"
    -#include "hb-ot-color-cpal-table.hh"
    -#include "hb-ot-color-sbix-table.hh"
    -#include "hb-ot-color-svg-table.hh"
    +#include "OT/Color/CBDT/CBDT.hh"
    +#include "OT/Color/COLR/COLR.hh"
    +#include "OT/Color/CPAL/CPAL.hh"
    +#include "OT/Color/sbix/sbix.hh"
    +#include "OT/Color/svg/svg.hh"
     
     
     /**
    @@ -61,7 +61,7 @@
      *
      * Tests whether a face includes a `CPAL` color-palette table.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 2.1.0
      */
    @@ -167,6 +167,10 @@ hb_ot_color_palette_get_flags (hb_face_t *face,
      * for allocating a buffer of suitable size before calling
      * hb_ot_color_palette_get_colors() a second time.
      *
    + * The RGBA values in the palette are unpremultiplied. See the
    + * OpenType spec [CPAL](https://learn.microsoft.com/en-us/typography/opentype/spec/cpal)
    + * section for details.
    + *
      * Return value: the total number of colors in the palette
      *
      * Since: 2.1.0
    @@ -190,16 +194,53 @@ hb_ot_color_palette_get_colors (hb_face_t     *face,
      * hb_ot_color_has_layers:
      * @face: #hb_face_t to work upon
      *
    - * Tests whether a face includes any `COLR` color layers.
    + * Tests whether a face includes a `COLR` table
    + * with data according to COLRv0.
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 2.1.0
      */
     hb_bool_t
     hb_ot_color_has_layers (hb_face_t *face)
     {
    -  return face->table.COLR->has_data ();
    +  return face->table.COLR->has_v0_data ();
    +}
    +
    +/**
    + * hb_ot_color_has_paint:
    + * @face: #hb_face_t to work upon
    + *
    + * Tests where a face includes a `COLR` table
    + * with data according to COLRv1.
    + *
    + * Return value: `true` if data found, `false` otherwise
    + *
    + * Since: 7.0.0
    + */
    +hb_bool_t
    +hb_ot_color_has_paint (hb_face_t *face)
    +{
    +  return face->table.COLR->has_v1_data ();
    +}
    +
    +/**
    + * hb_ot_color_glyph_has_paint:
    + * @face: #hb_face_t to work upon
    + * @glyph: The glyph index to query
    + *
    + * Tests where a face includes COLRv1 paint
    + * data for @glyph.
    + *
    + * Return value: `true` if data found, `false` otherwise
    + *
    + * Since: 7.0.0
    + */
    +hb_bool_t
    +hb_ot_color_glyph_has_paint (hb_face_t      *face,
    +                             hb_codepoint_t  glyph)
    +{
    +  return face->table.COLR->has_paint_for_glyph (glyph);
     }
     
     /**
    @@ -239,7 +280,7 @@ hb_ot_color_glyph_get_layers (hb_face_t           *face,
      *
      * Tests whether a face includes any `SVG` glyph images.
      *
    - * Return value: %true if data found, %false otherwise.
    + * Return value: `true` if data found, `false` otherwise.
      *
      * Since: 2.1.0
      */
    @@ -279,7 +320,7 @@ hb_ot_color_glyph_reference_svg (hb_face_t *face, hb_codepoint_t glyph)
      *
      * Tests whether a face has PNG glyph images (either in `CBDT` or `sbix` tables).
      *
    - * Return value: %true if data found, %false otherwise
    + * Return value: `true` if data found, `false` otherwise
      *
      * Since: 2.1.0
      */
    @@ -295,8 +336,8 @@ hb_ot_color_has_png (hb_face_t *face)
      * @glyph: a glyph index
      *
      * Fetches the PNG image for a glyph. This function takes a font object, not a face object,
    - * as input. To get an optimally sized PNG blob, the UPEM value must be set on the @font
    - * object. If UPEM is unset, the blob returned will be the largest PNG available.
    + * as input. To get an optimally sized PNG blob, the PPEM values must be set on the @font
    + * object. If PPEM is unset, the blob returned will be the largest PNG available.
      *
      * If the glyph has no PNG image, the singleton empty blob is returned.
      *
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-color.h b/src/java.desktop/share/native/libharfbuzz/hb-ot-color.h
    index ff659c7a45888..7fa810bd96f2b 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-color.h
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-ot-color.h
    @@ -102,6 +102,10 @@ hb_ot_color_has_layers (hb_face_t *face);
      *
      * Pairs of glyph and color index.
      *
    + * A color index of 0xFFFF does not refer to a palette
    + * color, but indicates that the foreground color should
    + * be used.
    + *
      * Since: 2.1.0
      **/
     typedef struct hb_ot_color_layer_t {
    @@ -116,6 +120,15 @@ hb_ot_color_glyph_get_layers (hb_face_t           *face,
                                   unsigned int        *layer_count, /* IN/OUT.  May be NULL. */
                                   hb_ot_color_layer_t *layers /* OUT.     May be NULL. */);
     
    +/* COLRv1 */
    +
    +HB_EXTERN hb_bool_t
    +hb_ot_color_has_paint (hb_face_t *face);
    +
    +HB_EXTERN hb_bool_t
    +hb_ot_color_glyph_has_paint (hb_face_t      *face,
    +                             hb_codepoint_t  glyph);
    +
     /*
      * SVG
      */
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-deprecated.h b/src/java.desktop/share/native/libharfbuzz/hb-ot-deprecated.h
    index da205d6713dfc..24656af53bbee 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-deprecated.h
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-ot-deprecated.h
    @@ -67,26 +67,30 @@ HB_BEGIN_DECLS
     
     
     /* Like hb_ot_layout_table_find_script, but takes zero-terminated array of scripts to test */
    -HB_EXTERN HB_DEPRECATED_FOR (hb_ot_layout_table_select_script) hb_bool_t
    +HB_DEPRECATED_FOR (hb_ot_layout_table_select_script)
    +HB_EXTERN hb_bool_t
     hb_ot_layout_table_choose_script (hb_face_t      *face,
                                       hb_tag_t        table_tag,
                                       const hb_tag_t *script_tags,
                                       unsigned int   *script_index,
                                       hb_tag_t       *chosen_script);
     
    -HB_EXTERN HB_DEPRECATED_FOR (hb_ot_layout_script_select_language) hb_bool_t
    +HB_DEPRECATED_FOR (hb_ot_layout_script_select_language)
    +HB_EXTERN hb_bool_t
     hb_ot_layout_script_find_language (hb_face_t    *face,
                                        hb_tag_t      table_tag,
                                        unsigned int  script_index,
                                        hb_tag_t      language_tag,
                                        unsigned int *language_index);
     
    -HB_EXTERN HB_DEPRECATED_FOR (hb_ot_tags_from_script_and_language) void
    +HB_DEPRECATED_FOR (hb_ot_tags_from_script_and_language)
    +HB_EXTERN void
     hb_ot_tags_from_script (hb_script_t  script,
                             hb_tag_t    *script_tag_1,
                             hb_tag_t    *script_tag_2);
     
    -HB_EXTERN HB_DEPRECATED_FOR (hb_ot_tags_from_script_and_language) hb_tag_t
    +HB_DEPRECATED_FOR (hb_ot_tags_from_script_and_language)
    +HB_EXTERN hb_tag_t
     hb_ot_tag_from_language (hb_language_t language);
     
     
    @@ -121,13 +125,15 @@ typedef struct hb_ot_var_axis_t {
       float max_value;
     } hb_ot_var_axis_t;
     
    -HB_EXTERN HB_DEPRECATED_FOR (hb_ot_var_get_axis_infos) unsigned int
    +HB_DEPRECATED_FOR (hb_ot_var_get_axis_infos)
    +HB_EXTERN unsigned int
     hb_ot_var_get_axes (hb_face_t        *face,
                         unsigned int      start_offset,
                         unsigned int     *axes_count /* IN/OUT */,
                         hb_ot_var_axis_t *axes_array /* OUT */);
     
    -HB_EXTERN HB_DEPRECATED_FOR (hb_ot_var_find_axis_info) hb_bool_t
    +HB_DEPRECATED_FOR (hb_ot_var_find_axis_info)
    +HB_EXTERN hb_bool_t
     hb_ot_var_find_axis (hb_face_t        *face,
                          hb_tag_t          axis_tag,
                          unsigned int     *axis_index,
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-face-table-list.hh b/src/java.desktop/share/native/libharfbuzz/hb-ot-face-table-list.hh
    index c05034b3bb5ae..b552dfdd9daed 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-face-table-list.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-ot-face-table-list.hh
    @@ -56,9 +56,9 @@ HB_OT_CORE_TABLE (OT, maxp)
     #if !defined(HB_NO_FACE_COLLECT_UNICODES) || !defined(HB_NO_OT_FONT)
     HB_OT_ACCELERATOR (OT, cmap)
     #endif
    -HB_OT_TABLE (OT, hhea)
    +HB_OT_CORE_TABLE (OT, hhea)
     HB_OT_ACCELERATOR (OT, hmtx)
    -HB_OT_TABLE (OT, OS2)
    +HB_OT_CORE_TABLE (OT, OS2)
     #if !defined(HB_NO_OT_FONT_GLYPH_NAMES) || !defined(HB_NO_METRICS) || !defined(HB_NO_STYLE)
     HB_OT_ACCELERATOR (OT, post)
     #endif
    @@ -66,7 +66,7 @@ HB_OT_ACCELERATOR (OT, post)
     HB_OT_ACCELERATOR (OT, name)
     #endif
     #ifndef HB_NO_STYLE
    -HB_OT_TABLE (OT, STAT)
    +HB_OT_CORE_TABLE (OT, STAT)
     #endif
     #ifndef HB_NO_META
     HB_OT_ACCELERATOR (OT, meta)
    @@ -74,9 +74,9 @@ HB_OT_ACCELERATOR (OT, meta)
     
     /* Vertical layout. */
     #ifndef HB_NO_VERTICAL
    -HB_OT_TABLE (OT, vhea)
    +HB_OT_CORE_TABLE (OT, vhea)
     HB_OT_ACCELERATOR (OT, vmtx)
    -HB_OT_TABLE (OT, VORG)
    +HB_OT_CORE_TABLE (OT, VORG)
     #endif
     
     /* TrueType outlines. */
    @@ -91,15 +91,16 @@ HB_OT_ACCELERATOR (OT, cff2)
     
     /* OpenType variations. */
     #ifndef HB_NO_VAR
    -HB_OT_TABLE (OT, fvar)
    -HB_OT_TABLE (OT, avar)
    +HB_OT_CORE_TABLE (OT, fvar)
    +HB_OT_CORE_TABLE (OT, avar)
    +HB_OT_CORE_TABLE (OT, cvar)
     HB_OT_ACCELERATOR (OT, gvar)
    -HB_OT_TABLE (OT, MVAR)
    +HB_OT_CORE_TABLE (OT, MVAR)
     #endif
     
     /* Legacy kern. */
     #ifndef HB_NO_OT_KERN
    -HB_OT_TABLE (OT, kern)
    +HB_OT_CORE_TABLE (OT, kern)
     #endif
     
     /* OpenType shaping. */
    @@ -107,12 +108,12 @@ HB_OT_TABLE (OT, kern)
     HB_OT_ACCELERATOR (OT, GDEF)
     HB_OT_ACCELERATOR (OT, GSUB)
     HB_OT_ACCELERATOR (OT, GPOS)
    -//HB_OT_TABLE (OT, JSTF)
    +//HB_OT_CORE_TABLE (OT, JSTF)
     #endif
     
     /* OpenType baseline. */
     #ifndef HB_NO_BASE
    -HB_OT_TABLE (OT, BASE)
    +HB_OT_CORE_TABLE (OT, BASE)
     #endif
     
     /* AAT shaping. */
    @@ -129,8 +130,8 @@ HB_OT_TABLE (AAT, feat)
     
     /* OpenType color fonts. */
     #ifndef HB_NO_COLOR
    -HB_OT_TABLE (OT, COLR)
    -HB_OT_TABLE (OT, CPAL)
    +HB_OT_CORE_TABLE (OT, COLR)
    +HB_OT_CORE_TABLE (OT, CPAL)
     HB_OT_ACCELERATOR (OT, CBDT)
     HB_OT_ACCELERATOR (OT, sbix)
     HB_OT_ACCELERATOR (OT, SVG)
    @@ -138,7 +139,7 @@ HB_OT_ACCELERATOR (OT, SVG)
     
     /* OpenType math. */
     #ifndef HB_NO_MATH
    -HB_OT_TABLE (OT, MATH)
    +HB_OT_CORE_TABLE (OT, MATH)
     #endif
     
     
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-face.cc b/src/java.desktop/share/native/libharfbuzz/hb-ot-face.cc
    index 5ef8df43ce7c0..2243ee0287414 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-face.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-ot-face.cc
    @@ -35,9 +35,9 @@
     #include "hb-ot-meta-table.hh"
     #include "hb-ot-name-table.hh"
     #include "hb-ot-post-table.hh"
    -#include "hb-ot-color-cbdt-table.hh"
    -#include "hb-ot-color-sbix-table.hh"
    -#include "hb-ot-color-svg-table.hh"
    +#include "OT/Color/CBDT/CBDT.hh"
    +#include "OT/Color/sbix/sbix.hh"
    +#include "OT/Color/svg/svg.hh"
     #include "hb-ot-layout-gdef-table.hh"
     #include "hb-ot-layout-gsub-table.hh"
     #include "hb-ot-layout-gpos-table.hh"
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-font.cc b/src/java.desktop/share/native/libharfbuzz/hb-ot-font.cc
    index 2f3d47591182f..884cea0f6b3da 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-font.cc
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-ot-font.cc
    @@ -34,18 +34,20 @@
     #include "hb-font.hh"
     #include "hb-machinery.hh"
     #include "hb-ot-face.hh"
    +#include "hb-outline.hh"
     
     #include "hb-ot-cmap-table.hh"
     #include "hb-ot-glyf-table.hh"
     #include "hb-ot-cff1-table.hh"
     #include "hb-ot-cff2-table.hh"
     #include "hb-ot-hmtx-table.hh"
    -#include "hb-ot-os2-table.hh"
     #include "hb-ot-post-table.hh"
     #include "hb-ot-stat-table.hh" // Just so we compile it; unused otherwise.
     #include "hb-ot-vorg-table.hh"
    -#include "hb-ot-color-cbdt-table.hh"
    -#include "hb-ot-color-sbix-table.hh"
    +#include "OT/Color/CBDT/CBDT.hh"
    +#include "OT/Color/COLR/COLR.hh"
    +#include "OT/Color/sbix/sbix.hh"
    +#include "OT/Color/svg/svg.hh"
     
     
     /**
    @@ -59,13 +61,20 @@
      * never need to call these functions directly.
      **/
     
    +using hb_ot_font_cmap_cache_t    = hb_cache_t<21, 16, 8, true>;
    +using hb_ot_font_advance_cache_t = hb_cache_t<24, 16, 8, true>;
    +
    +static hb_user_data_key_t hb_ot_font_cmap_cache_user_data_key;
    +
     struct hb_ot_font_t
     {
       const hb_ot_face_t *ot_face;
     
    +  hb_ot_font_cmap_cache_t *cmap_cache;
    +
       /* h_advance caching */
       mutable hb_atomic_int_t cached_coords_serial;
    -  mutable hb_atomic_ptr_t advance_cache;
    +  mutable hb_atomic_ptr_t advance_cache;
     };
     
     static hb_ot_font_t *
    @@ -77,6 +86,33 @@ _hb_ot_font_create (hb_font_t *font)
     
       ot_font->ot_face = &font->face->table;
     
    +  // retry:
    +  auto *cmap_cache  = (hb_ot_font_cmap_cache_t *) hb_face_get_user_data (font->face,
    +                                                                         &hb_ot_font_cmap_cache_user_data_key);
    +  if (!cmap_cache)
    +  {
    +    cmap_cache = (hb_ot_font_cmap_cache_t *) hb_malloc (sizeof (hb_ot_font_cmap_cache_t));
    +    if (unlikely (!cmap_cache)) goto out;
    +    cmap_cache->init ();
    +    if (unlikely (!hb_face_set_user_data (font->face,
    +                                          &hb_ot_font_cmap_cache_user_data_key,
    +                                          cmap_cache,
    +                                          hb_free,
    +                                          false)))
    +    {
    +      hb_free (cmap_cache);
    +      cmap_cache = nullptr;
    +      /* Normally we would retry here, but that would
    +       * infinite-loop if the face is the empty-face.
    +       * Just let it go and this font will be uncached if it
    +       * happened to collide with another thread creating the
    +       * cache at the same time. */
    +      // goto retry;
    +    }
    +  }
    +  out:
    +  ot_font->cmap_cache = cmap_cache;
    +
       return ot_font;
     }
     
    @@ -86,11 +122,7 @@ _hb_ot_font_destroy (void *font_data)
       hb_ot_font_t *ot_font = (hb_ot_font_t *) font_data;
     
       auto *cache = ot_font->advance_cache.get_relaxed ();
    -  if (cache)
    -  {
    -    cache->fini ();
    -    hb_free (cache);
    -  }
    +  hb_free (cache);
     
       hb_free (ot_font);
     }
    @@ -104,7 +136,7 @@ hb_ot_get_nominal_glyph (hb_font_t *font HB_UNUSED,
     {
       const hb_ot_font_t *ot_font = (const hb_ot_font_t *) font_data;
       const hb_ot_face_t *ot_face = ot_font->ot_face;
    -  return ot_face->cmap->get_nominal_glyph (unicode, glyph);
    +  return ot_face->cmap->get_nominal_glyph (unicode, glyph, ot_font->cmap_cache);
     }
     
     static unsigned int
    @@ -121,7 +153,8 @@ hb_ot_get_nominal_glyphs (hb_font_t *font HB_UNUSED,
       const hb_ot_face_t *ot_face = ot_font->ot_face;
       return ot_face->cmap->get_nominal_glyphs (count,
                                                 first_unicode, unicode_stride,
    -                                            first_glyph, glyph_stride);
    +                                            first_glyph, glyph_stride,
    +                                            ot_font->cmap_cache);
     }
     
     static hb_bool_t
    @@ -134,7 +167,9 @@ hb_ot_get_variation_glyph (hb_font_t *font HB_UNUSED,
     {
       const hb_ot_font_t *ot_font = (const hb_ot_font_t *) font_data;
       const hb_ot_face_t *ot_face = ot_font->ot_face;
    -  return ot_face->cmap->get_variation_glyph (unicode, variation_selector, glyph);
    +  return ot_face->cmap->get_variation_glyph (unicode,
    +                                             variation_selector, glyph,
    +                                             ot_font->cmap_cache);
     }
     
     static void
    @@ -146,12 +181,15 @@ hb_ot_get_glyph_h_advances (hb_font_t* font, void* font_data,
                                 unsigned advance_stride,
                                 void *user_data HB_UNUSED)
     {
    +
       const hb_ot_font_t *ot_font = (const hb_ot_font_t *) font_data;
       const hb_ot_face_t *ot_face = ot_font->ot_face;
       const OT::hmtx_accelerator_t &hmtx = *ot_face->hmtx;
     
    +  hb_position_t *orig_first_advance = first_advance;
    +
     #ifndef HB_NO_VAR
    -  const OT::HVARVVAR &HVAR = *hmtx.var_table;
    +  const OT::HVAR &HVAR = *hmtx.var_table;
       const OT::VariationStore &varStore = &HVAR + HVAR.varStore;
       OT::VariationStore::cache_t *varStore_cache = font->num_coords * count >= 128 ? varStore.create_cache () : nullptr;
     
    @@ -161,14 +199,14 @@ hb_ot_get_glyph_h_advances (hb_font_t* font, void* font_data,
       bool use_cache = false;
     #endif
     
    -  hb_advance_cache_t *cache = nullptr;
    +  hb_ot_font_advance_cache_t *cache = nullptr;
       if (use_cache)
       {
       retry:
    -    cache = ot_font->advance_cache.get ();
    +    cache = ot_font->advance_cache.get_acquire ();
         if (unlikely (!cache))
         {
    -      cache = (hb_advance_cache_t *) hb_malloc (sizeof (hb_advance_cache_t));
    +      cache = (hb_ot_font_advance_cache_t *) hb_malloc (sizeof (hb_ot_font_advance_cache_t));
           if (unlikely (!cache))
           {
             use_cache = false;
    @@ -181,7 +219,7 @@ hb_ot_get_glyph_h_advances (hb_font_t* font, void* font_data,
             hb_free (cache);
             goto retry;
           }
    -      ot_font->cached_coords_serial.set (font->serial_coords);
    +      ot_font->cached_coords_serial.set_release (font->serial_coords);
         }
       }
       out:
    @@ -190,17 +228,17 @@ hb_ot_get_glyph_h_advances (hb_font_t* font, void* font_data,
       {
         for (unsigned int i = 0; i < count; i++)
         {
    -      *first_advance = font->em_scale_x (hmtx.get_advance (*first_glyph, font, varStore_cache));
    +      *first_advance = font->em_scale_x (hmtx.get_advance_with_var_unscaled (*first_glyph, font, varStore_cache));
           first_glyph = &StructAtOffsetUnaligned (first_glyph, glyph_stride);
           first_advance = &StructAtOffsetUnaligned (first_advance, advance_stride);
         }
       }
       else
       { /* Use cache. */
    -    if (ot_font->cached_coords_serial.get () != (int) font->serial_coords)
    +    if (ot_font->cached_coords_serial.get_acquire () != (int) font->serial_coords)
         {
           ot_font->advance_cache->init ();
    -      ot_font->cached_coords_serial.set (font->serial_coords);
    +      ot_font->cached_coords_serial.set_release (font->serial_coords);
         }
     
         for (unsigned int i = 0; i < count; i++)
    @@ -211,7 +249,7 @@ hb_ot_get_glyph_h_advances (hb_font_t* font, void* font_data,
             v = cv;
           else
           {
    -        v = hmtx.get_advance (*first_glyph, font, varStore_cache);
    +        v = hmtx.get_advance_with_var_unscaled (*first_glyph, font, varStore_cache);
             ot_font->advance_cache->set (*first_glyph, v);
           }
           *first_advance = font->em_scale_x (v);
    @@ -223,6 +261,18 @@ hb_ot_get_glyph_h_advances (hb_font_t* font, void* font_data,
     #ifndef HB_NO_VAR
       OT::VariationStore::destroy_cache (varStore_cache);
     #endif
    +
    +  if (font->x_strength && !font->embolden_in_place)
    +  {
    +    /* Emboldening. */
    +    hb_position_t x_strength = font->x_scale >= 0 ? font->x_strength : -font->x_strength;
    +    first_advance = orig_first_advance;
    +    for (unsigned int i = 0; i < count; i++)
    +    {
    +      *first_advance += *first_advance ? x_strength : 0;
    +      first_advance = &StructAtOffsetUnaligned (first_advance, advance_stride);
    +    }
    +  }
     }
     
     #ifndef HB_NO_VERTICAL
    @@ -239,10 +289,12 @@ hb_ot_get_glyph_v_advances (hb_font_t* font, void* font_data,
       const hb_ot_face_t *ot_face = ot_font->ot_face;
       const OT::vmtx_accelerator_t &vmtx = *ot_face->vmtx;
     
    +  hb_position_t *orig_first_advance = first_advance;
    +
       if (vmtx.has_data ())
       {
     #ifndef HB_NO_VAR
    -    const OT::HVARVVAR &VVAR = *vmtx.var_table;
    +    const OT::VVAR &VVAR = *vmtx.var_table;
         const OT::VariationStore &varStore = &VVAR + VVAR.varStore;
         OT::VariationStore::cache_t *varStore_cache = font->num_coords ? varStore.create_cache () : nullptr;
     #else
    @@ -251,7 +303,7 @@ hb_ot_get_glyph_v_advances (hb_font_t* font, void* font_data,
     
         for (unsigned int i = 0; i < count; i++)
         {
    -      *first_advance = font->em_scale_y (-(int) vmtx.get_advance (*first_glyph, font, varStore_cache));
    +      *first_advance = font->em_scale_y (-(int) vmtx.get_advance_with_var_unscaled (*first_glyph, font, varStore_cache));
           first_glyph = &StructAtOffsetUnaligned (first_glyph, glyph_stride);
           first_advance = &StructAtOffsetUnaligned (first_advance, advance_stride);
         }
    @@ -273,6 +325,18 @@ hb_ot_get_glyph_v_advances (hb_font_t* font, void* font_data,
           first_advance = &StructAtOffsetUnaligned (first_advance, advance_stride);
         }
       }
    +
    +  if (font->y_strength && !font->embolden_in_place)
    +  {
    +    /* Emboldening. */
    +    hb_position_t y_strength = font->y_scale >= 0 ? font->y_strength : -font->y_strength;
    +    first_advance = orig_first_advance;
    +    for (unsigned int i = 0; i < count; i++)
    +    {
    +      *first_advance += *first_advance ? y_strength : 0;
    +      first_advance = &StructAtOffsetUnaligned (first_advance, advance_stride);
    +    }
    +  }
     }
     #endif
     
    @@ -293,17 +357,28 @@ hb_ot_get_glyph_v_origin (hb_font_t *font,
       const OT::VORG &VORG = *ot_face->VORG;
       if (VORG.has_data ())
       {
    -    *y = font->em_scale_y (VORG.get_y_origin (glyph));
    +    float delta = 0;
    +
    +#ifndef HB_NO_VAR
    +    const OT::vmtx_accelerator_t &vmtx = *ot_face->vmtx;
    +    const OT::VVAR &VVAR = *vmtx.var_table;
    +    if (font->num_coords)
    +      VVAR.get_vorg_delta_unscaled (glyph,
    +                                    font->coords, font->num_coords,
    +                                    &delta);
    +#endif
    +
    +    *y = font->em_scalef_y (VORG.get_y_origin (glyph) + delta);
         return true;
       }
     
       hb_glyph_extents_t extents = {0};
       if (ot_face->glyf->get_extents (font, glyph, &extents))
       {
    -    if (ot_face->vmtx->has_data ())
    +    const OT::vmtx_accelerator_t &vmtx = *ot_face->vmtx;
    +    int tsb = 0;
    +    if (vmtx.get_leading_bearing_with_var_unscaled (font, glyph, &tsb))
         {
    -      const OT::vmtx_accelerator_t &vmtx = *ot_face->vmtx;
    -      hb_position_t tsb = vmtx.get_side_bearing (font, glyph);
           *y = extents.y_bearing + font->em_scale_y (tsb);
           return true;
         }
    @@ -336,17 +411,17 @@ hb_ot_get_glyph_extents (hb_font_t *font,
     
     #if !defined(HB_NO_OT_FONT_BITMAP) && !defined(HB_NO_COLOR)
       if (ot_face->sbix->get_extents (font, glyph, extents)) return true;
    +  if (ot_face->CBDT->get_extents (font, glyph, extents)) return true;
    +#endif
    +#if !defined(HB_NO_COLOR) && !defined(HB_NO_PAINT)
    +  if (ot_face->COLR->get_extents (font, glyph, extents)) return true;
     #endif
       if (ot_face->glyf->get_extents (font, glyph, extents)) return true;
     #ifndef HB_NO_OT_FONT_CFF
       if (ot_face->cff1->get_extents (font, glyph, extents)) return true;
       if (ot_face->cff2->get_extents (font, glyph, extents)) return true;
     #endif
    -#if !defined(HB_NO_OT_FONT_BITMAP) && !defined(HB_NO_COLOR)
    -  if (ot_face->CBDT->get_extents (font, glyph, extents)) return true;
    -#endif
     
    -  // TODO Hook up side-bearings variations.
       return false;
     }
     
    @@ -391,9 +466,16 @@ hb_ot_get_font_h_extents (hb_font_t *font,
                               hb_font_extents_t *metrics,
                               void *user_data HB_UNUSED)
     {
    -  return _hb_ot_metrics_get_position_common (font, HB_OT_METRICS_TAG_HORIZONTAL_ASCENDER, &metrics->ascender) &&
    -         _hb_ot_metrics_get_position_common (font, HB_OT_METRICS_TAG_HORIZONTAL_DESCENDER, &metrics->descender) &&
    -         _hb_ot_metrics_get_position_common (font, HB_OT_METRICS_TAG_HORIZONTAL_LINE_GAP, &metrics->line_gap);
    +  bool ret = _hb_ot_metrics_get_position_common (font, HB_OT_METRICS_TAG_HORIZONTAL_ASCENDER, &metrics->ascender) &&
    +             _hb_ot_metrics_get_position_common (font, HB_OT_METRICS_TAG_HORIZONTAL_DESCENDER, &metrics->descender) &&
    +             _hb_ot_metrics_get_position_common (font, HB_OT_METRICS_TAG_HORIZONTAL_LINE_GAP, &metrics->line_gap);
    +
    +  /* Embolden */
    +  int y_shift = font->y_strength;
    +  if (font->y_scale < 0) y_shift = -y_shift;
    +  metrics->ascender += y_shift;
    +
    +  return ret;
     }
     
     #ifndef HB_NO_VERTICAL
    @@ -411,17 +493,62 @@ hb_ot_get_font_v_extents (hb_font_t *font,
     
     #ifndef HB_NO_DRAW
     static void
    -hb_ot_get_glyph_shape (hb_font_t *font,
    -                       void *font_data HB_UNUSED,
    -                       hb_codepoint_t glyph,
    -                       hb_draw_funcs_t *draw_funcs, void *draw_data,
    -                       void *user_data)
    +hb_ot_draw_glyph (hb_font_t *font,
    +                  void *font_data HB_UNUSED,
    +                  hb_codepoint_t glyph,
    +                  hb_draw_funcs_t *draw_funcs, void *draw_data,
    +                  void *user_data)
    +{
    +  bool embolden = font->x_strength || font->y_strength;
    +  hb_outline_t outline;
    +
    +  { // Need draw_session to be destructed before emboldening.
    +    hb_draw_session_t draw_session (embolden ? hb_outline_recording_pen_get_funcs () : draw_funcs,
    +                                    embolden ? &outline : draw_data, font->slant_xy);
    +    if (!font->face->table.glyf->get_path (font, glyph, draw_session))
    +#ifndef HB_NO_CFF
    +    if (!font->face->table.cff1->get_path (font, glyph, draw_session))
    +    if (!font->face->table.cff2->get_path (font, glyph, draw_session))
    +#endif
    +    {}
    +  }
    +
    +  if (embolden)
    +  {
    +    float x_shift = font->embolden_in_place ? 0 : (float) font->x_strength / 2;
    +    float y_shift = (float) font->y_strength / 2;
    +    if (font->x_scale < 0) x_shift = -x_shift;
    +    if (font->y_scale < 0) y_shift = -y_shift;
    +    outline.embolden (font->x_strength, font->y_strength,
    +                      x_shift, y_shift);
    +
    +    outline.replay (draw_funcs, draw_data);
    +  }
    +}
    +#endif
    +
    +#ifndef HB_NO_PAINT
    +static void
    +hb_ot_paint_glyph (hb_font_t *font,
    +                   void *font_data,
    +                   hb_codepoint_t glyph,
    +                   hb_paint_funcs_t *paint_funcs, void *paint_data,
    +                   unsigned int palette,
    +                   hb_color_t foreground,
    +                   void *user_data)
     {
    -  hb_draw_session_t draw_session (draw_funcs, draw_data, font->slant_xy);
    -  if (font->face->table.glyf->get_path (font, glyph, draw_session)) return;
    +#ifndef HB_NO_COLOR
    +  if (font->face->table.COLR->paint_glyph (font, glyph, paint_funcs, paint_data, palette, foreground)) return;
    +  if (font->face->table.SVG->paint_glyph (font, glyph, paint_funcs, paint_data)) return;
    +#ifndef HB_NO_OT_FONT_BITMAP
    +  if (font->face->table.CBDT->paint_glyph (font, glyph, paint_funcs, paint_data)) return;
    +  if (font->face->table.sbix->paint_glyph (font, glyph, paint_funcs, paint_data)) return;
    +#endif
    +#endif
    +  if (font->face->table.glyf->paint_glyph (font, glyph, paint_funcs, paint_data, foreground)) return;
     #ifndef HB_NO_CFF
    -  if (font->face->table.cff1->get_path (font, glyph, draw_session)) return;
    -  if (font->face->table.cff2->get_path (font, glyph, draw_session)) return;
    +  if (font->face->table.cff1->paint_glyph (font, glyph, paint_funcs, paint_data, foreground)) return;
    +  if (font->face->table.cff2->paint_glyph (font, glyph, paint_funcs, paint_data, foreground)) return;
     #endif
     }
     #endif
    @@ -449,7 +576,11 @@ static struct hb_ot_font_funcs_lazy_loader_t : hb_font_funcs_lazy_loader_tface->table.glyf->get_side_bearing_var (font, glyph, is_vertical);
    -}
    -
    -unsigned
    -_glyf_get_advance_var (hb_font_t *font, hb_codepoint_t glyph, bool is_vertical)
    -{
    -  return font->face->table.glyf->get_advance_var (font, glyph, is_vertical);
    -}
    -#endif
    -
    -
     #endif
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-hdmx-table.hh b/src/java.desktop/share/native/libharfbuzz/hb-ot-hdmx-table.hh
    index 84154467ed139..e13321ee6fa1a 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-hdmx-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-ot-hdmx-table.hh
    @@ -76,7 +76,7 @@ struct DeviceRecord
       HBUINT8                       maxWidth;       /* Maximum width. */
       UnsizedArrayOf       widthsZ;        /* Array of widths (numGlyphs is from the 'maxp' table). */
       public:
    -  DEFINE_SIZE_ARRAY (2, widthsZ);
    +  DEFINE_SIZE_UNBOUNDED (2);
     };
     
     
    @@ -87,14 +87,6 @@ struct hdmx
       unsigned int get_size () const
       { return min_size + numRecords * sizeDeviceRecord; }
     
    -  const DeviceRecord& operator [] (unsigned int i) const
    -  {
    -    /* XXX Null(DeviceRecord) is NOT safe as it's num-glyphs lengthed.
    -     * https://github.com/harfbuzz/harfbuzz/issues/1300 */
    -    if (unlikely (i >= numRecords)) return Null (DeviceRecord);
    -    return StructAtOffset (&this->firstDeviceRecord, i * sizeDeviceRecord);
    -  }
    -
       template
       bool serialize (hb_serialize_context_t *c, unsigned version, Iterator it)
    @@ -156,6 +148,7 @@ struct hdmx
         TRACE_SANITIZE (this);
         return_trace (c->check_struct (this) &&
                       !hb_unsigned_mul_overflows (numRecords, sizeDeviceRecord) &&
    +                  min_size + numRecords * sizeDeviceRecord > numRecords * sizeDeviceRecord &&
                       sizeDeviceRecord >= DeviceRecord::min_size &&
                       c->check_range (this, get_size ()));
       }
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-head-table.hh b/src/java.desktop/share/native/libharfbuzz/hb-ot-head-table.hh
    index c87356ae16c92..ab057fcfe4f10 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-head-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-ot-head-table.hh
    @@ -63,7 +63,25 @@ struct head
       bool subset (hb_subset_context_t *c) const
       {
         TRACE_SUBSET (this);
    -    return_trace (serialize (c->serializer));
    +    head *out = c->serializer->embed (this);
    +    if (unlikely (!out)) return_trace (false);
    +
    +    if (c->plan->normalized_coords)
    +    {
    +      if (unlikely (!c->serializer->check_assign (out->xMin, c->plan->head_maxp_info.xMin,
    +                                                  HB_SERIALIZE_ERROR_INT_OVERFLOW)))
    +        return_trace (false);
    +      if (unlikely (!c->serializer->check_assign (out->xMax, c->plan->head_maxp_info.xMax,
    +                                                  HB_SERIALIZE_ERROR_INT_OVERFLOW)))
    +        return_trace (false);
    +      if (unlikely (!c->serializer->check_assign (out->yMin, c->plan->head_maxp_info.yMin,
    +                                                  HB_SERIALIZE_ERROR_INT_OVERFLOW)))
    +        return_trace (false);
    +      if (unlikely (!c->serializer->check_assign (out->yMax, c->plan->head_maxp_info.yMax,
    +                                                  HB_SERIALIZE_ERROR_INT_OVERFLOW)))
    +        return_trace (false);
    +    }
    +    return_trace (true);
       }
     
       enum mac_style_flag_t {
    @@ -97,6 +115,7 @@ struct head
                                              * entire font as HBUINT32, then store
                                              * 0xB1B0AFBAu - sum. */
       HBUINT32      magicNumber;            /* Set to 0x5F0F3CF5u. */
    +  public:
       HBUINT16      flags;                  /* Bit 0: Baseline for font at y=0;
                                              * Bit 1: Left sidebearing point at x=0;
                                              * Bit 2: Instructions may depend on point size;
    @@ -141,6 +160,7 @@ struct head
                                              * encoded in the cmap subtables represent proper
                                              * support for those code points.
                                              * Bit 15: Reserved, set to 0. */
    +  protected:
       HBUINT16      unitsPerEm;             /* Valid range is from 16 to 16384. This value
                                              * should be a power of 2 for fonts that have
                                              * TrueType outlines. */
    @@ -148,10 +168,12 @@ struct head
                                                January 1, 1904. 64-bit integer */
       LONGDATETIME  modified;               /* Number of seconds since 12:00 midnight,
                                                January 1, 1904. 64-bit integer */
    +  public:
       HBINT16       xMin;                   /* For all glyph bounding boxes. */
       HBINT16       yMin;                   /* For all glyph bounding boxes. */
       HBINT16       xMax;                   /* For all glyph bounding boxes. */
       HBINT16       yMax;                   /* For all glyph bounding boxes. */
    +  protected:
       HBUINT16      macStyle;               /* Bit 0: Bold (if set to 1);
                                              * Bit 1: Italic (if set to 1)
                                              * Bit 2: Underline (if set to 1)
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-hmtx-table.hh b/src/java.desktop/share/native/libharfbuzz/hb-ot-hmtx-table.hh
    index 9883df20860ba..e830fd09cf0da 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-hmtx-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-ot-hmtx-table.hh
    @@ -31,6 +31,7 @@
     #include "hb-ot-maxp-table.hh"
     #include "hb-ot-hhea-table.hh"
     #include "hb-ot-var-hvar-table.hh"
    +#include "hb-ot-var-mvar-table.hh"
     #include "hb-ot-metrics.hh"
     
     /*
    @@ -43,11 +44,14 @@
     #define HB_OT_TAG_vmtx HB_TAG('v','m','t','x')
     
     
    -HB_INTERNAL int
    -_glyf_get_side_bearing_var (hb_font_t *font, hb_codepoint_t glyph, bool is_vertical);
    +HB_INTERNAL bool
    +_glyf_get_leading_bearing_with_var_unscaled (hb_font_t *font, hb_codepoint_t glyph, bool is_vertical, int *lsb);
     
     HB_INTERNAL unsigned
    -_glyf_get_advance_var (hb_font_t *font, hb_codepoint_t glyph, bool is_vertical);
    +_glyf_get_advance_with_var_unscaled (hb_font_t *font, hb_codepoint_t glyph, bool is_vertical);
    +
    +HB_INTERNAL bool
    +_glyf_get_leading_bearing_without_var_unscaled (hb_face_t *face, hb_codepoint_t gid, bool is_vertical, int *lsb);
     
     
     namespace OT {
    @@ -62,7 +66,7 @@ struct LongMetric
     };
     
     
    -template 
    +template 
     struct hmtxvmtx
     {
       bool sanitize (hb_sanitize_context_t *c HB_UNUSED) const
    @@ -73,11 +77,15 @@ struct hmtxvmtx
         return_trace (true);
       }
     
    +  const hb_hashmap_t>* get_mtx_map (const hb_subset_plan_t *plan) const
    +  { return T::is_horizontal ? &plan->hmtx_map : &plan->vmtx_map; }
     
    -  bool subset_update_header (hb_subset_plan_t *plan,
    -                             unsigned int num_hmetrics) const
    +  bool subset_update_header (hb_subset_context_t *c,
    +                             unsigned int num_hmetrics,
    +                             const hb_hashmap_t> *mtx_map,
    +                             const hb_map_t *bounds_map) const
       {
    -    hb_blob_t *src_blob = hb_sanitize_context_t ().reference_table (plan->source, H::tableTag);
    +    hb_blob_t *src_blob = hb_sanitize_context_t ().reference_table (c->plan->source, H::tableTag);
         hb_blob_t *dest_blob = hb_blob_copy_writable_or_fail (src_blob);
         hb_blob_destroy (src_blob);
     
    @@ -87,9 +95,58 @@ struct hmtxvmtx
     
         unsigned int length;
         H *table = (H *) hb_blob_get_data (dest_blob, &length);
    -    table->numberOfLongMetrics = num_hmetrics;
    +    c->serializer->check_assign (table->numberOfLongMetrics, num_hmetrics, HB_SERIALIZE_ERROR_INT_OVERFLOW);
    +
    +#ifndef HB_NO_VAR
    +    if (c->plan->normalized_coords)
    +    {
    +      auto &MVAR = *c->plan->source->table.MVAR;
    +      if (T::is_horizontal)
    +      {
    +        HB_ADD_MVAR_VAR (HB_OT_METRICS_TAG_HORIZONTAL_CARET_RISE,   caretSlopeRise);
    +        HB_ADD_MVAR_VAR (HB_OT_METRICS_TAG_HORIZONTAL_CARET_RUN,    caretSlopeRun);
    +        HB_ADD_MVAR_VAR (HB_OT_METRICS_TAG_HORIZONTAL_CARET_OFFSET, caretOffset);
    +      }
    +      else
    +      {
    +        HB_ADD_MVAR_VAR (HB_OT_METRICS_TAG_VERTICAL_CARET_RISE,     caretSlopeRise);
    +        HB_ADD_MVAR_VAR (HB_OT_METRICS_TAG_VERTICAL_CARET_RUN,      caretSlopeRun);
    +        HB_ADD_MVAR_VAR (HB_OT_METRICS_TAG_VERTICAL_CARET_OFFSET,   caretOffset);
    +      }
    +
    +      int min_lsb = 0x7FFF;
    +      int min_rsb = 0x7FFF;
    +      int max_extent = -0x7FFF;
    +      unsigned max_adv = 0;
    +      for (const auto _ : *mtx_map)
    +      {
    +        hb_codepoint_t gid = _.first;
    +        unsigned adv = _.second.first;
    +        int lsb = _.second.second;
    +        max_adv = hb_max (max_adv, adv);
    +
    +        if (bounds_map->has (gid))
    +        {
    +          unsigned bound_width = bounds_map->get (gid);
    +          int rsb = adv - lsb - bound_width;
    +          int extent = lsb + bound_width;
    +          min_lsb = hb_min (min_lsb, lsb);
    +          min_rsb = hb_min (min_rsb, rsb);
    +          max_extent = hb_max (max_extent, extent);
    +        }
    +      }
    +
    +      table->advanceMax = max_adv;
    +      if (!bounds_map->is_empty ())
    +      {
    +        table->minLeadingBearing = min_lsb;
    +        table->minTrailingBearing = min_rsb;
    +        table->maxExtent = max_extent;
    +      }
    +    }
    +#endif
     
    -    bool result = plan->add_table (H::tableTag, dest_blob);
    +    bool result = c->plan->add_table (H::tableTag, dest_blob);
         hb_blob_destroy (dest_blob);
     
         return result;
    @@ -111,12 +168,19 @@ struct hmtxvmtx
             lm.sb = _.second;
             if (unlikely (!c->embed (&lm))) return;
           }
    -      else
    +      else if (idx < 0x10000u)
           {
             FWORD *sb = c->allocate_size (FWORD::static_size);
             if (unlikely (!sb)) return;
             *sb = _.second;
           }
    +      else
    +      {
    +        // TODO: This does not do tail optimization.
    +        UFWORD *adv = c->allocate_size (UFWORD::static_size);
    +        if (unlikely (!adv)) return;
    +        *adv = _.first;
    +      }
           idx++;
         }
       }
    @@ -130,14 +194,15 @@ struct hmtxvmtx
     
         accelerator_t _mtx (c->plan->source);
         unsigned num_long_metrics;
    +    const hb_hashmap_t> *mtx_map = get_mtx_map (c->plan);
         {
           /* Determine num_long_metrics to encode. */
           auto& plan = c->plan;
    -      num_long_metrics = plan->num_output_glyphs ();
    -      hb_codepoint_t old_gid = 0;
    -      unsigned int last_advance = plan->old_gid_for_new_gid (num_long_metrics - 1, &old_gid) ? _mtx.get_advance (old_gid) : 0;
    +
    +      num_long_metrics = hb_min (plan->num_output_glyphs (), 0xFFFFu);
    +      unsigned int last_advance = get_new_gid_advance_unscaled (plan, mtx_map, num_long_metrics - 1, _mtx);
           while (num_long_metrics > 1 &&
    -             last_advance == (plan->old_gid_for_new_gid (num_long_metrics - 2, &old_gid) ? _mtx.get_advance (old_gid) : 0))
    +             last_advance == get_new_gid_advance_unscaled (plan, mtx_map, num_long_metrics - 2, _mtx))
           {
             num_long_metrics--;
           }
    @@ -145,12 +210,19 @@ struct hmtxvmtx
     
         auto it =
         + hb_range (c->plan->num_output_glyphs ())
    -    | hb_map ([c, &_mtx] (unsigned _)
    +    | hb_map ([c, &_mtx, mtx_map] (unsigned _)
                   {
    -                hb_codepoint_t old_gid;
    -                if (!c->plan->old_gid_for_new_gid (_, &old_gid))
    -                  return hb_pair (0u, 0);
    -                return hb_pair (_mtx.get_advance (old_gid), _mtx.get_side_bearing (old_gid));
    +                if (!mtx_map->has (_))
    +                {
    +                  hb_codepoint_t old_gid;
    +                  if (!c->plan->old_gid_for_new_gid (_, &old_gid))
    +                    return hb_pair (0u, 0);
    +                  int lsb = 0;
    +                  if (!_mtx.get_leading_bearing_without_var_unscaled (old_gid, &lsb))
    +                    (void) _glyf_get_leading_bearing_without_var_unscaled (c->plan->source, old_gid, !T::is_horizontal, &lsb);
    +                  return hb_pair (_mtx.get_advance_without_var_unscaled (old_gid), +lsb);
    +                }
    +                return mtx_map->get (_);
                   })
         ;
     
    @@ -160,7 +232,8 @@ struct hmtxvmtx
           return_trace (false);
     
         // Amend header num hmetrics
    -    if (unlikely (!subset_update_header (c->plan, num_long_metrics)))
    +    if (unlikely (!subset_update_header (c, num_long_metrics, mtx_map,
    +                                         T::is_horizontal ? &c->plan->bounds_width_map : &c->plan->bounds_height_map)))
           return_trace (false);
     
         return_trace (true);
    @@ -173,7 +246,7 @@ struct hmtxvmtx
         accelerator_t (hb_face_t *face)
         {
           table = hb_sanitize_context_t ().reference_table (face, T::tableTag);
    -      var_table = hb_sanitize_context_t ().reference_table (face, T::variationsTag);
    +      var_table = hb_sanitize_context_t ().reference_table (face, T::variationsTag);
     
           default_advance = T::is_horizontal ? hb_face_get_upem (face) / 2 : hb_face_get_upem (face);
     
    @@ -221,36 +294,46 @@ struct hmtxvmtx
     
         bool has_data () const { return (bool) num_bearings; }
     
    -    int get_side_bearing (hb_codepoint_t glyph) const
    +    bool get_leading_bearing_without_var_unscaled (hb_codepoint_t glyph,
    +                                                   int *lsb) const
         {
           if (glyph < num_long_metrics)
    -        return table->longMetricZ[glyph].sb;
    +      {
    +        *lsb = table->longMetricZ[glyph].sb;
    +        return true;
    +      }
     
           if (unlikely (glyph >= num_bearings))
    -        return 0;
    +        return false;
     
           const FWORD *bearings = (const FWORD *) &table->longMetricZ[num_long_metrics];
    -      return bearings[glyph - num_long_metrics];
    +      *lsb = bearings[glyph - num_long_metrics];
    +      return true;
         }
     
    -    int get_side_bearing (hb_font_t *font, hb_codepoint_t glyph) const
    +    bool get_leading_bearing_with_var_unscaled (hb_font_t *font,
    +                                                hb_codepoint_t glyph,
    +                                                int *lsb) const
         {
    -      int side_bearing = get_side_bearing (glyph);
    +      if (!font->num_coords)
    +        return get_leading_bearing_without_var_unscaled (glyph, lsb);
     
     #ifndef HB_NO_VAR
    -      if (unlikely (glyph >= num_bearings) || !font->num_coords)
    -        return side_bearing;
    -
    -      if (var_table.get_length ())
    -        return side_bearing + var_table->get_side_bearing_var (glyph, font->coords, font->num_coords);
    +      float delta;
    +      if (var_table->get_lsb_delta_unscaled (glyph, font->coords, font->num_coords, &delta) &&
    +          get_leading_bearing_without_var_unscaled (glyph, lsb))
    +      {
    +        *lsb += roundf (delta);
    +        return true;
    +      }
     
    -      return _glyf_get_side_bearing_var (font, glyph, T::tableTag == HB_OT_TAG_vmtx);
    +      return _glyf_get_leading_bearing_with_var_unscaled (font, glyph, T::tableTag == HB_OT_TAG_vmtx, lsb);
     #else
    -      return side_bearing;
    +      return false;
     #endif
         }
     
    -    unsigned int get_advance (hb_codepoint_t glyph) const
    +    unsigned int get_advance_without_var_unscaled (hb_codepoint_t glyph) const
         {
           /* OpenType case. */
           if (glyph < num_bearings)
    @@ -262,7 +345,7 @@ struct hmtxvmtx
           if (unlikely (!num_advances))
             return default_advance;
     
    -#ifdef HB_NO_BORING_EXPANSION
    +#ifdef HB_NO_BEYOND_64K
           return 0;
     #endif
     
    @@ -272,10 +355,8 @@ struct hmtxvmtx
           /* num_bearings <= glyph < num_glyphs;
            * num_bearings <= num_advances */
     
    -      /* TODO Optimize */
    -
           if (num_bearings == num_advances)
    -        return get_advance (num_bearings - 1);
    +        return get_advance_without_var_unscaled (num_bearings - 1);
     
           const FWORD *bearings = (const FWORD *) &table->longMetricZ[num_long_metrics];
           const UFWORD *advances = (const UFWORD *) &bearings[num_bearings - num_long_metrics];
    @@ -283,20 +364,22 @@ struct hmtxvmtx
           return advances[hb_min (glyph - num_bearings, num_advances - num_bearings - 1)];
         }
     
    -    unsigned int get_advance (hb_codepoint_t  glyph,
    -                              hb_font_t      *font,
    -                              VariationStore::cache_t *store_cache = nullptr) const
    +    unsigned get_advance_with_var_unscaled (hb_codepoint_t  glyph,
    +                                            hb_font_t      *font,
    +                                            VariationStore::cache_t *store_cache = nullptr) const
         {
    -      unsigned int advance = get_advance (glyph);
    +      unsigned int advance = get_advance_without_var_unscaled (glyph);
     
     #ifndef HB_NO_VAR
           if (unlikely (glyph >= num_bearings) || !font->num_coords)
             return advance;
     
           if (var_table.get_length ())
    -        return advance + roundf (var_table->get_advance_var (glyph, font, store_cache)); // TODO Optimize?!
    +        return advance + roundf (var_table->get_advance_delta_unscaled (glyph,
    +                                                                        font->coords, font->num_coords,
    +                                                                        store_cache));
     
    -      return _glyf_get_advance_var (font, glyph, T::tableTag == HB_OT_TAG_vmtx);
    +      return _glyf_get_advance_with_var_unscaled (font, glyph, T::tableTag == HB_OT_TAG_vmtx);
     #else
           return advance;
     #endif
    @@ -313,9 +396,25 @@ struct hmtxvmtx
     
         public:
         hb_blob_ptr_t table;
    -    hb_blob_ptr_t var_table;
    +    hb_blob_ptr_t var_table;
       };
     
    +  /* get advance: when no variations, call get_advance_without_var_unscaled.
    +   * when there're variations, get advance value from mtx_map in subset_plan*/
    +  unsigned get_new_gid_advance_unscaled (const hb_subset_plan_t *plan,
    +                                         const hb_hashmap_t> *mtx_map,
    +                                         unsigned new_gid,
    +                                         const accelerator_t &_mtx) const
    +  {
    +    if (mtx_map->is_empty ())
    +    {
    +      hb_codepoint_t old_gid = 0;
    +      return plan->old_gid_for_new_gid (new_gid, &old_gid) ?
    +             _mtx.get_advance_without_var_unscaled (old_gid) : 0;
    +    }
    +    return mtx_map->get (new_gid).first;
    +  }
    +
       protected:
       UnsizedArrayOf
                     longMetricZ;    /* Paired advance width and leading
    @@ -346,12 +445,12 @@ struct hmtxvmtx
       DEFINE_SIZE_ARRAY (0, longMetricZ);
     };
     
    -struct hmtx : hmtxvmtx {
    +struct hmtx : hmtxvmtx {
       static constexpr hb_tag_t tableTag = HB_OT_TAG_hmtx;
       static constexpr hb_tag_t variationsTag = HB_OT_TAG_HVAR;
       static constexpr bool is_horizontal = true;
     };
    -struct vmtx : hmtxvmtx {
    +struct vmtx : hmtxvmtx {
       static constexpr hb_tag_t tableTag = HB_OT_TAG_vmtx;
       static constexpr hb_tag_t variationsTag = HB_OT_TAG_VVAR;
       static constexpr bool is_horizontal = false;
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-layout-base-table.hh b/src/java.desktop/share/native/libharfbuzz/hb-ot-layout-base-table.hh
    index b9b5b76a97cc1..bcf12221619f9 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-layout-base-table.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-ot-layout-base-table.hh
    @@ -49,7 +49,7 @@ struct BaseCoordFormat1
       bool sanitize (hb_sanitize_context_t *c) const
       {
         TRACE_SANITIZE (this);
    -    return_trace (likely (c->check_struct (this)));
    +    return_trace (c->check_struct (this));
       }
     
       protected:
    diff --git a/src/java.desktop/share/native/libharfbuzz/hb-ot-layout-common.hh b/src/java.desktop/share/native/libharfbuzz/hb-ot-layout-common.hh
    index ed8acdbd62ab4..1ff697b1b632d 100644
    --- a/src/java.desktop/share/native/libharfbuzz/hb-ot-layout-common.hh
    +++ b/src/java.desktop/share/native/libharfbuzz/hb-ot-layout-common.hh
    @@ -35,69 +35,42 @@
     #include "hb-set.hh"
     #include "hb-bimap.hh"
     
    +#include "OT/Layout/Common/Coverage.hh"
    +#include "OT/Layout/types.hh"
     
    -#ifndef HB_MAX_NESTING_LEVEL
    -#define HB_MAX_NESTING_LEVEL    64
    -#endif
    -#ifndef HB_MAX_CONTEXT_LENGTH
    -#define HB_MAX_CONTEXT_LENGTH   64
    -#endif
    -#ifndef HB_CLOSURE_MAX_STAGES
    -/*
    - * The maximum number of times a lookup can be applied during shaping.
    - * Used to limit the number of iterations of the closure algorithm.
    - * This must be larger than the number of times add_pause() is
    - * called in a collect_features call of any shaper.
    - */
    -#define HB_CLOSURE_MAX_STAGES   32
    -#endif
    -
    -#ifndef HB_MAX_SCRIPTS
    -#define HB_MAX_SCRIPTS  500
    -#endif
    -
    -#ifndef HB_MAX_LANGSYS
    -#define HB_MAX_LANGSYS  2000
    -#endif
    -
    -#ifndef HB_MAX_LANGSYS_FEATURE_COUNT
    -#define HB_MAX_LANGSYS_FEATURE_COUNT 50000
    -#endif
    -
    -#ifndef HB_MAX_FEATURES
    -#define HB_MAX_FEATURES 750
    -#endif
    -
    -#ifndef HB_MAX_FEATURE_INDICES
    -#define HB_MAX_FEATURE_INDICES  1500
    -#endif
    -
    -#ifndef HB_MAX_LOOKUP_VISIT_COUNT
    -#define HB_MAX_LOOKUP_VISIT_COUNT       35000
    -#endif
    +// TODO(garretrieger): cleanup these after migration.
    +using OT::Layout::Common::Coverage;
    +using OT::Layout::Common::RangeRecord;
    +using OT::Layout::SmallTypes;
    +using OT::Layout::MediumTypes;
     
     
     namespace OT {
     
    -
    -#define NOT_COVERED             ((unsigned int) -1)
    -
    -
     template
    -static inline void Coverage_serialize (hb_serialize_context_t *c,
    +static inline bool ClassDef_serialize (hb_serialize_context_t *c,
                                            Iterator it);
     
    -template
    -static inline void ClassDef_serialize (hb_serialize_context_t *c,
    -                                       Iterator it);
    -
    -static void ClassDef_remap_and_serialize (
    +static bool ClassDef_remap_and_serialize (
         hb_serialize_context_t *c,
         const hb_set_t &klasses,
         bool use_class_zero,
         hb_sorted_vector_t> &glyph_and_klass, /* IN/OUT */
         hb_map_t *klass_map /*IN/OUT*/);
     
    +struct hb_collect_feature_substitutes_with_var_context_t
    +{
    +  const hb_map_t *axes_index_tag_map;
    +  const hb_hashmap_t *axes_location;
    +  hb_hashmap_t> *record_cond_idx_map;
    +  hb_hashmap_t *feature_substitutes_map;
    +
    +  // not stored in subset_plan
    +  hb_set_t *feature_indices;
    +  bool apply;
    +  unsigned cur_record_idx;
    +  hb_hashmap_t, unsigned> *conditionset_map;
    +};
     
     struct hb_prune_langsys_context_t
     {
    @@ -164,24 +137,40 @@ struct hb_subset_layout_context_t :
       const hb_map_t *lookup_index_map;
       const hb_hashmap_t> *script_langsys_map;
       const hb_map_t *feature_index_map;
    +  const hb_hashmap_t *feature_substitutes_map;
    +  hb_hashmap_t> *feature_record_cond_idx_map;
    +
       unsigned cur_script_index;
    +  unsigned cur_feature_var_record_idx;
     
       hb_subset_layout_context_t (hb_subset_context_t *c_,
    -                              hb_tag_t tag_,
    -                              hb_map_t *lookup_map_,
    -                              hb_hashmap_t> *script_langsys_map_,
    -                              hb_map_t *feature_index_map_) :
    +                              hb_tag_t tag_) :
                                     subset_context (c_),
                                     table_tag (tag_),
    -                                lookup_index_map (lookup_map_),
    -                                script_langsys_map (script_langsys_map_),
    -                                feature_index_map (feature_index_map_),
                                     cur_script_index (0xFFFFu),
    +                                cur_feature_var_record_idx (0u),
                                     script_count (0),
                                     langsys_count (0),
                                     feature_index_count (0),
                                     lookup_index_count (0)
    -  {}
    +  {
    +    if (tag_ == HB_OT_TAG_GSUB)
    +    {
    +      lookup_index_map = &c_->plan->gsub_lookups;
    +      script_langsys_map = &c_->plan->gsub_langsys;
    +      feature_index_map = &c_->plan->gsub_features;
    +      feature_substitutes_map = &c_->plan->gsub_feature_substitutes_map;
    +      feature_record_cond_idx_map = c_->plan->user_axes_location.is_empty () ? nullptr : &c_->plan->gsub_feature_record_cond_idx_map;
    +    }
    +    else
    +    {
    +      lookup_index_map = &c_->plan->gpos_lookups;
    +      script_langsys_map = &c_->plan->gpos_langsys;
    +      feature_index_map = &c_->plan->gpos_features;
    +      feature_substitutes_map = &c_->plan->gpos_feature_substitutes_map;
    +      feature_record_cond_idx_map = c_->plan->user_axes_location.is_empty () ? nullptr : &c_->plan->gpos_feature_record_cond_idx_map;
    +    }
    +  }
     
       private:
       unsigned script_count;
    @@ -190,6 +179,7 @@ struct hb_subset_layout_context_t :
       unsigned lookup_index_count;
     };
     
    +struct VariationStore;
     struct hb_collect_variation_indices_context_t :
            hb_dispatch_context_t
     {
    @@ -198,15 +188,27 @@ struct hb_collect_variation_indices_context_t :
       static return_t default_return_value () { return hb_empty_t (); }
     
       hb_set_t *layout_variation_indices;
    +  hb_hashmap_t> *varidx_delta_map;
    +  hb_font_t *font;
    +  const VariationStore *var_store;
       const hb_set_t *glyph_set;
       const hb_map_t *gpos_lookups;
    +  float *store_cache;
     
       hb_collect_variation_indices_context_t (hb_set_t *layout_variation_indices_,
    +                                          hb_hashmap_t> *varidx_delta_map_,
    +                                          hb_font_t *font_,
    +                                          const VariationStore *var_store_,
                                               const hb_set_t *glyph_set_,
    -                                          const hb_map_t *gpos_lookups_) :
    +                                          const hb_map_t *gpos_lookups_,
    +                                          float *store_cache_) :
                                             layout_variation_indices (layout_variation_indices_),
    +                                        varidx_delta_map (varidx_delta_map_),
    +                                        font (font_),
    +                                        var_store (var_store_),
                                             glyph_set (glyph_set_),
    -                                        gpos_lookups (gpos_lookups_) {}
    +                                        gpos_lookups (gpos_lookups_),
    +                                        store_cache (store_cache_) {}
     };
     
     template
    @@ -315,6 +317,31 @@ struct subset_record_array_t
       const void *base;
     };
     
    +template
    +struct subset_record_array_arg_t
    +{
    +  subset_record_array_arg_t (hb_subset_layout_context_t *c_, OutputArray* out_,
    +                             const void *base_,
    +                             Arg &&arg_) : subset_layout_context (c_),
    +                                           out (out_), base (base_), arg (arg_) {}
    +
    +  template 
    +  void
    +  operator () (T&& record)
    +  {
    +    auto snap = subset_layout_context->subset_context->serializer->snapshot ();
    +    bool ret = record.subset (subset_layout_context, base, arg);
    +    if (!ret) subset_layout_context->subset_context->serializer->revert (snap);
    +    else out->len++;
    +  }
    +
    +  private:
    +  hb_subset_layout_context_t *subset_layout_context;
    +  OutputArray *out;
    +  const void *base;
    +  Arg &&arg;
    +};
    +
     /*
      * Helper to subset a RecordList/record array. Subsets each Record in the array and
      * discards the record if the subset operation returns false.
    @@ -326,6 +353,13 @@ struct
       operator () (hb_subset_layout_context_t *c, OutputArray* out,
                    const void *base) const
       { return subset_record_array_t (c, out, base); }
    +
    +  /* Variant with one extra argument passed to subset */
    +  template
    +  subset_record_array_arg_t
    +  operator () (hb_subset_layout_context_t *c, OutputArray* out,
    +               const void *base, Arg &&arg) const
    +  { return subset_record_array_arg_t (c, out, base, arg); }
     }
     HB_FUNCOBJ (subset_record_array);
     
    @@ -377,166 +411,6 @@ HB_FUNCOBJ (serialize_math_record_array);
      * Script, ScriptList, LangSys, Feature, FeatureList, Lookup, LookupList
      */
     
    -struct Record_sanitize_closure_t {
    -  hb_tag_t tag;
    -  const void *list_base;
    -};
    -
    -template 
    -struct Record
    -{
    -  int cmp (hb_tag_t a) const { return tag.cmp (a); }
    -
    -  bool subset (hb_subset_layout_context_t *c, const void *base) const
    -  {
    -    TRACE_SUBSET (this);
    -    auto *out = c->subset_context->serializer->embed (this);
    -    if (unlikely (!out)) return_trace (false);
    -    bool ret = out->offset.serialize_subset (c->subset_context, offset, base, c, &tag);
    -    return_trace (ret);
    -  }
    -
    -  bool sanitize (hb_sanitize_context_t *c, const void *base) const
    -  {
    -    TRACE_SANITIZE (this);
    -    const Record_sanitize_closure_t closure = {tag, base};
    -    return_trace (c->check_struct (this) && offset.sanitize (c, base, &closure));
    -  }
    -
    -  Tag           tag;            /* 4-byte Tag identifier */
    -  Offset16To
    -                offset;         /* Offset from beginning of object holding
    -                                 * the Record */
    -  public:
    -  DEFINE_SIZE_STATIC (6);
    -};
    -
    -template 
    -struct RecordArrayOf : SortedArray16Of>
    -{
    -  const Offset16To& get_offset (unsigned int i) const
    -  { return (*this)[i].offset; }
    -  Offset16To& get_offset (unsigned int i)
    -  { return (*this)[i].offset; }
    -  const Tag& get_tag (unsigned int i) const
    -  { return (*this)[i].tag; }
    -  unsigned int get_tags (unsigned int start_offset,
    -                         unsigned int *record_count /* IN/OUT */,
    -                         hb_tag_t     *record_tags /* OUT */) const
    -  {
    -    if (record_count)
    -    {
    -      + this->sub_array (start_offset, record_count)
    -      | hb_map (&Record::tag)
    -      | hb_sink (hb_array (record_tags, *record_count))
    -      ;
    -    }
    -    return this->len;
    -  }
    -  bool find_index (hb_tag_t tag, unsigned int *index) const
    -  {
    -    return this->bfind (tag, index, HB_NOT_FOUND_STORE, Index::NOT_FOUND_INDEX);
    -  }
    -};
    -
    -template 
    -struct RecordListOf : RecordArrayOf
    -{
    -  const Type& operator [] (unsigned int i) const
    -  { return this+this->get_offset (i); }
    -
    -  bool subset (hb_subset_context_t *c,
    -               hb_subset_layout_context_t *l) const
    -  {
    -    TRACE_SUBSET (this);
    -    auto *out = c->serializer->start_embed (*this);
    -    if (unlikely (!c->serializer->extend_min (out))) return_trace (false);
    -
    -    + this->iter ()
    -    | hb_apply (subset_record_array (l, out, this))
    -    ;
    -    return_trace (true);
    -  }
    -
    -  bool sanitize (hb_sanitize_context_t *c) const
    -  {
    -    TRACE_SANITIZE (this);
    -    return_trace (RecordArrayOf::sanitize (c, this));
    -  }
    -};
    -
    -struct Feature;
    -
    -struct RecordListOfFeature : RecordListOf
    -{
    -  bool subset (hb_subset_context_t *c,
    -               hb_subset_layout_context_t *l) const
    -  {
    -    TRACE_SUBSET (this);
    -    auto *out = c->serializer->start_embed (*this);
    -    if (unlikely (!out || !c->serializer->extend_min (out))) return_trace (false);
    -
    -    unsigned count = this->len;
    -    + hb_zip (*this, hb_range (count))
    -    | hb_filter (l->feature_index_map, hb_second)
    -    | hb_map (hb_first)
    -    | hb_apply (subset_record_array (l, out, this))
    -    ;
    -    return_trace (true);
    -  }
    -};
    -
    -struct Script;
    -struct RecordListOfScript : RecordListOf