diff --git a/.github/actions/build-jtreg/action.yml b/.github/actions/build-jtreg/action.yml
new file mode 100644
index 0000000000000..3abfa260c1767
--- /dev/null
+++ b/.github/actions/build-jtreg/action.yml
@@ -0,0 +1,68 @@
+#
+# Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+name: 'Build JTReg'
+description: 'Build JTReg'
+
+runs:
+ using: composite
+ steps:
+ - name: 'Get JTReg version configuration'
+ id: version
+ uses: ./.github/actions/config
+ with:
+ var: JTREG_VERSION
+
+ - name: 'Check cache for already built JTReg'
+ id: get-cached
+ uses: actions/cache@v4
+ with:
+ path: jtreg/installed
+ key: jtreg-${{ steps.version.outputs.value }}
+
+ - name: 'Checkout the JTReg source'
+ uses: actions/checkout@v4
+ with:
+ repository: openjdk/jtreg
+ ref: jtreg-${{ steps.version.outputs.value }}
+ path: jtreg/src
+ if: (steps.get-cached.outputs.cache-hit != 'true')
+
+ - name: 'Build JTReg'
+ run: |
+ # Build JTReg and move files to the proper locations
+ bash make/build.sh --jdk "$JAVA_HOME_17_X64"
+ mkdir ../installed
+ mv build/images/jtreg/* ../installed
+ working-directory: jtreg/src
+ shell: bash
+ if: (steps.get-cached.outputs.cache-hit != 'true')
+
+ - name: 'Upload JTReg artifact'
+ uses: actions/upload-artifact@v4
+ with:
+ name: bundles-jtreg-${{ steps.version.outputs.value }}
+ path: jtreg/installed
+ retention-days: 1
diff --git a/.github/actions/get-jtreg/action.yml b/.github/actions/get-jtreg/action.yml
index faedcc18807c0..78a3a4c9edddd 100644
--- a/.github/actions/get-jtreg/action.yml
+++ b/.github/actions/get-jtreg/action.yml
@@ -24,7 +24,7 @@
#
name: 'Get JTReg'
-description: 'Download JTReg from cache or source location'
+description: 'Get JTReg'
outputs:
path:
description: 'Path to the installed JTReg'
@@ -39,36 +39,12 @@ runs:
with:
var: JTREG_VERSION
- - name: 'Check cache for JTReg'
- id: get-cached-jtreg
- uses: actions/cache@v4
+ - name: 'Download JTReg artifact'
+ id: download-jtreg
+ uses: actions/download-artifact@v4
with:
+ name: bundles-jtreg-${{ steps.version.outputs.value }}
path: jtreg/installed
- key: jtreg-${{ steps.version.outputs.value }}
-
- - name: 'Checkout the JTReg source'
- uses: actions/checkout@v4
- with:
- repository: openjdk/jtreg
- ref: jtreg-${{ steps.version.outputs.value }}
- path: jtreg/src
- if: steps.get-cached-jtreg.outputs.cache-hit != 'true'
-
- - name: 'Build JTReg'
- run: |
- # If runner architecture is x64 set JAVA_HOME_17_X64 otherwise set to JAVA_HOME_17_arm64
- if [[ '${{ runner.arch }}' == 'X64' ]]; then
- JDK="$JAVA_HOME_17_X64"
- else
- JDK="$JAVA_HOME_17_arm64"
- fi
- # Build JTReg and move files to the proper locations
- bash make/build.sh --jdk "$JDK"
- mkdir ../installed
- mv build/images/jtreg/* ../installed
- working-directory: jtreg/src
- shell: bash
- if: steps.get-cached-jtreg.outputs.cache-hit != 'true'
- name: 'Export path to where JTReg is installed'
id: path-name
diff --git a/.github/scripts/gen-build-failure-report.sh b/.github/scripts/gen-build-failure-report.sh
index fd3215fc7fe2d..2dda69a3f331b 100644
--- a/.github/scripts/gen-build-failure-report.sh
+++ b/.github/scripts/gen-build-failure-report.sh
@@ -24,12 +24,19 @@
# questions.
#
+# Import common utils
+. .github/scripts/report-utils.sh
+
GITHUB_STEP_SUMMARY="$1"
BUILD_DIR="$(ls -d build/*)"
# Send signal to the do-build action that we failed
touch "$BUILD_DIR/build-failure"
+# Collect hs_errs for build-time crashes, e.g. javac, jmod, jlink, CDS.
+# These usually land in make/
+hs_err_files=$(ls make/hs_err*.log 2> /dev/null || true)
+
(
echo '### :boom: Build failure summary'
echo ''
@@ -46,6 +53,20 @@ touch "$BUILD_DIR/build-failure"
echo ''
echo ''
+ for hs_err in $hs_err_files; do
+ echo "View HotSpot error log: "$hs_err"
"
+ echo ''
+ echo '```'
+ echo "$hs_err:"
+ echo ''
+ cat "$hs_err"
+ echo '```'
+ echo ' '
+ echo ''
+ done
+
echo ''
echo ':arrow_right: To see the entire test log, click the job in the list to the left. To download logs, see the `failure-logs` [artifact above](#artifacts).'
) >> $GITHUB_STEP_SUMMARY
+
+truncate_summary
diff --git a/.github/scripts/gen-test-results.sh b/.github/scripts/gen-test-results.sh
index 9e85eef4dc08d..bdf3eb3b9cbc5 100644
--- a/.github/scripts/gen-test-results.sh
+++ b/.github/scripts/gen-test-results.sh
@@ -24,6 +24,9 @@
# questions.
#
+# Import common utils
+. .github/scripts/report-utils.sh
+
GITHUB_STEP_SUMMARY="$1"
test_suite_name=$(cat build/run-test-prebuilt/test-support/test-last-ids.txt)
@@ -89,18 +92,6 @@ for test in $failures $errors; do
fi
done >> $GITHUB_STEP_SUMMARY
-# With many failures, the summary can easily exceed 1024 kB, the limit set by Github
-# Trim it down if so.
-summary_size=$(wc -c < $GITHUB_STEP_SUMMARY)
-if [[ $summary_size -gt 1000000 ]]; then
- # Trim to below 1024 kB, and cut off after the last detail group
- head -c 1000000 $GITHUB_STEP_SUMMARY | tac | sed -n -e '/<\/details>/,$ p' | tac > $GITHUB_STEP_SUMMARY.tmp
- mv $GITHUB_STEP_SUMMARY.tmp $GITHUB_STEP_SUMMARY
- (
- echo ''
- echo ':x: **WARNING: Summary is too large and has been truncated.**'
- echo ''
- ) >> $GITHUB_STEP_SUMMARY
-fi
-
echo ':arrow_right: To see the entire test log, click the job in the list to the left.' >> $GITHUB_STEP_SUMMARY
+
+truncate_summary
diff --git a/.github/scripts/report-utils.sh b/.github/scripts/report-utils.sh
new file mode 100644
index 0000000000000..da5b6c04b3cbe
--- /dev/null
+++ b/.github/scripts/report-utils.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+function truncate_summary() {
+ # With large hs_errs, the summary can easily exceed 1024 kB, the limit set by Github
+ # Trim it down if so.
+ summary_size=$(wc -c < $GITHUB_STEP_SUMMARY)
+ if [[ $summary_size -gt 1000000 ]]; then
+ # Trim to below 1024 kB, and cut off after the last detail group
+ head -c 1000000 $GITHUB_STEP_SUMMARY | tac | sed -n -e '/<\/details>/,$ p' | tac > $GITHUB_STEP_SUMMARY.tmp
+ mv $GITHUB_STEP_SUMMARY.tmp $GITHUB_STEP_SUMMARY
+ (
+ echo ''
+ echo ':x: **WARNING: Summary is too large and has been truncated.**'
+ echo ''
+ ) >> $GITHUB_STEP_SUMMARY
+ fi
+}
diff --git a/.github/workflows/build-cross-compile.yml b/.github/workflows/build-cross-compile.yml
index 29750b485e4b4..8e97ff4439f67 100644
--- a/.github/workflows/build-cross-compile.yml
+++ b/.github/workflows/build-cross-compile.yml
@@ -84,7 +84,7 @@ jobs:
- target-cpu: riscv64
gnu-arch: riscv64
debian-arch: riscv64
- debian-repository: https://snapshot.debian.org/archive/debian/20240228T034848Z/
+ debian-repository: https://httpredir.debian.org/debian/
debian-version: sid
tolerate-sysroot-errors: true
@@ -131,6 +131,7 @@ jobs:
id: create-sysroot
run: >
sudo debootstrap
+ --no-merged-usr
--arch=${{ matrix.debian-arch }}
--verbose
--include=fakeroot,symlinks,build-essential,libx11-dev,libxext-dev,libxrender-dev,libxrandr-dev,libxtst-dev,libxt-dev,libcups2-dev,libfontconfig1-dev,libasound2-dev,libfreetype-dev,libpng-dev
@@ -151,6 +152,9 @@ jobs:
rm -rf sysroot/usr/{sbin,bin,share}
rm -rf sysroot/usr/lib/{apt,gcc,udev,systemd}
rm -rf sysroot/usr/libexec/gcc
+ # /{bin,sbin,lib}/ are not symbolic links to /usr/{bin,sbin,lib}/ when debootstrap with --no-merged-usr
+ rm -rf sysroot/{sbin,bin}
+ rm -rf sysroot/lib/{udev,systemd}
if: steps.create-sysroot.outcome == 'success' && steps.get-cached-sysroot.outputs.cache-hit != 'true'
- name: 'Remove broken sysroot'
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index d5958853701ca..00f64d2aedf59 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -54,8 +54,8 @@ jobs:
### Determine platforms to include
###
- select:
- name: 'Select platforms'
+ prepare:
+ name: 'Prepare the run'
runs-on: ubuntu-22.04
env:
# List of platforms to exclude by default
@@ -73,7 +73,19 @@ jobs:
docs: ${{ steps.include.outputs.docs }}
steps:
- # This function must be inlined in main.yml, or we'd be forced to checkout the repo
+ - name: 'Checkout the scripts'
+ uses: actions/checkout@v4
+ with:
+ sparse-checkout: |
+ .github
+ make/conf/github-actions.conf
+
+ - name: 'Build JTReg'
+ id: jtreg
+ uses: ./.github/actions/build-jtreg
+
+ # TODO: Now that we are checking out the repo scripts, we can put the following code
+ # into a separate file
- name: 'Check what jobs to run'
id: include
run: |
@@ -149,18 +161,18 @@ jobs:
build-linux-x64:
name: linux-x64
- needs: select
+ needs: prepare
uses: ./.github/workflows/build-linux.yml
with:
platform: linux-x64
gcc-major-version: '10'
configure-arguments: ${{ github.event.inputs.configure-arguments }}
make-arguments: ${{ github.event.inputs.make-arguments }}
- if: needs.select.outputs.linux-x64 == 'true'
+ if: needs.prepare.outputs.linux-x64 == 'true'
build-linux-x86-hs:
name: linux-x86-hs
- needs: select
+ needs: prepare
uses: ./.github/workflows/build-linux.yml
with:
platform: linux-x86
@@ -174,11 +186,11 @@ jobs:
extra-conf-options: '--with-target-bits=32 --enable-fallback-linker --enable-libffi-bundling'
configure-arguments: ${{ github.event.inputs.configure-arguments }}
make-arguments: ${{ github.event.inputs.make-arguments }}
- if: needs.select.outputs.linux-x86-hs == 'true'
+ if: needs.prepare.outputs.linux-x86-hs == 'true'
build-linux-x64-hs-nopch:
name: linux-x64-hs-nopch
- needs: select
+ needs: prepare
uses: ./.github/workflows/build-linux.yml
with:
platform: linux-x64
@@ -188,11 +200,11 @@ jobs:
extra-conf-options: '--disable-precompiled-headers'
configure-arguments: ${{ github.event.inputs.configure-arguments }}
make-arguments: ${{ github.event.inputs.make-arguments }}
- if: needs.select.outputs.linux-x64-variants == 'true'
+ if: needs.prepare.outputs.linux-x64-variants == 'true'
build-linux-x64-hs-zero:
name: linux-x64-hs-zero
- needs: select
+ needs: prepare
uses: ./.github/workflows/build-linux.yml
with:
platform: linux-x64
@@ -202,11 +214,11 @@ jobs:
extra-conf-options: '--with-jvm-variants=zero --disable-precompiled-headers'
configure-arguments: ${{ github.event.inputs.configure-arguments }}
make-arguments: ${{ github.event.inputs.make-arguments }}
- if: needs.select.outputs.linux-x64-variants == 'true'
+ if: needs.prepare.outputs.linux-x64-variants == 'true'
build-linux-x64-hs-minimal:
name: linux-x64-hs-minimal
- needs: select
+ needs: prepare
uses: ./.github/workflows/build-linux.yml
with:
platform: linux-x64
@@ -216,11 +228,11 @@ jobs:
extra-conf-options: '--with-jvm-variants=minimal --disable-precompiled-headers'
configure-arguments: ${{ github.event.inputs.configure-arguments }}
make-arguments: ${{ github.event.inputs.make-arguments }}
- if: needs.select.outputs.linux-x64-variants == 'true'
+ if: needs.prepare.outputs.linux-x64-variants == 'true'
build-linux-x64-hs-optimized:
name: linux-x64-hs-optimized
- needs: select
+ needs: prepare
uses: ./.github/workflows/build-linux.yml
with:
platform: linux-x64
@@ -231,32 +243,31 @@ jobs:
extra-conf-options: '--with-debug-level=optimized --disable-precompiled-headers'
configure-arguments: ${{ github.event.inputs.configure-arguments }}
make-arguments: ${{ github.event.inputs.make-arguments }}
- if: needs.select.outputs.linux-x64-variants == 'true'
+ if: needs.prepare.outputs.linux-x64-variants == 'true'
build-linux-cross-compile:
name: linux-cross-compile
- needs:
- - select
+ needs: prepare
uses: ./.github/workflows/build-cross-compile.yml
with:
gcc-major-version: '10'
configure-arguments: ${{ github.event.inputs.configure-arguments }}
make-arguments: ${{ github.event.inputs.make-arguments }}
- if: needs.select.outputs.linux-cross-compile == 'true'
+ if: needs.prepare.outputs.linux-cross-compile == 'true'
build-alpine-linux-x64:
name: alpine-linux-x64
- needs: select
+ needs: prepare
uses: ./.github/workflows/build-alpine-linux.yml
with:
platform: alpine-linux-x64
configure-arguments: ${{ github.event.inputs.configure-arguments }}
make-arguments: ${{ github.event.inputs.make-arguments }}
- if: needs.select.outputs.alpine-linux-x64 == 'true'
+ if: needs.prepare.outputs.alpine-linux-x64 == 'true'
build-macos-x64:
name: macos-x64
- needs: select
+ needs: prepare
uses: ./.github/workflows/build-macos.yml
with:
platform: macos-x64
@@ -264,11 +275,11 @@ jobs:
xcode-toolset-version: '14.3.1'
configure-arguments: ${{ github.event.inputs.configure-arguments }}
make-arguments: ${{ github.event.inputs.make-arguments }}
- if: needs.select.outputs.macos-x64 == 'true'
+ if: needs.prepare.outputs.macos-x64 == 'true'
build-macos-aarch64:
name: macos-aarch64
- needs: select
+ needs: prepare
uses: ./.github/workflows/build-macos.yml
with:
platform: macos-aarch64
@@ -276,11 +287,11 @@ jobs:
xcode-toolset-version: '14.3.1'
configure-arguments: ${{ github.event.inputs.configure-arguments }}
make-arguments: ${{ github.event.inputs.make-arguments }}
- if: needs.select.outputs.macos-aarch64 == 'true'
+ if: needs.prepare.outputs.macos-aarch64 == 'true'
build-windows-x64:
name: windows-x64
- needs: select
+ needs: prepare
uses: ./.github/workflows/build-windows.yml
with:
platform: windows-x64
@@ -288,11 +299,11 @@ jobs:
msvc-toolset-architecture: 'x86.x64'
configure-arguments: ${{ github.event.inputs.configure-arguments }}
make-arguments: ${{ github.event.inputs.make-arguments }}
- if: needs.select.outputs.windows-x64 == 'true'
+ if: needs.prepare.outputs.windows-x64 == 'true'
build-windows-aarch64:
name: windows-aarch64
- needs: select
+ needs: prepare
uses: ./.github/workflows/build-windows.yml
with:
platform: windows-aarch64
@@ -302,11 +313,11 @@ jobs:
extra-conf-options: '--openjdk-target=aarch64-unknown-cygwin'
configure-arguments: ${{ github.event.inputs.configure-arguments }}
make-arguments: ${{ github.event.inputs.make-arguments }}
- if: needs.select.outputs.windows-aarch64 == 'true'
+ if: needs.prepare.outputs.windows-aarch64 == 'true'
build-docs:
name: docs
- needs: select
+ needs: prepare
uses: ./.github/workflows/build-linux.yml
with:
platform: linux-x64
@@ -318,7 +329,7 @@ jobs:
gcc-major-version: '10'
configure-arguments: ${{ github.event.inputs.configure-arguments }}
make-arguments: ${{ github.event.inputs.make-arguments }}
- if: needs.select.outputs.docs == 'true'
+ if: needs.prepare.outputs.docs == 'true'
###
### Test jobs
@@ -363,48 +374,3 @@ jobs:
platform: windows-x64
bootjdk-platform: windows-x64
runs-on: windows-2019
-
- # Remove bundles so they are not misconstrued as binary distributions from the JDK project
- remove-bundles:
- name: 'Remove bundle artifacts'
- runs-on: ubuntu-22.04
- if: always()
- needs:
- - build-linux-x64
- - build-linux-x86-hs
- - build-linux-x64-hs-nopch
- - build-linux-x64-hs-zero
- - build-linux-x64-hs-minimal
- - build-linux-x64-hs-optimized
- - build-linux-cross-compile
- - build-alpine-linux-x64
- - build-macos-x64
- - build-macos-aarch64
- - build-windows-x64
- - build-windows-aarch64
- - test-linux-x64
- - test-macos-x64
- - test-macos-aarch64
- - test-windows-x64
-
- steps:
- - name: 'Remove bundle artifacts'
- run: |
- # Find and remove all bundle artifacts
- # See: https://docs.github.com/en/rest/actions/artifacts?apiVersion=2022-11-28
- ALL_ARTIFACT_IDS="$(curl -sL \
- -H 'Accept: application/vnd.github+json' \
- -H 'Authorization: Bearer ${{ github.token }}' \
- -H 'X-GitHub-Api-Version: 2022-11-28' \
- '${{ github.api_url }}/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts?per_page=100')"
- BUNDLE_ARTIFACT_IDS="$(echo "$ALL_ARTIFACT_IDS" | jq -r -c '.artifacts | map(select(.name|startswith("bundles-"))) | .[].id')"
- for id in $BUNDLE_ARTIFACT_IDS; do
- echo "Removing $id"
- curl -sL \
- -X DELETE \
- -H 'Accept: application/vnd.github+json' \
- -H 'Authorization: Bearer ${{ github.token }}' \
- -H 'X-GitHub-Api-Version: 2022-11-28' \
- "${{ github.api_url }}/repos/${{ github.repository }}/actions/artifacts/$id" \
- || echo "Failed to remove bundle"
- done
diff --git a/doc/building.html b/doc/building.html
index c91d876246cde..63af224584ac3 100644
--- a/doc/building.html
+++ b/doc/building.html
@@ -2016,10 +2016,18 @@
Spaces in Path
have short
paths. You can run fsutil file setshortname
in
-cmd
on certain directories, such as
-Microsoft Visual Studio
or Windows Kits
, to
-assign arbitrary short paths so configure
can access
-them.
+cmd
on directories to assign arbitrary short paths so
+configure
can access them. If the result says "Access
+denied", it may be that there are processes running in that directory;
+in this case, you can reboot Windows in safe mode and run the command on
+those directories again.
+The only directories required to have short paths are
+Microsoft Visual Studio
and Windows Kits
; the
+rest of the "contains space" warnings from configure
, such
+as IntelliJ IDEA
, can be ignored. You can choose any short
+name; once it is set, configure
's tools like
+cygpath
can convert the directory with spaces to your
+chosen short name and pass it to the build system.
Getting Help
If none of the suggestions in this document helps you, or if you find
what you believe is a bug in the build system, please contact the Build
diff --git a/doc/building.md b/doc/building.md
index 47ad9e7c72b4c..466e8d7edf89a 100644
--- a/doc/building.md
+++ b/doc/building.md
@@ -1800,9 +1800,17 @@ temporarily.
On Windows, when configuring, `fixpath.sh` may report that some directory names
have spaces. Usually, it assumes those directories have [short
paths](https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/fsutil-8dot3name).
-You can run `fsutil file setshortname` in `cmd` on certain directories, such as
-`Microsoft Visual Studio` or `Windows Kits`, to assign arbitrary short paths so
-`configure` can access them.
+You can run `fsutil file setshortname` in `cmd` on directories to assign
+arbitrary short paths so `configure` can access them. If the result says "Access
+denied", it may be that there are processes running in that directory; in this
+case, you can reboot Windows in safe mode and run the command on those directories
+again.
+
+The only directories required to have short paths are `Microsoft Visual Studio`
+and `Windows Kits`; the rest of the "contains space" warnings from `configure`,
+such as `IntelliJ IDEA`, can be ignored. You can choose any short name; once it
+is set, `configure`'s tools like `cygpath` can convert the directory with spaces
+to your chosen short name and pass it to the build system.
### Getting Help
diff --git a/doc/testing.html b/doc/testing.html
index c56dfa3118833..6285fab1682e4 100644
--- a/doc/testing.html
+++ b/doc/testing.html
@@ -72,6 +72,9 @@
Testing the JDK
Non-US
locale
PKCS11 Tests
+Testing with
+alternative security providers
Client UI
Tests
@@ -434,7 +437,7 @@ FAILURE_HANDLER_TIMEOUT
Sets the argument -timeoutHandlerTimeout
for JTReg. The
default value is 0. This is only valid if the failure handler is
built.
-JTREG_TEST_THREAD_FACTORY
+TEST_THREAD_FACTORY
Sets the -testThreadFactory
for JTReg. It should be the
fully qualified classname of a class which implements
java.util.concurrent.ThreadFactory
. One such implementation
@@ -586,6 +589,18 @@
PKCS11 Tests
JTREG="JAVA_OPTIONS=-Djdk.test.lib.artifacts.nsslib-linux_aarch64=/path/to/NSS-libs"
For more notes about the PKCS11 tests, please refer to
test/jdk/sun/security/pkcs11/README.
+Testing with
+alternative security providers
+Some security tests use a hardcoded provider for
+KeyFactory
, Cipher
,
+KeyPairGenerator
, KeyGenerator
,
+AlgorithmParameterGenerator
, KeyAgreement
,
+Mac
, MessageDigest
, SecureRandom
,
+Signature
, AlgorithmParameters
,
+Configuration
, Policy
, or
+SecretKeyFactory
objects. Specify the
+-Dtest.provider.name=NAME
property to use a different
+provider for the service(s).
Client UI Tests
System key shortcuts
Some Client UI tests use key sequences which may be reserved by the
diff --git a/doc/testing.md b/doc/testing.md
index 351745c9293c5..351690c5e601c 100644
--- a/doc/testing.md
+++ b/doc/testing.md
@@ -380,7 +380,7 @@ Defaults to 4.
Sets the argument `-timeoutHandlerTimeout` for JTReg. The default value is 0.
This is only valid if the failure handler is built.
-#### JTREG_TEST_THREAD_FACTORY
+#### TEST_THREAD_FACTORY
Sets the `-testThreadFactory` for JTReg. It should be the fully qualified
classname of a class which implements `java.util.concurrent.ThreadFactory`. One
@@ -603,6 +603,15 @@ $ make test TEST="jtreg:sun/security/pkcs11/Secmod/AddTrustedCert.java" \
For more notes about the PKCS11 tests, please refer to
test/jdk/sun/security/pkcs11/README.
+### Testing with alternative security providers
+
+Some security tests use a hardcoded provider for `KeyFactory`, `Cipher`,
+`KeyPairGenerator`, `KeyGenerator`, `AlgorithmParameterGenerator`,
+`KeyAgreement`, `Mac`, `MessageDigest`, `SecureRandom`, `Signature`,
+`AlgorithmParameters`, `Configuration`, `Policy`, or `SecretKeyFactory` objects.
+Specify the `-Dtest.provider.name=NAME` property to use a different provider for
+the service(s).
+
### Client UI Tests
#### System key shortcuts
diff --git a/make/Global.gmk b/make/Global.gmk
index 6b97b7ad05940..16a5b05cccbe3 100644
--- a/make/Global.gmk
+++ b/make/Global.gmk
@@ -102,6 +102,7 @@ help:
$(info $(_) # method is 'auto', 'ignore' or 'fail' (default))
$(info $(_) TEST="test1 ..." # Use the given test descriptor(s) for testing, e.g.)
$(info $(_) # make test TEST="jdk_lang gtest:all")
+ $(info $(_) TEST_DEPS="dependency1 ..." # Specify additional dependencies for running tests, e.g docs-jdk
$(info $(_) JTREG="OPT1=x;OPT2=y" # Control the JTREG test harness, use 'make test-only JTREG=help' to list)
$(info $(_) GTEST="OPT1=x;OPT2=y" # Control the GTEST test harness, use 'make test-only GTEST=help' to list)
$(info $(_) MICRO="OPT1=x;OPT2=y" # Control the MICRO test harness, use 'make test-only MICRO=help' to list)
diff --git a/make/InitSupport.gmk b/make/InitSupport.gmk
index 2471d82d6dbfb..eea593a80db95 100644
--- a/make/InitSupport.gmk
+++ b/make/InitSupport.gmk
@@ -50,7 +50,7 @@ ifeq ($(HAS_SPEC), )
# Make control variables, handled by Init.gmk
INIT_CONTROL_VARIABLES += LOG CONF CONF_NAME SPEC JOBS TEST_JOBS CONF_CHECK \
- COMPARE_BUILD JTREG GTEST MICRO TEST_OPTS TEST_VM_OPTS
+ COMPARE_BUILD JTREG GTEST MICRO TEST_OPTS TEST_VM_OPTS TEST_DEPS
# All known make control variables
MAKE_CONTROL_VARIABLES := $(INIT_CONTROL_VARIABLES) TEST JDK_FILTER SPEC_FILTER
diff --git a/make/Main.gmk b/make/Main.gmk
index 4b3efaf651ec9..25dfba7d36c08 100644
--- a/make/Main.gmk
+++ b/make/Main.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -785,13 +785,13 @@ $(eval $(call SetupTarget, build-microbenchmark, \
$(eval $(call SetupTarget, test, \
MAKEFILE := RunTests, \
ARGS := TEST="$(TEST)", \
- DEPS := jdk-image test-image, \
+ DEPS := jdk-image test-image $(TEST_DEPS), \
))
$(eval $(call SetupTarget, exploded-test, \
MAKEFILE := RunTests, \
ARGS := TEST="$(TEST)" JDK_IMAGE_DIR=$(JDK_OUTPUTDIR), \
- DEPS := exploded-image test-image, \
+ DEPS := exploded-image test-image $(TEST_DEPS), \
))
ifeq ($(JCOV_ENABLED), true)
@@ -1110,8 +1110,8 @@ else
test-make-compile-commands: compile-commands
# Declare dependency for all generated test targets
- $(foreach t, $(filter-out test-make%, $(ALL_TEST_TARGETS)), $(eval $t: jdk-image test-image))
- $(foreach t, $(ALL_EXPLODED_TEST_TARGETS), $(eval $t: exploded-image test-image))
+ $(foreach t, $(filter-out test-make%, $(ALL_TEST_TARGETS)), $(eval $t: jdk-image test-image $(TEST_DEPS)))
+ $(foreach t, $(ALL_EXPLODED_TEST_TARGETS), $(eval $t: exploded-image test-image $(TEST_DEPS)))
interim-image: $(INTERIM_JMOD_TARGETS)
diff --git a/make/RunTests.gmk b/make/RunTests.gmk
index a6a4bd343d5ba..2d6392d783bd2 100644
--- a/make/RunTests.gmk
+++ b/make/RunTests.gmk
@@ -264,6 +264,7 @@ jaxp_JTREG_PROBLEM_LIST += $(TOPDIR)/test/jaxp/ProblemList.txt
langtools_JTREG_PROBLEM_LIST += $(TOPDIR)/test/langtools/ProblemList.txt
hotspot_JTREG_PROBLEM_LIST += $(TOPDIR)/test/hotspot/jtreg/ProblemList.txt
lib-test_JTREG_PROBLEM_LIST += $(TOPDIR)/test/lib-test/ProblemList.txt
+docs_JTREG_PROBLEM_LIST += $(TOPDIR)/test/docs/ProblemList.txt
################################################################################
# Parse test selection
@@ -603,7 +604,7 @@ define SetupRunMicroTestBody
$1_JMH_JVM_ARGS += $$(MICRO_VM_OPTIONS) $$(MICRO_JAVA_OPTIONS)
endif
- $1_MICRO_VM_OPTIONS := -jvmArgs $(call ShellQuote,$$($1_JMH_JVM_ARGS))
+ $1_MICRO_VM_OPTIONS := -jvmArgsPrepend $(call ShellQuote,$$($1_JMH_JVM_ARGS))
ifneq ($$(MICRO_ITER), )
$1_MICRO_ITER := -i $$(MICRO_ITER)
@@ -739,6 +740,11 @@ define SetupRunJtregTestBody
# Only the problem list for the current test root should be used.
$1_JTREG_PROBLEM_LIST := $$(filter $$($1_TEST_ROOT)%, $$($1_JTREG_PROBLEM_LIST))
+ # Pass along the path to the tidy html checker
+ ifneq ($$(TIDY), )
+ $1_JTREG_BASIC_OPTIONS += -Dtidy=$$(TIDY)
+ endif
+
ifneq ($(TEST_JOBS), 0)
$$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(TEST_JOBS)))
else
@@ -847,11 +853,7 @@ define SetupRunJtregTestBody
endif
ifneq ($$(findstring -XX:+UseZGC, $$(JTREG_ALL_OPTIONS)), )
- ifneq ($$(findstring -XX:-ZGenerational, $$(JTREG_ALL_OPTIONS)), )
- JTREG_AUTO_PROBLEM_LISTS += ProblemList-zgc.txt
- else
- JTREG_AUTO_PROBLEM_LISTS += ProblemList-generational-zgc.txt
- endif
+ JTREG_AUTO_PROBLEM_LISTS += ProblemList-zgc.txt
endif
ifneq ($$(JTREG_EXTRA_PROBLEM_LISTS), )
@@ -868,6 +870,8 @@ define SetupRunJtregTestBody
$1_JTREG_BASIC_OPTIONS += -e:TEST_IMAGE_DIR=$(TEST_IMAGE_DIR)
+ $1_JTREG_BASIC_OPTIONS += -e:DOCS_JDK_IMAGE_DIR=$$(DOCS_JDK_IMAGE_DIR)
+
ifneq ($$(JTREG_FAILURE_HANDLER_OPTIONS), )
$1_JTREG_LAUNCHER_OPTIONS += -Djava.library.path="$(JTREG_FAILURE_HANDLER_DIR)"
endif
diff --git a/make/autoconf/configure.ac b/make/autoconf/configure.ac
index 3ec95ba8181de..21bf61a3fa1e9 100644
--- a/make/autoconf/configure.ac
+++ b/make/autoconf/configure.ac
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -202,6 +202,7 @@ FLAGS_POST_TOOLCHAIN
LIB_TESTS_SETUP_JTREG
LIB_TESTS_SETUP_JMH
LIB_TESTS_SETUP_JIB
+LIB_TESTS_SETUP_TIDY
# Now we can test some aspects on the target using configure macros.
PLATFORM_SETUP_OPENJDK_TARGET_BITS
diff --git a/make/autoconf/flags-cflags.m4 b/make/autoconf/flags-cflags.m4
index 992a0282c043b..bd8f2ec9a7f2e 100644
--- a/make/autoconf/flags-cflags.m4
+++ b/make/autoconf/flags-cflags.m4
@@ -936,15 +936,13 @@ AC_DEFUN_ONCE([FLAGS_SETUP_BRANCH_PROTECTION],
if test "x$OPENJDK_TARGET_CPU" = xaarch64; then
if test "x$TOOLCHAIN_TYPE" = xgcc || test "x$TOOLCHAIN_TYPE" = xclang; then
- FLAGS_COMPILER_CHECK_ARGUMENTS(ARGUMENT: [${BRANCH_PROTECTION_FLAG}],
+ FLAGS_COMPILER_CHECK_ARGUMENTS(ARGUMENT: [$BRANCH_PROTECTION_FLAG],
IF_TRUE: [BRANCH_PROTECTION_AVAILABLE=true])
fi
fi
- BRANCH_PROTECTION_CFLAGS=""
UTIL_ARG_ENABLE(NAME: branch-protection, DEFAULT: false,
- RESULT: USE_BRANCH_PROTECTION, AVAILABLE: $BRANCH_PROTECTION_AVAILABLE,
+ RESULT: BRANCH_PROTECTION_ENABLED, AVAILABLE: $BRANCH_PROTECTION_AVAILABLE,
DESC: [enable branch protection when compiling C/C++],
- IF_ENABLED: [ BRANCH_PROTECTION_CFLAGS=${BRANCH_PROTECTION_FLAG}])
- AC_SUBST(BRANCH_PROTECTION_CFLAGS)
+ IF_ENABLED: [BRANCH_PROTECTION_CFLAGS=$BRANCH_PROTECTION_FLAG])
])
diff --git a/make/autoconf/flags-other.m4 b/make/autoconf/flags-other.m4
index 8d4d405b07639..f0fa82489df30 100644
--- a/make/autoconf/flags-other.m4
+++ b/make/autoconf/flags-other.m4
@@ -150,5 +150,9 @@ AC_DEFUN([FLAGS_SETUP_ASFLAGS_CPU_DEP],
$2JVM_ASFLAGS="${$2JVM_ASFLAGS} $ARM_ARCH_TYPE_ASFLAGS $ARM_FLOAT_TYPE_ASFLAGS"
fi
+ if test "x$BRANCH_PROTECTION_ENABLED" = "xtrue"; then
+ $2JVM_ASFLAGS="${$2JVM_ASFLAGS} $BRANCH_PROTECTION_FLAG"
+ fi
+
AC_SUBST($2JVM_ASFLAGS)
])
diff --git a/make/autoconf/jvm-features.m4 b/make/autoconf/jvm-features.m4
index bd33315090fce..9695644bafe63 100644
--- a/make/autoconf/jvm-features.m4
+++ b/make/autoconf/jvm-features.m4
@@ -479,6 +479,22 @@ AC_DEFUN([JVM_FEATURES_CALCULATE_ACTIVE],
$JVM_FEATURES_ENABLED, $JVM_FEATURES_DISABLED)
])
+################################################################################
+# Filter the unsupported feature combinations.
+# This is called after JVM_FEATURES_ACTIVE are fully populated.
+#
+AC_DEFUN([JVM_FEATURES_FILTER_UNSUPPORTED],
+[
+ # G1 late barrier expansion in C2 is not implemented for some platforms.
+ # Choose not to support G1 in this configuration.
+ if JVM_FEATURES_IS_ACTIVE(compiler2); then
+ if test "x$OPENJDK_TARGET_CPU" = "xx86"; then
+ AC_MSG_NOTICE([G1 cannot be used with C2 on this platform, disabling G1])
+ UTIL_GET_NON_MATCHING_VALUES(JVM_FEATURES_ACTIVE, $JVM_FEATURES_ACTIVE, "g1gc")
+ fi
+ fi
+])
+
################################################################################
# Helper function for JVM_FEATURES_VERIFY. Check if the specified JVM
# feature is active. To be used in shell if constructs, like this:
@@ -554,6 +570,9 @@ AC_DEFUN_ONCE([JVM_FEATURES_SETUP],
# The result is stored in JVM_FEATURES_ACTIVE.
JVM_FEATURES_CALCULATE_ACTIVE($variant)
+ # Filter unsupported feature combinations from JVM_FEATURES_ACTIVE.
+ JVM_FEATURES_FILTER_UNSUPPORTED
+
# Verify consistency for JVM_FEATURES_ACTIVE.
JVM_FEATURES_VERIFY($variant)
diff --git a/make/autoconf/lib-hsdis.m4 b/make/autoconf/lib-hsdis.m4
index bd78768d03e48..a4d2c5f81f3f3 100644
--- a/make/autoconf/lib-hsdis.m4
+++ b/make/autoconf/lib-hsdis.m4
@@ -266,8 +266,10 @@ AC_DEFUN([LIB_SETUP_HSDIS_BINUTILS],
HSDIS_CFLAGS="-DLIBARCH_$OPENJDK_TARGET_CPU_LEGACY_LIB"
elif test "x$BINUTILS_INSTALL_DIR" != x; then
disasm_header="\"$BINUTILS_INSTALL_DIR/include/dis-asm.h\""
- if test -e $BINUTILS_INSTALL_DIR/lib/libbfd.a && \
- test -e $BINUTILS_INSTALL_DIR/lib/libopcodes.a && \
+ if (test -e $BINUTILS_INSTALL_DIR/lib/libbfd.a || \
+ test -e $BINUTILS_INSTALL_DIR/lib64/libbfd.a) && \
+ (test -e $BINUTILS_INSTALL_DIR/lib/libopcodes.a || \
+ test -e $BINUTILS_INSTALL_DIR/lib64/libopcodes.a) && \
(test -e $BINUTILS_INSTALL_DIR/lib/libiberty.a || \
test -e $BINUTILS_INSTALL_DIR/lib64/libiberty.a || \
test -e $BINUTILS_INSTALL_DIR/lib32/libiberty.a); then
@@ -275,7 +277,19 @@ AC_DEFUN([LIB_SETUP_HSDIS_BINUTILS],
# libiberty ignores --libdir and may be installed in $BINUTILS_INSTALL_DIR/lib, $BINUTILS_INSTALL_DIR/lib32
# or $BINUTILS_INSTALL_DIR/lib64, depending on system setup
+ LIBOPCODES_LIB=""
+ LIBBFD_LIB=""
LIBIBERTY_LIB=""
+ if test -e $BINUTILS_INSTALL_DIR/lib/libbfd.a; then
+ LIBBFD_LIB="$BINUTILS_INSTALL_DIR/lib/libbfd.a"
+ else
+ LIBBFD_LIB="$BINUTILS_INSTALL_DIR/lib64/libbfd.a"
+ fi
+ if test -e $BINUTILS_INSTALL_DIR/lib/libopcodes.a; then
+ LIBOPCODES_LIB="$BINUTILS_INSTALL_DIR/lib/libopcodes.a"
+ else
+ LIBOPCODES_LIB="$BINUTILS_INSTALL_DIR/lib64/libopcodes.a"
+ fi
if test -e $BINUTILS_INSTALL_DIR/lib/libiberty.a; then
LIBIBERTY_LIB="$BINUTILS_INSTALL_DIR/lib/libiberty.a"
elif test -e $BINUTILS_INSTALL_DIR/lib32/libiberty.a; then
@@ -283,7 +297,7 @@ AC_DEFUN([LIB_SETUP_HSDIS_BINUTILS],
else
LIBIBERTY_LIB="$BINUTILS_INSTALL_DIR/lib64/libiberty.a"
fi
- HSDIS_LIBS="$BINUTILS_INSTALL_DIR/lib/libbfd.a $BINUTILS_INSTALL_DIR/lib/libopcodes.a $LIBIBERTY_LIB"
+ HSDIS_LIBS="$LIBBFD_LIB $LIBOPCODES_LIB $LIBIBERTY_LIB"
# If we have libsframe add it.
if test -e $BINUTILS_INSTALL_DIR/lib/libsframe.a; then
HSDIS_LIBS="$HSDIS_LIBS $BINUTILS_INSTALL_DIR/lib/libsframe.a"
diff --git a/make/autoconf/lib-tests.m4 b/make/autoconf/lib-tests.m4
index ffbc7d0531041..afb9403080871 100644
--- a/make/autoconf/lib-tests.m4
+++ b/make/autoconf/lib-tests.m4
@@ -308,6 +308,32 @@ AC_DEFUN_ONCE([LIB_TESTS_SETUP_JIB],
AC_SUBST(JIB_HOME)
])
+# Setup the tidy html checker
+AC_DEFUN_ONCE([LIB_TESTS_SETUP_TIDY],
+[
+ UTIL_LOOKUP_PROGS(TIDY, tidy)
+
+ if test "x$TIDY" != x; then
+ AC_MSG_CHECKING([if tidy is working properly])
+ tidy_output=`$TIDY --version 2>&1`
+ if ! $ECHO "$tidy_output" | $GREP -q "HTML Tidy" 2>&1 > /dev/null; then
+ AC_MSG_RESULT([no])
+ AC_MSG_NOTICE([$TIDY is not a valid tidy executable and will be ignored. Output from --version: $tidy_output])
+ TIDY=
+ elif ! $ECHO "$tidy_output" | $GREP -q "version" 2>&1 > /dev/null; then
+ AC_MSG_RESULT([no])
+ AC_MSG_NOTICE([$TIDY is missing a proper version number and will be ignored. Output from --version: $tidy_output])
+ TIDY=
+ else
+ AC_MSG_RESULT([yes])
+ AC_MSG_CHECKING([for tidy version])
+ tidy_version=`$ECHO $tidy_output | $SED -e 's/.*version //g'`
+ AC_MSG_RESULT([$tidy_version])
+ fi
+ fi
+ AC_SUBST(TIDY)
+])
+
################################################################################
#
# Check if building of the jtreg failure handler should be enabled.
diff --git a/make/autoconf/spec.gmk.template b/make/autoconf/spec.gmk.template
index 20b1d00aa893f..a5f4143c8885f 100644
--- a/make/autoconf/spec.gmk.template
+++ b/make/autoconf/spec.gmk.template
@@ -429,7 +429,6 @@ ENABLE_LIBFFI_BUNDLING := @ENABLE_LIBFFI_BUNDLING@
LIBFFI_LIB_FILE := @LIBFFI_LIB_FILE@
FILE_MACRO_CFLAGS := @FILE_MACRO_CFLAGS@
REPRODUCIBLE_CFLAGS := @REPRODUCIBLE_CFLAGS@
-BRANCH_PROTECTION_CFLAGS := @BRANCH_PROTECTION_CFLAGS@
STATIC_LIBS_CFLAGS := @STATIC_LIBS_CFLAGS@
@@ -744,6 +743,7 @@ MKDIR := @MKDIR@
MV := @MV@
NICE := @NICE@
PANDOC := @PANDOC@
+TIDY := @TIDY@
PATCH := @PATCH@
PRINTF := @PRINTF@
READLINK := @READLINK@
diff --git a/make/autoconf/toolchain.m4 b/make/autoconf/toolchain.m4
index 75c8d2b61d084..d84ae447e541a 100644
--- a/make/autoconf/toolchain.m4
+++ b/make/autoconf/toolchain.m4
@@ -307,7 +307,7 @@ AC_DEFUN_ONCE([TOOLCHAIN_POST_DETECTION],
[
# Restore old path, except for the microsoft toolchain, which requires the
# toolchain path to remain in place. Otherwise the compiler will not work in
- # some siutations in later configure checks.
+ # some situations in later configure checks.
if test "x$TOOLCHAIN_TYPE" != "xmicrosoft"; then
PATH="$OLD_PATH"
fi
@@ -316,10 +316,6 @@ AC_DEFUN_ONCE([TOOLCHAIN_POST_DETECTION],
# This is necessary since AC_PROG_CC defaults CFLAGS to "-g -O2"
CFLAGS="$ORG_CFLAGS"
CXXFLAGS="$ORG_CXXFLAGS"
-
- # filter out some unwanted additions autoconf may add to CXX; we saw this on macOS with autoconf 2.72
- UTIL_GET_NON_MATCHING_VALUES(cxx_filtered, $CXX, -std=c++11 -std=gnu++11)
- CXX="$cxx_filtered"
])
# Check if a compiler is of the toolchain type we expect, and save the version
@@ -358,6 +354,11 @@ AC_DEFUN([TOOLCHAIN_EXTRACT_COMPILER_VERSION],
# Copyright (C) 2013 Free Software Foundation, Inc.
# This is free software; see the source for copying conditions. There is NO
# warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ # or look like
+ # gcc (GCC) 10.2.1 20200825 (Alibaba 10.2.1-3.8 2.32)
+ # Copyright (C) 2020 Free Software Foundation, Inc.
+ # This is free software; see the source for copying conditions. There is NO
+ # warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
COMPILER_VERSION_OUTPUT=`$COMPILER --version 2>&1`
# Check that this is likely to be GCC.
$ECHO "$COMPILER_VERSION_OUTPUT" | $GREP "Free Software Foundation" > /dev/null
@@ -371,7 +372,8 @@ AC_DEFUN([TOOLCHAIN_EXTRACT_COMPILER_VERSION],
COMPILER_VERSION_STRING=`$ECHO $COMPILER_VERSION_OUTPUT | \
$SED -e 's/ *Copyright .*//'`
COMPILER_VERSION_NUMBER=`$ECHO $COMPILER_VERSION_OUTPUT | \
- $SED -e 's/^.* \(@<:@1-9@:>@<:@0-9@:>@*\.@<:@0-9.@:>@*\)@<:@^0-9.@:>@.*$/\1/'`
+ $AWK -F ')' '{print [$]2}' | \
+ $AWK '{print [$]1}'`
elif test "x$TOOLCHAIN_TYPE" = xclang; then
# clang --version output typically looks like
# Apple clang version 15.0.0 (clang-1500.3.9.4)
diff --git a/make/autoconf/util.m4 b/make/autoconf/util.m4
index 6beadb4c942c7..5a6142d509202 100644
--- a/make/autoconf/util.m4
+++ b/make/autoconf/util.m4
@@ -25,6 +25,70 @@
m4_include([util_paths.m4])
+###############################################################################
+# Overwrite the existing version of AC_PROG_CC with our own custom variant.
+# Unlike the regular AC_PROG_CC, the compiler list must always be passed.
+AC_DEFUN([AC_PROG_CC],
+[
+ AC_LANG_PUSH(C)
+ AC_ARG_VAR([CC], [C compiler command])
+ AC_ARG_VAR([CFLAGS], [C compiler flags])
+
+ _AC_ARG_VAR_LDFLAGS()
+ _AC_ARG_VAR_LIBS()
+ _AC_ARG_VAR_CPPFLAGS()
+
+ AC_CHECK_TOOLS(CC, [$1])
+
+ test -z "$CC" && AC_MSG_FAILURE([no acceptable C compiler found in \$PATH])
+
+ # Provide some information about the compiler.
+ _AS_ECHO_LOG([checking for _AC_LANG compiler version])
+ set X $ac_compile
+ ac_compiler=$[2]
+ for ac_option in --version -v -V -qversion -version; do
+ _AC_DO_LIMIT([$ac_compiler $ac_option >&AS_MESSAGE_LOG_FD])
+ done
+
+ m4_expand_once([_AC_COMPILER_EXEEXT])
+ m4_expand_once([_AC_COMPILER_OBJEXT])
+
+ _AC_PROG_CC_G
+
+ AC_LANG_POP(C)
+])
+
+###############################################################################
+# Overwrite the existing version of AC_PROG_CXX with our own custom variant.
+# Unlike the regular AC_PROG_CXX, the compiler list must always be passed.
+AC_DEFUN([AC_PROG_CXX],
+[
+ AC_LANG_PUSH(C++)
+ AC_ARG_VAR([CXX], [C++ compiler command])
+ AC_ARG_VAR([CXXFLAGS], [C++ compiler flags])
+
+ _AC_ARG_VAR_LDFLAGS()
+ _AC_ARG_VAR_LIBS()
+ _AC_ARG_VAR_CPPFLAGS()
+
+ AC_CHECK_TOOLS(CXX, [$1])
+
+ # Provide some information about the compiler.
+ _AS_ECHO_LOG([checking for _AC_LANG compiler version])
+ set X $ac_compile
+ ac_compiler=$[2]
+ for ac_option in --version -v -V -qversion; do
+ _AC_DO_LIMIT([$ac_compiler $ac_option >&AS_MESSAGE_LOG_FD])
+ done
+
+ m4_expand_once([_AC_COMPILER_EXEEXT])
+ m4_expand_once([_AC_COMPILER_OBJEXT])
+
+ _AC_PROG_CXX_G
+
+ AC_LANG_POP(C++)
+])
+
################################################################################
# Create a function/macro that takes a series of named arguments. The call is
# similar to AC_DEFUN, but the setup of the function looks like this:
diff --git a/make/common/FileUtils.gmk b/make/common/FileUtils.gmk
index 89cbe872721af..d3cc4872ebb8f 100644
--- a/make/common/FileUtils.gmk
+++ b/make/common/FileUtils.gmk
@@ -136,7 +136,7 @@ ifeq ($(call isTargetOs, macosx), true)
$(CP) -fRP '$(call DecodeSpace, $<)' '$(call DecodeSpace, $@)'; \
fi
if [ -n "`$(XATTR) -ls '$(call DecodeSpace, $@)'`" ]; then \
- $(CHMOD) u+w '$(call DecodeSpace, $@)'; \
+ $(CHMOD) -h u+w '$(call DecodeSpace, $@)'; \
$(XATTR) -cs '$(call DecodeSpace, $@)'; \
fi
endef
diff --git a/make/common/FindTests.gmk b/make/common/FindTests.gmk
index 14f0ef0b431e7..db9cfe774defe 100644
--- a/make/common/FindTests.gmk
+++ b/make/common/FindTests.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -43,7 +43,7 @@ $(eval $(call IncludeCustomExtension, common/FindTests.gmk))
TEST_BASEDIRS += $(TOPDIR)/test $(TOPDIR)
# JTREG_TESTROOTS might have been set by a custom extension
-JTREG_TESTROOTS += $(addprefix $(TOPDIR)/test/, hotspot/jtreg jdk langtools jaxp lib-test)
+JTREG_TESTROOTS += $(addprefix $(TOPDIR)/test/, hotspot/jtreg jdk langtools jaxp lib-test docs)
# Extract the names of the Jtreg group files from the TEST.ROOT files. The
# TEST.ROOT files being properties files can be interpreted as makefiles so
diff --git a/make/conf/github-actions.conf b/make/conf/github-actions.conf
index eca6c05033d88..a6b383daa8fd4 100644
--- a/make/conf/github-actions.conf
+++ b/make/conf/github-actions.conf
@@ -29,21 +29,21 @@ GTEST_VERSION=1.14.0
JTREG_VERSION=7.4+1
LINUX_X64_BOOT_JDK_EXT=tar.gz
-LINUX_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk22.0.2/c9ecb94cd31b495da20a27d4581645e8/9/GPL/openjdk-22.0.2_linux-x64_bin.tar.gz
-LINUX_X64_BOOT_JDK_SHA256=41536f115668308ecf4eba92aaf6acaeb0936225828b741efd83b6173ba82963
+LINUX_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk23/3c5b90190c68498b986a97f276efd28a/37/GPL/openjdk-23_linux-x64_bin.tar.gz
+LINUX_X64_BOOT_JDK_SHA256=08fea92724127c6fa0f2e5ea0b07ff4951ccb1e2f22db3c21eebbd7347152a67
ALPINE_LINUX_X64_BOOT_JDK_EXT=tar.gz
-ALPINE_LINUX_X64_BOOT_JDK_URL=https://github.com/adoptium/temurin22-binaries/releases/download/jdk-22.0.2%2B9/OpenJDK22U-jdk_x64_alpine-linux_hotspot_22.0.2_9.tar.gz
-ALPINE_LINUX_X64_BOOT_JDK_SHA256=49f73414824b1a7c268a611225fa4d7ce5e25600201e0f1cd59f94d1040b5264
+ALPINE_LINUX_X64_BOOT_JDK_URL=https://github.com/adoptium/temurin23-binaries/releases/download/jdk-23%2B37/OpenJDK23U-jdk_x64_alpine-linux_hotspot_23_37.tar.gz
+ALPINE_LINUX_X64_BOOT_JDK_SHA256=bff4c78f30d8d173e622bf2f40c36113df47337fc6d1ee5105ed2459841165aa
MACOS_AARCH64_BOOT_JDK_EXT=tar.gz
-MACOS_AARCH64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk22.0.2/c9ecb94cd31b495da20a27d4581645e8/9/GPL/openjdk-22.0.2_macos-aarch64_bin.tar.gz
-MACOS_AARCH64_BOOT_JDK_SHA256=3dab98730234e1a87aec14bcb8171d2cae101e96ff4eed1dab96abbb08e843fd
+MACOS_AARCH64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk23/3c5b90190c68498b986a97f276efd28a/37/GPL/openjdk-23_macos-aarch64_bin.tar.gz
+MACOS_AARCH64_BOOT_JDK_SHA256=9527bf080a74ae6dca51df413aa826f0c011c6048885e4c8ad112172be8815f3
MACOS_X64_BOOT_JDK_EXT=tar.gz
-MACOS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk22.0.2/c9ecb94cd31b495da20a27d4581645e8/9/GPL/openjdk-22.0.2_macos-x64_bin.tar.gz
-MACOS_X64_BOOT_JDK_SHA256=e8b3ec7a7077711223d31156e771f11723cd7af31c2017f1bd2eda20855940fb
+MACOS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk23/3c5b90190c68498b986a97f276efd28a/37/GPL/openjdk-23_macos-x64_bin.tar.gz
+MACOS_X64_BOOT_JDK_SHA256=5c3a909fd2079d0e376dd43c85c4f7d02d08914866f196480bd47784b2a0121e
WINDOWS_X64_BOOT_JDK_EXT=zip
-WINDOWS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk22.0.2/c9ecb94cd31b495da20a27d4581645e8/9/GPL/openjdk-22.0.2_windows-x64_bin.zip
-WINDOWS_X64_BOOT_JDK_SHA256=f2a9b9ab944e71a64637fcdc6b13a1188cf02d4eb9ecf71dc927e98b3e45f5dc
+WINDOWS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk23/3c5b90190c68498b986a97f276efd28a/37/GPL/openjdk-23_windows-x64_bin.zip
+WINDOWS_X64_BOOT_JDK_SHA256=cba5013874ba50cae543c86fe6423453816c77281e2751a8a9a633d966f1dc04
diff --git a/make/conf/jib-profiles.js b/make/conf/jib-profiles.js
index 30c45d4cde161..0785d340f48b6 100644
--- a/make/conf/jib-profiles.js
+++ b/make/conf/jib-profiles.js
@@ -390,8 +390,8 @@ var getJibProfilesCommon = function (input, data) {
};
};
- common.boot_jdk_version = "22";
- common.boot_jdk_build_number = "36";
+ common.boot_jdk_version = "23";
+ common.boot_jdk_build_number = "37";
common.boot_jdk_home = input.get("boot_jdk", "install_path") + "/jdk-"
+ common.boot_jdk_version
+ (input.build_os == "macosx" ? ".jdk/Contents/Home" : "");
@@ -415,7 +415,7 @@ var getJibProfilesProfiles = function (input, common, data) {
"linux-x64": {
target_os: "linux",
target_cpu: "x64",
- dependencies: ["devkit", "gtest", "build_devkit", "graphviz", "pandoc"],
+ dependencies: ["devkit", "gtest", "build_devkit", "graphviz", "pandoc", "tidy"],
configure_args: concat(
(input.build_cpu == "x64" ? common.configure_args_64bit
: "--openjdk-target=x86_64-linux-gnu"),
@@ -441,7 +441,7 @@ var getJibProfilesProfiles = function (input, common, data) {
"macosx-x64": {
target_os: "macosx",
target_cpu: "x64",
- dependencies: ["devkit", "gtest", "graphviz", "pandoc"],
+ dependencies: ["devkit", "gtest", "graphviz", "pandoc", "tidy"],
configure_args: concat(common.configure_args_64bit, "--with-zlib=system",
"--with-macosx-version-max=11.00.00",
"--enable-compatible-cds-alignment",
@@ -453,7 +453,7 @@ var getJibProfilesProfiles = function (input, common, data) {
"macosx-aarch64": {
target_os: "macosx",
target_cpu: "aarch64",
- dependencies: ["devkit", "gtest", "graphviz", "pandoc"],
+ dependencies: ["devkit", "gtest", "graphviz", "pandoc", "tidy"],
configure_args: concat(common.configure_args_64bit,
"--with-macosx-version-max=11.00.00"),
},
@@ -486,7 +486,7 @@ var getJibProfilesProfiles = function (input, common, data) {
"linux-aarch64": {
target_os: "linux",
target_cpu: "aarch64",
- dependencies: ["devkit", "gtest", "build_devkit", "graphviz", "pandoc"],
+ dependencies: ["devkit", "gtest", "build_devkit", "graphviz", "pandoc", "tidy"],
configure_args: [
"--with-zlib=system",
"--disable-dtrace",
@@ -957,7 +957,7 @@ var getJibProfilesProfiles = function (input, common, data) {
// Profiles used to run tests using Jib for internal dependencies.
var testedProfile = input.testedProfile;
- if (testedProfile == null) {
+ if (testedProfile == null || testedProfile == "docs") {
testedProfile = input.build_os + "-" + input.build_cpu;
}
var testedProfileJdk = testedProfile + ".jdk";
@@ -999,25 +999,38 @@ var getJibProfilesProfiles = function (input, common, data) {
testOnlyProfilesPrebuilt["run-test-prebuilt"]["dependencies"].push(testedProfile + ".jdk_symbols");
}
+ var testOnlyProfilesPrebuiltDocs = {
+ "run-test-prebuilt-docs": clone(testOnlyProfilesPrebuilt["run-test-prebuilt"])
+ };
+
+ testOnlyProfilesPrebuiltDocs["run-test-prebuilt-docs"].dependencies.push("docs.doc_api_spec", "tidy");
+ testOnlyProfilesPrebuiltDocs["run-test-prebuilt-docs"].environment["DOCS_JDK_IMAGE_DIR"]
+ = input.get("docs.doc_api_spec", "install_path");
+ testOnlyProfilesPrebuiltDocs["run-test-prebuilt-docs"].environment["TIDY"]
+ = input.get("tidy", "home_path") + "/bin/tidy";
+ testOnlyProfilesPrebuiltDocs["run-test-prebuilt-docs"].labels = "test-docs";
+
// If actually running the run-test-prebuilt profile, verify that the input
// variable is valid and if so, add the appropriate target_* values from
// the tested profile. Use testImageProfile value as backup.
- if (input.profile == "run-test-prebuilt") {
+ if (input.profile == "run-test-prebuilt" || input.profile == "run-test-prebuilt-docs") {
if (profiles[testedProfile] == null && profiles[testImageProfile] == null) {
error("testedProfile is not defined: " + testedProfile + " " + testImageProfile);
}
}
- if (profiles[testedProfile] != null) {
- testOnlyProfilesPrebuilt["run-test-prebuilt"]["target_os"]
- = profiles[testedProfile]["target_os"];
- testOnlyProfilesPrebuilt["run-test-prebuilt"]["target_cpu"]
- = profiles[testedProfile]["target_cpu"];
- } else if (profiles[testImageProfile] != null) {
- testOnlyProfilesPrebuilt["run-test-prebuilt"]["target_os"]
- = profiles[testImageProfile]["target_os"];
- testOnlyProfilesPrebuilt["run-test-prebuilt"]["target_cpu"]
- = profiles[testImageProfile]["target_cpu"];
+ function updateProfileTargets(profiles, testedProfile, testImageProfile, targetProfile, runTestProfile) {
+ var profileToCheck = profiles[testedProfile] || profiles[testImageProfile];
+
+ if (profileToCheck != null) {
+ targetProfile[runTestProfile]["target_os"] = profileToCheck["target_os"];
+ targetProfile[runTestProfile]["target_cpu"] = profileToCheck["target_cpu"];
+ }
}
+
+ updateProfileTargets(profiles, testedProfile, testImageProfile, testOnlyProfilesPrebuilt, "run-test-prebuilt");
+ updateProfileTargets(profiles, testedProfile, testImageProfile, testOnlyProfilesPrebuiltDocs, "run-test-prebuilt-docs");
+
+ profiles = concatObjects(profiles, testOnlyProfilesPrebuiltDocs);
profiles = concatObjects(profiles, testOnlyProfilesPrebuilt);
// On macosx add the devkit bin dir to the path in all the run-test profiles.
@@ -1067,6 +1080,8 @@ var getJibProfilesProfiles = function (input, common, data) {
}
profiles["run-test-prebuilt"] = concatObjects(profiles["run-test-prebuilt"],
runTestPrebuiltSrcFullExtra);
+ profiles["run-test-prebuilt-docs"] = concatObjects(profiles["run-test-prebuilt-docs"],
+ runTestPrebuiltSrcFullExtra);
}
// Generate the missing platform attributes
@@ -1275,6 +1290,14 @@ var getJibProfilesDependencies = function (input, common) {
ext: "tar.gz",
revision: "3.4.2+1.0"
},
+ tidy: {
+ organization: common.organization,
+ ext: "tar.gz",
+ revision: "5.9.20+1",
+ environment_path: input.get("tidy", "home_path") + "/bin/tidy",
+ configure_args: "TIDY=" + input.get("tidy", "home_path") + "/bin/tidy",
+ module: "tidy-html-" + (input.target_os === "macosx" ? input.target_os : input.target_platform),
+ },
};
return dependencies;
diff --git a/make/conf/module-loader-map.conf b/make/conf/module-loader-map.conf
index 1062b780a79ff..b628bfbf2da30 100644
--- a/make/conf/module-loader-map.conf
+++ b/make/conf/module-loader-map.conf
@@ -62,6 +62,7 @@ UPGRADEABLE_PLATFORM_MODULES= \
java.compiler \
jdk.graal.compiler \
jdk.graal.compiler.management \
+ jdk.jsobject \
#
PLATFORM_MODULES= \
@@ -79,7 +80,6 @@ PLATFORM_MODULES= \
jdk.crypto.cryptoki \
jdk.dynalink \
jdk.httpserver \
- jdk.jsobject \
jdk.localedata \
jdk.naming.dns \
jdk.security.auth \
diff --git a/make/conf/version-numbers.conf b/make/conf/version-numbers.conf
index 1d47c2cddd001..055f9ca886618 100644
--- a/make/conf/version-numbers.conf
+++ b/make/conf/version-numbers.conf
@@ -37,6 +37,6 @@ DEFAULT_VERSION_DATE=2025-03-18
DEFAULT_VERSION_CLASSFILE_MAJOR=68 # "`$EXPR $DEFAULT_VERSION_FEATURE + 44`"
DEFAULT_VERSION_CLASSFILE_MINOR=0
DEFAULT_VERSION_DOCS_API_SINCE=11
-DEFAULT_ACCEPTABLE_BOOT_VERSIONS="22 23 24"
+DEFAULT_ACCEPTABLE_BOOT_VERSIONS="23 24"
DEFAULT_JDK_SOURCE_TARGET_VERSION=24
DEFAULT_PROMOTED_VERSION_PRE=ea
diff --git a/make/devkit/createTidyBundle.sh b/make/devkit/createTidyBundle.sh
new file mode 100644
index 0000000000000..b3f8e6bf07805
--- /dev/null
+++ b/make/devkit/createTidyBundle.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+#
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+# Creates a tidy bundle in the build directory. A dependency that can be
+# used to validate and correct HTML.
+
+# wget, cmake and gcc are required to build tidy.
+
+set -e
+
+GITHUB_USER="htacg"
+REPO_NAME="tidy-html5"
+COMMIT_HASH="d08ddc2860aa95ba8e301343a30837f157977cba"
+SCRIPT_DIR="$(cd "$(dirname $0)" > /dev/null && pwd)"
+INSTALL_PREFIX="${SCRIPT_DIR}/../../build/tidy/tidy/"
+BUILD_DIR="build/cmake"
+
+OS_NAME=$(uname -s)
+OS_ARCH=$(uname -m)
+
+DOWNLOAD_URL="https://github.com/$GITHUB_USER/$REPO_NAME/archive/$COMMIT_HASH.tar.gz"
+OUTPUT_FILE="$REPO_NAME-$COMMIT_HASH.tar.gz"
+
+wget "$DOWNLOAD_URL" -O "$OUTPUT_FILE"
+
+tar -xzf "$OUTPUT_FILE"
+rm -rf "$OUTPUT_FILE"
+
+SRC_DIR="$REPO_NAME-$COMMIT_HASH"
+
+mkdir -p "$SRC_DIR/$BUILD_DIR"
+cd "$SRC_DIR/$BUILD_DIR"
+
+case $OS_NAME in
+ Linux|Darwin)
+ echo "Building Tidy HTML5 for Unix-like platform ($OS_NAME)..."
+
+ CMAKE_ARCH_OPTIONS=""
+ if [ "$OS_NAME" == "Darwin" ]; then
+ if [[ "$OS_ARCH" == "arm64" || "$OS_ARCH" == "x86_64" ]]; then
+ CMAKE_ARCH_OPTIONS="-DCMAKE_OSX_ARCHITECTURES=x86_64;arm64"
+ fi
+ fi
+
+ cmake ../.. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$INSTALL_PREFIX" $CMAKE_ARCH_OPTIONS
+ make install
+ ;;
+
+ *)
+ echo "Unsupported OS: $OS_NAME"
+ exit 1
+ ;;
+esac
+
+cd "$SCRIPT_DIR"
+rm -rf "$SRC_DIR"
+
+cd "$INSTALL_PREFIX.."
+PACKAGED_FILE="tidy-html5.tar.gz"
+
+tar -czvf "$PACKAGED_FILE" -C "$INSTALL_PREFIX.." tidy
+
+echo "Created $INSTALL_PREFIX..$PACKAGED_FILE"
diff --git a/make/hotspot/gensrc/GensrcAdlc.gmk b/make/hotspot/gensrc/GensrcAdlc.gmk
index 8dada3cec0a1d..ce3f268402672 100644
--- a/make/hotspot/gensrc/GensrcAdlc.gmk
+++ b/make/hotspot/gensrc/GensrcAdlc.gmk
@@ -193,13 +193,18 @@ ifeq ($(call check-jvm-feature, compiler2), true)
ifeq ($(call check-jvm-feature, zgc), true)
AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \
- $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/x/x_$(HOTSPOT_TARGET_CPU).ad \
- $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/x/x_$(HOTSPOT_TARGET_CPU_ARCH).ad \
$d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/z/z_$(HOTSPOT_TARGET_CPU).ad \
$d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/z/z_$(HOTSPOT_TARGET_CPU_ARCH).ad \
)))
endif
+ ifeq ($(call check-jvm-feature, g1gc), true)
+ AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \
+ $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/g1/g1_$(HOTSPOT_TARGET_CPU).ad \
+ $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/g1/g1_$(HOTSPOT_TARGET_CPU_ARCH).ad \
+ )))
+ endif
+
SINGLE_AD_SRCFILE := $(ADLC_SUPPORT_DIR)/all-ad-src.ad
INSERT_FILENAME_AWK_SCRIPT := \
diff --git a/make/hotspot/lib/JvmFeatures.gmk b/make/hotspot/lib/JvmFeatures.gmk
index c4c030810fc44..b94031515f79e 100644
--- a/make/hotspot/lib/JvmFeatures.gmk
+++ b/make/hotspot/lib/JvmFeatures.gmk
@@ -150,7 +150,6 @@ endif
ifneq ($(call check-jvm-feature, zgc), true)
JVM_CFLAGS_FEATURES += -DINCLUDE_ZGC=0
JVM_EXCLUDE_PATTERNS += gc/z
- JVM_EXCLUDE_PATTERNS += gc/x
endif
ifneq ($(call check-jvm-feature, shenandoahgc), true)
diff --git a/make/modules/jdk.hotspot.agent/Lib.gmk b/make/modules/jdk.hotspot.agent/Lib.gmk
index f0ede594d0ce8..12f1c1f2a9077 100644
--- a/make/modules/jdk.hotspot.agent/Lib.gmk
+++ b/make/modules/jdk.hotspot.agent/Lib.gmk
@@ -59,9 +59,7 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBSAPROC, \
OPTIMIZATION := HIGH, \
EXTRA_HEADER_DIRS := java.base:libjvm, \
DISABLED_WARNINGS_gcc := sign-compare, \
- DISABLED_WARNINGS_gcc_LinuxDebuggerLocal.cpp := unused-variable, \
DISABLED_WARNINGS_gcc_ps_core.c := pointer-arith, \
- DISABLED_WARNINGS_gcc_symtab.c := unused-but-set-variable, \
DISABLED_WARNINGS_clang := sign-compare, \
DISABLED_WARNINGS_clang_libproc_impl.c := format-nonliteral, \
DISABLED_WARNINGS_clang_MacosxDebuggerLocal.m := unused-variable, \
diff --git a/make/modules/jdk.incubator.vector/Lib.gmk b/make/modules/jdk.incubator.vector/Lib.gmk
index 0620549f05cd7..bf6ace6f97f7c 100644
--- a/make/modules/jdk.incubator.vector/Lib.gmk
+++ b/make/modules/jdk.incubator.vector/Lib.gmk
@@ -37,3 +37,21 @@ ifeq ($(call isTargetOs, linux windows)+$(call isTargetCpu, x86_64)+$(INCLUDE_CO
TARGETS += $(BUILD_LIBJSVML)
endif
+
+################################################################################
+## Build libsleef
+################################################################################
+
+ifeq ($(call isTargetOs, linux)+$(call isTargetCpu, riscv64)+$(INCLUDE_COMPILER2), true+true+true)
+ $(eval $(call SetupJdkLibrary, BUILD_LIBSLEEF, \
+ NAME := sleef, \
+ OPTIMIZATION := HIGH, \
+ SRC := libsleef/lib, \
+ EXTRA_SRC := libsleef/generated, \
+ DISABLED_WARNINGS_gcc := unused-function sign-compare tautological-compare ignored-qualifiers, \
+ DISABLED_WARNINGS_clang := unused-function sign-compare tautological-compare ignored-qualifiers, \
+ CFLAGS := -march=rv64gcv, \
+ ))
+
+ TARGETS += $(BUILD_LIBSLEEF)
+endif
diff --git a/make/test/BuildMicrobenchmark.gmk b/make/test/BuildMicrobenchmark.gmk
index 700aac3c9c6dc..1b2eb40b252cf 100644
--- a/make/test/BuildMicrobenchmark.gmk
+++ b/make/test/BuildMicrobenchmark.gmk
@@ -49,7 +49,6 @@ MICROBENCHMARK_CLASSES := $(MICROBENCHMARK_OUTPUT)/classes
MICROBENCHMARK_JAR_BIN := $(MICROBENCHMARK_OUTPUT)/jar
MICROBENCHMARK_TOOLS_CLASSES := $(MICROBENCHMARK_OUTPUT)/tools-classes
-MICROBENCHMARK_INDIFY_DONE := $(MICROBENCHMARK_CLASSES)/_indify.marker
JMH_UNPACKED_DIR := $(MICROBENCHMARK_OUTPUT)/jmh_jars
JMH_UNPACKED_JARS_DONE := $(JMH_UNPACKED_DIR)/_unpacked.marker
@@ -71,17 +70,6 @@ MICROBENCHMARK_MANIFEST := Build: $(FULL_VERSION)\n\
\nJMH-Version: $(JMH_VERSION)\n\
\nName: OpenJDK Microbenchmark Suite
-#### Compile Indify tool
-
-$(eval $(call SetupJavaCompilation, BUILD_INDIFY, \
- TARGET_RELEASE := $(TARGET_RELEASE_BOOTJDK), \
- SRC := $(TOPDIR)/test/jdk/java/lang/invoke, \
- INCLUDE_FILES := indify/Indify.java, \
- DISABLED_WARNINGS := this-escape rawtypes serial options, \
- BIN := $(MICROBENCHMARK_TOOLS_CLASSES), \
- JAVAC_FLAGS := -XDstringConcat=inline -Xprefer:newer, \
-))
-
#### Compile Targets
# Building microbenchmark requires the jdk.unsupported and java.management modules.
@@ -124,14 +112,6 @@ $(eval $(call SetupJavaCompilation, BUILD_JDK_MICROBENCHMARK, \
$(BUILD_JDK_MICROBENCHMARK): $(JMH_COMPILE_JARS)
-# Run Indify
-$(MICROBENCHMARK_INDIFY_DONE): $(BUILD_INDIFY) $(BUILD_JDK_MICROBENCHMARK)
- $(call LogWarn, Running Indify on microbenchmark classes)
- $(JAVA_SMALL) -cp $(MICROBENCHMARK_TOOLS_CLASSES) \
- indify.Indify --overwrite $(MICROBENCHMARK_CLASSES) \
- $(LOG_DEBUG) 2>&1
- $(TOUCH) $@
-
# Unpacking dependencies for inclusion in the benchmark JARs
$(JMH_UNPACKED_JARS_DONE): $(JMH_RUNTIME_JARS)
$(RM) -r $(JMH_UNPACKED_DIR)
@@ -144,8 +124,7 @@ $(JMH_UNPACKED_JARS_DONE): $(JMH_RUNTIME_JARS)
# Create benchmarks JAR file with benchmarks for both the old and new JDK
$(eval $(call SetupJarArchive, BUILD_JDK_JAR, \
- DEPENDENCIES := $(BUILD_JDK_MICROBENCHMARK) $(JMH_UNPACKED_JARS_DONE) \
- $(MICROBENCHMARK_INDIFY_DONE), \
+ DEPENDENCIES := $(BUILD_JDK_MICROBENCHMARK) $(JMH_UNPACKED_JARS_DONE), \
SRCS := $(MICROBENCHMARK_CLASSES) $(JMH_UNPACKED_DIR), \
BIN := $(MICROBENCHMARK_JAR_BIN), \
SUFFIXES := .*, \
diff --git a/make/test/JtregNativeJdk.gmk b/make/test/JtregNativeJdk.gmk
index d9f1e334a5cf8..90055cb5c0114 100644
--- a/make/test/JtregNativeJdk.gmk
+++ b/make/test/JtregNativeJdk.gmk
@@ -115,6 +115,8 @@ ifeq ($(call isTargetOs, linux), true)
# stripping during the test libraries' build.
BUILD_JDK_JTREG_LIBRARIES_CFLAGS_libFib := -g
BUILD_JDK_JTREG_LIBRARIES_STRIP_SYMBOLS_libFib := false
+ # nio tests' libCreationTimeHelper native needs -ldl linker flag
+ BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libCreationTimeHelper := -ldl
endif
ifeq ($(ASAN_ENABLED), true)
diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad
index 39eae43a287e7..d9c77a2f52926 100644
--- a/src/hotspot/cpu/aarch64/aarch64.ad
+++ b/src/hotspot/cpu/aarch64/aarch64.ad
@@ -2307,10 +2307,6 @@ const RegMask* Matcher::predicate_reg_mask(void) {
return &_PR_REG_mask;
}
-const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
- return new TypeVectMask(elemTy, length);
-}
-
// Vector calling convention not yet implemented.
bool Matcher::supports_vector_calling_convention(void) {
return false;
@@ -2620,7 +2616,8 @@ static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
if (is_vshift_con_pattern(n, m) ||
is_vector_bitwise_not_pattern(n, m) ||
- is_valid_sve_arith_imm_pattern(n, m)) {
+ is_valid_sve_arith_imm_pattern(n, m) ||
+ is_encode_and_store_pattern(n, m)) {
mstack.push(m, Visit);
return true;
}
@@ -2720,7 +2717,7 @@ typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
{
Address addr = mem2address(opcode, base, index, scale, disp);
if (addr.getMode() == Address::base_plus_offset) {
- // Fix up any out-of-range offsets.
+ /* Fix up any out-of-range offsets. */
assert_different_registers(rscratch1, base);
assert_different_registers(rscratch1, reg);
addr = __ legitimize_address(addr, size_in_memory, rscratch1);
@@ -2761,11 +2758,7 @@ typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
int opcode, Register base, int index, int size, int disp)
{
if (index == -1) {
- // Fix up any out-of-range offsets.
- assert_different_registers(rscratch1, base);
- Address addr = Address(base, disp);
- addr = __ legitimize_address(addr, (1 << T), rscratch1);
- (masm->*insn)(reg, T, addr);
+ (masm->*insn)(reg, T, Address(base, disp));
} else {
assert(disp == 0, "unsupported address mode");
(masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
@@ -2820,7 +2813,7 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_ldrsbw(iRegI dst, memory mem) %{
+ enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
Register dst_reg = as_Register($dst$$reg);
loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
@@ -2828,7 +2821,7 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_ldrsb(iRegI dst, memory mem) %{
+ enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
Register dst_reg = as_Register($dst$$reg);
loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
@@ -2836,7 +2829,7 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_ldrb(iRegI dst, memory mem) %{
+ enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
Register dst_reg = as_Register($dst$$reg);
loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
@@ -2844,7 +2837,7 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_ldrb(iRegL dst, memory mem) %{
+ enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
Register dst_reg = as_Register($dst$$reg);
loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
@@ -2852,7 +2845,7 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_ldrshw(iRegI dst, memory mem) %{
+ enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
Register dst_reg = as_Register($dst$$reg);
loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
@@ -2860,7 +2853,7 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_ldrsh(iRegI dst, memory mem) %{
+ enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
Register dst_reg = as_Register($dst$$reg);
loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
@@ -2868,7 +2861,7 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_ldrh(iRegI dst, memory mem) %{
+ enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
Register dst_reg = as_Register($dst$$reg);
loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
@@ -2876,7 +2869,7 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_ldrh(iRegL dst, memory mem) %{
+ enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
Register dst_reg = as_Register($dst$$reg);
loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
@@ -2884,7 +2877,7 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_ldrw(iRegI dst, memory mem) %{
+ enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
Register dst_reg = as_Register($dst$$reg);
loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
@@ -2892,7 +2885,7 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_ldrw(iRegL dst, memory mem) %{
+ enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
Register dst_reg = as_Register($dst$$reg);
loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
@@ -2900,7 +2893,7 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_ldrsw(iRegL dst, memory mem) %{
+ enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
Register dst_reg = as_Register($dst$$reg);
loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
@@ -2908,7 +2901,7 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_ldr(iRegL dst, memory mem) %{
+ enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
Register dst_reg = as_Register($dst$$reg);
loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
@@ -2916,7 +2909,7 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_ldrs(vRegF dst, memory mem) %{
+ enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
FloatRegister dst_reg = as_FloatRegister($dst$$reg);
loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
@@ -2924,7 +2917,7 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_ldrd(vRegD dst, memory mem) %{
+ enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
FloatRegister dst_reg = as_FloatRegister($dst$$reg);
loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
@@ -2932,7 +2925,7 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_strb(iRegI src, memory mem) %{
+ enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
Register src_reg = as_Register($src$$reg);
loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
@@ -2940,14 +2933,14 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_strb0(memory mem) %{
+ enc_class aarch64_enc_strb0(memory1 mem) %{
loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
%}
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_strh(iRegI src, memory mem) %{
+ enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
Register src_reg = as_Register($src$$reg);
loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
@@ -2955,14 +2948,14 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_strh0(memory mem) %{
+ enc_class aarch64_enc_strh0(memory2 mem) %{
loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
%}
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_strw(iRegI src, memory mem) %{
+ enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
Register src_reg = as_Register($src$$reg);
loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
@@ -2970,14 +2963,14 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_strw0(memory mem) %{
+ enc_class aarch64_enc_strw0(memory4 mem) %{
loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
%}
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_str(iRegL src, memory mem) %{
+ enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
Register src_reg = as_Register($src$$reg);
// we sometimes get asked to store the stack pointer into the
// current thread -- we cannot do that directly on AArch64
@@ -2992,14 +2985,14 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_str0(memory mem) %{
+ enc_class aarch64_enc_str0(memory8 mem) %{
loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
%}
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_strs(vRegF src, memory mem) %{
+ enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
FloatRegister src_reg = as_FloatRegister($src$$reg);
loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
@@ -3007,7 +3000,7 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_strd(vRegD src, memory mem) %{
+ enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
FloatRegister src_reg = as_FloatRegister($src$$reg);
loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
@@ -3015,7 +3008,7 @@ encode %{
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_strb0_ordered(memory mem) %{
+ enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
__ membar(Assembler::StoreStore);
loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
@@ -3217,7 +3210,7 @@ encode %{
// synchronized read/update encodings
- enc_class aarch64_enc_ldaxr(iRegL dst, memory mem) %{
+ enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
Register dst_reg = as_Register($dst$$reg);
Register base = as_Register($mem$$base);
int index = $mem$$index;
@@ -3245,7 +3238,7 @@ encode %{
}
%}
- enc_class aarch64_enc_stlxr(iRegLNoSp src, memory mem) %{
+ enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
Register src_reg = as_Register($src$$reg);
Register base = as_Register($mem$$base);
int index = $mem$$index;
@@ -4173,10 +4166,60 @@ operand immIU7()
interface(CONST_INTER);
%}
-// Offset for immediate loads and stores
+// Offset for scaled or unscaled immediate loads and stores
operand immIOffset()
%{
- predicate(n->get_int() >= -256 && n->get_int() <= 65520);
+ predicate(Address::offset_ok_for_immed(n->get_int(), 0));
+ match(ConI);
+
+ op_cost(0);
+ format %{ %}
+ interface(CONST_INTER);
+%}
+
+operand immIOffset1()
+%{
+ predicate(Address::offset_ok_for_immed(n->get_int(), 0));
+ match(ConI);
+
+ op_cost(0);
+ format %{ %}
+ interface(CONST_INTER);
+%}
+
+operand immIOffset2()
+%{
+ predicate(Address::offset_ok_for_immed(n->get_int(), 1));
+ match(ConI);
+
+ op_cost(0);
+ format %{ %}
+ interface(CONST_INTER);
+%}
+
+operand immIOffset4()
+%{
+ predicate(Address::offset_ok_for_immed(n->get_int(), 2));
+ match(ConI);
+
+ op_cost(0);
+ format %{ %}
+ interface(CONST_INTER);
+%}
+
+operand immIOffset8()
+%{
+ predicate(Address::offset_ok_for_immed(n->get_int(), 3));
+ match(ConI);
+
+ op_cost(0);
+ format %{ %}
+ interface(CONST_INTER);
+%}
+
+operand immIOffset16()
+%{
+ predicate(Address::offset_ok_for_immed(n->get_int(), 4));
match(ConI);
op_cost(0);
@@ -4194,6 +4237,56 @@ operand immLOffset()
interface(CONST_INTER);
%}
+operand immLoffset1()
+%{
+ predicate(Address::offset_ok_for_immed(n->get_long(), 0));
+ match(ConL);
+
+ op_cost(0);
+ format %{ %}
+ interface(CONST_INTER);
+%}
+
+operand immLoffset2()
+%{
+ predicate(Address::offset_ok_for_immed(n->get_long(), 1));
+ match(ConL);
+
+ op_cost(0);
+ format %{ %}
+ interface(CONST_INTER);
+%}
+
+operand immLoffset4()
+%{
+ predicate(Address::offset_ok_for_immed(n->get_long(), 2));
+ match(ConL);
+
+ op_cost(0);
+ format %{ %}
+ interface(CONST_INTER);
+%}
+
+operand immLoffset8()
+%{
+ predicate(Address::offset_ok_for_immed(n->get_long(), 3));
+ match(ConL);
+
+ op_cost(0);
+ format %{ %}
+ interface(CONST_INTER);
+%}
+
+operand immLoffset16()
+%{
+ predicate(Address::offset_ok_for_immed(n->get_long(), 4));
+ match(ConL);
+
+ op_cost(0);
+ format %{ %}
+ interface(CONST_INTER);
+%}
+
// 5 bit signed long integer
operand immL5()
%{
@@ -5106,7 +5199,105 @@ operand indIndex(iRegP reg, iRegL lreg)
%}
%}
-operand indOffI(iRegP reg, immIOffset off)
+operand indOffI1(iRegP reg, immIOffset1 off)
+%{
+ constraint(ALLOC_IN_RC(ptr_reg));
+ match(AddP reg off);
+ op_cost(0);
+ format %{ "[$reg, $off]" %}
+ interface(MEMORY_INTER) %{
+ base($reg);
+ index(0xffffffff);
+ scale(0x0);
+ disp($off);
+ %}
+%}
+
+operand indOffI2(iRegP reg, immIOffset2 off)
+%{
+ constraint(ALLOC_IN_RC(ptr_reg));
+ match(AddP reg off);
+ op_cost(0);
+ format %{ "[$reg, $off]" %}
+ interface(MEMORY_INTER) %{
+ base($reg);
+ index(0xffffffff);
+ scale(0x0);
+ disp($off);
+ %}
+%}
+
+operand indOffI4(iRegP reg, immIOffset4 off)
+%{
+ constraint(ALLOC_IN_RC(ptr_reg));
+ match(AddP reg off);
+ op_cost(0);
+ format %{ "[$reg, $off]" %}
+ interface(MEMORY_INTER) %{
+ base($reg);
+ index(0xffffffff);
+ scale(0x0);
+ disp($off);
+ %}
+%}
+
+operand indOffI8(iRegP reg, immIOffset8 off)
+%{
+ constraint(ALLOC_IN_RC(ptr_reg));
+ match(AddP reg off);
+ op_cost(0);
+ format %{ "[$reg, $off]" %}
+ interface(MEMORY_INTER) %{
+ base($reg);
+ index(0xffffffff);
+ scale(0x0);
+ disp($off);
+ %}
+%}
+
+operand indOffI16(iRegP reg, immIOffset16 off)
+%{
+ constraint(ALLOC_IN_RC(ptr_reg));
+ match(AddP reg off);
+ op_cost(0);
+ format %{ "[$reg, $off]" %}
+ interface(MEMORY_INTER) %{
+ base($reg);
+ index(0xffffffff);
+ scale(0x0);
+ disp($off);
+ %}
+%}
+
+operand indOffL1(iRegP reg, immLoffset1 off)
+%{
+ constraint(ALLOC_IN_RC(ptr_reg));
+ match(AddP reg off);
+ op_cost(0);
+ format %{ "[$reg, $off]" %}
+ interface(MEMORY_INTER) %{
+ base($reg);
+ index(0xffffffff);
+ scale(0x0);
+ disp($off);
+ %}
+%}
+
+operand indOffL2(iRegP reg, immLoffset2 off)
+%{
+ constraint(ALLOC_IN_RC(ptr_reg));
+ match(AddP reg off);
+ op_cost(0);
+ format %{ "[$reg, $off]" %}
+ interface(MEMORY_INTER) %{
+ base($reg);
+ index(0xffffffff);
+ scale(0x0);
+ disp($off);
+ %}
+%}
+
+operand indOffL4(iRegP reg, immLoffset4 off)
%{
constraint(ALLOC_IN_RC(ptr_reg));
match(AddP reg off);
@@ -5120,7 +5311,21 @@ operand indOffI(iRegP reg, immIOffset off)
%}
%}
-operand indOffL(iRegP reg, immLOffset off)
+operand indOffL8(iRegP reg, immLoffset8 off)
+%{
+ constraint(ALLOC_IN_RC(ptr_reg));
+ match(AddP reg off);
+ op_cost(0);
+ format %{ "[$reg, $off]" %}
+ interface(MEMORY_INTER) %{
+ base($reg);
+ index(0xffffffff);
+ scale(0x0);
+ disp($off);
+ %}
+%}
+
+operand indOffL16(iRegP reg, immLoffset16 off)
%{
constraint(ALLOC_IN_RC(ptr_reg));
match(AddP reg off);
@@ -5496,7 +5701,10 @@ operand iRegL2P(iRegL reg) %{
interface(REG_INTER)
%}
-opclass vmem(indirect, indIndex, indOffI, indOffL, indOffIN, indOffLN);
+opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
+opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
+opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
+opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
//----------OPERAND CLASSES----------------------------------------------------
// Operand Classes are groups of operands that are used as to simplify
@@ -5508,9 +5716,23 @@ opclass vmem(indirect, indIndex, indOffI, indOffL, indOffIN, indOffLN);
// memory is used to define read/write location for load/store
// instruction defs. we can turn a memory op into an Address
-opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI, indOffL,
- indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN,
- indOffLN, indirectX2P, indOffX2P);
+opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
+ indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
+
+opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
+ indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
+
+opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
+ indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
+
+opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
+ indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
+
+// All of the memory operands. For the pipeline description.
+opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
+ indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
+ indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
+
// iRegIorL2I is used for src inputs in rules for 32 bit int (I)
// operations. it allows the src to be either an iRegI or a (ConvL2I
@@ -6212,7 +6434,7 @@ define %{
// Load Instructions
// Load Byte (8 bit signed)
-instruct loadB(iRegINoSp dst, memory mem)
+instruct loadB(iRegINoSp dst, memory1 mem)
%{
match(Set dst (LoadB mem));
predicate(!needs_acquiring_load(n));
@@ -6226,7 +6448,7 @@ instruct loadB(iRegINoSp dst, memory mem)
%}
// Load Byte (8 bit signed) into long
-instruct loadB2L(iRegLNoSp dst, memory mem)
+instruct loadB2L(iRegLNoSp dst, memory1 mem)
%{
match(Set dst (ConvI2L (LoadB mem)));
predicate(!needs_acquiring_load(n->in(1)));
@@ -6240,7 +6462,7 @@ instruct loadB2L(iRegLNoSp dst, memory mem)
%}
// Load Byte (8 bit unsigned)
-instruct loadUB(iRegINoSp dst, memory mem)
+instruct loadUB(iRegINoSp dst, memory1 mem)
%{
match(Set dst (LoadUB mem));
predicate(!needs_acquiring_load(n));
@@ -6254,7 +6476,7 @@ instruct loadUB(iRegINoSp dst, memory mem)
%}
// Load Byte (8 bit unsigned) into long
-instruct loadUB2L(iRegLNoSp dst, memory mem)
+instruct loadUB2L(iRegLNoSp dst, memory1 mem)
%{
match(Set dst (ConvI2L (LoadUB mem)));
predicate(!needs_acquiring_load(n->in(1)));
@@ -6268,7 +6490,7 @@ instruct loadUB2L(iRegLNoSp dst, memory mem)
%}
// Load Short (16 bit signed)
-instruct loadS(iRegINoSp dst, memory mem)
+instruct loadS(iRegINoSp dst, memory2 mem)
%{
match(Set dst (LoadS mem));
predicate(!needs_acquiring_load(n));
@@ -6282,7 +6504,7 @@ instruct loadS(iRegINoSp dst, memory mem)
%}
// Load Short (16 bit signed) into long
-instruct loadS2L(iRegLNoSp dst, memory mem)
+instruct loadS2L(iRegLNoSp dst, memory2 mem)
%{
match(Set dst (ConvI2L (LoadS mem)));
predicate(!needs_acquiring_load(n->in(1)));
@@ -6296,7 +6518,7 @@ instruct loadS2L(iRegLNoSp dst, memory mem)
%}
// Load Char (16 bit unsigned)
-instruct loadUS(iRegINoSp dst, memory mem)
+instruct loadUS(iRegINoSp dst, memory2 mem)
%{
match(Set dst (LoadUS mem));
predicate(!needs_acquiring_load(n));
@@ -6310,7 +6532,7 @@ instruct loadUS(iRegINoSp dst, memory mem)
%}
// Load Short/Char (16 bit unsigned) into long
-instruct loadUS2L(iRegLNoSp dst, memory mem)
+instruct loadUS2L(iRegLNoSp dst, memory2 mem)
%{
match(Set dst (ConvI2L (LoadUS mem)));
predicate(!needs_acquiring_load(n->in(1)));
@@ -6324,7 +6546,7 @@ instruct loadUS2L(iRegLNoSp dst, memory mem)
%}
// Load Integer (32 bit signed)
-instruct loadI(iRegINoSp dst, memory mem)
+instruct loadI(iRegINoSp dst, memory4 mem)
%{
match(Set dst (LoadI mem));
predicate(!needs_acquiring_load(n));
@@ -6338,7 +6560,7 @@ instruct loadI(iRegINoSp dst, memory mem)
%}
// Load Integer (32 bit signed) into long
-instruct loadI2L(iRegLNoSp dst, memory mem)
+instruct loadI2L(iRegLNoSp dst, memory4 mem)
%{
match(Set dst (ConvI2L (LoadI mem)));
predicate(!needs_acquiring_load(n->in(1)));
@@ -6352,7 +6574,7 @@ instruct loadI2L(iRegLNoSp dst, memory mem)
%}
// Load Integer (32 bit unsigned) into long
-instruct loadUI2L(iRegLNoSp dst, memory mem, immL_32bits mask)
+instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
%{
match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
@@ -6366,7 +6588,7 @@ instruct loadUI2L(iRegLNoSp dst, memory mem, immL_32bits mask)
%}
// Load Long (64 bit signed)
-instruct loadL(iRegLNoSp dst, memory mem)
+instruct loadL(iRegLNoSp dst, memory8 mem)
%{
match(Set dst (LoadL mem));
predicate(!needs_acquiring_load(n));
@@ -6380,7 +6602,7 @@ instruct loadL(iRegLNoSp dst, memory mem)
%}
// Load Range
-instruct loadRange(iRegINoSp dst, memory mem)
+instruct loadRange(iRegINoSp dst, memory4 mem)
%{
match(Set dst (LoadRange mem));
@@ -6393,7 +6615,7 @@ instruct loadRange(iRegINoSp dst, memory mem)
%}
// Load Pointer
-instruct loadP(iRegPNoSp dst, memory mem)
+instruct loadP(iRegPNoSp dst, memory8 mem)
%{
match(Set dst (LoadP mem));
predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
@@ -6407,10 +6629,10 @@ instruct loadP(iRegPNoSp dst, memory mem)
%}
// Load Compressed Pointer
-instruct loadN(iRegNNoSp dst, memory mem)
+instruct loadN(iRegNNoSp dst, memory4 mem)
%{
match(Set dst (LoadN mem));
- predicate(!needs_acquiring_load(n));
+ predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
ins_cost(4 * INSN_COST);
format %{ "ldrw $dst, $mem\t# compressed ptr" %}
@@ -6421,7 +6643,7 @@ instruct loadN(iRegNNoSp dst, memory mem)
%}
// Load Klass Pointer
-instruct loadKlass(iRegPNoSp dst, memory mem)
+instruct loadKlass(iRegPNoSp dst, memory8 mem)
%{
match(Set dst (LoadKlass mem));
predicate(!needs_acquiring_load(n));
@@ -6435,7 +6657,7 @@ instruct loadKlass(iRegPNoSp dst, memory mem)
%}
// Load Narrow Klass Pointer
-instruct loadNKlass(iRegNNoSp dst, memory mem)
+instruct loadNKlass(iRegNNoSp dst, memory4 mem)
%{
match(Set dst (LoadNKlass mem));
predicate(!needs_acquiring_load(n));
@@ -6449,7 +6671,7 @@ instruct loadNKlass(iRegNNoSp dst, memory mem)
%}
// Load Float
-instruct loadF(vRegF dst, memory mem)
+instruct loadF(vRegF dst, memory4 mem)
%{
match(Set dst (LoadF mem));
predicate(!needs_acquiring_load(n));
@@ -6463,7 +6685,7 @@ instruct loadF(vRegF dst, memory mem)
%}
// Load Double
-instruct loadD(vRegD dst, memory mem)
+instruct loadD(vRegD dst, memory8 mem)
%{
match(Set dst (LoadD mem));
predicate(!needs_acquiring_load(n));
@@ -6666,38 +6888,8 @@ instruct loadConD(vRegD dst, immD con) %{
// Store Instructions
-// Store CMS card-mark Immediate
-instruct storeimmCM0(immI0 zero, memory mem)
-%{
- match(Set mem (StoreCM mem zero));
-
- ins_cost(INSN_COST);
- format %{ "storestore (elided)\n\t"
- "strb zr, $mem\t# byte" %}
-
- ins_encode(aarch64_enc_strb0(mem));
-
- ins_pipe(istore_mem);
-%}
-
-// Store CMS card-mark Immediate with intervening StoreStore
-// needed when using CMS with no conditional card marking
-instruct storeimmCM0_ordered(immI0 zero, memory mem)
-%{
- match(Set mem (StoreCM mem zero));
-
- ins_cost(INSN_COST * 2);
- format %{ "storestore\n\t"
- "dmb ishst"
- "\n\tstrb zr, $mem\t# byte" %}
-
- ins_encode(aarch64_enc_strb0_ordered(mem));
-
- ins_pipe(istore_mem);
-%}
-
// Store Byte
-instruct storeB(iRegIorL2I src, memory mem)
+instruct storeB(iRegIorL2I src, memory1 mem)
%{
match(Set mem (StoreB mem src));
predicate(!needs_releasing_store(n));
@@ -6711,7 +6903,7 @@ instruct storeB(iRegIorL2I src, memory mem)
%}
-instruct storeimmB0(immI0 zero, memory mem)
+instruct storeimmB0(immI0 zero, memory1 mem)
%{
match(Set mem (StoreB mem zero));
predicate(!needs_releasing_store(n));
@@ -6725,7 +6917,7 @@ instruct storeimmB0(immI0 zero, memory mem)
%}
// Store Char/Short
-instruct storeC(iRegIorL2I src, memory mem)
+instruct storeC(iRegIorL2I src, memory2 mem)
%{
match(Set mem (StoreC mem src));
predicate(!needs_releasing_store(n));
@@ -6738,7 +6930,7 @@ instruct storeC(iRegIorL2I src, memory mem)
ins_pipe(istore_reg_mem);
%}
-instruct storeimmC0(immI0 zero, memory mem)
+instruct storeimmC0(immI0 zero, memory2 mem)
%{
match(Set mem (StoreC mem zero));
predicate(!needs_releasing_store(n));
@@ -6753,7 +6945,7 @@ instruct storeimmC0(immI0 zero, memory mem)
// Store Integer
-instruct storeI(iRegIorL2I src, memory mem)
+instruct storeI(iRegIorL2I src, memory4 mem)
%{
match(Set mem(StoreI mem src));
predicate(!needs_releasing_store(n));
@@ -6766,7 +6958,7 @@ instruct storeI(iRegIorL2I src, memory mem)
ins_pipe(istore_reg_mem);
%}
-instruct storeimmI0(immI0 zero, memory mem)
+instruct storeimmI0(immI0 zero, memory4 mem)
%{
match(Set mem(StoreI mem zero));
predicate(!needs_releasing_store(n));
@@ -6780,7 +6972,7 @@ instruct storeimmI0(immI0 zero, memory mem)
%}
// Store Long (64 bit signed)
-instruct storeL(iRegL src, memory mem)
+instruct storeL(iRegL src, memory8 mem)
%{
match(Set mem (StoreL mem src));
predicate(!needs_releasing_store(n));
@@ -6794,7 +6986,7 @@ instruct storeL(iRegL src, memory mem)
%}
// Store Long (64 bit signed)
-instruct storeimmL0(immL0 zero, memory mem)
+instruct storeimmL0(immL0 zero, memory8 mem)
%{
match(Set mem (StoreL mem zero));
predicate(!needs_releasing_store(n));
@@ -6808,7 +7000,7 @@ instruct storeimmL0(immL0 zero, memory mem)
%}
// Store Pointer
-instruct storeP(iRegP src, memory mem)
+instruct storeP(iRegP src, memory8 mem)
%{
match(Set mem (StoreP mem src));
predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
@@ -6822,7 +7014,7 @@ instruct storeP(iRegP src, memory mem)
%}
// Store Pointer
-instruct storeimmP0(immP0 zero, memory mem)
+instruct storeimmP0(immP0 zero, memory8 mem)
%{
match(Set mem (StoreP mem zero));
predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
@@ -6836,10 +7028,10 @@ instruct storeimmP0(immP0 zero, memory mem)
%}
// Store Compressed Pointer
-instruct storeN(iRegN src, memory mem)
+instruct storeN(iRegN src, memory4 mem)
%{
match(Set mem (StoreN mem src));
- predicate(!needs_releasing_store(n));
+ predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
ins_cost(INSN_COST);
format %{ "strw $src, $mem\t# compressed ptr" %}
@@ -6849,10 +7041,10 @@ instruct storeN(iRegN src, memory mem)
ins_pipe(istore_reg_mem);
%}
-instruct storeImmN0(immN0 zero, memory mem)
+instruct storeImmN0(immN0 zero, memory4 mem)
%{
match(Set mem (StoreN mem zero));
- predicate(!needs_releasing_store(n));
+ predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
ins_cost(INSN_COST);
format %{ "strw zr, $mem\t# compressed ptr" %}
@@ -6863,7 +7055,7 @@ instruct storeImmN0(immN0 zero, memory mem)
%}
// Store Float
-instruct storeF(vRegF src, memory mem)
+instruct storeF(vRegF src, memory4 mem)
%{
match(Set mem (StoreF mem src));
predicate(!needs_releasing_store(n));
@@ -6880,7 +7072,7 @@ instruct storeF(vRegF src, memory mem)
// implement storeImmF0 and storeFImmPacked
// Store Double
-instruct storeD(vRegD src, memory mem)
+instruct storeD(vRegD src, memory8 mem)
%{
match(Set mem (StoreD mem src));
predicate(!needs_releasing_store(n));
@@ -6894,7 +7086,7 @@ instruct storeD(vRegD src, memory mem)
%}
// Store Compressed Klass Pointer
-instruct storeNKlass(iRegN src, memory mem)
+instruct storeNKlass(iRegN src, memory4 mem)
%{
predicate(!needs_releasing_store(n));
match(Set mem (StoreNKlass mem src));
@@ -6913,7 +7105,7 @@ instruct storeNKlass(iRegN src, memory mem)
// prefetch instructions
// Must be safe to execute with invalid address (cannot fault).
-instruct prefetchalloc( memory mem ) %{
+instruct prefetchalloc( memory8 mem ) %{
match(PrefetchAllocation mem);
ins_cost(INSN_COST);
@@ -7086,6 +7278,7 @@ instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
%{
match(Set dst (LoadN mem));
+ predicate(n->as_Load()->barrier_data() == 0);
ins_cost(VOLATILE_REF_COST);
format %{ "ldarw $dst, $mem\t# compressed ptr" %}
@@ -7253,6 +7446,7 @@ instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
%{
match(Set mem (StoreN mem src));
+ predicate(n->as_Store()->barrier_data() == 0);
ins_cost(VOLATILE_REF_COST);
format %{ "stlrw $src, $mem\t# compressed ptr" %}
@@ -7265,6 +7459,7 @@ instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
%{
match(Set mem (StoreN mem zero));
+ predicate(n->as_Store()->barrier_data() == 0);
ins_cost(VOLATILE_REF_COST);
format %{ "stlrw zr, $mem\t# compressed ptr" %}
@@ -7482,7 +7677,7 @@ instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
ins_pipe(pipe_class_default);
%}
-instruct popCountI_mem(iRegINoSp dst, memory mem, vRegF tmp) %{
+instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
match(Set dst (PopCountI (LoadI mem)));
effect(TEMP tmp);
ins_cost(INSN_COST * 13);
@@ -7523,7 +7718,7 @@ instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
ins_pipe(pipe_class_default);
%}
-instruct popCountL_mem(iRegINoSp dst, memory mem, vRegD tmp) %{
+instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
match(Set dst (PopCountL (LoadL mem)));
effect(TEMP tmp);
ins_cost(INSN_COST * 13);
@@ -8061,6 +8256,7 @@ instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval
instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
match(Set res (CompareAndSwapN mem (Binary oldval newval)));
+ predicate(n->as_LoadStore()->barrier_data() == 0);
ins_cost(2 * VOLATILE_REF_COST);
effect(KILL cr);
@@ -8175,7 +8371,7 @@ instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP new
instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
- predicate(needs_acquiring_load_exclusive(n));
+ predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
match(Set res (CompareAndSwapN mem (Binary oldval newval)));
ins_cost(VOLATILE_REF_COST);
@@ -8280,6 +8476,7 @@ instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL ne
// This pattern is generated automatically from cas.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
+ predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr);
@@ -8389,7 +8586,7 @@ instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL
// This pattern is generated automatically from cas.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
- predicate(needs_acquiring_load_exclusive(n));
+ predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
ins_cost(VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr);
@@ -8501,6 +8698,7 @@ instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL ne
// This pattern is generated automatically from cas.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
+ predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
ins_cost(2 * VOLATILE_REF_COST);
effect(KILL cr);
@@ -8620,7 +8818,7 @@ instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL
// This pattern is generated automatically from cas.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
- predicate(needs_acquiring_load_exclusive(n));
+ predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
ins_cost(VOLATILE_REF_COST);
effect(KILL cr);
@@ -8681,6 +8879,7 @@ instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
%}
instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
+ predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set prev (GetAndSetN mem newv));
ins_cost(2 * VOLATILE_REF_COST);
format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
@@ -8724,7 +8923,7 @@ instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
%}
instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
- predicate(needs_acquiring_load_exclusive(n));
+ predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
match(Set prev (GetAndSetN mem newv));
ins_cost(VOLATILE_REF_COST);
format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
@@ -16672,7 +16871,7 @@ instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
ins_pipe(pipe_slow);
%}
-instruct compressBitsI_memcon(iRegINoSp dst, memory mem, immI mask,
+instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
vRegF tdst, vRegF tsrc, vRegF tmask) %{
match(Set dst (CompressBits (LoadI mem) mask));
effect(TEMP tdst, TEMP tsrc, TEMP tmask);
@@ -16709,7 +16908,7 @@ instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
ins_pipe(pipe_slow);
%}
-instruct compressBitsL_memcon(iRegLNoSp dst, memory mem, immL mask,
+instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
vRegF tdst, vRegF tsrc, vRegF tmask) %{
match(Set dst (CompressBits (LoadL mem) mask));
effect(TEMP tdst, TEMP tsrc, TEMP tmask);
@@ -16746,7 +16945,7 @@ instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
ins_pipe(pipe_slow);
%}
-instruct expandBitsI_memcon(iRegINoSp dst, memory mem, immI mask,
+instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
vRegF tdst, vRegF tsrc, vRegF tmask) %{
match(Set dst (ExpandBits (LoadI mem) mask));
effect(TEMP tdst, TEMP tsrc, TEMP tmask);
@@ -16784,7 +16983,7 @@ instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
%}
-instruct expandBitsL_memcon(iRegINoSp dst, memory mem, immL mask,
+instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
vRegF tdst, vRegF tsrc, vRegF tmask) %{
match(Set dst (ExpandBits (LoadL mem) mask));
effect(TEMP tdst, TEMP tsrc, TEMP tmask);
diff --git a/src/hotspot/cpu/aarch64/aarch64_vector.ad b/src/hotspot/cpu/aarch64/aarch64_vector.ad
index cdbc4103df89a..0d3a240cecfd3 100644
--- a/src/hotspot/cpu/aarch64/aarch64_vector.ad
+++ b/src/hotspot/cpu/aarch64/aarch64_vector.ad
@@ -345,7 +345,7 @@ source %{
// ------------------------------ Vector load/store ----------------------------
// Load Vector (16 bits)
-instruct loadV2(vReg dst, vmem mem) %{
+instruct loadV2(vReg dst, vmem2 mem) %{
predicate(n->as_LoadVector()->memory_size() == 2);
match(Set dst (LoadVector mem));
format %{ "loadV2 $dst, $mem\t# vector (16 bits)" %}
@@ -354,7 +354,7 @@ instruct loadV2(vReg dst, vmem mem) %{
%}
// Store Vector (16 bits)
-instruct storeV2(vReg src, vmem mem) %{
+instruct storeV2(vReg src, vmem2 mem) %{
predicate(n->as_StoreVector()->memory_size() == 2);
match(Set mem (StoreVector mem src));
format %{ "storeV2 $mem, $src\t# vector (16 bits)" %}
@@ -363,7 +363,7 @@ instruct storeV2(vReg src, vmem mem) %{
%}
// Load Vector (32 bits)
-instruct loadV4(vReg dst, vmem mem) %{
+instruct loadV4(vReg dst, vmem4 mem) %{
predicate(n->as_LoadVector()->memory_size() == 4);
match(Set dst (LoadVector mem));
format %{ "loadV4 $dst, $mem\t# vector (32 bits)" %}
@@ -372,7 +372,7 @@ instruct loadV4(vReg dst, vmem mem) %{
%}
// Store Vector (32 bits)
-instruct storeV4(vReg src, vmem mem) %{
+instruct storeV4(vReg src, vmem4 mem) %{
predicate(n->as_StoreVector()->memory_size() == 4);
match(Set mem (StoreVector mem src));
format %{ "storeV4 $mem, $src\t# vector (32 bits)" %}
@@ -381,7 +381,7 @@ instruct storeV4(vReg src, vmem mem) %{
%}
// Load Vector (64 bits)
-instruct loadV8(vReg dst, vmem mem) %{
+instruct loadV8(vReg dst, vmem8 mem) %{
predicate(n->as_LoadVector()->memory_size() == 8);
match(Set dst (LoadVector mem));
format %{ "loadV8 $dst, $mem\t# vector (64 bits)" %}
@@ -390,7 +390,7 @@ instruct loadV8(vReg dst, vmem mem) %{
%}
// Store Vector (64 bits)
-instruct storeV8(vReg src, vmem mem) %{
+instruct storeV8(vReg src, vmem8 mem) %{
predicate(n->as_StoreVector()->memory_size() == 8);
match(Set mem (StoreVector mem src));
format %{ "storeV8 $mem, $src\t# vector (64 bits)" %}
@@ -399,7 +399,7 @@ instruct storeV8(vReg src, vmem mem) %{
%}
// Load Vector (128 bits)
-instruct loadV16(vReg dst, vmem mem) %{
+instruct loadV16(vReg dst, vmem16 mem) %{
predicate(n->as_LoadVector()->memory_size() == 16);
match(Set dst (LoadVector mem));
format %{ "loadV16 $dst, $mem\t# vector (128 bits)" %}
@@ -408,7 +408,7 @@ instruct loadV16(vReg dst, vmem mem) %{
%}
// Store Vector (128 bits)
-instruct storeV16(vReg src, vmem mem) %{
+instruct storeV16(vReg src, vmem16 mem) %{
predicate(n->as_StoreVector()->memory_size() == 16);
match(Set mem (StoreVector mem src));
format %{ "storeV16 $mem, $src\t# vector (128 bits)" %}
diff --git a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
index 020a75b51fa8f..99708e9ef317d 100644
--- a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
+++ b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
@@ -338,7 +338,7 @@ dnl VECTOR_LOAD_STORE($1, $2, $3, $4, $5 )
dnl VECTOR_LOAD_STORE(type, nbytes, arg_name, nbits, size)
define(`VECTOR_LOAD_STORE', `
// ifelse(load, $1, Load, Store) Vector ($4 bits)
-instruct $1V$2(vReg $3, vmem mem) %{
+instruct $1V$2(vReg $3, vmem$2 mem) %{
predicate(`n->as_'ifelse(load, $1, Load, Store)Vector()->memory_size() == $2);
match(Set ifelse(load, $1, dst (LoadVector mem), mem (StoreVector mem src)));
format %{ "$1V$2 ifelse(load, $1, `$dst, $mem', `$mem, $src')\t# vector ($4 bits)" %}
diff --git a/src/hotspot/cpu/aarch64/ad_encode.m4 b/src/hotspot/cpu/aarch64/ad_encode.m4
index e3d8ea661b60a..008dbd2c9369c 100644
--- a/src/hotspot/cpu/aarch64/ad_encode.m4
+++ b/src/hotspot/cpu/aarch64/ad_encode.m4
@@ -34,7 +34,7 @@ define(access, `
define(load,`
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_$2($1 dst, memory mem) %{dnl
+ enc_class aarch64_enc_$2($1 dst, memory$5 mem) %{dnl
access(dst,$2,$3,$4,$5)')dnl
load(iRegI,ldrsbw,,,1)
load(iRegI,ldrsb,,,1)
@@ -53,12 +53,12 @@ load(vRegD,ldrd,Float,,8)
define(STORE,`
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_$2($1 src, memory mem) %{dnl
+ enc_class aarch64_enc_$2($1 src, memory$5 mem) %{dnl
access(src,$2,$3,$4,$5)')dnl
define(STORE0,`
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_$2`'0(memory mem) %{
+ enc_class aarch64_enc_$2`'0(memory$4 mem) %{
choose(masm,zr,$2,$mem->opcode(),
as_$3Register($mem$$base),$mem$$index,$mem$$scale,$mem$$disp,$4)')dnl
STORE(iRegI,strb,,,1)
@@ -82,7 +82,7 @@ STORE(vRegD,strd,Float,,8)
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
- enc_class aarch64_enc_strb0_ordered(memory mem) %{
+ enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
__ membar(Assembler::StoreStore);
loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
index 5e116d82761ac..3d1be91e9b232 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
@@ -275,14 +275,14 @@ void LIR_Assembler::osr_entry() {
// verify the interpreter's monitor has a non-null object
{
Label L;
- __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
+ __ ldr(rscratch1, __ form_address(rscratch1, OSR_buf, slot_offset + 1*BytesPerWord, 0));
__ cbnz(rscratch1, L);
__ stop("locked object is null");
__ bind(L);
}
#endif
- __ ldr(r19, Address(OSR_buf, slot_offset));
- __ ldr(r20, Address(OSR_buf, slot_offset + BytesPerWord));
+ __ ldr(r19, __ form_address(rscratch1, OSR_buf, slot_offset, 0));
+ __ ldr(r20, __ form_address(rscratch1, OSR_buf, slot_offset + BytesPerWord, 0));
__ str(r19, frame_map()->address_for_monitor_lock(i));
__ str(r20, frame_map()->address_for_monitor_object(i));
}
@@ -990,10 +990,7 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch
__ decode_heap_oop(dest->as_register());
}
- if (!(UseZGC && !ZGenerational)) {
- // Load barrier has not yet been applied, so ZGC can't verify the oop here
- __ verify_oop(dest->as_register());
- }
+ __ verify_oop(dest->as_register());
}
}
@@ -1168,8 +1165,8 @@ void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
if (op->init_check()) {
- __ ldrb(rscratch1, Address(op->klass()->as_register(),
- InstanceKlass::init_state_offset()));
+ __ lea(rscratch1, Address(op->klass()->as_register(), InstanceKlass::init_state_offset()));
+ __ ldarb(rscratch1, rscratch1);
__ cmpw(rscratch1, InstanceKlass::fully_initialized);
add_debug_info_for_null_check_here(op->stub()->info());
__ br(Assembler::NE, *op->stub()->entry());
diff --git a/src/hotspot/cpu/aarch64/c2_CodeStubs_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_CodeStubs_aarch64.cpp
index dabafb9288b83..4bd509880f29c 100644
--- a/src/hotspot/cpu/aarch64/c2_CodeStubs_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c2_CodeStubs_aarch64.cpp
@@ -64,31 +64,4 @@ void C2EntryBarrierStub::emit(C2_MacroAssembler& masm) {
__ emit_int32(0); // nmethod guard value
}
-int C2HandleAnonOMOwnerStub::max_size() const {
- // Max size of stub has been determined by testing with 0, in which case
- // C2CodeStubList::emit() will throw an assertion and report the actual size that
- // is needed.
- return 24;
-}
-
-void C2HandleAnonOMOwnerStub::emit(C2_MacroAssembler& masm) {
- __ bind(entry());
- Register mon = monitor();
- Register t = tmp();
- assert(t != noreg, "need tmp register");
-
- // Fix owner to be the current thread.
- __ str(rthread, Address(mon, ObjectMonitor::owner_offset()));
-
- // Pop owner object from lock-stack.
- __ ldrw(t, Address(rthread, JavaThread::lock_stack_top_offset()));
- __ subw(t, t, oopSize);
-#ifdef ASSERT
- __ str(zr, Address(rthread, t));
-#endif
- __ strw(t, Address(rthread, JavaThread::lock_stack_top_offset()));
-
- __ b(continuation());
-}
-
#undef __
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
index 62831ee72ba05..b29be7213baf4 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
@@ -217,7 +217,7 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
// StoreLoad achieves this.
membar(StoreLoad);
- // Check if the entry lists are empty.
+ // Check if the entry lists are empty (EntryList first - by convention).
ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset()));
ldr(tmpReg, Address(tmp, ObjectMonitor::cxq_offset()));
orr(rscratch1, rscratch1, tmpReg);
@@ -538,7 +538,7 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box, Regi
// StoreLoad achieves this.
membar(StoreLoad);
- // Check if the entry lists are empty.
+ // Check if the entry lists are empty (EntryList first - by convention).
ldr(rscratch1, Address(t1_monitor, ObjectMonitor::EntryList_offset()));
ldr(t3_t, Address(t1_monitor, ObjectMonitor::cxq_offset()));
orr(rscratch1, rscratch1, t3_t);
diff --git a/src/hotspot/cpu/aarch64/cas.m4 b/src/hotspot/cpu/aarch64/cas.m4
index f8aac0c4939fa..7e13e153db18a 100644
--- a/src/hotspot/cpu/aarch64/cas.m4
+++ b/src/hotspot/cpu/aarch64/cas.m4
@@ -45,7 +45,9 @@ define(`CAS_INSN',
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct compareAndExchange$1$6(iReg$2NoSp res, indirect mem, iReg$2 oldval, iReg$2 newval, rFlagsReg cr) %{
ifelse($1$6,PAcq,INDENT(predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));),
+ $1$6,NAcq,INDENT(predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);),
$1,P,INDENT(predicate(n->as_LoadStore()->barrier_data() == 0);),
+ $1,N,INDENT(predicate(n->as_LoadStore()->barrier_data() == 0);),
$6,Acq,INDENT(predicate(needs_acquiring_load_exclusive(n));),
`dnl')
match(Set res (CompareAndExchange$1 mem (Binary oldval newval)));
@@ -122,7 +124,9 @@ define(`CAS_INSN3',
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct weakCompareAndSwap$1$6(iRegINoSp res, indirect mem, iReg$2 oldval, iReg$2 newval, rFlagsReg cr) %{
ifelse($1$6,PAcq,INDENT(predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));),
+ $1$6,NAcq,INDENT(predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);),
$1,P,INDENT(predicate(n->as_LoadStore()->barrier_data() == 0);),
+ $1,N,INDENT(predicate(n->as_LoadStore()->barrier_data() == 0);),
$6,Acq,INDENT(predicate(needs_acquiring_load_exclusive(n));),
`dnl')
match(Set res (WeakCompareAndSwap$1 mem (Binary oldval newval)));
diff --git a/src/hotspot/cpu/aarch64/copy_aarch64.hpp b/src/hotspot/cpu/aarch64/copy_aarch64.hpp
index 0ff9ace59cc2f..5ccf2cd835ee7 100644
--- a/src/hotspot/cpu/aarch64/copy_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/copy_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -64,28 +64,34 @@ static void pd_zero_to_bytes(void* to, size_t count) {
" br %[t0];\n" \
" .align 5;\n" \
"0:" \
+" hint #0x24; // bti j\n" \
" b 1f;\n" \
" .align 5;\n" \
+" hint #0x24; // bti j\n" \
" ldr %[t0], [%[s], #0];\n" \
" str %[t0], [%[d], #0];\n" \
" b 1f;\n" \
" .align 5;\n" \
+" hint #0x24; // bti j\n" \
" ldp %[t0], %[t1], [%[s], #0];\n" \
" stp %[t0], %[t1], [%[d], #0];\n" \
" b 1f;\n" \
" .align 5;\n" \
+" hint #0x24; // bti j\n" \
" ldp %[t0], %[t1], [%[s], #0];\n" \
" ldr %[t2], [%[s], #16];\n" \
" stp %[t0], %[t1], [%[d], #0];\n" \
" str %[t2], [%[d], #16];\n" \
" b 1f;\n" \
" .align 5;\n" \
+" hint #0x24; // bti j\n" \
" ldp %[t0], %[t1], [%[s], #0];\n" \
" ldp %[t2], %[t3], [%[s], #16];\n" \
" stp %[t0], %[t1], [%[d], #0];\n" \
" stp %[t2], %[t3], [%[d], #16];\n" \
" b 1f;\n" \
" .align 5;\n" \
+" hint #0x24; // bti j\n" \
" ldp %[t0], %[t1], [%[s], #0];\n" \
" ldp %[t2], %[t3], [%[s], #16];\n" \
" ldr %[t4], [%[s], #32];\n" \
@@ -94,6 +100,7 @@ static void pd_zero_to_bytes(void* to, size_t count) {
" str %[t4], [%[d], #32];\n" \
" b 1f;\n" \
" .align 5;\n" \
+" hint #0x24; // bti j\n" \
" ldp %[t0], %[t1], [%[s], #0];\n" \
" ldp %[t2], %[t3], [%[s], #16];\n" \
" ldp %[t4], %[t5], [%[s], #32];\n" \
@@ -103,6 +110,7 @@ static void pd_zero_to_bytes(void* to, size_t count) {
" stp %[t4], %[t5], [%[d], #32];\n" \
" b 1f;\n" \
" .align 5;\n" \
+" hint #0x24; // bti j\n" \
" ldr %[t6], [%[s], #0];\n" \
" ldp %[t0], %[t1], [%[s], #8];\n" \
" ldp %[t2], %[t3], [%[s], #24];\n" \
@@ -110,6 +118,7 @@ static void pd_zero_to_bytes(void* to, size_t count) {
" str %[t6], [%[d]], #8;\n" \
" b 2b;\n" \
" .align 5;\n" \
+" hint #0x24; // bti j\n" \
" ldp %[t0], %[t1], [%[s], #0];\n" \
" ldp %[t2], %[t3], [%[s], #16];\n" \
" ldp %[t4], %[t5], [%[s], #32];\n" \
diff --git a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
index d02038b6e9193..b978c350ce131 100644
--- a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
@@ -38,7 +38,10 @@
#include "c1/c1_LIRAssembler.hpp"
#include "c1/c1_MacroAssembler.hpp"
#include "gc/g1/c1/g1BarrierSetC1.hpp"
-#endif
+#endif // COMPILER1
+#ifdef COMPILER2
+#include "gc/g1/c2/g1BarrierSetC2.hpp"
+#endif // COMPILER2
#define __ masm->
@@ -95,6 +98,54 @@ void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* mas
__ pop(saved_regs, sp);
}
+static void generate_queue_test_and_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
+ const Register thread, const Register value, const Register temp1, const Register temp2) {
+ // Can we store a value in the given thread's buffer?
+ // (The index field is typed as size_t.)
+ __ ldr(temp1, Address(thread, in_bytes(index_offset))); // temp1 := *(index address)
+ __ cbz(temp1, runtime); // jump to runtime if index == 0 (full buffer)
+ // The buffer is not full, store value into it.
+ __ sub(temp1, temp1, wordSize); // temp1 := next index
+ __ str(temp1, Address(thread, in_bytes(index_offset))); // *(index address) := next index
+ __ ldr(temp2, Address(thread, in_bytes(buffer_offset))); // temp2 := buffer address
+ __ str(value, Address(temp2, temp1)); // *(buffer address + next index) := value
+}
+
+static void generate_pre_barrier_fast_path(MacroAssembler* masm,
+ const Register thread,
+ const Register tmp1) {
+ Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
+ // Is marking active?
+ if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
+ __ ldrw(tmp1, in_progress);
+ } else {
+ assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
+ __ ldrb(tmp1, in_progress);
+ }
+}
+
+static void generate_pre_barrier_slow_path(MacroAssembler* masm,
+ const Register obj,
+ const Register pre_val,
+ const Register thread,
+ const Register tmp1,
+ const Register tmp2,
+ Label& done,
+ Label& runtime) {
+ // Do we need to load the previous value?
+ if (obj != noreg) {
+ __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
+ }
+ // Is the previous value null?
+ __ cbz(pre_val, done);
+ generate_queue_test_and_insertion(masm,
+ G1ThreadLocalData::satb_mark_queue_index_offset(),
+ G1ThreadLocalData::satb_mark_queue_buffer_offset(),
+ runtime,
+ thread, pre_val, tmp1, tmp2);
+ __ b(done);
+}
+
void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
Register obj,
Register pre_val,
@@ -115,43 +166,10 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
assert_different_registers(obj, pre_val, tmp1, tmp2);
assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
- Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
- Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
- Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
-
- // Is marking active?
- if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
- __ ldrw(tmp1, in_progress);
- } else {
- assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
- __ ldrb(tmp1, in_progress);
- }
+ generate_pre_barrier_fast_path(masm, thread, tmp1);
+ // If marking is not active (*(mark queue active address) == 0), jump to done
__ cbzw(tmp1, done);
-
- // Do we need to load the previous value?
- if (obj != noreg) {
- __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
- }
-
- // Is the previous value null?
- __ cbz(pre_val, done);
-
- // Can we store original value in the thread's buffer?
- // Is index == 0?
- // (The index field is typed as size_t.)
-
- __ ldr(tmp1, index); // tmp := *index_adr
- __ cbz(tmp1, runtime); // tmp == 0?
- // If yes, goto runtime
-
- __ sub(tmp1, tmp1, wordSize); // tmp := tmp - wordSize
- __ str(tmp1, index); // *index_adr := tmp
- __ ldr(tmp2, buffer);
- __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr
-
- // Record the previous value
- __ str(pre_val, Address(tmp1, 0));
- __ b(done);
+ generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp1, tmp2, done, runtime);
__ bind(runtime);
@@ -182,6 +200,50 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
}
+static void generate_post_barrier_fast_path(MacroAssembler* masm,
+ const Register store_addr,
+ const Register new_val,
+ const Register tmp1,
+ const Register tmp2,
+ Label& done,
+ bool new_val_may_be_null) {
+ // Does store cross heap regions?
+ __ eor(tmp1, store_addr, new_val); // tmp1 := store address ^ new value
+ __ lsr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
+ __ cbz(tmp1, done);
+ // Crosses regions, storing null?
+ if (new_val_may_be_null) {
+ __ cbz(new_val, done);
+ }
+ // Storing region crossing non-null, is card young?
+ __ lsr(tmp1, store_addr, CardTable::card_shift()); // tmp1 := card address relative to card table base
+ __ load_byte_map_base(tmp2); // tmp2 := card table base address
+ __ add(tmp1, tmp1, tmp2); // tmp1 := card address
+ __ ldrb(tmp2, Address(tmp1)); // tmp2 := card
+ __ cmpw(tmp2, (int)G1CardTable::g1_young_card_val()); // tmp2 := card == young_card_val?
+}
+
+static void generate_post_barrier_slow_path(MacroAssembler* masm,
+ const Register thread,
+ const Register tmp1,
+ const Register tmp2,
+ Label& done,
+ Label& runtime) {
+ __ membar(Assembler::StoreLoad); // StoreLoad membar
+ __ ldrb(tmp2, Address(tmp1)); // tmp2 := card
+ __ cbzw(tmp2, done);
+ // Storing a region crossing, non-null oop, card is clean.
+ // Dirty card and log.
+ STATIC_ASSERT(CardTable::dirty_card_val() == 0);
+ __ strb(zr, Address(tmp1)); // *(card address) := dirty_card_val
+ generate_queue_test_and_insertion(masm,
+ G1ThreadLocalData::dirty_card_queue_index_offset(),
+ G1ThreadLocalData::dirty_card_queue_buffer_offset(),
+ runtime,
+ thread, tmp1, tmp2, rscratch1);
+ __ b(done);
+}
+
void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
Register store_addr,
Register new_val,
@@ -194,70 +256,116 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
assert(store_addr != noreg && new_val != noreg && tmp1 != noreg
&& tmp2 != noreg, "expecting a register");
- Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
- Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
-
- BarrierSet* bs = BarrierSet::barrier_set();
- CardTableBarrierSet* ctbs = barrier_set_cast(bs);
- CardTable* ct = ctbs->card_table();
-
Label done;
Label runtime;
- // Does store cross heap regions?
+ generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, done, true /* new_val_may_be_null */);
+ // If card is young, jump to done
+ __ br(Assembler::EQ, done);
+ generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, done, runtime);
- __ eor(tmp1, store_addr, new_val);
- __ lsr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes);
- __ cbz(tmp1, done);
+ __ bind(runtime);
+ // save the live input values
+ RegSet saved = RegSet::of(store_addr);
+ __ push(saved, sp);
+ __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp1, thread);
+ __ pop(saved, sp);
- // crosses regions, storing null?
+ __ bind(done);
+}
- __ cbz(new_val, done);
+#if defined(COMPILER2)
- // storing region crossing non-null, is card already dirty?
+static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
+ SaveLiveRegisters save_registers(masm, stub);
+ if (c_rarg0 != arg) {
+ __ mov(c_rarg0, arg);
+ }
+ __ mov(c_rarg1, rthread);
+ __ mov(rscratch1, runtime_path);
+ __ blr(rscratch1);
+}
- const Register card_addr = tmp1;
+void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2,
+ G1PreBarrierStubC2* stub) {
+ assert(thread == rthread, "must be");
+ assert_different_registers(obj, pre_val, tmp1, tmp2);
+ assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
- __ lsr(card_addr, store_addr, CardTable::card_shift());
+ stub->initialize_registers(obj, pre_val, thread, tmp1, tmp2);
- // get the address of the card
- __ load_byte_map_base(tmp2);
- __ add(card_addr, card_addr, tmp2);
- __ ldrb(tmp2, Address(card_addr));
- __ cmpw(tmp2, (int)G1CardTable::g1_young_card_val());
- __ br(Assembler::EQ, done);
+ generate_pre_barrier_fast_path(masm, thread, tmp1);
+ // If marking is active (*(mark queue active address) != 0), jump to stub (slow path)
+ __ cbnzw(tmp1, *stub->entry());
- assert((int)CardTable::dirty_card_val() == 0, "must be 0");
+ __ bind(*stub->continuation());
+}
- __ membar(Assembler::StoreLoad);
+void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm,
+ G1PreBarrierStubC2* stub) const {
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ Label runtime;
+ Register obj = stub->obj();
+ Register pre_val = stub->pre_val();
+ Register thread = stub->thread();
+ Register tmp1 = stub->tmp1();
+ Register tmp2 = stub->tmp2();
- __ ldrb(tmp2, Address(card_addr));
- __ cbzw(tmp2, done);
+ __ bind(*stub->entry());
+ generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp1, tmp2, *stub->continuation(), runtime);
- // storing a region crossing, non-null oop, card is clean.
- // dirty card and log.
+ __ bind(runtime);
+ generate_c2_barrier_runtime_call(masm, stub, pre_val, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry));
+ __ b(*stub->continuation());
+}
- __ strb(zr, Address(card_addr));
+void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2,
+ G1PostBarrierStubC2* stub) {
+ assert(thread == rthread, "must be");
+ assert_different_registers(store_addr, new_val, thread, tmp1, tmp2,
+ rscratch1);
+ assert(store_addr != noreg && new_val != noreg && tmp1 != noreg
+ && tmp2 != noreg, "expecting a register");
- __ ldr(rscratch1, queue_index);
- __ cbz(rscratch1, runtime);
- __ sub(rscratch1, rscratch1, wordSize);
- __ str(rscratch1, queue_index);
+ stub->initialize_registers(thread, tmp1, tmp2);
- __ ldr(tmp2, buffer);
- __ str(card_addr, Address(tmp2, rscratch1));
- __ b(done);
+ bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
+ generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, *stub->continuation(), new_val_may_be_null);
+ // If card is not young, jump to stub (slow path)
+ __ br(Assembler::NE, *stub->entry());
- __ bind(runtime);
- // save the live input values
- RegSet saved = RegSet::of(store_addr);
- __ push(saved, sp);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
- __ pop(saved, sp);
+ __ bind(*stub->continuation());
+}
- __ bind(done);
+void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
+ G1PostBarrierStubC2* stub) const {
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ Label runtime;
+ Register thread = stub->thread();
+ Register tmp1 = stub->tmp1(); // tmp1 holds the card address.
+ Register tmp2 = stub->tmp2();
+ assert(stub->tmp3() == noreg, "not needed in this platform");
+
+ __ bind(*stub->entry());
+ generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, *stub->continuation(), runtime);
+
+ __ bind(runtime);
+ generate_c2_barrier_runtime_call(masm, stub, tmp1, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
+ __ b(*stub->continuation());
}
+#endif // COMPILER2
+
void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp2) {
bool on_oop = is_reference_type(type);
diff --git a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.hpp
index 7b4bc8cdc49de..4baa18cb94544 100644
--- a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.hpp
@@ -33,6 +33,8 @@ class LIR_Assembler;
class StubAssembler;
class G1PreBarrierStub;
class G1PostBarrierStub;
+class G1PreBarrierStubC2;
+class G1PostBarrierStubC2;
class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
protected:
@@ -69,6 +71,27 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm);
#endif
+#ifdef COMPILER2
+ void g1_write_barrier_pre_c2(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2,
+ G1PreBarrierStubC2* c2_stub);
+ void generate_c2_pre_barrier_stub(MacroAssembler* masm,
+ G1PreBarrierStubC2* stub) const;
+ void g1_write_barrier_post_c2(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2,
+ G1PostBarrierStubC2* c2_stub);
+ void generate_c2_post_barrier_stub(MacroAssembler* masm,
+ G1PostBarrierStubC2* stub) const;
+#endif
+
void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp2);
};
diff --git a/src/hotspot/cpu/aarch64/gc/g1/g1_aarch64.ad b/src/hotspot/cpu/aarch64/gc/g1/g1_aarch64.ad
new file mode 100644
index 0000000000000..081a67d68807b
--- /dev/null
+++ b/src/hotspot/cpu/aarch64/gc/g1/g1_aarch64.ad
@@ -0,0 +1,680 @@
+//
+// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+
+source_hpp %{
+
+#include "gc/g1/c2/g1BarrierSetC2.hpp"
+#include "gc/shared/gc_globals.hpp"
+
+%}
+
+source %{
+
+#include "gc/g1/g1BarrierSetAssembler_aarch64.hpp"
+#include "gc/g1/g1BarrierSetRuntime.hpp"
+
+static void write_barrier_pre(MacroAssembler* masm,
+ const MachNode* node,
+ Register obj,
+ Register pre_val,
+ Register tmp1,
+ Register tmp2,
+ RegSet preserve = RegSet(),
+ RegSet no_preserve = RegSet()) {
+ if (!G1PreBarrierStubC2::needs_barrier(node)) {
+ return;
+ }
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
+ G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
+ for (RegSetIterator reg = preserve.begin(); *reg != noreg; ++reg) {
+ stub->preserve(*reg);
+ }
+ for (RegSetIterator reg = no_preserve.begin(); *reg != noreg; ++reg) {
+ stub->dont_preserve(*reg);
+ }
+ g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, rthread, tmp1, tmp2, stub);
+}
+
+static void write_barrier_post(MacroAssembler* masm,
+ const MachNode* node,
+ Register store_addr,
+ Register new_val,
+ Register tmp1,
+ Register tmp2) {
+ if (!G1PostBarrierStubC2::needs_barrier(node)) {
+ return;
+ }
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
+ G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
+ g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, rthread, tmp1, tmp2, stub);
+}
+
+%}
+
+// BEGIN This section of the file is automatically generated. Do not edit --------------
+
+// This section is generated from g1_aarch64.m4
+
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1StoreP(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreP mem src));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(INSN_COST);
+ format %{ "str $src, $mem\t# ptr" %}
+ ins_encode %{
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $src$$Register) /* preserve */);
+ __ str($src$$Register, $mem$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $src$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(istore_reg_mem);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1StorePVolatile(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreP mem src));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(VOLATILE_REF_COST);
+ format %{ "stlr $src, $mem\t# ptr" %}
+ ins_encode %{
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $src$$Register) /* preserve */);
+ __ stlr($src$$Register, $mem$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $src$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_class_memory);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1StoreN(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreN mem src));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(INSN_COST);
+ format %{ "strw $src, $mem\t# compressed ptr" %}
+ ins_encode %{
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $src$$Register) /* preserve */);
+ __ strw($src$$Register, $mem$$Register);
+ if ((barrier_data() & G1C2BarrierPost) != 0) {
+ if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
+ __ decode_heap_oop($tmp1$$Register, $src$$Register);
+ } else {
+ __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register);
+ }
+ }
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(istore_reg_mem);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1StoreNVolatile(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreN mem src));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(VOLATILE_REF_COST);
+ format %{ "stlrw $src, $mem\t# compressed ptr" %}
+ ins_encode %{
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $src$$Register) /* preserve */);
+ __ stlrw($src$$Register, $mem$$Register);
+ if ((barrier_data() & G1C2BarrierPost) != 0) {
+ if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
+ __ decode_heap_oop($tmp1$$Register, $src$$Register);
+ } else {
+ __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register);
+ }
+ }
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_class_memory);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1EncodePAndStoreN(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreN mem (EncodeP src)));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(INSN_COST);
+ format %{ "encode_heap_oop $tmp1, $src\n\t"
+ "strw $tmp1, $mem\t# compressed ptr" %}
+ ins_encode %{
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $src$$Register) /* preserve */);
+ if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
+ __ encode_heap_oop($tmp1$$Register, $src$$Register);
+ } else {
+ __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register);
+ }
+ __ strw($tmp1$$Register, $mem$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $src$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(istore_reg_mem);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1EncodePAndStoreNVolatile(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreN mem (EncodeP src)));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(VOLATILE_REF_COST);
+ format %{ "encode_heap_oop $tmp1, $src\n\t"
+ "stlrw $tmp1, $mem\t# compressed ptr" %}
+ ins_encode %{
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $src$$Register) /* preserve */);
+ if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
+ __ encode_heap_oop($tmp1$$Register, $src$$Register);
+ } else {
+ __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register);
+ }
+ __ stlrw($tmp1$$Register, $mem$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $src$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_class_memory);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1CompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
+%{
+ predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(2 * VOLATILE_REF_COST);
+ format %{ "cmpxchg $res = $mem, $oldval, $newval\t# ptr" %}
+ ins_encode %{
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ // Pass $oldval to the pre-barrier (instead of loading from $mem), because
+ // $oldval is the only value that can be overwritten.
+ // The same holds for g1CompareAndSwapP and its Acq variant.
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $oldval$$Register /* pre_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+ false /* acquire */, true /* release */, false /* weak */, $res$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $newval$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1CompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
+%{
+ predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(VOLATILE_REF_COST);
+ format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# ptr" %}
+ ins_encode %{
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ // Pass $oldval to the pre-barrier (instead of loading from $mem), because
+ // $oldval is the only value that can be overwritten.
+ // The same holds for g1CompareAndSwapP and its Acq variant.
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $oldval$$Register /* pre_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+ true /* acquire */, true /* release */, false /* weak */, $res$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $newval$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1CompareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(2 * VOLATILE_REF_COST);
+ format %{ "cmpxchg $res = $mem, $oldval, $newval\t# narrow oop" %}
+ ins_encode %{
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
+ false /* acquire */, true /* release */, false /* weak */, $res$$Register);
+ __ decode_heap_oop($tmp1$$Register, $newval$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1CompareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(VOLATILE_REF_COST);
+ format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# narrow oop" %}
+ ins_encode %{
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
+ true /* acquire */, true /* release */, false /* weak */, $res$$Register);
+ __ decode_heap_oop($tmp1$$Register, $newval$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1CompareAndSwapP(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(2 * VOLATILE_REF_COST);
+ format %{ "cmpxchg $mem, $oldval, $newval\t# (ptr)\n\t"
+ "cset $res, EQ" %}
+ ins_encode %{
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $oldval$$Register /* pre_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+ false /* acquire */, true /* release */, false /* weak */, noreg);
+ __ cset($res$$Register, Assembler::EQ);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $newval$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1CompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(VOLATILE_REF_COST);
+ format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (ptr)\n\t"
+ "cset $res, EQ" %}
+ ins_encode %{
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $oldval$$Register /* pre_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+ true /* acquire */, true /* release */, false /* weak */, noreg);
+ __ cset($res$$Register, Assembler::EQ);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $newval$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1CompareAndSwapN(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndSwapN mem (Binary oldval newval)));
+ match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(2 * VOLATILE_REF_COST);
+ format %{ "cmpxchg $mem, $oldval, $newval\t# (narrow oop)\n\t"
+ "cset $res, EQ" %}
+ ins_encode %{
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
+ false /* acquire */, true /* release */, false /* weak */, noreg);
+ __ cset($res$$Register, Assembler::EQ);
+ __ decode_heap_oop($tmp1$$Register, $newval$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1CompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndSwapN mem (Binary oldval newval)));
+ match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(VOLATILE_REF_COST);
+ format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop)\n\t"
+ "cset $res, EQ" %}
+ ins_encode %{
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
+ true /* acquire */, true /* release */, false /* weak */, noreg);
+ __ cset($res$$Register, Assembler::EQ);
+ __ decode_heap_oop($tmp1$$Register, $newval$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1GetAndSetP(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set preval (GetAndSetP mem newval));
+ effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(2 * VOLATILE_REF_COST);
+ format %{ "atomic_xchg $preval, $newval, [$mem]" %}
+ ins_encode %{
+ assert_different_registers($mem$$Register, $newval$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $preval$$Register /* pre_val (as a temporary register) */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
+ __ atomic_xchg($preval$$Register, $newval$$Register, $mem$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $newval$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_serial);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1GetAndSetPAcq(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set preval (GetAndSetP mem newval));
+ effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(VOLATILE_REF_COST);
+ format %{ "atomic_xchg_acq $preval, $newval, [$mem]" %}
+ ins_encode %{
+ assert_different_registers($mem$$Register, $newval$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $preval$$Register /* pre_val (as a temporary register) */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
+ __ atomic_xchgal($preval$$Register, $newval$$Register, $mem$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $newval$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_serial);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1GetAndSetN(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set preval (GetAndSetN mem newval));
+ effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(2 * VOLATILE_REF_COST);
+ format %{ "atomic_xchgw $preval, $newval, [$mem]" %}
+ ins_encode %{
+ assert_different_registers($mem$$Register, $newval$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
+ __ atomic_xchgw($preval$$Register, $newval$$Register, $mem$$Register);
+ __ decode_heap_oop($tmp1$$Register, $newval$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_serial);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1GetAndSetNAcq(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set preval (GetAndSetN mem newval));
+ effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(VOLATILE_REF_COST);
+ format %{ "atomic_xchgw_acq $preval, $newval, [$mem]" %}
+ ins_encode %{
+ assert_different_registers($mem$$Register, $newval$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
+ __ atomic_xchgalw($preval$$Register, $newval$$Register, $mem$$Register);
+ __ decode_heap_oop($tmp1$$Register, $newval$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_serial);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1LoadP(iRegPNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
+%{
+ // This instruction does not need an acquiring counterpart because it is only
+ // used for reference loading (Reference::get()). The same holds for g1LoadN.
+ predicate(UseG1GC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
+ match(Set dst (LoadP mem));
+ effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(4 * INSN_COST);
+ format %{ "ldr $dst, $mem\t# ptr" %}
+ ins_encode %{
+ __ ldr($dst$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $dst$$Register /* pre_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(iload_reg_mem);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1LoadN(iRegNNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
+ match(Set dst (LoadN mem));
+ effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(4 * INSN_COST);
+ format %{ "ldrw $dst, $mem\t# compressed ptr" %}
+ ins_encode %{
+ __ ldrw($dst$$Register, $mem$$Register);
+ if ((barrier_data() & G1C2BarrierPre) != 0) {
+ __ decode_heap_oop($tmp1$$Register, $dst$$Register);
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ }
+ %}
+ ins_pipe(iload_reg_mem);
+%}
+
+// END This section of the file is automatically generated. Do not edit --------------
diff --git a/src/hotspot/cpu/aarch64/gc/g1/g1_aarch64.m4 b/src/hotspot/cpu/aarch64/gc/g1/g1_aarch64.m4
new file mode 100644
index 0000000000000..8fb1f7e8e428b
--- /dev/null
+++ b/src/hotspot/cpu/aarch64/gc/g1/g1_aarch64.m4
@@ -0,0 +1,384 @@
+dnl Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+dnl DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+dnl
+dnl This code is free software; you can redistribute it and/or modify it
+dnl under the terms of the GNU General Public License version 2 only, as
+dnl published by the Free Software Foundation.
+dnl
+dnl This code is distributed in the hope that it will be useful, but WITHOUT
+dnl ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+dnl FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+dnl version 2 for more details (a copy is included in the LICENSE file that
+dnl accompanied this code).
+dnl
+dnl You should have received a copy of the GNU General Public License version
+dnl 2 along with this work; if not, write to the Free Software Foundation,
+dnl Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+dnl
+dnl Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+dnl or visit www.oracle.com if you need additional information or have any
+dnl questions.
+dnl
+// BEGIN This section of the file is automatically generated. Do not edit --------------
+
+// This section is generated from g1_aarch64.m4
+
+define(`STOREP_INSN',
+`
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1StoreP$1(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && ifelse($1,Volatile,'needs_releasing_store(n)`,'!needs_releasing_store(n)`) && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreP mem src));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(ifelse($1,Volatile,VOLATILE_REF_COST,INSN_COST));
+ format %{ "$2 $src, $mem\t# ptr" %}
+ ins_encode %{
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $src$$Register) /* preserve */);
+ __ $2($src$$Register, $mem$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $src$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(ifelse($1,Volatile,pipe_class_memory,istore_reg_mem));
+%}')dnl
+STOREP_INSN(,str)
+STOREP_INSN(Volatile,stlr)
+dnl
+define(`STOREN_INSN',
+`
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1StoreN$1(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && ifelse($1,Volatile,'needs_releasing_store(n)`,'!needs_releasing_store(n)`) && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreN mem src));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(ifelse($1,Volatile,VOLATILE_REF_COST,INSN_COST));
+ format %{ "$2 $src, $mem\t# compressed ptr" %}
+ ins_encode %{
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $src$$Register) /* preserve */);
+ __ $2($src$$Register, $mem$$Register);
+ if ((barrier_data() & G1C2BarrierPost) != 0) {
+ if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
+ __ decode_heap_oop($tmp1$$Register, $src$$Register);
+ } else {
+ __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register);
+ }
+ }
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(ifelse($1,Volatile,pipe_class_memory,istore_reg_mem));
+%}')dnl
+STOREN_INSN(,strw)
+STOREN_INSN(Volatile,stlrw)
+dnl
+define(`ENCODESTOREN_INSN',
+`
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1EncodePAndStoreN$1(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && ifelse($1,Volatile,'needs_releasing_store(n)`,'!needs_releasing_store(n)`) && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreN mem (EncodeP src)));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(ifelse($1,Volatile,VOLATILE_REF_COST,INSN_COST));
+ format %{ "encode_heap_oop $tmp1, $src\n\t"
+ "$2 $tmp1, $mem\t# compressed ptr" %}
+ ins_encode %{
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $src$$Register) /* preserve */);
+ if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
+ __ encode_heap_oop($tmp1$$Register, $src$$Register);
+ } else {
+ __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register);
+ }
+ __ $2($tmp1$$Register, $mem$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $src$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(ifelse($1,Volatile,pipe_class_memory,istore_reg_mem));
+%}')dnl
+ENCODESTOREN_INSN(,strw)
+ENCODESTOREN_INSN(Volatile,stlrw)
+dnl
+define(`CAEP_INSN',
+`
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1CompareAndExchangeP$1(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
+%{
+ predicate(UseG1GC && ifelse($1,Acq,'needs_acquiring_load_exclusive(n)`,'!needs_acquiring_load_exclusive(n)`) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(ifelse($1,Acq,VOLATILE_REF_COST,2 * VOLATILE_REF_COST));
+ format %{ "cmpxchg$2 $res = $mem, $oldval, $newval\t# ptr" %}
+ ins_encode %{
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ // Pass $oldval to the pre-barrier (instead of loading from $mem), because
+ // $oldval is the only value that can be overwritten.
+ // The same holds for g1CompareAndSwapP and its Acq variant.
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $oldval$$Register /* pre_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+ $3 /* acquire */, true /* release */, false /* weak */, $res$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $newval$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}')dnl
+CAEP_INSN(,,false)
+CAEP_INSN(Acq,_acq,true)
+dnl
+define(`CAEN_INSN',
+`
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1CompareAndExchangeN$1(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && ifelse($1,Acq,'needs_acquiring_load_exclusive(n)`,'!needs_acquiring_load_exclusive(n)`) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(ifelse($1,Acq,VOLATILE_REF_COST,2 * VOLATILE_REF_COST));
+ format %{ "cmpxchg$2 $res = $mem, $oldval, $newval\t# narrow oop" %}
+ ins_encode %{
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
+ $3 /* acquire */, true /* release */, false /* weak */, $res$$Register);
+ __ decode_heap_oop($tmp1$$Register, $newval$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}')dnl
+CAEN_INSN(,,false)
+CAEN_INSN(Acq,_acq,true)
+dnl
+define(`CASP_INSN',
+`
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1CompareAndSwapP$1(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && ifelse($1,Acq,'needs_acquiring_load_exclusive(n)`,'!needs_acquiring_load_exclusive(n)`) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(ifelse($1,Acq,VOLATILE_REF_COST,2 * VOLATILE_REF_COST));
+ format %{ "cmpxchg$2 $mem, $oldval, $newval\t# (ptr)\n\t"
+ "cset $res, EQ" %}
+ ins_encode %{
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $oldval$$Register /* pre_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+ $3 /* acquire */, true /* release */, false /* weak */, noreg);
+ __ cset($res$$Register, Assembler::EQ);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $newval$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}')dnl
+CASP_INSN(,,false)
+CASP_INSN(Acq,_acq,true)
+dnl
+define(`CASN_INSN',
+`
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1CompareAndSwapN$1(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && ifelse($1,Acq,'needs_acquiring_load_exclusive(n)`,'!needs_acquiring_load_exclusive(n)`) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndSwapN mem (Binary oldval newval)));
+ match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(ifelse($1,Acq,VOLATILE_REF_COST,2 * VOLATILE_REF_COST));
+ format %{ "cmpxchg$2 $mem, $oldval, $newval\t# (narrow oop)\n\t"
+ "cset $res, EQ" %}
+ ins_encode %{
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
+ $3 /* acquire */, true /* release */, false /* weak */, noreg);
+ __ cset($res$$Register, Assembler::EQ);
+ __ decode_heap_oop($tmp1$$Register, $newval$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}')dnl
+CASN_INSN(,,false)
+CASN_INSN(Acq,_acq,true)
+dnl
+define(`XCHGP_INSN',
+`
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1GetAndSetP$1(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && ifelse($1,Acq,'needs_acquiring_load_exclusive(n)`,'!needs_acquiring_load_exclusive(n)`) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set preval (GetAndSetP mem newval));
+ effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(ifelse($1,Acq,VOLATILE_REF_COST,2 * VOLATILE_REF_COST));
+ format %{ "atomic_xchg$2 $preval, $newval, [$mem]" %}
+ ins_encode %{
+ assert_different_registers($mem$$Register, $newval$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $preval$$Register /* pre_val (as a temporary register) */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
+ __ $3($preval$$Register, $newval$$Register, $mem$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $newval$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_serial);
+%}')dnl
+XCHGP_INSN(,,atomic_xchg)
+XCHGP_INSN(Acq,_acq,atomic_xchgal)
+dnl
+define(`XCHGN_INSN',
+`
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1GetAndSetN$1(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && ifelse($1,Acq,'needs_acquiring_load_exclusive(n)`,'!needs_acquiring_load_exclusive(n)`) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set preval (GetAndSetN mem newval));
+ effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(ifelse($1,Acq,VOLATILE_REF_COST,2 * VOLATILE_REF_COST));
+ format %{ "$2 $preval, $newval, [$mem]" %}
+ ins_encode %{
+ assert_different_registers($mem$$Register, $newval$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
+ __ $3($preval$$Register, $newval$$Register, $mem$$Register);
+ __ decode_heap_oop($tmp1$$Register, $newval$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_serial);
+%}')dnl
+XCHGN_INSN(,atomic_xchgw,atomic_xchgw)
+XCHGN_INSN(Acq,atomic_xchgw_acq,atomic_xchgalw)
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1LoadP(iRegPNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
+%{
+ // This instruction does not need an acquiring counterpart because it is only
+ // used for reference loading (Reference::get()). The same holds for g1LoadN.
+ predicate(UseG1GC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
+ match(Set dst (LoadP mem));
+ effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(4 * INSN_COST);
+ format %{ "ldr $dst, $mem\t# ptr" %}
+ ins_encode %{
+ __ ldr($dst$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $dst$$Register /* pre_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(iload_reg_mem);
+%}
+
+// This pattern is generated automatically from g1_aarch64.m4.
+// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
+instruct g1LoadN(iRegNNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
+ match(Set dst (LoadN mem));
+ effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(4 * INSN_COST);
+ format %{ "ldrw $dst, $mem\t# compressed ptr" %}
+ ins_encode %{
+ __ ldrw($dst$$Register, $mem$$Register);
+ if ((barrier_data() & G1C2BarrierPre) != 0) {
+ __ decode_heap_oop($tmp1$$Register, $dst$$Register);
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ }
+ %}
+ ins_pipe(iload_reg_mem);
+%}
+
+// END This section of the file is automatically generated. Do not edit --------------
diff --git a/src/hotspot/cpu/aarch64/gc/x/xBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/x/xBarrierSetAssembler_aarch64.cpp
deleted file mode 100644
index 5c891e8c170fb..0000000000000
--- a/src/hotspot/cpu/aarch64/gc/x/xBarrierSetAssembler_aarch64.cpp
+++ /dev/null
@@ -1,462 +0,0 @@
-/*
- * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "asm/macroAssembler.inline.hpp"
-#include "code/codeBlob.hpp"
-#include "code/vmreg.inline.hpp"
-#include "gc/x/xBarrier.inline.hpp"
-#include "gc/x/xBarrierSet.hpp"
-#include "gc/x/xBarrierSetAssembler.hpp"
-#include "gc/x/xBarrierSetRuntime.hpp"
-#include "gc/x/xThreadLocalData.hpp"
-#include "memory/resourceArea.hpp"
-#include "runtime/sharedRuntime.hpp"
-#include "utilities/macros.hpp"
-#ifdef COMPILER1
-#include "c1/c1_LIRAssembler.hpp"
-#include "c1/c1_MacroAssembler.hpp"
-#include "gc/x/c1/xBarrierSetC1.hpp"
-#endif // COMPILER1
-#ifdef COMPILER2
-#include "gc/x/c2/xBarrierSetC2.hpp"
-#endif // COMPILER2
-
-#ifdef PRODUCT
-#define BLOCK_COMMENT(str) /* nothing */
-#else
-#define BLOCK_COMMENT(str) __ block_comment(str)
-#endif
-
-#undef __
-#define __ masm->
-
-void XBarrierSetAssembler::load_at(MacroAssembler* masm,
- DecoratorSet decorators,
- BasicType type,
- Register dst,
- Address src,
- Register tmp1,
- Register tmp2) {
- if (!XBarrierSet::barrier_needed(decorators, type)) {
- // Barrier not needed
- BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
- return;
- }
-
- assert_different_registers(rscratch1, rscratch2, src.base());
- assert_different_registers(rscratch1, rscratch2, dst);
-
- Label done;
-
- // Load bad mask into scratch register.
- __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
- __ lea(rscratch2, src);
- __ ldr(dst, src);
-
- // Test reference against bad mask. If mask bad, then we need to fix it up.
- __ tst(dst, rscratch1);
- __ br(Assembler::EQ, done);
-
- __ enter(/*strip_ret_addr*/true);
-
- __ push_call_clobbered_registers_except(RegSet::of(dst));
-
- if (c_rarg0 != dst) {
- __ mov(c_rarg0, dst);
- }
- __ mov(c_rarg1, rscratch2);
-
- __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
-
- // Make sure dst has the return value.
- if (dst != r0) {
- __ mov(dst, r0);
- }
-
- __ pop_call_clobbered_registers_except(RegSet::of(dst));
- __ leave();
-
- __ bind(done);
-}
-
-#ifdef ASSERT
-
-void XBarrierSetAssembler::store_at(MacroAssembler* masm,
- DecoratorSet decorators,
- BasicType type,
- Address dst,
- Register val,
- Register tmp1,
- Register tmp2,
- Register tmp3) {
- // Verify value
- if (is_reference_type(type)) {
- // Note that src could be noreg, which means we
- // are storing null and can skip verification.
- if (val != noreg) {
- Label done;
-
- // tmp1, tmp2 and tmp3 are often set to noreg.
- RegSet savedRegs = RegSet::of(rscratch1);
- __ push(savedRegs, sp);
-
- __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
- __ tst(val, rscratch1);
- __ br(Assembler::EQ, done);
- __ stop("Verify oop store failed");
- __ should_not_reach_here();
- __ bind(done);
- __ pop(savedRegs, sp);
- }
- }
-
- // Store value
- BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, noreg);
-}
-
-#endif // ASSERT
-
-void XBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
- DecoratorSet decorators,
- bool is_oop,
- Register src,
- Register dst,
- Register count,
- RegSet saved_regs) {
- if (!is_oop) {
- // Barrier not needed
- return;
- }
-
- BLOCK_COMMENT("XBarrierSetAssembler::arraycopy_prologue {");
-
- assert_different_registers(src, count, rscratch1);
-
- __ push(saved_regs, sp);
-
- if (count == c_rarg0) {
- if (src == c_rarg1) {
- // exactly backwards!!
- __ mov(rscratch1, c_rarg0);
- __ mov(c_rarg0, c_rarg1);
- __ mov(c_rarg1, rscratch1);
- } else {
- __ mov(c_rarg1, count);
- __ mov(c_rarg0, src);
- }
- } else {
- __ mov(c_rarg0, src);
- __ mov(c_rarg1, count);
- }
-
- __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_array_addr(), 2);
-
- __ pop(saved_regs, sp);
-
- BLOCK_COMMENT("} XBarrierSetAssembler::arraycopy_prologue");
-}
-
-void XBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
- Register jni_env,
- Register robj,
- Register tmp,
- Label& slowpath) {
- BLOCK_COMMENT("XBarrierSetAssembler::try_resolve_jobject_in_native {");
-
- assert_different_registers(jni_env, robj, tmp);
-
- // Resolve jobject
- BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, robj, tmp, slowpath);
-
- // The Address offset is too large to direct load - -784. Our range is +127, -128.
- __ mov(tmp, (int64_t)(in_bytes(XThreadLocalData::address_bad_mask_offset()) -
- in_bytes(JavaThread::jni_environment_offset())));
-
- // Load address bad mask
- __ add(tmp, jni_env, tmp);
- __ ldr(tmp, Address(tmp));
-
- // Check address bad mask
- __ tst(robj, tmp);
- __ br(Assembler::NE, slowpath);
-
- BLOCK_COMMENT("} XBarrierSetAssembler::try_resolve_jobject_in_native");
-}
-
-#ifdef COMPILER1
-
-#undef __
-#define __ ce->masm()->
-
-void XBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
- LIR_Opr ref) const {
- assert_different_registers(rscratch1, rthread, ref->as_register());
-
- __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
- __ tst(ref->as_register(), rscratch1);
-}
-
-void XBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
- XLoadBarrierStubC1* stub) const {
- // Stub entry
- __ bind(*stub->entry());
-
- Register ref = stub->ref()->as_register();
- Register ref_addr = noreg;
- Register tmp = noreg;
-
- if (stub->tmp()->is_valid()) {
- // Load address into tmp register
- ce->leal(stub->ref_addr(), stub->tmp());
- ref_addr = tmp = stub->tmp()->as_pointer_register();
- } else {
- // Address already in register
- ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
- }
-
- assert_different_registers(ref, ref_addr, noreg);
-
- // Save r0 unless it is the result or tmp register
- // Set up SP to accommodate parameters and maybe r0..
- if (ref != r0 && tmp != r0) {
- __ sub(sp, sp, 32);
- __ str(r0, Address(sp, 16));
- } else {
- __ sub(sp, sp, 16);
- }
-
- // Setup arguments and call runtime stub
- ce->store_parameter(ref_addr, 1);
- ce->store_parameter(ref, 0);
-
- __ far_call(stub->runtime_stub());
-
- // Verify result
- __ verify_oop(r0);
-
- // Move result into place
- if (ref != r0) {
- __ mov(ref, r0);
- }
-
- // Restore r0 unless it is the result or tmp register
- if (ref != r0 && tmp != r0) {
- __ ldr(r0, Address(sp, 16));
- __ add(sp, sp, 32);
- } else {
- __ add(sp, sp, 16);
- }
-
- // Stub exit
- __ b(*stub->continuation());
-}
-
-#undef __
-#define __ sasm->
-
-void XBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
- DecoratorSet decorators) const {
- __ prologue("zgc_load_barrier stub", false);
-
- __ push_call_clobbered_registers_except(RegSet::of(r0));
-
- // Setup arguments
- __ load_parameter(0, c_rarg0);
- __ load_parameter(1, c_rarg1);
-
- __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
-
- __ pop_call_clobbered_registers_except(RegSet::of(r0));
-
- __ epilogue();
-}
-#endif // COMPILER1
-
-#ifdef COMPILER2
-
-OptoReg::Name XBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
- if (!OptoReg::is_reg(opto_reg)) {
- return OptoReg::Bad;
- }
-
- const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
- if (vm_reg->is_FloatRegister()) {
- return opto_reg & ~1;
- }
-
- return opto_reg;
-}
-
-#undef __
-#define __ _masm->
-
-class XSaveLiveRegisters {
-private:
- MacroAssembler* const _masm;
- RegSet _gp_regs;
- FloatRegSet _fp_regs;
- PRegSet _p_regs;
-
-public:
- void initialize(XLoadBarrierStubC2* stub) {
- // Record registers that needs to be saved/restored
- RegMaskIterator rmi(stub->live());
- while (rmi.has_next()) {
- const OptoReg::Name opto_reg = rmi.next();
- if (OptoReg::is_reg(opto_reg)) {
- const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
- if (vm_reg->is_Register()) {
- _gp_regs += RegSet::of(vm_reg->as_Register());
- } else if (vm_reg->is_FloatRegister()) {
- _fp_regs += FloatRegSet::of(vm_reg->as_FloatRegister());
- } else if (vm_reg->is_PRegister()) {
- _p_regs += PRegSet::of(vm_reg->as_PRegister());
- } else {
- fatal("Unknown register type");
- }
- }
- }
-
- // Remove C-ABI SOE registers, scratch regs and _ref register that will be updated
- _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9, stub->ref());
- }
-
- XSaveLiveRegisters(MacroAssembler* masm, XLoadBarrierStubC2* stub) :
- _masm(masm),
- _gp_regs(),
- _fp_regs(),
- _p_regs() {
-
- // Figure out what registers to save/restore
- initialize(stub);
-
- // Save registers
- __ push(_gp_regs, sp);
- __ push_fp(_fp_regs, sp);
- __ push_p(_p_regs, sp);
- }
-
- ~XSaveLiveRegisters() {
- // Restore registers
- __ pop_p(_p_regs, sp);
- __ pop_fp(_fp_regs, sp);
-
- // External runtime call may clobber ptrue reg
- __ reinitialize_ptrue();
-
- __ pop(_gp_regs, sp);
- }
-};
-
-#undef __
-#define __ _masm->
-
-class XSetupArguments {
-private:
- MacroAssembler* const _masm;
- const Register _ref;
- const Address _ref_addr;
-
-public:
- XSetupArguments(MacroAssembler* masm, XLoadBarrierStubC2* stub) :
- _masm(masm),
- _ref(stub->ref()),
- _ref_addr(stub->ref_addr()) {
-
- // Setup arguments
- if (_ref_addr.base() == noreg) {
- // No self healing
- if (_ref != c_rarg0) {
- __ mov(c_rarg0, _ref);
- }
- __ mov(c_rarg1, 0);
- } else {
- // Self healing
- if (_ref == c_rarg0) {
- // _ref is already at correct place
- __ lea(c_rarg1, _ref_addr);
- } else if (_ref != c_rarg1) {
- // _ref is in wrong place, but not in c_rarg1, so fix it first
- __ lea(c_rarg1, _ref_addr);
- __ mov(c_rarg0, _ref);
- } else if (_ref_addr.base() != c_rarg0 && _ref_addr.index() != c_rarg0) {
- assert(_ref == c_rarg1, "Mov ref first, vacating c_rarg0");
- __ mov(c_rarg0, _ref);
- __ lea(c_rarg1, _ref_addr);
- } else {
- assert(_ref == c_rarg1, "Need to vacate c_rarg1 and _ref_addr is using c_rarg0");
- if (_ref_addr.base() == c_rarg0 || _ref_addr.index() == c_rarg0) {
- __ mov(rscratch2, c_rarg1);
- __ lea(c_rarg1, _ref_addr);
- __ mov(c_rarg0, rscratch2);
- } else {
- ShouldNotReachHere();
- }
- }
- }
- }
-
- ~XSetupArguments() {
- // Transfer result
- if (_ref != r0) {
- __ mov(_ref, r0);
- }
- }
-};
-
-#undef __
-#define __ masm->
-
-void XBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, XLoadBarrierStubC2* stub) const {
- BLOCK_COMMENT("XLoadBarrierStubC2");
-
- // Stub entry
- __ bind(*stub->entry());
-
- {
- XSaveLiveRegisters save_live_registers(masm, stub);
- XSetupArguments setup_arguments(masm, stub);
- __ mov(rscratch1, stub->slow_path());
- __ blr(rscratch1);
- }
- // Stub exit
- __ b(*stub->continuation());
-}
-
-#endif // COMPILER2
-
-#undef __
-#define __ masm->
-
-void XBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
- // Check if mask is good.
- // verifies that XAddressBadMask & r0 == 0
- __ ldr(tmp2, Address(rthread, XThreadLocalData::address_bad_mask_offset()));
- __ andr(tmp1, obj, tmp2);
- __ cbnz(tmp1, error);
-
- BarrierSetAssembler::check_oop(masm, obj, tmp1, tmp2, error);
-}
-
-#undef __
diff --git a/src/hotspot/cpu/aarch64/gc/x/xBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/x/xBarrierSetAssembler_aarch64.hpp
deleted file mode 100644
index 8c1e9521757b4..0000000000000
--- a/src/hotspot/cpu/aarch64/gc/x/xBarrierSetAssembler_aarch64.hpp
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#ifndef CPU_AARCH64_GC_X_XBARRIERSETASSEMBLER_AARCH64_HPP
-#define CPU_AARCH64_GC_X_XBARRIERSETASSEMBLER_AARCH64_HPP
-
-#include "code/vmreg.hpp"
-#include "oops/accessDecorators.hpp"
-#ifdef COMPILER2
-#include "opto/optoreg.hpp"
-#endif // COMPILER2
-
-#ifdef COMPILER1
-class LIR_Assembler;
-class LIR_Opr;
-class StubAssembler;
-#endif // COMPILER1
-
-#ifdef COMPILER2
-class Node;
-#endif // COMPILER2
-
-#ifdef COMPILER1
-class XLoadBarrierStubC1;
-#endif // COMPILER1
-
-#ifdef COMPILER2
-class XLoadBarrierStubC2;
-#endif // COMPILER2
-
-class XBarrierSetAssembler : public XBarrierSetAssemblerBase {
-public:
- virtual void load_at(MacroAssembler* masm,
- DecoratorSet decorators,
- BasicType type,
- Register dst,
- Address src,
- Register tmp1,
- Register tmp2);
-
-#ifdef ASSERT
- virtual void store_at(MacroAssembler* masm,
- DecoratorSet decorators,
- BasicType type,
- Address dst,
- Register val,
- Register tmp1,
- Register tmp2,
- Register tmp3);
-#endif // ASSERT
-
- virtual void arraycopy_prologue(MacroAssembler* masm,
- DecoratorSet decorators,
- bool is_oop,
- Register src,
- Register dst,
- Register count,
- RegSet saved_regs);
-
- virtual void try_resolve_jobject_in_native(MacroAssembler* masm,
- Register jni_env,
- Register robj,
- Register tmp,
- Label& slowpath);
-
- virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_data_patch; }
-
-#ifdef COMPILER1
- void generate_c1_load_barrier_test(LIR_Assembler* ce,
- LIR_Opr ref) const;
-
- void generate_c1_load_barrier_stub(LIR_Assembler* ce,
- XLoadBarrierStubC1* stub) const;
-
- void generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
- DecoratorSet decorators) const;
-#endif // COMPILER1
-
-#ifdef COMPILER2
- OptoReg::Name refine_register(const Node* node,
- OptoReg::Name opto_reg);
-
- void generate_c2_load_barrier_stub(MacroAssembler* masm,
- XLoadBarrierStubC2* stub) const;
-#endif // COMPILER2
-
- void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);
-};
-
-#endif // CPU_AARCH64_GC_X_XBARRIERSETASSEMBLER_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/gc/x/xGlobals_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/x/xGlobals_aarch64.cpp
deleted file mode 100644
index a9c53da3d0186..0000000000000
--- a/src/hotspot/cpu/aarch64/gc/x/xGlobals_aarch64.cpp
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/shared/gcLogPrecious.hpp"
-#include "gc/shared/gc_globals.hpp"
-#include "gc/x/xGlobals.hpp"
-#include "runtime/globals.hpp"
-#include "runtime/os.hpp"
-#include "utilities/globalDefinitions.hpp"
-#include "utilities/powerOfTwo.hpp"
-
-#ifdef LINUX
-#include
-#endif // LINUX
-
-//
-// The heap can have three different layouts, depending on the max heap size.
-//
-// Address Space & Pointer Layout 1
-// --------------------------------
-//
-// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB)
-// . .
-// . .
-// . .
-// +--------------------------------+ 0x0000014000000000 (20TB)
-// | Remapped View |
-// +--------------------------------+ 0x0000010000000000 (16TB)
-// . .
-// +--------------------------------+ 0x00000c0000000000 (12TB)
-// | Marked1 View |
-// +--------------------------------+ 0x0000080000000000 (8TB)
-// | Marked0 View |
-// +--------------------------------+ 0x0000040000000000 (4TB)
-// . .
-// +--------------------------------+ 0x0000000000000000
-//
-// 6 4 4 4 4
-// 3 6 5 2 1 0
-// +--------------------+----+-----------------------------------------------+
-// |00000000 00000000 00|1111|11 11111111 11111111 11111111 11111111 11111111|
-// +--------------------+----+-----------------------------------------------+
-// | | |
-// | | * 41-0 Object Offset (42-bits, 4TB address space)
-// | |
-// | * 45-42 Metadata Bits (4-bits) 0001 = Marked0 (Address view 4-8TB)
-// | 0010 = Marked1 (Address view 8-12TB)
-// | 0100 = Remapped (Address view 16-20TB)
-// | 1000 = Finalizable (Address view N/A)
-// |
-// * 63-46 Fixed (18-bits, always zero)
-//
-//
-// Address Space & Pointer Layout 2
-// --------------------------------
-//
-// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB)
-// . .
-// . .
-// . .
-// +--------------------------------+ 0x0000280000000000 (40TB)
-// | Remapped View |
-// +--------------------------------+ 0x0000200000000000 (32TB)
-// . .
-// +--------------------------------+ 0x0000180000000000 (24TB)
-// | Marked1 View |
-// +--------------------------------+ 0x0000100000000000 (16TB)
-// | Marked0 View |
-// +--------------------------------+ 0x0000080000000000 (8TB)
-// . .
-// +--------------------------------+ 0x0000000000000000
-//
-// 6 4 4 4 4
-// 3 7 6 3 2 0
-// +------------------+-----+------------------------------------------------+
-// |00000000 00000000 0|1111|111 11111111 11111111 11111111 11111111 11111111|
-// +-------------------+----+------------------------------------------------+
-// | | |
-// | | * 42-0 Object Offset (43-bits, 8TB address space)
-// | |
-// | * 46-43 Metadata Bits (4-bits) 0001 = Marked0 (Address view 8-16TB)
-// | 0010 = Marked1 (Address view 16-24TB)
-// | 0100 = Remapped (Address view 32-40TB)
-// | 1000 = Finalizable (Address view N/A)
-// |
-// * 63-47 Fixed (17-bits, always zero)
-//
-//
-// Address Space & Pointer Layout 3
-// --------------------------------
-//
-// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB)
-// . .
-// . .
-// . .
-// +--------------------------------+ 0x0000500000000000 (80TB)
-// | Remapped View |
-// +--------------------------------+ 0x0000400000000000 (64TB)
-// . .
-// +--------------------------------+ 0x0000300000000000 (48TB)
-// | Marked1 View |
-// +--------------------------------+ 0x0000200000000000 (32TB)
-// | Marked0 View |
-// +--------------------------------+ 0x0000100000000000 (16TB)
-// . .
-// +--------------------------------+ 0x0000000000000000
-//
-// 6 4 4 4 4
-// 3 8 7 4 3 0
-// +------------------+----+-------------------------------------------------+
-// |00000000 00000000 |1111|1111 11111111 11111111 11111111 11111111 11111111|
-// +------------------+----+-------------------------------------------------+
-// | | |
-// | | * 43-0 Object Offset (44-bits, 16TB address space)
-// | |
-// | * 47-44 Metadata Bits (4-bits) 0001 = Marked0 (Address view 16-32TB)
-// | 0010 = Marked1 (Address view 32-48TB)
-// | 0100 = Remapped (Address view 64-80TB)
-// | 1000 = Finalizable (Address view N/A)
-// |
-// * 63-48 Fixed (16-bits, always zero)
-//
-
-// Default value if probing is not implemented for a certain platform
-// Max address bit is restricted by implicit assumptions in the code, for instance
-// the bit layout of XForwardingEntry or Partial array entry (see XMarkStackEntry) in mark stack
-static const size_t DEFAULT_MAX_ADDRESS_BIT = 46;
-// Minimum value returned, if probing fails
-static const size_t MINIMUM_MAX_ADDRESS_BIT = 36;
-
-static size_t probe_valid_max_address_bit() {
-#ifdef LINUX
- size_t max_address_bit = 0;
- const size_t page_size = os::vm_page_size();
- for (size_t i = DEFAULT_MAX_ADDRESS_BIT; i > MINIMUM_MAX_ADDRESS_BIT; --i) {
- const uintptr_t base_addr = ((uintptr_t) 1U) << i;
- if (msync((void*)base_addr, page_size, MS_ASYNC) == 0) {
- // msync succeeded, the address is valid, and maybe even already mapped.
- max_address_bit = i;
- break;
- }
- if (errno != ENOMEM) {
- // Some error occurred. This should never happen, but msync
- // has some undefined behavior, hence ignore this bit.
-#ifdef ASSERT
- fatal("Received '%s' while probing the address space for the highest valid bit", os::errno_name(errno));
-#else // ASSERT
- log_warning_p(gc)("Received '%s' while probing the address space for the highest valid bit", os::errno_name(errno));
-#endif // ASSERT
- continue;
- }
- // Since msync failed with ENOMEM, the page might not be mapped.
- // Try to map it, to see if the address is valid.
- void* const result_addr = mmap((void*) base_addr, page_size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0);
- if (result_addr != MAP_FAILED) {
- munmap(result_addr, page_size);
- }
- if ((uintptr_t) result_addr == base_addr) {
- // address is valid
- max_address_bit = i;
- break;
- }
- }
- if (max_address_bit == 0) {
- // probing failed, allocate a very high page and take that bit as the maximum
- const uintptr_t high_addr = ((uintptr_t) 1U) << DEFAULT_MAX_ADDRESS_BIT;
- void* const result_addr = mmap((void*) high_addr, page_size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0);
- if (result_addr != MAP_FAILED) {
- max_address_bit = BitsPerSize_t - count_leading_zeros((size_t) result_addr) - 1;
- munmap(result_addr, page_size);
- }
- }
- log_info_p(gc, init)("Probing address space for the highest valid bit: " SIZE_FORMAT, max_address_bit);
- return MAX2(max_address_bit, MINIMUM_MAX_ADDRESS_BIT);
-#else // LINUX
- return DEFAULT_MAX_ADDRESS_BIT;
-#endif // LINUX
-}
-
-size_t XPlatformAddressOffsetBits() {
- const static size_t valid_max_address_offset_bits = probe_valid_max_address_bit() + 1;
- const size_t max_address_offset_bits = valid_max_address_offset_bits - 3;
- const size_t min_address_offset_bits = max_address_offset_bits - 2;
- const size_t address_offset = round_up_power_of_2(MaxHeapSize * XVirtualToPhysicalRatio);
- const size_t address_offset_bits = log2i_exact(address_offset);
- return clamp(address_offset_bits, min_address_offset_bits, max_address_offset_bits);
-}
-
-size_t XPlatformAddressMetadataShift() {
- return XPlatformAddressOffsetBits();
-}
diff --git a/src/hotspot/cpu/aarch64/gc/x/xGlobals_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/x/xGlobals_aarch64.hpp
deleted file mode 100644
index 870b0d74d5766..0000000000000
--- a/src/hotspot/cpu/aarch64/gc/x/xGlobals_aarch64.hpp
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#ifndef CPU_AARCH64_GC_X_XGLOBALS_AARCH64_HPP
-#define CPU_AARCH64_GC_X_XGLOBALS_AARCH64_HPP
-
-const size_t XPlatformHeapViews = 3;
-const size_t XPlatformCacheLineSize = 64;
-
-size_t XPlatformAddressOffsetBits();
-size_t XPlatformAddressMetadataShift();
-
-#endif // CPU_AARCH64_GC_X_XGLOBALS_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/gc/x/x_aarch64.ad b/src/hotspot/cpu/aarch64/gc/x/x_aarch64.ad
deleted file mode 100644
index 5e690a8e47b94..0000000000000
--- a/src/hotspot/cpu/aarch64/gc/x/x_aarch64.ad
+++ /dev/null
@@ -1,249 +0,0 @@
-//
-// Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
-// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-//
-// This code is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License version 2 only, as
-// published by the Free Software Foundation.
-//
-// This code is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-// version 2 for more details (a copy is included in the LICENSE file that
-// accompanied this code).
-//
-// You should have received a copy of the GNU General Public License version
-// 2 along with this work; if not, write to the Free Software Foundation,
-// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-//
-// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-// or visit www.oracle.com if you need additional information or have any
-// questions.
-//
-
-source_hpp %{
-
-#include "gc/shared/gc_globals.hpp"
-#include "gc/x/c2/xBarrierSetC2.hpp"
-#include "gc/x/xThreadLocalData.hpp"
-
-%}
-
-source %{
-
-static void x_load_barrier(MacroAssembler* masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data) {
- if (barrier_data == XLoadBarrierElided) {
- return;
- }
- XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data);
- __ ldr(tmp, Address(rthread, XThreadLocalData::address_bad_mask_offset()));
- __ andr(tmp, tmp, ref);
- __ cbnz(tmp, *stub->entry());
- __ bind(*stub->continuation());
-}
-
-static void x_load_barrier_slow_path(MacroAssembler* masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {
- XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, XLoadBarrierStrong);
- __ b(*stub->entry());
- __ bind(*stub->continuation());
-}
-
-%}
-
-// Load Pointer
-instruct xLoadP(iRegPNoSp dst, memory mem, rFlagsReg cr)
-%{
- match(Set dst (LoadP mem));
- predicate(UseZGC && !ZGenerational && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() != 0));
- effect(TEMP dst, KILL cr);
-
- ins_cost(4 * INSN_COST);
-
- format %{ "ldr $dst, $mem" %}
-
- ins_encode %{
- Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
- if (ref_addr.getMode() == Address::base_plus_offset) {
- // Fix up any out-of-range offsets.
- assert_different_registers(rscratch1, as_Register($mem$$base));
- assert_different_registers(rscratch1, $dst$$Register);
- ref_addr = __ legitimize_address(ref_addr, 8, rscratch1);
- }
- __ ldr($dst$$Register, ref_addr);
- x_load_barrier(masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, barrier_data());
- %}
-
- ins_pipe(iload_reg_mem);
-%}
-
-// Load Pointer Volatile
-instruct xLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg cr)
-%{
- match(Set dst (LoadP mem));
- predicate(UseZGC && !ZGenerational && needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
- effect(TEMP dst, KILL cr);
-
- ins_cost(VOLATILE_REF_COST);
-
- format %{ "ldar $dst, $mem\t" %}
-
- ins_encode %{
- __ ldar($dst$$Register, $mem$$Register);
- x_load_barrier(masm, this, Address($mem$$Register), $dst$$Register, rscratch2 /* tmp */, barrier_data());
- %}
-
- ins_pipe(pipe_serial);
-%}
-
-instruct xCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
- match(Set res (CompareAndSwapP mem (Binary oldval newval)));
- match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
- predicate(UseZGC && !ZGenerational && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
- effect(KILL cr, TEMP_DEF res);
-
- ins_cost(2 * VOLATILE_REF_COST);
-
- format %{ "cmpxchg $mem, $oldval, $newval\n\t"
- "cset $res, EQ" %}
-
- ins_encode %{
- guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
- false /* acquire */, true /* release */, false /* weak */, rscratch2);
- __ cset($res$$Register, Assembler::EQ);
- if (barrier_data() != XLoadBarrierElided) {
- Label good;
- __ ldr(rscratch1, Address(rthread, XThreadLocalData::address_bad_mask_offset()));
- __ andr(rscratch1, rscratch1, rscratch2);
- __ cbz(rscratch1, good);
- x_load_barrier_slow_path(masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */);
- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
- false /* acquire */, true /* release */, false /* weak */, rscratch2);
- __ cset($res$$Register, Assembler::EQ);
- __ bind(good);
- }
- %}
-
- ins_pipe(pipe_slow);
-%}
-
-instruct xCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
- match(Set res (CompareAndSwapP mem (Binary oldval newval)));
- match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
- predicate(UseZGC && !ZGenerational && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == XLoadBarrierStrong));
- effect(KILL cr, TEMP_DEF res);
-
- ins_cost(2 * VOLATILE_REF_COST);
-
- format %{ "cmpxchg $mem, $oldval, $newval\n\t"
- "cset $res, EQ" %}
-
- ins_encode %{
- guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
- true /* acquire */, true /* release */, false /* weak */, rscratch2);
- __ cset($res$$Register, Assembler::EQ);
- if (barrier_data() != XLoadBarrierElided) {
- Label good;
- __ ldr(rscratch1, Address(rthread, XThreadLocalData::address_bad_mask_offset()));
- __ andr(rscratch1, rscratch1, rscratch2);
- __ cbz(rscratch1, good);
- x_load_barrier_slow_path(masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */ );
- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
- true /* acquire */, true /* release */, false /* weak */, rscratch2);
- __ cset($res$$Register, Assembler::EQ);
- __ bind(good);
- }
- %}
-
- ins_pipe(pipe_slow);
-%}
-
-instruct xCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
- match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
- predicate(UseZGC && !ZGenerational && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
- effect(TEMP_DEF res, KILL cr);
-
- ins_cost(2 * VOLATILE_REF_COST);
-
- format %{ "cmpxchg $res = $mem, $oldval, $newval" %}
-
- ins_encode %{
- guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
- false /* acquire */, true /* release */, false /* weak */, $res$$Register);
- if (barrier_data() != XLoadBarrierElided) {
- Label good;
- __ ldr(rscratch1, Address(rthread, XThreadLocalData::address_bad_mask_offset()));
- __ andr(rscratch1, rscratch1, $res$$Register);
- __ cbz(rscratch1, good);
- x_load_barrier_slow_path(masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */);
- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
- false /* acquire */, true /* release */, false /* weak */, $res$$Register);
- __ bind(good);
- }
- %}
-
- ins_pipe(pipe_slow);
-%}
-
-instruct xCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
- match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
- predicate(UseZGC && !ZGenerational && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
- effect(TEMP_DEF res, KILL cr);
-
- ins_cost(2 * VOLATILE_REF_COST);
-
- format %{ "cmpxchg $res = $mem, $oldval, $newval" %}
-
- ins_encode %{
- guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
- true /* acquire */, true /* release */, false /* weak */, $res$$Register);
- if (barrier_data() != XLoadBarrierElided) {
- Label good;
- __ ldr(rscratch1, Address(rthread, XThreadLocalData::address_bad_mask_offset()));
- __ andr(rscratch1, rscratch1, $res$$Register);
- __ cbz(rscratch1, good);
- x_load_barrier_slow_path(masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */);
- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
- true /* acquire */, true /* release */, false /* weak */, $res$$Register);
- __ bind(good);
- }
- %}
-
- ins_pipe(pipe_slow);
-%}
-
-instruct xGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
- match(Set prev (GetAndSetP mem newv));
- predicate(UseZGC && !ZGenerational && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
- effect(TEMP_DEF prev, KILL cr);
-
- ins_cost(2 * VOLATILE_REF_COST);
-
- format %{ "atomic_xchg $prev, $newv, [$mem]" %}
-
- ins_encode %{
- __ atomic_xchg($prev$$Register, $newv$$Register, $mem$$Register);
- x_load_barrier(masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, barrier_data());
- %}
-
- ins_pipe(pipe_serial);
-%}
-
-instruct xGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
- match(Set prev (GetAndSetP mem newv));
- predicate(UseZGC && !ZGenerational && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() != 0));
- effect(TEMP_DEF prev, KILL cr);
-
- ins_cost(VOLATILE_REF_COST);
-
- format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
-
- ins_encode %{
- __ atomic_xchgal($prev$$Register, $newv$$Register, $mem$$Register);
- x_load_barrier(masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, barrier_data());
- %}
- ins_pipe(pipe_serial);
-%}
diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
index 466e77a4460d0..3f1898b6742e1 100644
--- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
@@ -1189,6 +1189,8 @@ void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm,
__ lea(rscratch1, RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_native_oop_field_without_healing_addr()));
} else if (stub->is_atomic()) {
__ lea(rscratch1, RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr()));
+ } else if (stub->is_nokeepalive()) {
+ __ lea(rscratch1, RuntimeAddress(ZBarrierSetRuntime::no_keepalive_store_barrier_on_oop_field_without_healing_addr()));
} else {
__ lea(rscratch1, RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr()));
}
@@ -1307,11 +1309,11 @@ Label* ZLoadBarrierStubC2Aarch64::entry() {
return ZBarrierStubC2::entry();
}
-ZStoreBarrierStubC2Aarch64::ZStoreBarrierStubC2Aarch64(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic)
- : ZStoreBarrierStubC2(node, ref_addr, new_zaddress, new_zpointer, is_native, is_atomic), _deferred_emit(false) {}
+ZStoreBarrierStubC2Aarch64::ZStoreBarrierStubC2Aarch64(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic, bool is_nokeepalive)
+ : ZStoreBarrierStubC2(node, ref_addr, new_zaddress, new_zpointer, is_native, is_atomic, is_nokeepalive), _deferred_emit(false) {}
-ZStoreBarrierStubC2Aarch64* ZStoreBarrierStubC2Aarch64::create(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic) {
- ZStoreBarrierStubC2Aarch64* const stub = new (Compile::current()->comp_arena()) ZStoreBarrierStubC2Aarch64(node, ref_addr, new_zaddress, new_zpointer, is_native, is_atomic);
+ZStoreBarrierStubC2Aarch64* ZStoreBarrierStubC2Aarch64::create(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic, bool is_nokeepalive) {
+ ZStoreBarrierStubC2Aarch64* const stub = new (Compile::current()->comp_arena()) ZStoreBarrierStubC2Aarch64(node, ref_addr, new_zaddress, new_zpointer, is_native, is_atomic, is_nokeepalive);
register_stub(stub);
return stub;
}
diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp
index 2f716140ed19d..ad3a171c10370 100644
--- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp
@@ -280,10 +280,10 @@ class ZStoreBarrierStubC2Aarch64 : public ZStoreBarrierStubC2 {
private:
bool _deferred_emit;
- ZStoreBarrierStubC2Aarch64(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic);
+ ZStoreBarrierStubC2Aarch64(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic, bool is_nokeepalive);
public:
- static ZStoreBarrierStubC2Aarch64* create(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic);
+ static ZStoreBarrierStubC2Aarch64* create(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic, bool is_nokeepalive);
virtual void emit_code(MacroAssembler& masm);
};
diff --git a/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad
index 1510b42bfe97d..47abaae3d5b77 100644
--- a/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad
+++ b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad
@@ -91,7 +91,8 @@ static void z_store_barrier(MacroAssembler* masm, const MachNode* node, Address
z_color(masm, node, rnew_zpointer, rnew_zaddress);
} else {
bool is_native = (node->barrier_data() & ZBarrierNative) != 0;
- ZStoreBarrierStubC2Aarch64* const stub = ZStoreBarrierStubC2Aarch64::create(node, ref_addr, rnew_zaddress, rnew_zpointer, is_native, is_atomic);
+ bool is_nokeepalive = (node->barrier_data() & ZBarrierNoKeepalive) != 0;
+ ZStoreBarrierStubC2Aarch64* const stub = ZStoreBarrierStubC2Aarch64::create(node, ref_addr, rnew_zaddress, rnew_zpointer, is_native, is_atomic, is_nokeepalive);
ZBarrierSetAssembler* bs_asm = ZBarrierSet::assembler();
bs_asm->store_barrier_fast(masm, ref_addr, rnew_zaddress, rnew_zpointer, tmp, true /* in_nmethod */, is_atomic, *stub->entry(), *stub->continuation());
}
@@ -100,10 +101,10 @@ static void z_store_barrier(MacroAssembler* masm, const MachNode* node, Address
%}
// Load Pointer
-instruct zLoadP(iRegPNoSp dst, memory mem, rFlagsReg cr)
+instruct zLoadP(iRegPNoSp dst, memory8 mem, rFlagsReg cr)
%{
match(Set dst (LoadP mem));
- predicate(UseZGC && ZGenerational && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
+ predicate(UseZGC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
effect(TEMP dst, KILL cr);
ins_cost(4 * INSN_COST);
@@ -129,7 +130,7 @@ instruct zLoadP(iRegPNoSp dst, memory mem, rFlagsReg cr)
instruct zLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg cr)
%{
match(Set dst (LoadP mem));
- predicate(UseZGC && ZGenerational && needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
+ predicate(UseZGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
effect(TEMP dst, KILL cr);
ins_cost(VOLATILE_REF_COST);
@@ -148,7 +149,7 @@ instruct zLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg
// Store Pointer
instruct zStoreP(memory mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
%{
- predicate(UseZGC && ZGenerational && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
+ predicate(UseZGC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
match(Set mem (StoreP mem src));
effect(TEMP tmp, KILL cr);
@@ -165,7 +166,7 @@ instruct zStoreP(memory mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
// Store Pointer Volatile
instruct zStorePVolatile(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
%{
- predicate(UseZGC && ZGenerational && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
+ predicate(UseZGC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
match(Set mem (StoreP mem src));
effect(TEMP tmp, KILL cr);
@@ -182,7 +183,7 @@ instruct zStorePVolatile(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
match(Set res (CompareAndSwapP mem (Binary oldval newval)));
match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
- predicate(UseZGC && ZGenerational && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP res, KILL cr);
ins_cost(2 * VOLATILE_REF_COST);
@@ -206,7 +207,7 @@ instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newva
instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
match(Set res (CompareAndSwapP mem (Binary oldval newval)));
match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
- predicate(UseZGC && ZGenerational && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+ predicate(UseZGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP res, KILL cr);
ins_cost(2 * VOLATILE_REF_COST);
@@ -230,7 +231,7 @@ instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
- predicate(UseZGC && ZGenerational && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP res, KILL cr);
ins_cost(2 * VOLATILE_REF_COST);
@@ -253,7 +254,7 @@ instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP n
instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
- predicate(UseZGC && ZGenerational && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+ predicate(UseZGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP res, KILL cr);
ins_cost(2 * VOLATILE_REF_COST);
@@ -276,7 +277,7 @@ instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
match(Set prev (GetAndSetP mem newv));
- predicate(UseZGC && ZGenerational && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
effect(TEMP prev, KILL cr);
ins_cost(2 * VOLATILE_REF_COST);
@@ -294,7 +295,7 @@ instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
match(Set prev (GetAndSetP mem newv));
- predicate(UseZGC && ZGenerational && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+ predicate(UseZGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
effect(TEMP prev, KILL cr);
ins_cost(2 * VOLATILE_REF_COST);
diff --git a/src/hotspot/cpu/aarch64/globals_aarch64.hpp b/src/hotspot/cpu/aarch64/globals_aarch64.hpp
index 9c20e3737c868..800e771892109 100644
--- a/src/hotspot/cpu/aarch64/globals_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/globals_aarch64.hpp
@@ -38,7 +38,7 @@ define_pd_global(bool, UncommonNullCast, true); // Uncommon-trap nulls
define_pd_global(bool, DelayCompilerStubsGeneration, COMPILER2_OR_JVMCI);
-define_pd_global(uintx, CodeCacheSegmentSize, 64 COMPILER1_AND_COMPILER2_PRESENT(+64)); // Tiered compilation has large code-entry alignment.
+define_pd_global(uintx, CodeCacheSegmentSize, 64);
define_pd_global(intx, CodeEntryAlignment, 64);
define_pd_global(intx, OptoLoopAlignment, 16);
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
index c5c02619d446e..252f423211580 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
@@ -1838,7 +1838,8 @@ void MacroAssembler::clinit_barrier(Register klass, Register scratch, Label* L_f
L_slow_path = &L_fallthrough;
}
// Fast path check: class is fully initialized
- ldrb(scratch, Address(klass, InstanceKlass::init_state_offset()));
+ lea(scratch, Address(klass, InstanceKlass::init_state_offset()));
+ ldarb(scratch, scratch);
subs(zr, scratch, InstanceKlass::fully_initialized);
br(Assembler::EQ, *L_fast_path);
@@ -4781,23 +4782,6 @@ void MacroAssembler::kernel_crc32_common_fold_using_crypto_pmull(Register crc, R
mov(tmp1, v0, D, 1);
}
-SkipIfEqual::SkipIfEqual(
- MacroAssembler* masm, const bool* flag_addr, bool value) {
- _masm = masm;
- uint64_t offset;
- _masm->adrp(rscratch1, ExternalAddress((address)flag_addr), offset);
- _masm->ldrb(rscratch1, Address(rscratch1, offset));
- if (value) {
- _masm->cbnzw(rscratch1, _label);
- } else {
- _masm->cbzw(rscratch1, _label);
- }
-}
-
-SkipIfEqual::~SkipIfEqual() {
- _masm->bind(_label);
-}
-
void MacroAssembler::addptr(const Address &dst, int32_t src) {
Address adr;
switch(dst.getMode()) {
@@ -5010,8 +4994,10 @@ void MacroAssembler::decode_heap_oop(Register d, Register s) {
verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
#endif
if (CompressedOops::base() == nullptr) {
- if (CompressedOops::shift() != 0 || d != s) {
+ if (CompressedOops::shift() != 0) {
lsl(d, s, CompressedOops::shift());
+ } else if (d != s) {
+ mov(d, s);
}
} else {
Label done;
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
index e49f0c49ef66f..48fb3c2b07138 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
@@ -1652,24 +1652,6 @@ class MacroAssembler: public Assembler {
inline bool AbstractAssembler::pd_check_instruction_mark() { return false; }
#endif
-/**
- * class SkipIfEqual:
- *
- * Instantiating this class will result in assembly code being output that will
- * jump around any code emitted between the creation of the instance and it's
- * automatic destruction at the end of a scope block, depending on the value of
- * the flag passed to the constructor, which will be checked at run-time.
- */
-class SkipIfEqual {
- private:
- MacroAssembler* _masm;
- Label _label;
-
- public:
- SkipIfEqual(MacroAssembler*, const bool* flag_addr, bool value);
- ~SkipIfEqual();
-};
-
struct tableswitch {
Register _reg;
int _insn_index; jint _first_key; jint _last_key;
diff --git a/src/hotspot/cpu/aarch64/methodHandles_aarch64.cpp b/src/hotspot/cpu/aarch64/methodHandles_aarch64.cpp
index 68800d04d69ba..aa6a9d14ff176 100644
--- a/src/hotspot/cpu/aarch64/methodHandles_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/methodHandles_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -27,6 +27,7 @@
#include "asm/macroAssembler.hpp"
#include "classfile/javaClasses.inline.hpp"
#include "classfile/vmClasses.hpp"
+#include "compiler/disassembler.hpp"
#include "interpreter/interpreter.hpp"
#include "interpreter/interpreterRuntime.hpp"
#include "memory/allocation.inline.hpp"
@@ -36,7 +37,7 @@
#include "runtime/frame.inline.hpp"
#include "runtime/stubRoutines.hpp"
-#define __ _masm->
+#define __ Disassembler::hook(__FILE__, __LINE__, _masm)->
#ifdef PRODUCT
#define BLOCK_COMMENT(str) /* nothing */
diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp
index 770ed2f6c3868..1f6d729238974 100644
--- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp
@@ -78,7 +78,7 @@ address NativeCall::destination() const {
//
// Used in the runtime linkage of calls; see class CompiledIC.
void NativeCall::set_destination_mt_safe(address dest) {
- assert((Patching_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) ||
+ assert((CodeCache_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) ||
CompiledICLocker::is_safe(addr_at(0)),
"concurrent code patching");
diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
index eb235f8472c1d..de7fc5b281bc2 100644
--- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
@@ -1374,7 +1374,9 @@ class StubGenerator: public StubCodeGenerator {
// r15 is the byte adjustment needed to align s.
__ cbz(r15, aligned);
int shift = exact_log2(granularity);
- if (shift) __ lsr(r15, r15, shift);
+ if (shift > 0) {
+ __ lsr(r15, r15, shift);
+ }
__ sub(count, count, r15);
#if 0
@@ -1402,9 +1404,15 @@ class StubGenerator: public StubCodeGenerator {
// s is now 2-word-aligned.
- // We have a count of units and some trailing bytes. Adjust the
- // count and do a bulk copy of words.
- __ lsr(r15, count, exact_log2(wordSize/granularity));
+ // We have a count of units and some trailing bytes. Adjust the
+ // count and do a bulk copy of words. If the shift is zero
+ // perform a move instead to benefit from zero latency moves.
+ int shift = exact_log2(wordSize/granularity);
+ if (shift > 0) {
+ __ lsr(r15, count, shift);
+ } else {
+ __ mov(r15, count);
+ }
if (direction == copy_forwards) {
if (type != T_OBJECT) {
__ bl(copy_f);
@@ -7320,6 +7328,28 @@ class StubGenerator: public StubCodeGenerator {
return start;
}
+ // load Method* target of MethodHandle
+ // j_rarg0 = jobject receiver
+ // rmethod = result
+ address generate_upcall_stub_load_target() {
+ StubCodeMark mark(this, "StubRoutines", "upcall_stub_load_target");
+ address start = __ pc();
+
+ __ resolve_global_jobject(j_rarg0, rscratch1, rscratch2);
+ // Load target method from receiver
+ __ load_heap_oop(rmethod, Address(j_rarg0, java_lang_invoke_MethodHandle::form_offset()), rscratch1, rscratch2);
+ __ load_heap_oop(rmethod, Address(rmethod, java_lang_invoke_LambdaForm::vmentry_offset()), rscratch1, rscratch2);
+ __ load_heap_oop(rmethod, Address(rmethod, java_lang_invoke_MemberName::method_offset()), rscratch1, rscratch2);
+ __ access_load_at(T_ADDRESS, IN_HEAP, rmethod,
+ Address(rmethod, java_lang_invoke_ResolvedMethodName::vmtarget_offset()),
+ noreg, noreg);
+ __ str(rmethod, Address(rthread, JavaThread::callee_target_offset())); // just in case callee is deoptimized
+
+ __ ret(lr);
+
+ return start;
+ }
+
#undef __
#define __ masm->
@@ -8241,6 +8271,7 @@ class StubGenerator: public StubCodeGenerator {
#endif
StubRoutines::_upcall_stub_exception_handler = generate_upcall_stub_exception_handler();
+ StubRoutines::_upcall_stub_load_target = generate_upcall_stub_load_target();
StubRoutines::aarch64::set_completed(); // Inidicate that arraycopy and zero_blocks stubs are generated
}
diff --git a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp
index 3210789bbbdfa..9894841e933d8 100644
--- a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp
@@ -26,6 +26,7 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "classfile/javaClasses.hpp"
+#include "compiler/disassembler.hpp"
#include "compiler/compiler_globals.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
#include "interpreter/bytecodeHistogram.hpp"
@@ -67,7 +68,7 @@
// Max size with JVMTI
int TemplateInterpreter::InterpreterCodeSize = 200 * 1024;
-#define __ _masm->
+#define __ Disassembler::hook(__FILE__, __LINE__, _masm)->
address TemplateInterpreterGenerator::generate_slow_signature_handler() {
address entry = __ pc();
@@ -1998,13 +1999,21 @@ void TemplateInterpreterGenerator::set_vtos_entry_points(Template* t,
address& vep) {
assert(t->is_valid() && t->tos_in() == vtos, "illegal template");
Label L;
- aep = __ pc(); __ push_ptr(); __ b(L);
- fep = __ pc(); __ push_f(); __ b(L);
- dep = __ pc(); __ push_d(); __ b(L);
- lep = __ pc(); __ push_l(); __ b(L);
- bep = cep = sep =
- iep = __ pc(); __ push_i();
- vep = __ pc();
+ aep = __ pc(); // atos entry point
+ __ push_ptr();
+ __ b(L);
+ fep = __ pc(); // ftos entry point
+ __ push_f();
+ __ b(L);
+ dep = __ pc(); // dtos entry point
+ __ push_d();
+ __ b(L);
+ lep = __ pc(); // ltos entry point
+ __ push_l();
+ __ b(L);
+ bep = cep = sep = iep = __ pc(); // [bcsi]tos entry point
+ __ push_i();
+ vep = __ pc(); // vtos entry point
__ bind(L);
generate_and_dispatch(t);
}
diff --git a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
index 25eb339bfce71..48ff356f9a558 100644
--- a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
@@ -25,6 +25,7 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "compiler/disassembler.hpp"
#include "compiler/compilerDefinitions.inline.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
#include "gc/shared/collectedHeap.hpp"
@@ -49,7 +50,7 @@
#include "runtime/synchronizer.hpp"
#include "utilities/powerOfTwo.hpp"
-#define __ _masm->
+#define __ Disassembler::hook(__FILE__, __LINE__, _masm)->
// Address computation: local variables
diff --git a/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp b/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp
index 28ec07815be5c..517fccb2d1aa5 100644
--- a/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp
@@ -24,6 +24,7 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
+#include "classfile/javaClasses.hpp"
#include "logging/logStream.hpp"
#include "memory/resourceArea.hpp"
#include "prims/upcallLinker.hpp"
@@ -117,7 +118,7 @@ static void restore_callee_saved_registers(MacroAssembler* _masm, const ABIDescr
static const int upcall_stub_code_base_size = 1024;
static const int upcall_stub_size_per_arg = 16;
-address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
+address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature,
BasicType* out_sig_bt, int total_out_args,
BasicType ret_type,
jobject jabi, jobject jconv,
@@ -222,7 +223,6 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
__ block_comment("{ on_entry");
__ lea(c_rarg0, Address(sp, frame_data_offset));
- __ movptr(c_rarg1, (intptr_t)receiver);
__ movptr(rscratch1, CAST_FROM_FN_PTR(uint64_t, UpcallLinker::on_entry));
__ blr(rscratch1);
__ mov(rthread, r0);
@@ -238,12 +238,10 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
arg_shuffle.generate(_masm, as_VMStorage(shuffle_reg), abi._shadow_space_bytes, 0);
__ block_comment("} argument shuffle");
- __ block_comment("{ receiver ");
- __ get_vm_result(j_rarg0, rthread);
- __ block_comment("} receiver ");
-
- __ mov_metadata(rmethod, entry);
- __ str(rmethod, Address(rthread, JavaThread::callee_target_offset())); // just in case callee is deoptimized
+ __ block_comment("{ load target ");
+ __ movptr(j_rarg0, (intptr_t)receiver);
+ __ far_call(RuntimeAddress(StubRoutines::upcall_stub_load_target()), rscratch1); // puts target Method* in rmethod
+ __ block_comment("} load target ");
__ push_cont_fastpath(rthread);
@@ -318,7 +316,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
#ifndef PRODUCT
stringStream ss;
- ss.print("upcall_stub_%s", entry->signature()->as_C_string());
+ ss.print("upcall_stub_%s", signature->as_C_string());
const char* name = _masm->code_string(ss.as_string());
#else // PRODUCT
const char* name = "upcall_stub";
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
index d71162ac568ea..f18cec16488ae 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
@@ -259,6 +259,9 @@ void VM_Version::initialize() {
if (FLAG_IS_DEFAULT(UseCryptoPmullForCRC32)) {
FLAG_SET_DEFAULT(UseCryptoPmullForCRC32, true);
}
+ if (FLAG_IS_DEFAULT(CodeEntryAlignment)) {
+ FLAG_SET_DEFAULT(CodeEntryAlignment, 32);
+ }
}
if (UseCryptoPmullForCRC32 && (!VM_Version::supports_pmull() || !VM_Version::supports_sha3() || !VM_Version::supports_crc32())) {
diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad
index 2c7de0a58a204..bfca986f350cb 100644
--- a/src/hotspot/cpu/arm/arm.ad
+++ b/src/hotspot/cpu/arm/arm.ad
@@ -1003,10 +1003,6 @@ const RegMask* Matcher::predicate_reg_mask(void) {
return nullptr;
}
-const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
- return nullptr;
-}
-
// Vector calling convention not yet implemented.
bool Matcher::supports_vector_calling_convention(void) {
return false;
@@ -3890,6 +3886,7 @@ instruct loadRange(iRegI dst, memoryI mem) %{
instruct loadP(iRegP dst, memoryP mem) %{
+ predicate(!(UseG1GC && n->as_Load()->barrier_data() != 0));
match(Set dst (LoadP mem));
ins_cost(MEMORY_REF_COST);
size(4);
@@ -4225,18 +4222,6 @@ instruct storeB(memoryB mem, store_RegI src) %{
ins_pipe(istore_mem_reg);
%}
-instruct storeCM(memoryB mem, store_RegI src) %{
- match(Set mem (StoreCM mem src));
- ins_cost(MEMORY_REF_COST);
-
- size(4);
- format %{ "STRB $src,$mem\t! CMS card-mark byte" %}
- ins_encode %{
- __ strb($src$$Register, $mem$$Address);
- %}
- ins_pipe(istore_mem_reg);
-%}
-
// Store Char/Short
@@ -4356,6 +4341,7 @@ instruct movSP(store_ptr_RegP dst, SPRegP src) %{
instruct storeP(memoryP mem, store_ptr_RegP src) %{
+ predicate(!(UseG1GC && n->as_Store()->barrier_data() != 0));
match(Set mem (StoreP mem src));
ins_cost(MEMORY_REF_COST);
size(4);
@@ -5390,6 +5376,7 @@ instruct compareAndSwapI_bool(memoryex mem, iRegI oldval, iRegI newval, iRegI re
%}
instruct compareAndSwapP_bool(memoryex mem, iRegP oldval, iRegP newval, iRegI res, iRegI tmp, flagsReg ccr ) %{
+ predicate(!(UseG1GC && n->as_LoadStore()->barrier_data() != 0));
match(Set res (CompareAndSwapP mem (Binary oldval newval)));
effect( KILL ccr, TEMP tmp);
size(28);
@@ -5659,6 +5646,7 @@ instruct xchgL(memoryex mem, iRegLd newval, iRegLd res, iRegI tmp, flagsReg ccr)
%}
instruct xchgP(memoryex mem, iRegP newval, iRegP res, iRegI tmp, flagsReg ccr) %{
+ predicate(!(UseG1GC && n->as_LoadStore()->barrier_data() != 0));
match(Set res (GetAndSetP mem newval));
effect(KILL ccr, TEMP tmp, TEMP res);
size(16);
diff --git a/src/hotspot/cpu/arm/assembler_arm_32.hpp b/src/hotspot/cpu/arm/assembler_arm_32.hpp
index dd04ad1ab3a3c..e53eefac097ef 100644
--- a/src/hotspot/cpu/arm/assembler_arm_32.hpp
+++ b/src/hotspot/cpu/arm/assembler_arm_32.hpp
@@ -119,8 +119,9 @@ class RegisterSet {
}
friend RegisterSet operator | (const RegisterSet set1, const RegisterSet set2) {
- assert((set1._encoding & set2._encoding) == 0,
- "encoding constraint");
+// why so strong constraint?
+// assert((set1._encoding & set2._encoding) == 0,
+// "encoding constraint");
return RegisterSet(set1._encoding | set2._encoding);
}
@@ -142,6 +143,11 @@ class RegisterSet {
}
return count;
}
+
+ static RegisterSet from(RegSet set) {
+ assert(set.size(), "RegSet must not be empty");
+ return RegisterSet(set.bits());
+ }
};
#if R9_IS_SCRATCHED
@@ -157,6 +163,10 @@ class FloatRegisterSet {
public:
+ FloatRegisterSet() {
+ _encoding = 0;
+ }
+
FloatRegisterSet(FloatRegister reg) {
if (reg->hi_bit() == 0) {
_encoding = reg->hi_bits() << 12 | reg->lo_bit() << 22 | 1;
@@ -185,6 +195,15 @@ class FloatRegisterSet {
return (_encoding & 0xFFFFFF00) | ((_encoding & 0xFF) << 1);
}
+ static FloatRegisterSet from(FloatRegSet set) {
+ assert(set.size(), "FloatRegSet must not be empty");
+ // the vector load/store instructions operate on a set of consecutive registers.
+ // for the sake of simplicity, write all registers between the first and last in the set
+ size_t range = (*set.rbegin())->encoding() - (*set.begin())->encoding() + 1;
+ // push_float stores float regisgters by pairs
+ return FloatRegisterSet(*set.begin(), (range+1)/2);
+ }
+
};
diff --git a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
index bb6a93e6f8da7..b14e6f0b4ca0c 100644
--- a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
@@ -948,6 +948,7 @@ void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
if (op->init_check()) {
Register tmp = op->tmp1()->as_register();
__ ldrb(tmp, Address(op->klass()->as_register(), InstanceKlass::init_state_offset()));
+ __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadLoad | MacroAssembler::LoadStore), Rtemp);
add_debug_info_for_null_check_here(op->stub()->info());
__ cmp(tmp, InstanceKlass::fully_initialized);
__ b(*op->stub()->entry(), ne);
diff --git a/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp b/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp
index 3c5e29aa8710f..56ae7707fbf38 100644
--- a/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp
@@ -39,8 +39,10 @@
#include "c1/c1_LIRAssembler.hpp"
#include "c1/c1_MacroAssembler.hpp"
#include "gc/g1/c1/g1BarrierSetC1.hpp"
-#endif
-
+#endif // COMPILER1
+#ifdef COMPILER2
+#include "gc/g1/c2/g1BarrierSetC2.hpp"
+#endif // COMPILER2
#define __ masm->
#ifdef PRODUCT
@@ -106,70 +108,87 @@ void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* mas
#endif // !R9_IS_SCRATCHED
}
-// G1 pre-barrier.
-// Blows all volatile registers R0-R3, Rtemp, LR).
-// If store_addr != noreg, then previous value is loaded from [store_addr];
-// in such case store_addr and new_val registers are preserved;
-// otherwise pre_val register is preserved.
-void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
- Register store_addr,
- Register new_val,
- Register pre_val,
- Register tmp1,
- Register tmp2) {
- Label done;
- Label runtime;
-
- if (store_addr != noreg) {
- assert_different_registers(store_addr, new_val, pre_val, tmp1, tmp2, noreg);
- } else {
- assert (new_val == noreg, "should be");
- assert_different_registers(pre_val, tmp1, tmp2, noreg);
- }
-
- Address in_progress(Rthread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
- Address index(Rthread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
- Address buffer(Rthread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
+static void generate_queue_test_and_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
+ const Register thread, const Register value, const Register temp1, const Register temp2) {
+ assert_different_registers(value, temp1, temp2);
+ // Can we store original value in the thread's buffer?
+ // (The index field is typed as size_t.)
+ __ ldr(temp1, Address(thread, in_bytes(index_offset))); // temp1 := *(index address)
+ __ cbz(temp1, runtime); // jump to runtime if index == 0 (full buffer)
+ // The buffer is not full, store value into it.
+ __ sub(temp1, temp1, wordSize); // temp1 := next index
+ __ str(temp1, Address(thread, in_bytes(index_offset))); // *(index address) := next index
+ __ ldr(temp2, Address(thread, in_bytes(buffer_offset))); // temp2 := buffer address
+ // Record the previous value
+ __ str(value, Address(temp2, temp1)); // *(buffer address + next index) := value
+ }
+static void generate_pre_barrier_fast_path(MacroAssembler* masm,
+ const Register thread,
+ const Register tmp1) {
+ Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
// Is marking active?
assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "adjust this code");
__ ldrb(tmp1, in_progress);
- __ cbz(tmp1, done);
+}
+static void generate_pre_barrier_slow_path(MacroAssembler* masm,
+ const Register obj,
+ const Register pre_val,
+ const Register thread,
+ const Register tmp1,
+ const Register tmp2,
+ Label& done,
+ Label& runtime) {
// Do we need to load the previous value?
- if (store_addr != noreg) {
- __ load_heap_oop(pre_val, Address(store_addr, 0));
+ if (obj != noreg) {
+ __ load_heap_oop(pre_val, Address(obj, 0));
}
// Is the previous value null?
__ cbz(pre_val, done);
- // Can we store original value in the thread's buffer?
- // Is index == 0?
- // (The index field is typed as size_t.)
+ generate_queue_test_and_insertion(masm,
+ G1ThreadLocalData::satb_mark_queue_index_offset(),
+ G1ThreadLocalData::satb_mark_queue_buffer_offset(),
+ runtime,
+ thread, pre_val, tmp1, tmp2);
+ __ b(done);
+}
- __ ldr(tmp1, index); // tmp1 := *index_adr
- __ ldr(tmp2, buffer);
+// G1 pre-barrier.
+// Blows all volatile registers R0-R3, LR).
+// If obj != noreg, then previous value is loaded from [obj];
+// in such case obj and pre_val registers is preserved;
+// otherwise pre_val register is preserved.
+void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register tmp1,
+ Register tmp2) {
+ Label done;
+ Label runtime;
- __ subs(tmp1, tmp1, wordSize); // tmp1 := tmp1 - wordSize
- __ b(runtime, lt); // If negative, goto runtime
+ assert_different_registers(obj, pre_val, tmp1, tmp2, noreg);
- __ str(tmp1, index); // *index_adr := tmp1
+ generate_pre_barrier_fast_path(masm, Rthread, tmp1);
+ // If marking is not active (*(mark queue active address) == 0), jump to done
+ __ cbz(tmp1, done);
- // Record the previous value
- __ str(pre_val, Address(tmp2, tmp1));
- __ b(done);
+ generate_pre_barrier_slow_path(masm, obj, pre_val, Rthread, tmp1, tmp2, done, runtime);
__ bind(runtime);
// save the live input values
- if (store_addr != noreg) {
- // avoid raw_push to support any ordering of store_addr and new_val
- __ push(RegisterSet(store_addr) | RegisterSet(new_val));
- } else {
- __ push(pre_val);
+ RegisterSet set = RegisterSet(pre_val) | RegisterSet(R0, R3) | RegisterSet(R12);
+ // save the live input values
+ if (obj != noreg) {
+ // avoid raw_push to support any ordering of store_addr and pre_val
+ set = set | RegisterSet(obj);
}
+ __ push(set);
+
if (pre_val != R0) {
__ mov(R0, pre_val);
}
@@ -177,33 +196,17 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
__ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), R0, R1);
- if (store_addr != noreg) {
- __ pop(RegisterSet(store_addr) | RegisterSet(new_val));
- } else {
- __ pop(pre_val);
- }
-
+ __ pop(set);
__ bind(done);
}
-// G1 post-barrier.
-// Blows all volatile registers R0-R3, Rtemp, LR).
-void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
- Register store_addr,
- Register new_val,
- Register tmp1,
- Register tmp2,
- Register tmp3) {
-
- Address queue_index(Rthread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
- Address buffer(Rthread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
-
- BarrierSet* bs = BarrierSet::barrier_set();
- CardTableBarrierSet* ctbs = barrier_set_cast(bs);
- CardTable* ct = ctbs->card_table();
- Label done;
- Label runtime;
-
+static void generate_post_barrier_fast_path(MacroAssembler* masm,
+ const Register store_addr,
+ const Register new_val,
+ const Register tmp1,
+ const Register tmp2,
+ Label& done,
+ bool new_val_may_be_null) {
// Does store cross heap regions?
__ eor(tmp1, store_addr, new_val);
@@ -211,22 +214,31 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
__ b(done, eq);
// crosses regions, storing null?
-
- __ cbz(new_val, done);
-
+ if (new_val_may_be_null) {
+ __ cbz(new_val, done);
+ }
// storing region crossing non-null, is card already dirty?
const Register card_addr = tmp1;
- __ mov_address(tmp2, (address)ct->byte_map_base());
+ CardTableBarrierSet* ct = barrier_set_cast(BarrierSet::barrier_set());
+ __ mov_address(tmp2, (address)ct->card_table()->byte_map_base());
__ add(card_addr, tmp2, AsmOperand(store_addr, lsr, CardTable::card_shift()));
__ ldrb(tmp2, Address(card_addr));
__ cmp(tmp2, (int)G1CardTable::g1_young_card_val());
- __ b(done, eq);
+}
+static void generate_post_barrier_slow_path(MacroAssembler* masm,
+ const Register thread,
+ const Register tmp1,
+ const Register tmp2,
+ const Register tmp3,
+ Label& done,
+ Label& runtime) {
__ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad), tmp2);
-
assert(CardTable::dirty_card_val() == 0, "adjust this code");
+ // card_addr is loaded by generate_post_barrier_fast_path
+ const Register card_addr = tmp1;
__ ldrb(tmp2, Address(card_addr));
__ cbz(tmp2, done);
@@ -234,29 +246,139 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
// dirty card and log.
__ strb(__ zero_register(tmp2), Address(card_addr));
+ generate_queue_test_and_insertion(masm,
+ G1ThreadLocalData::dirty_card_queue_index_offset(),
+ G1ThreadLocalData::dirty_card_queue_buffer_offset(),
+ runtime,
+ thread, card_addr, tmp2, tmp3);
+ __ b(done);
+}
- __ ldr(tmp2, queue_index);
- __ ldr(tmp3, buffer);
- __ subs(tmp2, tmp2, wordSize);
- __ b(runtime, lt); // go to runtime if now negative
-
- __ str(tmp2, queue_index);
+// G1 post-barrier.
+// Blows all volatile registers R0-R3, LR).
+void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register tmp1,
+ Register tmp2,
+ Register tmp3) {
+ Label done;
+ Label runtime;
- __ str(card_addr, Address(tmp3, tmp2));
- __ b(done);
+ generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, done, true /* new_val_may_be_null */);
+ // If card is young, jump to done
+ // card_addr and card are loaded by generate_post_barrier_fast_path
+ const Register card = tmp2;
+ const Register card_addr = tmp1;
+ __ b(done, eq);
+ generate_post_barrier_slow_path(masm, Rthread, card_addr, tmp2, tmp3, done, runtime);
__ bind(runtime);
+ RegisterSet set = RegisterSet(store_addr) | RegisterSet(R0, R3) | RegisterSet(R12);
+ __ push(set);
+
if (card_addr != R0) {
__ mov(R0, card_addr);
}
__ mov(R1, Rthread);
__ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), R0, R1);
+ __ pop(set);
+
__ bind(done);
}
+#if defined(COMPILER2)
+
+static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path, Register tmp1) {
+ SaveLiveRegisters save_registers(masm, stub);
+ if (c_rarg0 != arg) {
+ __ mov(c_rarg0, arg);
+ }
+ __ mov(c_rarg1, Rthread);
+ __ call_VM_leaf(runtime_path, R0, R1);
+}
+
+void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2,
+ G1PreBarrierStubC2* stub) {
+ assert(thread == Rthread, "must be");
+ assert_different_registers(obj, pre_val, tmp1, tmp2);
+ assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
+
+ stub->initialize_registers(obj, pre_val, thread, tmp1, tmp2);
+
+ generate_pre_barrier_fast_path(masm, thread, tmp1);
+ // If marking is active (*(mark queue active address) != 0), jump to stub (slow path)
+ __ cbnz(tmp1, *stub->entry());
+
+ __ bind(*stub->continuation());
+}
+
+void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm,
+ G1PreBarrierStubC2* stub) const {
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ Label runtime;
+ Register obj = stub->obj();
+ Register pre_val = stub->pre_val();
+ Register thread = stub->thread();
+ Register tmp1 = stub->tmp1();
+ Register tmp2 = stub->tmp2();
+
+ __ bind(*stub->entry());
+ generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp1, tmp2, *stub->continuation(), runtime);
+
+ __ bind(runtime);
+ generate_c2_barrier_runtime_call(masm, stub, pre_val, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), tmp1);
+ __ b(*stub->continuation());
+}
+
+void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2,
+ Register tmp3,
+ G1PostBarrierStubC2* stub) {
+ assert(thread == Rthread, "must be");
+ assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, noreg);
+
+ stub->initialize_registers(thread, tmp1, tmp2, tmp3);
+
+ bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
+ generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, *stub->continuation(), new_val_may_be_null);
+ // If card is not young, jump to stub (slow path)
+ __ b(*stub->entry(), ne);
+
+ __ bind(*stub->continuation());
+}
+
+void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
+ G1PostBarrierStubC2* stub) const {
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ Label runtime;
+ Register thread = stub->thread();
+ Register tmp1 = stub->tmp1(); // tmp1 holds the card address.
+ Register tmp2 = stub->tmp2();
+ Register tmp3 = stub->tmp3();
+
+ __ bind(*stub->entry());
+ generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, tmp3, *stub->continuation(), runtime);
+
+ __ bind(runtime);
+ generate_c2_barrier_runtime_call(masm, stub, tmp1, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp2);
+ __ b(*stub->continuation());
+}
+
+#endif // COMPILER2
+
void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp2, Register tmp3) {
bool on_oop = type == T_OBJECT || type == T_ARRAY;
@@ -268,7 +390,7 @@ void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorator
if (on_oop && on_reference) {
// Generate the G1 pre-barrier code to log the value of
// the referent field in an SATB buffer.
- g1_write_barrier_pre(masm, noreg, noreg, dst, tmp1, tmp2);
+ g1_write_barrier_pre(masm, noreg, dst, tmp1, tmp2);
}
}
@@ -295,7 +417,7 @@ void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet deco
}
if (needs_pre_barrier) {
- g1_write_barrier_pre(masm, store_addr, new_val, tmp1, tmp2, tmp3);
+ g1_write_barrier_pre(masm, store_addr, tmp3 /*pre_val*/, tmp1, tmp2);
}
if (is_null) {
diff --git a/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.hpp b/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.hpp
index 52932faa3e4de..aefde19142e40 100644
--- a/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.hpp
+++ b/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.hpp
@@ -33,6 +33,8 @@ class LIR_Assembler;
class StubAssembler;
class G1PreBarrierStub;
class G1PostBarrierStub;
+class G1PreBarrierStubC2;
+class G1PostBarrierStubC2;
class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
protected:
@@ -43,7 +45,6 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
void g1_write_barrier_pre(MacroAssembler* masm,
Register store_addr,
- Register new_val,
Register pre_val,
Register tmp1,
Register tmp2);
@@ -70,6 +71,29 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm);
#endif
+
+#ifdef COMPILER2
+ void g1_write_barrier_pre_c2(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2,
+ G1PreBarrierStubC2* c2_stub);
+ void generate_c2_pre_barrier_stub(MacroAssembler* masm,
+ G1PreBarrierStubC2* stub) const;
+ void g1_write_barrier_post_c2(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2,
+ Register tmp3,
+ G1PostBarrierStubC2* c2_stub);
+ void generate_c2_post_barrier_stub(MacroAssembler* masm,
+ G1PostBarrierStubC2* stub) const;
+#endif
+
};
#endif // CPU_ARM_GC_G1_G1BARRIERSETASSEMBLER_ARM_HPP
diff --git a/src/hotspot/cpu/arm/gc/g1/g1_arm.ad b/src/hotspot/cpu/arm/gc/g1/g1_arm.ad
new file mode 100644
index 0000000000000..8a0a9e1aa531a
--- /dev/null
+++ b/src/hotspot/cpu/arm/gc/g1/g1_arm.ad
@@ -0,0 +1,201 @@
+//
+// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+
+source_hpp %{
+
+#include "gc/g1/c2/g1BarrierSetC2.hpp"
+#include "gc/shared/gc_globals.hpp"
+
+%}
+
+source %{
+
+#include "gc/g1/g1BarrierSetAssembler_arm.hpp"
+#include "gc/g1/g1BarrierSetRuntime.hpp"
+
+static void write_barrier_pre(MacroAssembler* masm,
+ const MachNode* node,
+ Register obj,
+ Register pre_val,
+ Register tmp1,
+ Register tmp2,
+ RegSet preserve = RegSet(),
+ RegSet no_preserve = RegSet()) {
+ if (!G1PreBarrierStubC2::needs_barrier(node)) {
+ return;
+ }
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
+ G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
+ for (RegSetIterator reg = preserve.begin(); *reg != noreg; ++reg) {
+ stub->preserve(*reg);
+ }
+ for (RegSetIterator reg = no_preserve.begin(); *reg != noreg; ++reg) {
+ stub->dont_preserve(*reg);
+ }
+ g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, Rthread, tmp1, tmp2, stub);
+}
+
+static void write_barrier_post(MacroAssembler* masm,
+ const MachNode* node,
+ Register store_addr,
+ Register new_val,
+ Register tmp1,
+ Register tmp2,
+ Register tmp3) {
+ if (!G1PostBarrierStubC2::needs_barrier(node)) {
+ return;
+ }
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
+ G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
+ g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, Rthread, tmp1, tmp2, tmp3, stub);
+}
+
+%}
+
+instruct g1StoreP(indirect mem, iRegP src, iRegP tmp1, iRegP tmp2, iRegP tmp3, flagsReg icc)
+%{
+ predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreP mem src));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL icc);
+ ins_cost(2 * (MEMORY_REF_COST + BRANCH_COST));
+ format %{ "sd $src, $mem\t# ptr" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $src$$Register) /* preserve */);
+ __ str($src$$Register, Address($mem$$Register));
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $src$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ $tmp3$$Register /* tmp3 */);
+ %}
+ ins_pipe(istore_mem_reg);
+%}
+
+instruct g1CompareAndSwapP(iRegI res, indirect mem, iRegP newval, iRegP tmp1, iRegP tmp2, iRegP tmp3, iRegP oldval, flagsReg ccr )
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+ effect(KILL ccr, TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3);
+ ins_cost(4 * (MEMORY_REF_COST + BRANCH_COST));
+ format %{ "loop: \n\t"
+ "LDREX $tmp1, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
+ "CMP $tmp1, $oldval\n\t"
+ "STREX.eq $tmp1, $newval, $mem\n\t"
+ "MOV.ne $tmp1, 0 \n\t"
+ "EORS.eq $tmp1,$tmp1, 1 \n\t"
+ "B.eq loop \n\t"
+ "MOV $res, $tmp1" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $oldval$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ Label loop;
+ __ bind(loop);
+ __ ldrex($tmp1$$Register,$mem$$Address);
+ __ cmp($tmp1$$Register, $oldval$$Register);
+ __ strex($tmp1$$Register, $newval$$Register, $mem$$Address, eq);
+ __ mov($tmp1$$Register, 0, ne);
+ __ eors($tmp1$$Register, $tmp1$$Register, 1, eq);
+ __ b(loop, eq);
+ __ mov($res$$Register, $tmp1$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $newval$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ $tmp3$$Register /* tmp3 */);
+ %}
+ ins_pipe(long_memory_op);
+%}
+
+
+instruct g1GetAndSetP(indirect mem, iRegP newval, iRegP tmp1, iRegP tmp2, iRegP tmp3, iRegP preval, flagsReg ccr)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set preval (GetAndSetP mem newval));
+ effect(KILL ccr, TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
+ ins_cost(4 * (MEMORY_REF_COST + BRANCH_COST));
+ format %{ "loop: \n\t"
+ "LDREX $preval, $mem\n\t"
+ "STREX $tmp1, $newval, $mem\n\t"
+ "CMP $tmp1, 0 \n\t"
+ "B.ne loop \n\t" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ assert_different_registers($mem$$Register, $newval$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $preval$$Register /* pre_val (as a temporary register) */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
+ Label loop;
+ __ bind(loop);
+ __ ldrex($preval$$Register,$mem$$Address);
+ __ strex($tmp1$$Register, $newval$$Register, $mem$$Address);
+ __ cmp($tmp1$$Register, 0);
+ __ b(loop, ne);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $newval$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ $tmp3$$Register /* tmp3 */);
+ %}
+ ins_pipe(long_memory_op);
+%}
+
+instruct g1LoadP(iRegP dst, indirect mem, iRegP tmp1, iRegP tmp2, flagsReg icc)
+%{
+ predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
+ match(Set dst (LoadP mem));
+ effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL icc);
+ ins_cost(MEMORY_REF_COST + BRANCH_COST);
+ format %{ "ld $dst, $mem\t# ptr" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ __ ldr($dst$$Register, Address($mem$$Register));
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $dst$$Register /* pre_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(iload_mem);
+%}
diff --git a/src/hotspot/cpu/arm/gc/shared/barrierSetAssembler_arm.cpp b/src/hotspot/cpu/arm/gc/shared/barrierSetAssembler_arm.cpp
index ea19730673cb6..c13a259a1b960 100644
--- a/src/hotspot/cpu/arm/gc/shared/barrierSetAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/gc/shared/barrierSetAssembler_arm.cpp
@@ -31,6 +31,10 @@
#include "runtime/javaThread.hpp"
#include "runtime/stubRoutines.hpp"
+#ifdef COMPILER2
+#include "gc/shared/c2/barrierSetC2.hpp"
+#endif // COMPILER2
+
#define __ masm->
void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
@@ -206,7 +210,57 @@ void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) {
#ifdef COMPILER2
OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
- Unimplemented(); // This must be implemented to support late barrier expansion.
+ if (!OptoReg::is_reg(opto_reg)) {
+ return OptoReg::Bad;
+ }
+
+ const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+ if (!vm_reg->is_valid()){
+ // skip APSR and FPSCR
+ return OptoReg::Bad;
+ }
+
+ return opto_reg;
}
+void SaveLiveRegisters::initialize(BarrierStubC2* stub) {
+ // Record registers that needs to be saved/restored
+ RegMaskIterator rmi(stub->preserve_set());
+ while (rmi.has_next()) {
+ const OptoReg::Name opto_reg = rmi.next();
+ if (OptoReg::is_reg(opto_reg)) {
+ const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+ if (vm_reg->is_Register()) {
+ gp_regs += RegSet::of(vm_reg->as_Register());
+ } else if (vm_reg->is_FloatRegister()) {
+ fp_regs += FloatRegSet::of(vm_reg->as_FloatRegister());
+ } else {
+ fatal("Unknown register type");
+ }
+ }
+ }
+ // Remove C-ABI SOE registers that will be updated
+ gp_regs -= RegSet::range(R4, R11) + RegSet::of(R13, R15);
+
+ // Remove C-ABI SOE fp registers
+ fp_regs -= FloatRegSet::range(S16, S31);
+}
+
+SaveLiveRegisters::SaveLiveRegisters(MacroAssembler* masm, BarrierStubC2* stub)
+ : masm(masm),
+ gp_regs(),
+ fp_regs() {
+ // Figure out what registers to save/restore
+ initialize(stub);
+
+ // Save registers
+ if (gp_regs.size() > 0) __ push(RegisterSet::from(gp_regs));
+ if (fp_regs.size() > 0) __ fpush(FloatRegisterSet::from(fp_regs));
+}
+
+SaveLiveRegisters::~SaveLiveRegisters() {
+ // Restore registers
+ if (fp_regs.size() > 0) __ fpop(FloatRegisterSet::from(fp_regs));
+ if (gp_regs.size() > 0) __ pop(RegisterSet::from(gp_regs));
+}
#endif // COMPILER2
diff --git a/src/hotspot/cpu/arm/gc/shared/barrierSetAssembler_arm.hpp b/src/hotspot/cpu/arm/gc/shared/barrierSetAssembler_arm.hpp
index 60021390ea26f..054d172f46340 100644
--- a/src/hotspot/cpu/arm/gc/shared/barrierSetAssembler_arm.hpp
+++ b/src/hotspot/cpu/arm/gc/shared/barrierSetAssembler_arm.hpp
@@ -31,7 +31,9 @@
#ifdef COMPILER2
#include "code/vmreg.hpp"
#include "opto/optoreg.hpp"
+#include "opto/regmask.hpp"
+class BarrierStubC2;
class Node;
#endif // COMPILER2
@@ -69,4 +71,26 @@ class BarrierSetAssembler: public CHeapObj {
#endif // COMPILER2
};
+#ifdef COMPILER2
+// This class saves and restores the registers that need to be preserved across
+// the runtime call represented by a given C2 barrier stub. Use as follows:
+// {
+// SaveLiveRegisters save(masm, stub);
+// ..
+// __ bl(...);
+// ..
+// }
+class SaveLiveRegisters {
+private:
+ MacroAssembler* const masm;
+ RegSet gp_regs;
+ FloatRegSet fp_regs;
+
+public:
+ void initialize(BarrierStubC2* stub);
+ SaveLiveRegisters(MacroAssembler* masm, BarrierStubC2* stub);
+ ~SaveLiveRegisters();
+};
+
+#endif // COMPILER2
#endif // CPU_ARM_GC_SHARED_BARRIERSETASSEMBLER_ARM_HPP
diff --git a/src/hotspot/cpu/arm/globals_arm.hpp b/src/hotspot/cpu/arm/globals_arm.hpp
index 084d10beea1d8..9c4b8500e1873 100644
--- a/src/hotspot/cpu/arm/globals_arm.hpp
+++ b/src/hotspot/cpu/arm/globals_arm.hpp
@@ -36,7 +36,7 @@ define_pd_global(bool, TrapBasedNullChecks, false); // Not needed
define_pd_global(bool, DelayCompilerStubsGeneration, false); // No need - only few compiler's stubs
-define_pd_global(uintx, CodeCacheSegmentSize, 64 COMPILER1_AND_COMPILER2_PRESENT(+64)); // Tiered compilation has large code-entry alignment.
+define_pd_global(uintx, CodeCacheSegmentSize, 64);
define_pd_global(intx, CodeEntryAlignment, 16);
define_pd_global(intx, OptoLoopAlignment, 16);
diff --git a/src/hotspot/cpu/arm/register_arm.hpp b/src/hotspot/cpu/arm/register_arm.hpp
index 9f486d2a62586..d8961fd293578 100644
--- a/src/hotspot/cpu/arm/register_arm.hpp
+++ b/src/hotspot/cpu/arm/register_arm.hpp
@@ -303,6 +303,31 @@ class ConcreteRegisterImpl : public AbstractRegisterImpl {
static const int max_fpr;
};
+typedef AbstractRegSet RegSet;
+typedef AbstractRegSet FloatRegSet;
+
+template <>
+inline Register AbstractRegSet::first() {
+ if (_bitset == 0) { return noreg; }
+ return as_Register(count_trailing_zeros(_bitset));
+}
+
+
+template <>
+inline FloatRegister AbstractRegSet::first() {
+ uint32_t first = _bitset & -_bitset;
+ return first ? as_FloatRegister(exact_log2(first)) : fnoreg;
+}
+
+template <>
+inline FloatRegister AbstractRegSet::last() {
+ if (_bitset == 0) { return fnoreg; }
+ int last = max_size() - 1 - count_leading_zeros(_bitset);
+ return as_FloatRegister(last);
+}
+
+
+
class VFPSystemRegisterImpl;
typedef VFPSystemRegisterImpl* VFPSystemRegister;
class VFPSystemRegisterImpl : public AbstractRegisterImpl {
diff --git a/src/hotspot/cpu/arm/templateTable_arm.cpp b/src/hotspot/cpu/arm/templateTable_arm.cpp
index 80519fd89f426..0974ff1f9a9c3 100644
--- a/src/hotspot/cpu/arm/templateTable_arm.cpp
+++ b/src/hotspot/cpu/arm/templateTable_arm.cpp
@@ -3974,6 +3974,7 @@ void TemplateTable::_new() {
// make sure klass is initialized
// make sure klass is fully initialized
__ ldrb(Rtemp, Address(Rklass, InstanceKlass::init_state_offset()));
+ __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadLoad | MacroAssembler::LoadStore), Rtemp);
__ cmp(Rtemp, InstanceKlass::fully_initialized);
__ b(slow_case, ne);
diff --git a/src/hotspot/cpu/arm/upcallLinker_arm.cpp b/src/hotspot/cpu/arm/upcallLinker_arm.cpp
index c7645f4a03351..696b2001e6b7b 100644
--- a/src/hotspot/cpu/arm/upcallLinker_arm.cpp
+++ b/src/hotspot/cpu/arm/upcallLinker_arm.cpp
@@ -25,7 +25,7 @@
#include "prims/upcallLinker.hpp"
#include "utilities/debug.hpp"
-address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
+address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature,
BasicType* out_sig_bt, int total_out_args,
BasicType ret_type,
jobject jabi, jobject jconv,
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.hpp b/src/hotspot/cpu/ppc/assembler_ppc.hpp
index d445108098b86..b2711ac43b0b0 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.hpp
@@ -2502,19 +2502,19 @@ class Assembler : public AbstractAssembler {
// load the constant are emitted beforehand. Store instructions need a
// tmp reg if the constant is not encodable as immediate.
// Size unpredictable.
- void ld( Register d, RegisterOrConstant roc, Register s1 = noreg);
- void lwa( Register d, RegisterOrConstant roc, Register s1 = noreg);
- void lwz( Register d, RegisterOrConstant roc, Register s1 = noreg);
- void lha( Register d, RegisterOrConstant roc, Register s1 = noreg);
- void lhz( Register d, RegisterOrConstant roc, Register s1 = noreg);
- void lbz( Register d, RegisterOrConstant roc, Register s1 = noreg);
- void std( Register d, RegisterOrConstant roc, Register s1 = noreg, Register tmp = noreg);
- void stw( Register d, RegisterOrConstant roc, Register s1 = noreg, Register tmp = noreg);
- void sth( Register d, RegisterOrConstant roc, Register s1 = noreg, Register tmp = noreg);
- void stb( Register d, RegisterOrConstant roc, Register s1 = noreg, Register tmp = noreg);
- void add( Register d, Register s, RegisterOrConstant roc);
- void add( Register d, RegisterOrConstant roc, Register s) { add(d, s, roc); }
- void sub( Register d, Register s, RegisterOrConstant roc);
+ void ld( Register d, RegisterOrConstant roc, Register s1 = noreg);
+ void lwa(Register d, RegisterOrConstant roc, Register s1 = noreg);
+ void lwz(Register d, RegisterOrConstant roc, Register s1 = noreg);
+ void lha(Register d, RegisterOrConstant roc, Register s1 = noreg);
+ void lhz(Register d, RegisterOrConstant roc, Register s1 = noreg);
+ void lbz(Register d, RegisterOrConstant roc, Register s1 = noreg);
+ void std(Register d, RegisterOrConstant roc, Register s1 = noreg, Register tmp = noreg);
+ void stw(Register d, RegisterOrConstant roc, Register s1 = noreg, Register tmp = noreg);
+ void sth(Register d, RegisterOrConstant roc, Register s1 = noreg, Register tmp = noreg);
+ void stb(Register d, RegisterOrConstant roc, Register s1 = noreg, Register tmp = noreg);
+ void add(Register d, Register s, RegisterOrConstant roc);
+ void add(Register d, RegisterOrConstant roc, Register s) { add(d, s, roc); }
+ void sub(Register d, Register s, RegisterOrConstant roc);
void xorr(Register d, Register s, RegisterOrConstant roc);
void xorr(Register d, RegisterOrConstant roc, Register s) { xorr(d, s, roc); }
void cmpw(ConditionRegister d, Register s, RegisterOrConstant roc);
@@ -2522,6 +2522,17 @@ class Assembler : public AbstractAssembler {
// Load pointer d from s1+roc.
void ld_ptr(Register d, RegisterOrConstant roc, Register s1 = noreg) { ld(d, roc, s1); }
+ void ld( Register d, Address &a);
+ void lwa(Register d, Address &a);
+ void lwz(Register d, Address &a);
+ void lha(Register d, Address &a);
+ void lhz(Register d, Address &a);
+ void lbz(Register d, Address &a);
+ void std(Register d, Address &a, Register tmp = noreg);
+ void stw(Register d, Address &a, Register tmp = noreg);
+ void sth(Register d, Address &a, Register tmp = noreg);
+ void stb(Register d, Address &a, Register tmp = noreg);
+
// Emit several instructions to load a 64 bit constant. This issues a fixed
// instruction pattern so that the constant can be patched later on.
enum {
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
index 98c8b629844c9..b0eaaccf0d097 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
@@ -338,28 +338,46 @@ inline void Assembler::insrwi( Register a, Register s, int n, int b)
// PPC 1, section 3.3.2 Fixed-Point Load Instructions
inline void Assembler::lwzx( Register d, Register s1, Register s2) { emit_int32(LWZX_OPCODE | rt(d) | ra0mem(s1) | rb(s2));}
+inline void Assembler::lwz( Register d, Address &a) {
+ lwz(d, a.index() != noreg ? RegisterOrConstant(a.index()) : RegisterOrConstant(a.disp()), a.base());
+}
inline void Assembler::lwz( Register d, int si16, Register s1) { emit_int32(LWZ_OPCODE | rt(d) | d1(si16) | ra0mem(s1));}
inline void Assembler::lwzu( Register d, int si16, Register s1) { assert(d != s1, "according to ibm manual"); emit_int32(LWZU_OPCODE | rt(d) | d1(si16) | rta0mem(s1));}
inline void Assembler::lwax( Register d, Register s1, Register s2) { emit_int32(LWAX_OPCODE | rt(d) | ra0mem(s1) | rb(s2));}
+inline void Assembler::lwa( Register d, Address &a) {
+ lwa(d, a.index() != noreg ? RegisterOrConstant(a.index()) : RegisterOrConstant(a.disp()), a.base());
+}
inline void Assembler::lwa( Register d, int si16, Register s1) { emit_int32(LWA_OPCODE | rt(d) | ds(si16) | ra0mem(s1));}
inline void Assembler::lwbrx( Register d, Register s1, Register s2) { emit_int32(LWBRX_OPCODE | rt(d) | ra0mem(s1) | rb(s2));}
inline void Assembler::lhzx( Register d, Register s1, Register s2) { emit_int32(LHZX_OPCODE | rt(d) | ra0mem(s1) | rb(s2));}
+inline void Assembler::lhz( Register d, Address &a) {
+ lhz(d, a.index() != noreg ? RegisterOrConstant(a.index()) : RegisterOrConstant(a.disp()), a.base());
+}
inline void Assembler::lhz( Register d, int si16, Register s1) { emit_int32(LHZ_OPCODE | rt(d) | d1(si16) | ra0mem(s1));}
inline void Assembler::lhzu( Register d, int si16, Register s1) { assert(d != s1, "according to ibm manual"); emit_int32(LHZU_OPCODE | rt(d) | d1(si16) | rta0mem(s1));}
inline void Assembler::lhbrx( Register d, Register s1, Register s2) { emit_int32(LHBRX_OPCODE | rt(d) | ra0mem(s1) | rb(s2));}
inline void Assembler::lhax( Register d, Register s1, Register s2) { emit_int32(LHAX_OPCODE | rt(d) | ra0mem(s1) | rb(s2));}
+inline void Assembler::lha( Register d, Address &a) {
+ lha(d, a.index() != noreg ? RegisterOrConstant(a.index()) : RegisterOrConstant(a.disp()), a.base());
+}
inline void Assembler::lha( Register d, int si16, Register s1) { emit_int32(LHA_OPCODE | rt(d) | d1(si16) | ra0mem(s1));}
inline void Assembler::lhau( Register d, int si16, Register s1) { assert(d != s1, "according to ibm manual"); emit_int32(LHAU_OPCODE | rt(d) | d1(si16) | rta0mem(s1));}
inline void Assembler::lbzx( Register d, Register s1, Register s2) { emit_int32(LBZX_OPCODE | rt(d) | ra0mem(s1) | rb(s2));}
+inline void Assembler::lbz( Register d, Address &a) {
+ lbz(d, a.index() != noreg ? RegisterOrConstant(a.index()) : RegisterOrConstant(a.disp()), a.base());
+}
inline void Assembler::lbz( Register d, int si16, Register s1) { emit_int32(LBZ_OPCODE | rt(d) | d1(si16) | ra0mem(s1));}
inline void Assembler::lbzu( Register d, int si16, Register s1) { assert(d != s1, "according to ibm manual"); emit_int32(LBZU_OPCODE | rt(d) | d1(si16) | rta0mem(s1));}
+inline void Assembler::ld( Register d, Address &a) {
+ ld(d, a.index() != noreg ? RegisterOrConstant(a.index()) : RegisterOrConstant(a.disp()), a.base());
+}
inline void Assembler::ld( Register d, int si16, Register s1) { emit_int32(LD_OPCODE | rt(d) | ds(si16) | ra0mem(s1));}
inline void Assembler::ld( Register d, ByteSize si16, Register s1) { assert(in_bytes(si16) < 0x7fff, "overflow"); ld(d, in_bytes(si16), s1); }
inline void Assembler::ldx( Register d, Register s1, Register s2) { emit_int32(LDX_OPCODE | rt(d) | ra0mem(s1) | rb(s2));}
@@ -371,19 +389,31 @@ inline void Assembler::ld_ptr(Register d, ByteSize b, Register s1) { ld(d, in_by
// PPC 1, section 3.3.3 Fixed-Point Store Instructions
inline void Assembler::stwx( Register d, Register s1, Register s2) { emit_int32(STWX_OPCODE | rs(d) | ra0mem(s1) | rb(s2));}
+inline void Assembler::stw( Register d, Address &a, Register tmp) {
+ stw(d, a.index() != noreg ? RegisterOrConstant(a.index()) : RegisterOrConstant(a.disp()), a.base(), tmp);
+}
inline void Assembler::stw( Register d, int si16, Register s1) { emit_int32(STW_OPCODE | rs(d) | d1(si16) | ra0mem(s1));}
inline void Assembler::stwu( Register d, int si16, Register s1) { emit_int32(STWU_OPCODE | rs(d) | d1(si16) | rta0mem(s1));}
inline void Assembler::stwbrx( Register d, Register s1, Register s2) { emit_int32(STWBRX_OPCODE | rs(d) | ra0mem(s1) | rb(s2));}
inline void Assembler::sthx( Register d, Register s1, Register s2) { emit_int32(STHX_OPCODE | rs(d) | ra0mem(s1) | rb(s2));}
+inline void Assembler::sth( Register d, Address &a, Register tmp) {
+ sth(d, a.index() != noreg ? RegisterOrConstant(a.index()) : RegisterOrConstant(a.disp()), a.base(), tmp);
+}
inline void Assembler::sth( Register d, int si16, Register s1) { emit_int32(STH_OPCODE | rs(d) | d1(si16) | ra0mem(s1));}
inline void Assembler::sthu( Register d, int si16, Register s1) { emit_int32(STHU_OPCODE | rs(d) | d1(si16) | rta0mem(s1));}
inline void Assembler::sthbrx( Register d, Register s1, Register s2) { emit_int32(STHBRX_OPCODE | rs(d) | ra0mem(s1) | rb(s2));}
inline void Assembler::stbx( Register d, Register s1, Register s2) { emit_int32(STBX_OPCODE | rs(d) | ra0mem(s1) | rb(s2));}
+inline void Assembler::stb( Register d, Address &a, Register tmp) {
+ stb(d, a.index() != noreg ? RegisterOrConstant(a.index()) : RegisterOrConstant(a.disp()), a.base(), tmp);
+}
inline void Assembler::stb( Register d, int si16, Register s1) { emit_int32(STB_OPCODE | rs(d) | d1(si16) | ra0mem(s1));}
inline void Assembler::stbu( Register d, int si16, Register s1) { emit_int32(STBU_OPCODE | rs(d) | d1(si16) | rta0mem(s1));}
+inline void Assembler::std( Register d, Address &a, Register tmp) {
+ std(d, a.index() != noreg ? RegisterOrConstant(a.index()) : RegisterOrConstant(a.disp()), a.base(), tmp);
+}
inline void Assembler::std( Register d, int si16, Register s1) { emit_int32(STD_OPCODE | rs(d) | ds(si16) | ra0mem(s1));}
inline void Assembler::stdx( Register d, Register s1, Register s2) { emit_int32(STDX_OPCODE | rs(d) | ra0mem(s1) | rb(s2));}
inline void Assembler::stdu( Register d, int si16, Register s1) { emit_int32(STDU_OPCODE | rs(d) | ds(si16) | rta0mem(s1));}
diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
index 42934dc7c3179..36e1ac82334fd 100644
--- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
@@ -133,9 +133,20 @@ void LIR_Assembler::osr_entry() {
// copied into place by code emitted in the IR.
Register OSR_buf = osrBufferPointer()->as_register();
- { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
- int monitor_offset = BytesPerWord * method()->max_locals() +
- (2 * BytesPerWord) * (number_of_locks - 1);
+ {
+ assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
+
+ const int locals_space = BytesPerWord * method()->max_locals();
+ int monitor_offset = locals_space + (2 * BytesPerWord) * (number_of_locks - 1);
+ bool use_OSR_bias = false;
+
+ if (!Assembler::is_simm16(monitor_offset + BytesPerWord) && number_of_locks > 0) {
+ // Offsets too large for ld instructions. Use bias.
+ __ add_const_optimized(OSR_buf, OSR_buf, locals_space);
+ monitor_offset -= locals_space;
+ use_OSR_bias = true;
+ }
+
// SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
// the OSR buffer using 2 word entries: first the lock and then
// the oop.
@@ -157,9 +168,14 @@ void LIR_Assembler::osr_entry() {
mo = frame_map()->address_for_monitor_object(i);
assert(ml.index() == noreg && mo.index() == noreg, "sanity");
__ ld(R0, slot_offset + 0, OSR_buf);
- __ std(R0, ml.disp(), ml.base());
+ __ std(R0, ml);
__ ld(R0, slot_offset + 1*BytesPerWord, OSR_buf);
- __ std(R0, mo.disp(), mo.base());
+ __ std(R0, mo);
+ }
+
+ if (use_OSR_bias) {
+ // Restore.
+ __ sub_const_optimized(OSR_buf, OSR_buf, locals_space);
}
}
}
@@ -213,7 +229,11 @@ int LIR_Assembler::emit_unwind_handler() {
if (method()->is_synchronized()) {
monitor_address(0, FrameMap::R4_opr);
stub = new MonitorExitStub(FrameMap::R4_opr, true, 0);
- __ unlock_object(R5, R6, R4, *stub->entry());
+ if (LockingMode == LM_MONITOR) {
+ __ b(*stub->entry());
+ } else {
+ __ unlock_object(R5, R6, R4, *stub->entry());
+ }
__ bind(*stub->continuation());
}
@@ -581,7 +601,7 @@ void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
__ fcmpu(CCR0, rsrc, rsrc);
if (dst_in_memory) {
__ li(R0, 0); // 0 in case of NAN
- __ std(R0, addr.disp(), addr.base());
+ __ std(R0, addr);
} else {
__ li(dst->as_register(), 0);
}
@@ -605,7 +625,7 @@ void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
__ fcmpu(CCR0, rsrc, rsrc);
if (dst_in_memory) {
__ li(R0, 0); // 0 in case of NAN
- __ std(R0, addr.disp(), addr.base());
+ __ std(R0, addr);
} else {
__ li(dst->as_register_lo(), 0);
}
@@ -873,20 +893,20 @@ void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
int value = c->as_jint_bits();
__ load_const_optimized(src_reg, value);
Address addr = frame_map()->address_for_slot(dest->single_stack_ix());
- __ stw(src_reg, addr.disp(), addr.base());
+ __ stw(src_reg, addr);
break;
}
case T_ADDRESS: {
int value = c->as_jint_bits();
__ load_const_optimized(src_reg, value);
Address addr = frame_map()->address_for_slot(dest->single_stack_ix());
- __ std(src_reg, addr.disp(), addr.base());
+ __ std(src_reg, addr);
break;
}
case T_OBJECT: {
jobject2reg(c->as_jobject(), src_reg);
Address addr = frame_map()->address_for_slot(dest->single_stack_ix());
- __ std(src_reg, addr.disp(), addr.base());
+ __ std(src_reg, addr);
break;
}
case T_LONG:
@@ -894,7 +914,7 @@ void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
int value = c->as_jlong_bits();
__ load_const_optimized(src_reg, value);
Address addr = frame_map()->address_for_double_slot(dest->double_stack_ix());
- __ std(src_reg, addr.disp(), addr.base());
+ __ std(src_reg, addr);
break;
}
default:
@@ -1070,24 +1090,24 @@ void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
case T_FLOAT: {
Address from = frame_map()->address_for_slot(src->single_stack_ix());
Address to = frame_map()->address_for_slot(dest->single_stack_ix());
- __ lwz(tmp, from.disp(), from.base());
- __ stw(tmp, to.disp(), to.base());
+ __ lwz(tmp, from);
+ __ stw(tmp, to);
break;
}
case T_ADDRESS:
case T_OBJECT: {
Address from = frame_map()->address_for_slot(src->single_stack_ix());
Address to = frame_map()->address_for_slot(dest->single_stack_ix());
- __ ld(tmp, from.disp(), from.base());
- __ std(tmp, to.disp(), to.base());
+ __ ld(tmp, from);
+ __ std(tmp, to);
break;
}
case T_LONG:
case T_DOUBLE: {
Address from = frame_map()->address_for_double_slot(src->double_stack_ix());
Address to = frame_map()->address_for_double_slot(dest->double_stack_ix());
- __ ld(tmp, from.disp(), from.base());
- __ std(tmp, to.disp(), to.base());
+ __ ld(tmp, from);
+ __ std(tmp, to);
break;
}
@@ -2274,6 +2294,7 @@ void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
}
__ lbz(op->tmp1()->as_register(),
in_bytes(InstanceKlass::init_state_offset()), op->klass()->as_register());
+ // acquire barrier included in membar_storestore() which follows the allocation immediately.
__ cmpwi(CCR0, op->tmp1()->as_register(), InstanceKlass::fully_initialized);
__ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CCR0, Assembler::equal), *op->stub()->entry());
}
diff --git a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
index 83fad376d292a..ea4d76e200fc2 100644
--- a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
@@ -114,6 +114,8 @@ void C1_MacroAssembler::lock_object(Register Rmark, Register Roop, Register Rbox
/*check without membar and ldarx first*/true);
// If compare/exchange succeeded we found an unlocked object and we now have locked it
// hence we are done.
+ } else {
+ assert(false, "Unhandled LockingMode:%d", LockingMode);
}
b(done);
@@ -168,6 +170,8 @@ void C1_MacroAssembler::unlock_object(Register Rmark, Register Roop, Register Rb
MacroAssembler::cmpxchgx_hint_release_lock(),
noreg,
&slow_int);
+ } else {
+ assert(false, "Unhandled LockingMode:%d", LockingMode);
}
b(done);
bind(slow_int);
diff --git a/src/hotspot/cpu/ppc/frame_ppc.cpp b/src/hotspot/cpu/ppc/frame_ppc.cpp
index 4c1ffeb0d768e..eb16af5e9db1b 100644
--- a/src/hotspot/cpu/ppc/frame_ppc.cpp
+++ b/src/hotspot/cpu/ppc/frame_ppc.cpp
@@ -117,9 +117,9 @@ bool frame::safe_for_sender(JavaThread *thread) {
return false;
}
- common_abi* sender_abi = (common_abi*) fp;
+ volatile common_abi* sender_abi = (common_abi*) fp; // May get updated concurrently by deoptimization!
intptr_t* sender_sp = (intptr_t*) fp;
- address sender_pc = (address) sender_abi->lr;;
+ address sender_pc = (address) sender_abi->lr;
if (Continuation::is_return_barrier_entry(sender_pc)) {
// If our sender_pc is the return barrier, then our "real" sender is the continuation entry
@@ -134,9 +134,18 @@ bool frame::safe_for_sender(JavaThread *thread) {
return false;
}
+ intptr_t* unextended_sender_sp = is_interpreted_frame() ? interpreter_frame_sender_sp() : sender_sp;
+
+ // If the sender is a deoptimized nmethod we need to check if the original pc is valid.
+ nmethod* sender_nm = sender_blob->as_nmethod_or_null();
+ if (sender_nm != nullptr && sender_nm->is_deopt_pc(sender_pc)) {
+ address orig_pc = *(address*)((address)unextended_sender_sp + sender_nm->orig_pc_offset());
+ if (!sender_nm->insts_contains_inclusive(orig_pc)) return false;
+ }
+
// It should be safe to construct the sender though it might not be valid.
- frame sender(sender_sp, sender_pc, nullptr /* unextended_sp */, nullptr /* fp */, sender_blob);
+ frame sender(sender_sp, sender_pc, unextended_sender_sp, nullptr /* fp */, sender_blob);
// Do we have a valid fp?
address sender_fp = (address) sender.fp();
diff --git a/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp
index 7d230d301c22b..39693bdf925bf 100644
--- a/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp
@@ -41,10 +41,20 @@
#include "c1/c1_LIRAssembler.hpp"
#include "c1/c1_MacroAssembler.hpp"
#include "gc/g1/c1/g1BarrierSetC1.hpp"
-#endif
+#endif // COMPILER1
+#ifdef COMPILER2
+#include "gc/g1/c2/g1BarrierSetC2.hpp"
+#endif // COMPILER2
#define __ masm->
+static void generate_marking_inactive_test(MacroAssembler* masm) {
+ int active_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
+ assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
+ __ lbz(R0, active_offset, R16_thread); // tmp1 := *(mark queue active address)
+ __ cmpwi(CCR0, R0, 0);
+}
+
void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
Register from, Register to, Register count,
Register preserve1, Register preserve2) {
@@ -58,13 +68,7 @@ void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm
Label filtered;
// Is marking active?
- if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
- __ lwz(R0, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()), R16_thread);
- } else {
- guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
- __ lbz(R0, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()), R16_thread);
- }
- __ cmpdi(CCR0, R0, 0);
+ generate_marking_inactive_test(masm);
__ beq(CCR0, filtered);
__ save_LR(R0);
@@ -109,35 +113,48 @@ void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* mas
__ restore_LR(R0);
}
+static void generate_queue_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
+ const Register value, const Register temp) {
+ assert_different_registers(value, temp);
+ // Can we store a value in the given thread's buffer?
+ // (The index field is typed as size_t.)
+ __ ld(temp, in_bytes(index_offset), R16_thread); // temp := *(index address)
+ __ cmpdi(CCR0, temp, 0); // jump to runtime if index == 0 (full buffer)
+ __ beq(CCR0, runtime);
+ // The buffer is not full, store value into it.
+ __ ld(R0, in_bytes(buffer_offset), R16_thread); // R0 := buffer address
+ __ addi(temp, temp, -wordSize); // temp := next index
+ __ std(temp, in_bytes(index_offset), R16_thread); // *(index address) := next index
+ __ stdx(value, temp, R0); // *(buffer address + next index) := value
+}
+
void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, DecoratorSet decorators,
Register obj, RegisterOrConstant ind_or_offs, Register pre_val,
Register tmp1, Register tmp2,
MacroAssembler::PreservationLevel preservation_level) {
+ assert_different_registers(pre_val, tmp1, tmp2);
+
bool not_null = (decorators & IS_NOT_NULL) != 0,
preloaded = obj == noreg;
Register nv_save = noreg;
- if (preloaded) {
+ // Determine necessary runtime invocation preservation measures
+ const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
+ const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
+ const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
+ int nbytes_save = 0;
+
+ if (pre_val->is_volatile() && preloaded && !preserve_gp_registers) {
// We are not loading the previous value so make
// sure that we don't trash the value in pre_val
// with the code below.
- assert_different_registers(pre_val, tmp1, tmp2);
- if (pre_val->is_volatile()) {
- nv_save = !tmp1->is_volatile() ? tmp1 : tmp2;
- assert(!nv_save->is_volatile(), "need one nv temp register if pre_val lives in volatile register");
- }
+ nv_save = !tmp1->is_volatile() ? tmp1 : tmp2;
+ assert(!nv_save->is_volatile(), "need one nv temp register if pre_val lives in volatile register");
}
Label runtime, filtered;
- // Is marking active?
- if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
- __ lwz(tmp1, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()), R16_thread);
- } else {
- guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
- __ lbz(tmp1, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()), R16_thread);
- }
- __ cmpdi(CCR0, tmp1, 0);
+ generate_marking_inactive_test(masm);
__ beq(CCR0, filtered);
// Do we need to load the previous value?
@@ -175,28 +192,12 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, Decorator
// Can we store original value in the thread's buffer?
// Is index == 0?
// (The index field is typed as size_t.)
- const Register Rbuffer = tmp1, Rindex = tmp2;
-
- __ ld(Rindex, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
- __ cmpdi(CCR0, Rindex, 0);
- __ beq(CCR0, runtime); // If index == 0, goto runtime.
- __ ld(Rbuffer, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
-
- __ addi(Rindex, Rindex, -wordSize); // Decrement index.
- __ std(Rindex, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
-
- // Record the previous value.
- __ stdx(pre_val, Rbuffer, Rindex);
+ generate_queue_insertion(masm, G1ThreadLocalData::satb_mark_queue_index_offset(), G1ThreadLocalData::satb_mark_queue_buffer_offset(),
+ runtime, pre_val, tmp1);
__ b(filtered);
__ bind(runtime);
- // Determine necessary runtime invocation preservation measures
- const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
- const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
- const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
- int nbytes_save = 0;
-
// May need to preserve LR. Also needed if current frame is not compatible with C calling convention.
if (needs_frame) {
if (preserve_gp_registers) {
@@ -210,11 +211,11 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, Decorator
__ push_frame_reg_args(nbytes_save, tmp2);
}
- if (pre_val->is_volatile() && preloaded && !preserve_gp_registers) {
+ if (nv_save != noreg) {
__ mr(nv_save, pre_val); // Save pre_val across C call if it was preloaded.
}
__ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, R16_thread);
- if (pre_val->is_volatile() && preloaded && !preserve_gp_registers) {
+ if (nv_save != noreg) {
__ mr(pre_val, nv_save); // restore
}
@@ -230,6 +231,26 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, Decorator
__ bind(filtered);
}
+static void generate_region_crossing_test(MacroAssembler* masm, const Register store_addr, const Register new_val) {
+ __ xorr(R0, store_addr, new_val); // tmp1 := store address ^ new value
+ __ srdi_(R0, R0, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
+}
+
+static Address generate_card_young_test(MacroAssembler* masm, const Register store_addr, const Register tmp1, const Register tmp2) {
+ CardTableBarrierSet* ct = barrier_set_cast(BarrierSet::barrier_set());
+ __ load_const_optimized(tmp1, (address)(ct->card_table()->byte_map_base()), tmp2);
+ __ srdi(tmp2, store_addr, CardTable::card_shift()); // tmp1 := card address relative to card table base
+ __ lbzx(R0, tmp1, tmp2); // tmp1 := card address
+ __ cmpwi(CCR0, R0, (int)G1CardTable::g1_young_card_val());
+ return Address(tmp1, tmp2); // return card address
+}
+
+static void generate_card_dirty_test(MacroAssembler* masm, Address card_addr) {
+ __ membar(Assembler::StoreLoad); // Must reload after StoreLoad membar due to concurrent refinement
+ __ lbzx(R0, card_addr.base(), card_addr.index()); // tmp2 := card
+ __ cmpwi(CCR0, R0, (int)G1CardTable::dirty_card_val()); // tmp2 := card == dirty_card_val?
+}
+
void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, DecoratorSet decorators,
Register store_addr, Register new_val,
Register tmp1, Register tmp2, Register tmp3,
@@ -241,9 +262,7 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, Decorato
CardTableBarrierSet* ct = barrier_set_cast(BarrierSet::barrier_set());
- // Does store cross heap regions?
- __ xorr(tmp1, store_addr, new_val);
- __ srdi_(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes);
+ generate_region_crossing_test(masm, store_addr, new_val);
__ beq(CCR0, filtered);
// Crosses regions, storing null?
@@ -257,43 +276,22 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, Decorato
__ beq(CCR0, filtered);
}
- // Storing region crossing non-null, is card already dirty?
- const Register Rcard_addr = tmp1;
- Register Rbase = tmp2;
- __ load_const_optimized(Rbase, (address)(ct->card_table()->byte_map_base()), /*temp*/ tmp3);
-
- __ srdi(Rcard_addr, store_addr, CardTable::card_shift());
-
- // Get the address of the card.
- __ lbzx(/*card value*/ tmp3, Rbase, Rcard_addr);
- __ cmpwi(CCR0, tmp3, (int)G1CardTable::g1_young_card_val());
+ Address card_addr = generate_card_young_test(masm, store_addr, tmp1, tmp2);
__ beq(CCR0, filtered);
- __ membar(Assembler::StoreLoad);
- __ lbzx(/*card value*/ tmp3, Rbase, Rcard_addr); // Reload after membar.
- __ cmpwi(CCR0, tmp3 /* card value */, (int)G1CardTable::dirty_card_val());
+ generate_card_dirty_test(masm, card_addr);
__ beq(CCR0, filtered);
- // Storing a region crossing, non-null oop, card is clean.
- // Dirty card and log.
- __ li(tmp3, (int)G1CardTable::dirty_card_val());
- //release(); // G1: oops are allowed to get visible after dirty marking.
- __ stbx(tmp3, Rbase, Rcard_addr);
-
- __ add(Rcard_addr, Rbase, Rcard_addr); // This is the address which needs to get enqueued.
- Rbase = noreg; // end of lifetime
+ __ li(R0, (int)G1CardTable::dirty_card_val());
+ __ stbx(R0, card_addr.base(), card_addr.index()); // *(card address) := dirty_card_val
- const Register Rqueue_index = tmp2,
- Rqueue_buf = tmp3;
- __ ld(Rqueue_index, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()), R16_thread);
- __ cmpdi(CCR0, Rqueue_index, 0);
- __ beq(CCR0, runtime); // index == 0 then jump to runtime
- __ ld(Rqueue_buf, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()), R16_thread);
+ Register Rcard_addr = tmp3;
+ __ add(Rcard_addr, card_addr.base(), card_addr.index()); // This is the address which needs to get enqueued.
- __ addi(Rqueue_index, Rqueue_index, -wordSize); // decrement index
- __ std(Rqueue_index, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()), R16_thread);
-
- __ stdx(Rcard_addr, Rqueue_buf, Rqueue_index); // store card
+ generate_queue_insertion(masm,
+ G1ThreadLocalData::dirty_card_queue_index_offset(),
+ G1ThreadLocalData::dirty_card_queue_buffer_offset(),
+ runtime, Rcard_addr, tmp1);
__ b(filtered);
__ bind(runtime);
@@ -392,6 +390,142 @@ void G1BarrierSetAssembler::resolve_jobject(MacroAssembler* masm, Register value
__ bind(done);
}
+#ifdef COMPILER2
+
+static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
+ SaveLiveRegisters save_registers(masm, stub);
+ __ call_VM_leaf(runtime_path, arg, R16_thread);
+}
+
+void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register tmp1,
+ Register tmp2,
+ G1PreBarrierStubC2* stub) {
+ assert_different_registers(obj, tmp1, tmp2, R0);
+ assert_different_registers(pre_val, tmp1, R0);
+ assert(!UseCompressedOops || tmp2 != noreg, "tmp2 needed with CompressedOops");
+
+ stub->initialize_registers(obj, pre_val, R16_thread, tmp1, tmp2);
+
+ generate_marking_inactive_test(masm);
+ __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CCR0, Assembler::equal), *stub->entry());
+
+ __ bind(*stub->continuation());
+}
+
+void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm,
+ G1PreBarrierStubC2* stub) const {
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ Label runtime;
+ Register obj = stub->obj();
+ Register pre_val = stub->pre_val();
+ Register tmp1 = stub->tmp1();
+
+ __ bind(*stub->entry());
+
+ if (obj != noreg) {
+ // Note: C2 currently doesn't use implicit null checks with barriers.
+ // Otherwise, obj could be null and the following instruction would raise a SIGSEGV.
+ if (UseCompressedOops) {
+ __ lwz(pre_val, 0, obj);
+ } else {
+ __ ld(pre_val, 0, obj);
+ }
+ }
+ __ cmpdi(CCR0, pre_val, 0);
+ __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CCR0, Assembler::equal), *stub->continuation());
+
+ Register pre_val_decoded = pre_val;
+ if (UseCompressedOops) {
+ pre_val_decoded = __ decode_heap_oop_not_null(stub->tmp2(), pre_val);
+ }
+
+ generate_queue_insertion(masm,
+ G1ThreadLocalData::satb_mark_queue_index_offset(),
+ G1ThreadLocalData::satb_mark_queue_buffer_offset(),
+ runtime, pre_val_decoded, tmp1);
+ __ b(*stub->continuation());
+
+ __ bind(runtime);
+ generate_c2_barrier_runtime_call(masm, stub, pre_val_decoded, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry));
+ __ b(*stub->continuation());
+}
+
+void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register tmp1,
+ Register tmp2,
+ G1PostBarrierStubC2* stub,
+ bool decode_new_val) {
+ assert_different_registers(store_addr, new_val, tmp1, R0);
+ assert_different_registers(store_addr, tmp1, tmp2, R0);
+
+ stub->initialize_registers(R16_thread, tmp1, tmp2);
+
+ bool null_check_required = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
+ Register new_val_decoded = new_val;
+
+ if (decode_new_val) {
+ assert(UseCompressedOops, "or should not be here");
+ if (null_check_required && CompressedOops::base() != nullptr) {
+ // We prefer doing the null check after the region crossing check.
+ // Only compressed oop modes with base != null require a null check here.
+ __ cmpwi(CCR0, new_val, 0);
+ __ beq(CCR0, *stub->continuation());
+ null_check_required = false;
+ }
+ new_val_decoded = __ decode_heap_oop_not_null(tmp2, new_val);
+ }
+
+ generate_region_crossing_test(masm, store_addr, new_val_decoded);
+ __ beq(CCR0, *stub->continuation());
+
+ // crosses regions, storing null?
+ if (null_check_required) {
+ __ cmpdi(CCR0, new_val_decoded, 0);
+ __ beq(CCR0, *stub->continuation());
+ }
+
+ Address card_addr = generate_card_young_test(masm, store_addr, tmp1, tmp2);
+ assert(card_addr.base() == tmp1 && card_addr.index() == tmp2, "needed by post barrier stub");
+ __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CCR0, Assembler::equal), *stub->entry());
+
+ __ bind(*stub->continuation());
+}
+
+void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
+ G1PostBarrierStubC2* stub) const {
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ Label runtime;
+ Address card_addr(stub->tmp1(), stub->tmp2()); // See above.
+
+ __ bind(*stub->entry());
+
+ generate_card_dirty_test(masm, card_addr);
+ __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CCR0, Assembler::equal), *stub->continuation());
+
+ __ li(R0, (int)G1CardTable::dirty_card_val());
+ __ stbx(R0, card_addr.base(), card_addr.index()); // *(card address) := dirty_card_val
+
+ Register Rcard_addr = stub->tmp1();
+ __ add(Rcard_addr, card_addr.base(), card_addr.index()); // This is the address which needs to get enqueued.
+
+ generate_queue_insertion(masm,
+ G1ThreadLocalData::dirty_card_queue_index_offset(),
+ G1ThreadLocalData::dirty_card_queue_buffer_offset(),
+ runtime, Rcard_addr, stub->tmp2());
+ __ b(*stub->continuation());
+
+ __ bind(runtime);
+ generate_c2_barrier_runtime_call(masm, stub, Rcard_addr, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
+ __ b(*stub->continuation());
+}
+
+#endif // COMPILER2
+
#ifdef COMPILER1
#undef __
@@ -470,13 +604,7 @@ void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler*
__ std(tmp2, -24, R1_SP);
// Is marking still active?
- if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
- __ lwz(tmp, satb_q_active_byte_offset, R16_thread);
- } else {
- assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
- __ lbz(tmp, satb_q_active_byte_offset, R16_thread);
- }
- __ cmpdi(CCR0, tmp, 0);
+ generate_marking_inactive_test(sasm);
__ beq(CCR0, marking_not_active);
__ bind(restart);
diff --git a/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.hpp
index d9a252ff6eaee..1c9fe8a5d106f 100644
--- a/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.hpp
@@ -30,10 +30,16 @@
#include "gc/shared/modRefBarrierSetAssembler.hpp"
#include "utilities/macros.hpp"
+#ifdef COMPILER2
+#include "gc/g1/c2/g1BarrierSetC2.hpp"
+#endif
+
class LIR_Assembler;
class StubAssembler;
class G1PreBarrierStub;
class G1PostBarrierStub;
+class G1PreBarrierStubC2;
+class G1PostBarrierStubC2;
class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
protected:
@@ -59,6 +65,25 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
MacroAssembler::PreservationLevel preservation_level);
public:
+#ifdef COMPILER2
+ void g1_write_barrier_pre_c2(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register tmp1,
+ Register tmp2,
+ G1PreBarrierStubC2* c2_stub);
+ void generate_c2_pre_barrier_stub(MacroAssembler* masm,
+ G1PreBarrierStubC2* stub) const;
+ void g1_write_barrier_post_c2(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register tmp1,
+ Register tmp2,
+ G1PostBarrierStubC2* c2_stub,
+ bool decode_new_val);
+ void generate_c2_post_barrier_stub(MacroAssembler* masm,
+ G1PostBarrierStubC2* stub) const;
+#endif
#ifdef COMPILER1
void gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub);
void gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub);
diff --git a/src/hotspot/cpu/ppc/gc/g1/g1_ppc.ad b/src/hotspot/cpu/ppc/gc/g1/g1_ppc.ad
new file mode 100644
index 0000000000000..f4163242cad7b
--- /dev/null
+++ b/src/hotspot/cpu/ppc/gc/g1/g1_ppc.ad
@@ -0,0 +1,684 @@
+//
+// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2024 SAP SE. All rights reserved.
+// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+
+source_hpp %{
+
+#include "gc/g1/c2/g1BarrierSetC2.hpp"
+#include "gc/shared/gc_globals.hpp"
+
+%}
+
+source %{
+
+#include "gc/g1/g1BarrierSetAssembler_ppc.hpp"
+#include "gc/g1/g1BarrierSetRuntime.hpp"
+
+static void pre_write_barrier(MacroAssembler* masm,
+ const MachNode* node,
+ Register obj,
+ Register pre_val,
+ Register tmp1,
+ Register tmp2 = noreg, // only needed with CompressedOops when pre_val needs to be preserved
+ RegSet preserve = RegSet(),
+ RegSet no_preserve = RegSet()) {
+ if (!G1PreBarrierStubC2::needs_barrier(node)) {
+ return;
+ }
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
+ G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
+ for (RegSetIterator reg = preserve.begin(); *reg != noreg; ++reg) {
+ stub->preserve(*reg);
+ }
+ for (RegSetIterator reg = no_preserve.begin(); *reg != noreg; ++reg) {
+ stub->dont_preserve(*reg);
+ }
+ g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, tmp1, (tmp2 != noreg) ? tmp2 : pre_val, stub);
+}
+
+static void post_write_barrier(MacroAssembler* masm,
+ const MachNode* node,
+ Register store_addr,
+ Register new_val,
+ Register tmp1,
+ Register tmp2,
+ bool decode_new_val = false) {
+ if (!G1PostBarrierStubC2::needs_barrier(node)) {
+ return;
+ }
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
+ G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
+ g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, tmp1, tmp2, stub, decode_new_val);
+}
+
+%}
+
+instruct g1StoreP(indirect mem, iRegPsrc src, iRegPdst tmp1, iRegPdst tmp2, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreP mem src));
+ effect(TEMP tmp1, TEMP tmp2, KILL cr0);
+ ins_cost(2 * MEMORY_REF_COST);
+ format %{ "std $mem, $src\t# ptr" %}
+ ins_encode %{
+ pre_write_barrier(masm, this,
+ $mem$$Register,
+ $tmp1$$Register,
+ $tmp2$$Register,
+ noreg,
+ RegSet::of($mem$$Register, $src$$Register) /* preserve */);
+ __ std($src$$Register, 0, $mem$$Register);
+ post_write_barrier(masm, this,
+ $mem$$Register,
+ $src$$Register /* new_val */,
+ $tmp1$$Register,
+ $tmp2$$Register);
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct g1StoreN(indirect mem, iRegNsrc src, iRegPdst tmp1, iRegPdst tmp2, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreN mem src));
+ effect(TEMP tmp1, TEMP tmp2, KILL cr0);
+ ins_cost(2 * MEMORY_REF_COST);
+ format %{ "stw $mem, $src\t# ptr" %}
+ ins_encode %{
+ pre_write_barrier(masm, this,
+ $mem$$Register,
+ $tmp1$$Register,
+ $tmp2$$Register,
+ noreg,
+ RegSet::of($mem$$Register, $src$$Register) /* preserve */);
+ __ stw($src$$Register, 0, $mem$$Register);
+ post_write_barrier(masm, this,
+ $mem$$Register,
+ $src$$Register /* new_val */,
+ $tmp1$$Register,
+ $tmp2$$Register,
+ true /* decode_new_val */);
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct g1EncodePAndStoreN(indirect mem, iRegPsrc src, iRegPdst tmp1, iRegPdst tmp2, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreN mem (EncodeP src)));
+ effect(TEMP tmp1, TEMP tmp2, KILL cr0);
+ ins_cost(2 * MEMORY_REF_COST);
+ format %{ "encode_heap_oop $src\n\t"
+ "stw $mem, $src\t# ptr" %}
+ ins_encode %{
+ pre_write_barrier(masm, this,
+ $mem$$Register,
+ $tmp1$$Register,
+ $tmp2$$Register,
+ noreg,
+ RegSet::of($mem$$Register, $src$$Register) /* preserve */);
+ Register encoded_oop = noreg;
+ if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
+ encoded_oop = __ encode_heap_oop($tmp2$$Register, $src$$Register);
+ } else {
+ encoded_oop = __ encode_heap_oop_not_null($tmp2$$Register, $src$$Register);
+ }
+ __ stw(encoded_oop, 0, $mem$$Register);
+ post_write_barrier(masm, this,
+ $mem$$Register,
+ $src$$Register /* new_val */,
+ $tmp1$$Register,
+ $tmp2$$Register);
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct g1CompareAndExchangeP(iRegPdst res, indirect mem, iRegPsrc oldval, iRegPsrc newval, iRegPdst tmp1, iRegPdst tmp2, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0 &&
+ (((CompareAndExchangeNode*)n)->order() != MemNode::acquire && ((CompareAndExchangeNode*)n)->order() != MemNode::seqcst));
+ match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
+ effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr0);
+ format %{ "cmpxchgd $newval, $mem" %}
+ ins_encode %{
+ Label no_update;
+ __ cmpxchgd(CCR0, $res$$Register, $oldval$$Register, $newval$$Register, $mem$$Register,
+ MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
+ noreg, &no_update, true);
+ // Pass oldval to SATB which is the only value which can get overwritten.
+ // Can be done after cmpxchg because there's no safepoint here.
+ pre_write_barrier(masm, this,
+ noreg,
+ $oldval$$Register,
+ $tmp1$$Register,
+ $tmp2$$Register,
+ RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
+ post_write_barrier(masm, this,
+ $mem$$Register,
+ $newval$$Register,
+ $tmp1$$Register,
+ $tmp2$$Register);
+ __ bind(no_update);
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct g1CompareAndExchangeP_acq(iRegPdst res, indirect mem, iRegPsrc oldval, iRegPsrc newval, iRegPdst tmp1, iRegPdst tmp2, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0 &&
+ (((CompareAndExchangeNode*)n)->order() == MemNode::acquire || ((CompareAndExchangeNode*)n)->order() == MemNode::seqcst));
+ match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
+ effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr0);
+ format %{ "cmpxchgd acq $newval, $mem" %}
+ ins_encode %{
+ Label no_update;
+ __ cmpxchgd(CCR0, $res$$Register, $oldval$$Register, $newval$$Register, $mem$$Register,
+ MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
+ noreg, &no_update, true);
+ // Pass oldval to SATB which is the only value which can get overwritten.
+ // Can be done after cmpxchg because there's no safepoint here.
+ pre_write_barrier(masm, this,
+ noreg,
+ $oldval$$Register,
+ $tmp1$$Register,
+ $tmp2$$Register,
+ RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
+ post_write_barrier(masm, this,
+ $mem$$Register,
+ $newval$$Register,
+ $tmp1$$Register,
+ $tmp2$$Register);
+ __ bind(no_update);
+ if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
+ __ isync();
+ } else {
+ // isync would be sufficient in case of CompareAndExchangeAcquire, but we currently don't optimize for that.
+ __ sync();
+ }
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct g1CompareAndExchangeN(iRegNdst res, indirect mem, iRegNsrc oldval, iRegNsrc newval, iRegPdst tmp1, iRegPdst tmp2, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0 &&
+ (((CompareAndExchangeNode*)n)->order() != MemNode::acquire && ((CompareAndExchangeNode*)n)->order() != MemNode::seqcst));
+ match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
+ effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr0);
+ format %{ "cmpxchgw $newval, $mem" %}
+ ins_encode %{
+ Label no_update;
+ __ cmpxchgw(CCR0, $res$$Register, $oldval$$Register, $newval$$Register, $mem$$Register,
+ MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
+ noreg, &no_update, true);
+ // Pass oldval to SATB which is the only value which can get overwritten.
+ // Can be done after cmpxchg because there's no safepoint here.
+ pre_write_barrier(masm, this,
+ noreg,
+ $oldval$$Register,
+ $tmp1$$Register,
+ $tmp2$$Register,
+ RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
+ post_write_barrier(masm, this,
+ $mem$$Register,
+ $newval$$Register,
+ $tmp1$$Register,
+ $tmp2$$Register,
+ true /* decode_new_val */);
+ __ bind(no_update);
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct g1CompareAndExchangeN_acq(iRegNdst res, indirect mem, iRegNsrc oldval, iRegNsrc newval, iRegPdst tmp1, iRegPdst tmp2, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0 &&
+ (((CompareAndExchangeNode*)n)->order() == MemNode::acquire || ((CompareAndExchangeNode*)n)->order() == MemNode::seqcst));
+ match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
+ effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr0);
+ format %{ "cmpxchgw acq $newval, $mem" %}
+ ins_encode %{
+ Label no_update;
+ __ cmpxchgw(CCR0, $res$$Register, $oldval$$Register, $newval$$Register, $mem$$Register,
+ MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
+ noreg, &no_update, true);
+ // Pass oldval to SATB which is the only value which can get overwritten.
+ // Can be done after cmpxchg because there's no safepoint here.
+ pre_write_barrier(masm, this,
+ noreg,
+ $oldval$$Register,
+ $tmp1$$Register,
+ $tmp2$$Register,
+ RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
+ post_write_barrier(masm, this,
+ $mem$$Register,
+ $newval$$Register,
+ $tmp1$$Register,
+ $tmp2$$Register,
+ true /* decode_new_val */);
+ __ bind(no_update);
+ if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
+ __ isync();
+ } else {
+ // isync would be sufficient in case of CompareAndExchangeAcquire, but we currently don't optimize for that.
+ __ sync();
+ }
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct g1CompareAndSwapP(iRegIdst res, indirect mem, iRegPsrc oldval, iRegPsrc newval, iRegPdst tmp, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0 &&
+ (((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*)n)->order() != MemNode::seqcst));
+ match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+ effect(TEMP_DEF res, TEMP tmp, KILL cr0);
+ format %{ "CMPXCHGD $res, $mem, $oldval, $newval; as bool; ptr" %}
+ ins_encode %{
+ Label no_update;
+ __ li($res$$Register, 0);
+ __ cmpxchgd(CCR0, R0, $oldval$$Register, $newval$$Register, $mem$$Register,
+ MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
+ noreg, &no_update, true);
+ // Pass oldval to SATB which is the only value which can get overwritten.
+ // Can be done after cmpxchg because there's no safepoint here.
+ pre_write_barrier(masm, this,
+ noreg,
+ $oldval$$Register /* pre_val */,
+ $tmp$$Register,
+ $res$$Register /* temp */,
+ RegSet::of($mem$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ post_write_barrier(masm, this,
+ $mem$$Register,
+ $newval$$Register,
+ $tmp$$Register,
+ $res$$Register /* temp */);
+ __ li($res$$Register, 1);
+ __ bind(no_update);
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct g1CompareAndSwapP_acq(iRegIdst res, indirect mem, iRegPsrc oldval, iRegPsrc newval, iRegPdst tmp, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0 &&
+ (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst));
+ match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+ effect(TEMP_DEF res, TEMP tmp, KILL cr0);
+ format %{ "CMPXCHGD acq $res, $mem, $oldval, $newval; as bool; ptr" %}
+ ins_encode %{
+ Label no_update;
+ __ li($res$$Register, 0);
+ __ cmpxchgd(CCR0, R0, $oldval$$Register, $newval$$Register, $mem$$Register,
+ MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
+ noreg, &no_update, true);
+ // Pass oldval to SATB which is the only value which can get overwritten.
+ // Can be done after cmpxchg because there's no safepoint here.
+ pre_write_barrier(masm, this,
+ noreg,
+ $oldval$$Register /* pre_val */,
+ $tmp$$Register,
+ $res$$Register /* temp */,
+ RegSet::of($mem$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ post_write_barrier(masm, this,
+ $mem$$Register,
+ $newval$$Register,
+ $tmp$$Register,
+ $res$$Register /* temp */);
+ __ li($res$$Register, 1);
+ __ bind(no_update);
+ if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
+ __ isync();
+ } else {
+ // isync would be sufficient in case of CompareAndExchangeAcquire, but we currently don't optimize for that.
+ __ sync();
+ }
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct g1CompareAndSwapN(iRegIdst res, indirect mem, iRegNsrc oldval, iRegNsrc newval, iRegPdst tmp, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0 &&
+ (((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*)n)->order() != MemNode::seqcst));
+ match(Set res (CompareAndSwapN mem (Binary oldval newval)));
+ effect(TEMP_DEF res, TEMP tmp, KILL cr0);
+ format %{ "CMPXCHGW $res, $mem, $oldval, $newval; as bool; ptr" %}
+ ins_encode %{
+ Label no_update;
+ __ li($res$$Register, 0);
+ __ cmpxchgw(CCR0, R0, $oldval$$Register, $newval$$Register, $mem$$Register,
+ MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
+ noreg, &no_update, true);
+ // Pass oldval to SATB which is the only value which can get overwritten.
+ // Can be done after cmpxchg because there's no safepoint here.
+ pre_write_barrier(masm, this,
+ noreg,
+ $oldval$$Register /* pre_val */,
+ $tmp$$Register,
+ $res$$Register /* temp */,
+ RegSet::of($mem$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ post_write_barrier(masm, this,
+ $mem$$Register,
+ $newval$$Register,
+ $tmp$$Register,
+ $res$$Register /* temp */,
+ true /* decode_new_val */);
+ __ li($res$$Register, 1);
+ __ bind(no_update);
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct g1CompareAndSwapN_acq(iRegIdst res, indirect mem, iRegNsrc oldval, iRegNsrc newval, iRegPdst tmp, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0 &&
+ (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst));
+ match(Set res (CompareAndSwapN mem (Binary oldval newval)));
+ effect(TEMP_DEF res, TEMP tmp, KILL cr0);
+ format %{ "CMPXCHGW acq $res, $mem, $oldval, $newval; as bool; ptr" %}
+ ins_encode %{
+ Label no_update;
+ __ li($res$$Register, 0);
+ __ cmpxchgw(CCR0, R0, $oldval$$Register, $newval$$Register, $mem$$Register,
+ MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
+ noreg, &no_update, true);
+ // Pass oldval to SATB which is the only value which can get overwritten.
+ // Can be done after cmpxchg because there's no safepoint here.
+ pre_write_barrier(masm, this,
+ noreg,
+ $oldval$$Register /* pre_val */,
+ $tmp$$Register,
+ $res$$Register /* temp */,
+ RegSet::of($mem$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ post_write_barrier(masm, this,
+ $mem$$Register,
+ $newval$$Register,
+ $tmp$$Register,
+ $res$$Register /* temp */,
+ true /* decode_new_val */);
+ __ li($res$$Register, 1);
+ __ bind(no_update);
+ if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
+ __ isync();
+ } else {
+ // isync would be sufficient in case of CompareAndExchangeAcquire, but we currently don't optimize for that.
+ __ sync();
+ }
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct weakG1CompareAndSwapP(iRegIdst res, indirect mem, iRegPsrc oldval, iRegPsrc newval, iRegPdst tmp, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0 &&
+ (((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*)n)->order() != MemNode::seqcst));
+ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+ effect(TEMP_DEF res, TEMP tmp, KILL cr0);
+ format %{ "weak CMPXCHGD $res, $mem, $oldval, $newval; as bool; ptr" %}
+ ins_encode %{
+ Label no_update;
+ __ li($res$$Register, 0);
+ __ cmpxchgd(CCR0, R0, $oldval$$Register, $newval$$Register, $mem$$Register,
+ MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
+ noreg, &no_update, true, true);
+ // Pass oldval to SATB which is the only value which can get overwritten.
+ // Can be done after cmpxchg because there's no safepoint here.
+ pre_write_barrier(masm, this,
+ noreg,
+ $oldval$$Register /* pre_val */,
+ $tmp$$Register,
+ $res$$Register /* temp */,
+ RegSet::of($mem$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ post_write_barrier(masm, this,
+ $mem$$Register,
+ $newval$$Register,
+ $tmp$$Register,
+ $res$$Register /* temp */);
+ __ li($res$$Register, 1);
+ __ bind(no_update);
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct weakG1CompareAndSwapP_acq(iRegIdst res, indirect mem, iRegPsrc oldval, iRegPsrc newval, iRegPdst tmp, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0 &&
+ (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst));
+ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+ effect(TEMP_DEF res, TEMP tmp, KILL cr0);
+ format %{ "weak CMPXCHGD acq $res, $mem, $oldval, $newval; as bool; ptr" %}
+ ins_encode %{
+ Label no_update;
+ __ li($res$$Register, 0);
+ __ cmpxchgd(CCR0, R0, $oldval$$Register, $newval$$Register, $mem$$Register,
+ MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
+ noreg, &no_update, true, true);
+ // Pass oldval to SATB which is the only value which can get overwritten.
+ // Can be done after cmpxchg because there's no safepoint here.
+ pre_write_barrier(masm, this,
+ noreg,
+ $oldval$$Register /* pre_val */,
+ $tmp$$Register,
+ $res$$Register /* temp */,
+ RegSet::of($mem$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ post_write_barrier(masm, this,
+ $mem$$Register,
+ $newval$$Register,
+ $tmp$$Register,
+ $res$$Register /* temp */);
+ __ li($res$$Register, 1);
+ if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
+ __ isync();
+ } else {
+ // isync would be sufficient in case of CompareAndExchangeAcquire, but we currently don't optimize for that.
+ __ sync();
+ }
+ __ bind(no_update); // weak version requires no memory barrier on failure
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct weakG1CompareAndSwapN(iRegIdst res, indirect mem, iRegNsrc oldval, iRegNsrc newval, iRegPdst tmp, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0 &&
+ (((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*)n)->order() != MemNode::seqcst));
+ match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
+ effect(TEMP_DEF res, TEMP tmp, KILL cr0);
+ format %{ "weak CMPXCHGW $res, $mem, $oldval, $newval; as bool; ptr" %}
+ ins_encode %{
+ Label no_update;
+ __ li($res$$Register, 0);
+ __ cmpxchgw(CCR0, R0, $oldval$$Register, $newval$$Register, $mem$$Register,
+ MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
+ noreg, &no_update, true, true);
+ // Pass oldval to SATB which is the only value which can get overwritten.
+ // Can be done after cmpxchg because there's no safepoint here.
+ pre_write_barrier(masm, this,
+ noreg,
+ $oldval$$Register /* pre_val */,
+ $tmp$$Register,
+ $res$$Register /* temp */,
+ RegSet::of($mem$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ post_write_barrier(masm, this,
+ $mem$$Register,
+ $newval$$Register,
+ $tmp$$Register,
+ $res$$Register /* temp */,
+ true /* decode_new_val */);
+ __ li($res$$Register, 1);
+ __ bind(no_update);
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct weakG1CompareAndSwapN_acq(iRegIdst res, indirect mem, iRegNsrc oldval, iRegNsrc newval, iRegPdst tmp, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0 &&
+ (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst));
+ match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
+ effect(TEMP_DEF res, TEMP tmp, KILL cr0);
+ format %{ "weak CMPXCHGW acq $res, $mem, $oldval, $newval; as bool; ptr" %}
+ ins_encode %{
+ Label no_update;
+ __ li($res$$Register, 0);
+ __ cmpxchgw(CCR0, R0, $oldval$$Register, $newval$$Register, $mem$$Register,
+ MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
+ noreg, &no_update, true, true);
+ // Pass oldval to SATB which is the only value which can get overwritten.
+ // Can be done after cmpxchg because there's no safepoint here.
+ pre_write_barrier(masm, this,
+ noreg,
+ $oldval$$Register /* pre_val */,
+ $tmp$$Register,
+ $res$$Register /* temp */,
+ RegSet::of($mem$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ post_write_barrier(masm, this,
+ $mem$$Register,
+ $newval$$Register,
+ $tmp$$Register,
+ $res$$Register /* temp */,
+ true /* decode_new_val */);
+ __ li($res$$Register, 1);
+ if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
+ __ isync();
+ } else {
+ // isync would be sufficient in case of CompareAndExchangeAcquire, but we currently don't optimize for that.
+ __ sync();
+ }
+ __ bind(no_update); // weak version requires no memory barrier on failure
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct g1GetAndSetP(iRegPdst res, indirect mem, iRegPsrc newval, iRegPdst tmp1, iRegPdst tmp2, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (GetAndSetP mem newval));
+ effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr0);
+ format %{ "GetAndSetP $newval, $mem" %}
+ ins_encode %{
+ assert_different_registers($mem$$Register, $newval$$Register);
+ __ getandsetd($res$$Register, $newval$$Register, $mem$$Register,
+ MacroAssembler::cmpxchgx_hint_atomic_update());
+ // Can be done after cmpxchg because there's no safepoint here.
+ pre_write_barrier(masm, this,
+ noreg /* obj */,
+ $res$$Register /* res */,
+ $tmp1$$Register,
+ $tmp2$$Register,
+ RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
+ post_write_barrier(masm, this,
+ $mem$$Register,
+ $newval$$Register,
+ $tmp1$$Register,
+ $tmp2$$Register);
+ if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
+ __ isync();
+ } else {
+ __ sync();
+ }
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct g1GetAndSetN(iRegNdst res, indirect mem, iRegNsrc newval, iRegPdst tmp1, iRegPdst tmp2, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (GetAndSetN mem newval));
+ effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr0);
+ format %{ "GetAndSetN $newval, $mem" %}
+ ins_encode %{
+ assert_different_registers($mem$$Register, $newval$$Register);
+ __ getandsetw($res$$Register, $newval$$Register, $mem$$Register,
+ MacroAssembler::cmpxchgx_hint_atomic_update());
+ // Can be done after cmpxchg because there's no safepoint here.
+ pre_write_barrier(masm, this,
+ noreg /* obj */,
+ $res$$Register /* res */,
+ $tmp1$$Register,
+ $tmp2$$Register,
+ RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
+ post_write_barrier(masm, this,
+ $mem$$Register,
+ $newval$$Register,
+ $tmp1$$Register,
+ $tmp2$$Register,
+ true /* decode_new_val */);
+ if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
+ __ isync();
+ } else {
+ __ sync();
+ }
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct g1LoadP(iRegPdst dst, memoryAlg4 mem, iRegPdst tmp, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_Load()->is_unordered() && n->as_Load()->barrier_data() != 0);
+ // This instruction does not need an acquiring counterpart because it is only
+ // used for reference loading (Reference::get()).
+ match(Set dst (LoadP mem));
+ effect(TEMP_DEF dst, TEMP tmp, KILL cr0);
+ ins_cost(2 * MEMORY_REF_COST);
+ format %{ "ld $dst, $mem\t# ptr" %}
+ ins_encode %{
+ __ ld($dst$$Register, $mem$$disp, $mem$$base$$Register);
+ pre_write_barrier(masm, this,
+ noreg /* obj */,
+ $dst$$Register /* pre_val */,
+ $tmp$$Register);
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct g1LoadN(iRegNdst dst, memoryAlg4 mem, iRegPdst tmp1, iRegPdst tmp2, flagsRegCR0 cr0)
+%{
+ predicate(UseG1GC && n->as_Load()->is_unordered() && n->as_Load()->barrier_data() != 0);
+ // This instruction does not need an acquiring counterpart because it is only
+ // used for reference loading (Reference::get()).
+ match(Set dst (LoadN mem));
+ effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, KILL cr0);
+ ins_cost(2 * MEMORY_REF_COST);
+ format %{ "lwz $dst, $mem\t# ptr" %}
+ ins_encode %{
+ __ lwz($dst$$Register, $mem$$disp, $mem$$base$$Register);
+ pre_write_barrier(masm, this,
+ noreg /* obj */,
+ $dst$$Register,
+ $tmp1$$Register,
+ $tmp2$$Register);
+ %}
+ ins_pipe(pipe_class_default);
+%}
diff --git a/src/hotspot/cpu/ppc/gc/x/xBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/x/xBarrierSetAssembler_ppc.cpp
deleted file mode 100644
index ca826e47352b5..0000000000000
--- a/src/hotspot/cpu/ppc/gc/x/xBarrierSetAssembler_ppc.cpp
+++ /dev/null
@@ -1,585 +0,0 @@
-/*
- * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2021, 2024 SAP SE. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "asm/register.hpp"
-#include "asm/macroAssembler.inline.hpp"
-#include "code/codeBlob.hpp"
-#include "code/vmreg.inline.hpp"
-#include "gc/x/xBarrier.inline.hpp"
-#include "gc/x/xBarrierSet.hpp"
-#include "gc/x/xBarrierSetAssembler.hpp"
-#include "gc/x/xBarrierSetRuntime.hpp"
-#include "gc/x/xThreadLocalData.hpp"
-#include "memory/resourceArea.hpp"
-#include "register_ppc.hpp"
-#include "runtime/sharedRuntime.hpp"
-#include "utilities/globalDefinitions.hpp"
-#include "utilities/macros.hpp"
-#ifdef COMPILER1
-#include "c1/c1_LIRAssembler.hpp"
-#include "c1/c1_MacroAssembler.hpp"
-#include "gc/x/c1/xBarrierSetC1.hpp"
-#endif // COMPILER1
-#ifdef COMPILER2
-#include "gc/x/c2/xBarrierSetC2.hpp"
-#endif // COMPILER2
-
-#undef __
-#define __ masm->
-
-void XBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
- Register base, RegisterOrConstant ind_or_offs, Register dst,
- Register tmp1, Register tmp2,
- MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null) {
- __ block_comment("load_at (zgc) {");
-
- // Check whether a special gc barrier is required for this particular load
- // (e.g. whether it's a reference load or not)
- if (!XBarrierSet::barrier_needed(decorators, type)) {
- BarrierSetAssembler::load_at(masm, decorators, type, base, ind_or_offs, dst,
- tmp1, tmp2, preservation_level, L_handle_null);
- return;
- }
-
- if (ind_or_offs.is_register()) {
- assert_different_registers(base, ind_or_offs.as_register(), tmp1, tmp2, R0, noreg);
- assert_different_registers(dst, ind_or_offs.as_register(), tmp1, tmp2, R0, noreg);
- } else {
- assert_different_registers(base, tmp1, tmp2, R0, noreg);
- assert_different_registers(dst, tmp1, tmp2, R0, noreg);
- }
-
- /* ==== Load the pointer using the standard implementation for the actual heap access
- and the decompression of compressed pointers ==== */
- // Result of 'load_at' (standard implementation) will be written back to 'dst'.
- // As 'base' is required for the C-call, it must be reserved in case of a register clash.
- Register saved_base = base;
- if (base == dst) {
- __ mr(tmp2, base);
- saved_base = tmp2;
- }
-
- BarrierSetAssembler::load_at(masm, decorators, type, base, ind_or_offs, dst,
- tmp1, noreg, preservation_level, L_handle_null);
-
- /* ==== Check whether pointer is dirty ==== */
- Label skip_barrier;
-
- // Load bad mask into scratch register.
- __ ld(tmp1, (intptr_t) XThreadLocalData::address_bad_mask_offset(), R16_thread);
-
- // The color bits of the to-be-tested pointer do not have to be equivalent to the 'bad_mask' testing bits.
- // A pointer is classified as dirty if any of the color bits that also match the bad mask is set.
- // Conversely, it follows that the logical AND of the bad mask and the pointer must be zero
- // if the pointer is not dirty.
- // Only dirty pointers must be processed by this barrier, so we can skip it in case the latter condition holds true.
- __ and_(tmp1, tmp1, dst);
- __ beq(CCR0, skip_barrier);
-
- /* ==== Invoke barrier ==== */
- int nbytes_save = 0;
-
- const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
- const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
- const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
-
- const bool preserve_R3 = dst != R3_ARG1;
-
- if (needs_frame) {
- if (preserve_gp_registers) {
- nbytes_save = (preserve_fp_registers
- ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
- : MacroAssembler::num_volatile_gp_regs) * BytesPerWord;
- nbytes_save -= preserve_R3 ? 0 : BytesPerWord;
- __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers, preserve_R3);
- }
-
- __ save_LR(tmp1);
- __ push_frame_reg_args(nbytes_save, tmp1);
- }
-
- // Setup arguments
- if (saved_base != R3_ARG1) {
- __ mr_if_needed(R3_ARG1, dst);
- __ add(R4_ARG2, ind_or_offs, saved_base);
- } else if (dst != R4_ARG2) {
- __ add(R4_ARG2, ind_or_offs, saved_base);
- __ mr(R3_ARG1, dst);
- } else {
- __ add(R0, ind_or_offs, saved_base);
- __ mr(R3_ARG1, dst);
- __ mr(R4_ARG2, R0);
- }
-
- __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators));
-
- Register result = R3_RET;
- if (needs_frame) {
- __ pop_frame();
- __ restore_LR(tmp1);
-
- if (preserve_R3) {
- __ mr(R0, R3_RET);
- result = R0;
- }
-
- if (preserve_gp_registers) {
- __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers, preserve_R3);
- }
- }
- __ mr_if_needed(dst, result);
-
- __ bind(skip_barrier);
- __ block_comment("} load_at (zgc)");
-}
-
-#ifdef ASSERT
-// The Z store barrier only verifies the pointers it is operating on and is thus a sole debugging measure.
-void XBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
- Register base, RegisterOrConstant ind_or_offs, Register val,
- Register tmp1, Register tmp2, Register tmp3,
- MacroAssembler::PreservationLevel preservation_level) {
- __ block_comment("store_at (zgc) {");
-
- // If the 'val' register is 'noreg', the to-be-stored value is a null pointer.
- if (is_reference_type(type) && val != noreg) {
- __ ld(tmp1, in_bytes(XThreadLocalData::address_bad_mask_offset()), R16_thread);
- __ and_(tmp1, tmp1, val);
- __ asm_assert_eq("Detected dirty pointer on the heap in Z store barrier");
- }
-
- // Store value
- BarrierSetAssembler::store_at(masm, decorators, type, base, ind_or_offs, val, tmp1, tmp2, tmp3, preservation_level);
-
- __ block_comment("} store_at (zgc)");
-}
-#endif // ASSERT
-
-void XBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType component_type,
- Register src, Register dst, Register count,
- Register preserve1, Register preserve2) {
- __ block_comment("arraycopy_prologue (zgc) {");
-
- /* ==== Check whether a special gc barrier is required for this particular load ==== */
- if (!is_reference_type(component_type)) {
- return;
- }
-
- Label skip_barrier;
-
- // Fast path: Array is of length zero
- __ cmpdi(CCR0, count, 0);
- __ beq(CCR0, skip_barrier);
-
- /* ==== Ensure register sanity ==== */
- Register tmp_R11 = R11_scratch1;
-
- assert_different_registers(src, dst, count, tmp_R11, noreg);
- if (preserve1 != noreg) {
- // Not technically required, but unlikely being intended.
- assert_different_registers(preserve1, preserve2);
- }
-
- /* ==== Invoke barrier (slowpath) ==== */
- int nbytes_save = 0;
-
- {
- assert(!noreg->is_volatile(), "sanity");
-
- if (preserve1->is_volatile()) {
- __ std(preserve1, -BytesPerWord * ++nbytes_save, R1_SP);
- }
-
- if (preserve2->is_volatile() && preserve1 != preserve2) {
- __ std(preserve2, -BytesPerWord * ++nbytes_save, R1_SP);
- }
-
- __ std(src, -BytesPerWord * ++nbytes_save, R1_SP);
- __ std(dst, -BytesPerWord * ++nbytes_save, R1_SP);
- __ std(count, -BytesPerWord * ++nbytes_save, R1_SP);
-
- __ save_LR(tmp_R11);
- __ push_frame_reg_args(nbytes_save, tmp_R11);
- }
-
- // XBarrierSetRuntime::load_barrier_on_oop_array_addr(src, count)
- if (count == R3_ARG1) {
- if (src == R4_ARG2) {
- // Arguments are provided in reverse order
- __ mr(tmp_R11, count);
- __ mr(R3_ARG1, src);
- __ mr(R4_ARG2, tmp_R11);
- } else {
- __ mr(R4_ARG2, count);
- __ mr(R3_ARG1, src);
- }
- } else {
- __ mr_if_needed(R3_ARG1, src);
- __ mr_if_needed(R4_ARG2, count);
- }
-
- __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_array_addr());
-
- __ pop_frame();
- __ restore_LR(tmp_R11);
-
- {
- __ ld(count, -BytesPerWord * nbytes_save--, R1_SP);
- __ ld(dst, -BytesPerWord * nbytes_save--, R1_SP);
- __ ld(src, -BytesPerWord * nbytes_save--, R1_SP);
-
- if (preserve2->is_volatile() && preserve1 != preserve2) {
- __ ld(preserve2, -BytesPerWord * nbytes_save--, R1_SP);
- }
-
- if (preserve1->is_volatile()) {
- __ ld(preserve1, -BytesPerWord * nbytes_save--, R1_SP);
- }
- }
-
- __ bind(skip_barrier);
-
- __ block_comment("} arraycopy_prologue (zgc)");
-}
-
-void XBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
- Register obj, Register tmp, Label& slowpath) {
- __ block_comment("try_resolve_jobject_in_native (zgc) {");
-
- assert_different_registers(jni_env, obj, tmp);
-
- // Resolve the pointer using the standard implementation for weak tag handling and pointer verification.
- BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
-
- // Check whether pointer is dirty.
- __ ld(tmp,
- in_bytes(XThreadLocalData::address_bad_mask_offset() - JavaThread::jni_environment_offset()),
- jni_env);
-
- __ and_(tmp, obj, tmp);
- __ bne(CCR0, slowpath);
-
- __ block_comment("} try_resolve_jobject_in_native (zgc)");
-}
-
-#undef __
-
-#ifdef COMPILER1
-#define __ ce->masm()->
-
-// Code emitted by LIR node "LIR_OpXLoadBarrierTest" which in turn is emitted by XBarrierSetC1::load_barrier.
-// The actual compare and branch instructions are represented as stand-alone LIR nodes.
-void XBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
- LIR_Opr ref) const {
- __ block_comment("load_barrier_test (zgc) {");
-
- __ ld(R0, in_bytes(XThreadLocalData::address_bad_mask_offset()), R16_thread);
- __ andr(R0, R0, ref->as_pointer_register());
- __ cmpdi(CCR5 /* as mandated by LIR node */, R0, 0);
-
- __ block_comment("} load_barrier_test (zgc)");
-}
-
-// Code emitted by code stub "XLoadBarrierStubC1" which in turn is emitted by XBarrierSetC1::load_barrier.
-// Invokes the runtime stub which is defined just below.
-void XBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
- XLoadBarrierStubC1* stub) const {
- __ block_comment("c1_load_barrier_stub (zgc) {");
-
- __ bind(*stub->entry());
-
- /* ==== Determine relevant data registers and ensure register sanity ==== */
- Register ref = stub->ref()->as_register();
- Register ref_addr = noreg;
-
- // Determine reference address
- if (stub->tmp()->is_valid()) {
- // 'tmp' register is given, so address might have an index or a displacement.
- ce->leal(stub->ref_addr(), stub->tmp());
- ref_addr = stub->tmp()->as_pointer_register();
- } else {
- // 'tmp' register is not given, so address must have neither an index nor a displacement.
- // The address' base register is thus usable as-is.
- assert(stub->ref_addr()->as_address_ptr()->disp() == 0, "illegal displacement");
- assert(!stub->ref_addr()->as_address_ptr()->index()->is_valid(), "illegal index");
-
- ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
- }
-
- assert_different_registers(ref, ref_addr, R0, noreg);
-
- /* ==== Invoke stub ==== */
- // Pass arguments via stack. The stack pointer will be bumped by the stub.
- __ std(ref, (intptr_t) -1 * BytesPerWord, R1_SP);
- __ std(ref_addr, (intptr_t) -2 * BytesPerWord, R1_SP);
-
- __ load_const_optimized(R0, stub->runtime_stub());
- __ call_stub(R0);
-
- // The runtime stub passes the result via the R0 register, overriding the previously-loaded stub address.
- __ mr_if_needed(ref, R0);
- __ b(*stub->continuation());
-
- __ block_comment("} c1_load_barrier_stub (zgc)");
-}
-
-#undef __
-#define __ sasm->
-
-// Code emitted by runtime code stub which in turn is emitted by XBarrierSetC1::generate_c1_runtime_stubs.
-void XBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
- DecoratorSet decorators) const {
- __ block_comment("c1_load_barrier_runtime_stub (zgc) {");
-
- const int stack_parameters = 2;
- const int nbytes_save = (MacroAssembler::num_volatile_regs + stack_parameters) * BytesPerWord;
-
- __ save_volatile_gprs(R1_SP, -nbytes_save);
- __ save_LR(R0);
-
- // Load arguments back again from the stack.
- __ ld(R3_ARG1, (intptr_t) -1 * BytesPerWord, R1_SP); // ref
- __ ld(R4_ARG2, (intptr_t) -2 * BytesPerWord, R1_SP); // ref_addr
-
- __ push_frame_reg_args(nbytes_save, R0);
-
- __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators));
-
- __ verify_oop(R3_RET, "Bad pointer after barrier invocation");
- __ mr(R0, R3_RET);
-
- __ pop_frame();
- __ restore_LR(R3_RET);
- __ restore_volatile_gprs(R1_SP, -nbytes_save);
-
- __ blr();
-
- __ block_comment("} c1_load_barrier_runtime_stub (zgc)");
-}
-
-#undef __
-#endif // COMPILER1
-
-#ifdef COMPILER2
-
-OptoReg::Name XBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) const {
- if (!OptoReg::is_reg(opto_reg)) {
- return OptoReg::Bad;
- }
-
- VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
- if ((vm_reg->is_Register() || vm_reg ->is_FloatRegister()) && (opto_reg & 1) != 0) {
- return OptoReg::Bad;
- }
-
- return opto_reg;
-}
-
-#define __ _masm->
-
-class XSaveLiveRegisters {
- MacroAssembler* _masm;
- RegMask _reg_mask;
- Register _result_reg;
- int _frame_size;
-
- public:
- XSaveLiveRegisters(MacroAssembler *masm, XLoadBarrierStubC2 *stub)
- : _masm(masm), _reg_mask(stub->live()), _result_reg(stub->ref()) {
-
- const int register_save_size = iterate_over_register_mask(ACTION_COUNT_ONLY) * BytesPerWord;
- _frame_size = align_up(register_save_size, frame::alignment_in_bytes)
- + frame::native_abi_reg_args_size;
-
- __ save_LR_CR(R0);
- __ push_frame(_frame_size, R0);
-
- iterate_over_register_mask(ACTION_SAVE, _frame_size);
- }
-
- ~XSaveLiveRegisters() {
- iterate_over_register_mask(ACTION_RESTORE, _frame_size);
-
- __ addi(R1_SP, R1_SP, _frame_size);
- __ restore_LR_CR(R0);
- }
-
- private:
- enum IterationAction : int {
- ACTION_SAVE,
- ACTION_RESTORE,
- ACTION_COUNT_ONLY
- };
-
- int iterate_over_register_mask(IterationAction action, int offset = 0) {
- int reg_save_index = 0;
- RegMaskIterator live_regs_iterator(_reg_mask);
-
- while(live_regs_iterator.has_next()) {
- const OptoReg::Name opto_reg = live_regs_iterator.next();
-
- // Filter out stack slots (spilled registers, i.e., stack-allocated registers).
- if (!OptoReg::is_reg(opto_reg)) {
- continue;
- }
-
- const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
- if (vm_reg->is_Register()) {
- Register std_reg = vm_reg->as_Register();
-
- // '_result_reg' will hold the end result of the operation. Its content must thus not be preserved.
- if (std_reg == _result_reg) {
- continue;
- }
-
- if (std_reg->encoding() >= R2->encoding() && std_reg->encoding() <= R12->encoding()) {
- reg_save_index++;
-
- if (action == ACTION_SAVE) {
- _masm->std(std_reg, offset - reg_save_index * BytesPerWord, R1_SP);
- } else if (action == ACTION_RESTORE) {
- _masm->ld(std_reg, offset - reg_save_index * BytesPerWord, R1_SP);
- } else {
- assert(action == ACTION_COUNT_ONLY, "Sanity");
- }
- }
- } else if (vm_reg->is_FloatRegister()) {
- FloatRegister fp_reg = vm_reg->as_FloatRegister();
- if (fp_reg->encoding() >= F0->encoding() && fp_reg->encoding() <= F13->encoding()) {
- reg_save_index++;
-
- if (action == ACTION_SAVE) {
- _masm->stfd(fp_reg, offset - reg_save_index * BytesPerWord, R1_SP);
- } else if (action == ACTION_RESTORE) {
- _masm->lfd(fp_reg, offset - reg_save_index * BytesPerWord, R1_SP);
- } else {
- assert(action == ACTION_COUNT_ONLY, "Sanity");
- }
- }
- } else if (vm_reg->is_ConditionRegister()) {
- // NOP. Conditions registers are covered by save_LR_CR
- } else if (vm_reg->is_VectorSRegister()) {
- assert(SuperwordUseVSX, "or should not reach here");
- VectorSRegister vs_reg = vm_reg->as_VectorSRegister();
- if (vs_reg->encoding() >= VSR32->encoding() && vs_reg->encoding() <= VSR51->encoding()) {
- reg_save_index += 2;
-
- Register spill_addr = R0;
- if (action == ACTION_SAVE) {
- _masm->addi(spill_addr, R1_SP, offset - reg_save_index * BytesPerWord);
- _masm->stxvd2x(vs_reg, spill_addr);
- } else if (action == ACTION_RESTORE) {
- _masm->addi(spill_addr, R1_SP, offset - reg_save_index * BytesPerWord);
- _masm->lxvd2x(vs_reg, spill_addr);
- } else {
- assert(action == ACTION_COUNT_ONLY, "Sanity");
- }
- }
- } else {
- if (vm_reg->is_SpecialRegister()) {
- fatal("Special registers are unsupported. Found register %s", vm_reg->name());
- } else {
- fatal("Register type is not known");
- }
- }
- }
-
- return reg_save_index;
- }
-};
-
-#undef __
-#define __ _masm->
-
-class XSetupArguments {
- MacroAssembler* const _masm;
- const Register _ref;
- const Address _ref_addr;
-
- public:
- XSetupArguments(MacroAssembler* masm, XLoadBarrierStubC2* stub) :
- _masm(masm),
- _ref(stub->ref()),
- _ref_addr(stub->ref_addr()) {
-
- // Desired register/argument configuration:
- // _ref: R3_ARG1
- // _ref_addr: R4_ARG2
-
- // '_ref_addr' can be unspecified. In that case, the barrier will not heal the reference.
- if (_ref_addr.base() == noreg) {
- assert_different_registers(_ref, R0, noreg);
-
- __ mr_if_needed(R3_ARG1, _ref);
- __ li(R4_ARG2, 0);
- } else {
- assert_different_registers(_ref, _ref_addr.base(), R0, noreg);
- assert(!_ref_addr.index()->is_valid(), "reference addresses must not contain an index component");
-
- if (_ref != R4_ARG2) {
- // Calculate address first as the address' base register might clash with R4_ARG2
- __ addi(R4_ARG2, _ref_addr.base(), _ref_addr.disp());
- __ mr_if_needed(R3_ARG1, _ref);
- } else if (_ref_addr.base() != R3_ARG1) {
- __ mr(R3_ARG1, _ref);
- __ addi(R4_ARG2, _ref_addr.base(), _ref_addr.disp()); // Clobbering _ref
- } else {
- // Arguments are provided in inverse order (i.e. _ref == R4_ARG2, _ref_addr == R3_ARG1)
- __ mr(R0, _ref);
- __ addi(R4_ARG2, _ref_addr.base(), _ref_addr.disp());
- __ mr(R3_ARG1, R0);
- }
- }
- }
-};
-
-#undef __
-#define __ masm->
-
-void XBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, XLoadBarrierStubC2* stub) const {
- __ block_comment("generate_c2_load_barrier_stub (zgc) {");
-
- __ bind(*stub->entry());
-
- Register ref = stub->ref();
- Address ref_addr = stub->ref_addr();
-
- assert_different_registers(ref, ref_addr.base());
-
- {
- XSaveLiveRegisters save_live_registers(masm, stub);
- XSetupArguments setup_arguments(masm, stub);
-
- __ call_VM_leaf(stub->slow_path());
- __ mr_if_needed(ref, R3_RET);
- }
-
- __ b(*stub->continuation());
-
- __ block_comment("} generate_c2_load_barrier_stub (zgc)");
-}
-
-#undef __
-#endif // COMPILER2
diff --git a/src/hotspot/cpu/ppc/gc/x/xBarrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/x/xBarrierSetAssembler_ppc.hpp
deleted file mode 100644
index 8dfd4524dfe5a..0000000000000
--- a/src/hotspot/cpu/ppc/gc/x/xBarrierSetAssembler_ppc.hpp
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2021, 2022 SAP SE. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#ifndef CPU_PPC_GC_X_XBARRIERSETASSEMBLER_PPC_HPP
-#define CPU_PPC_GC_X_XBARRIERSETASSEMBLER_PPC_HPP
-
-#include "code/vmreg.hpp"
-#include "oops/accessDecorators.hpp"
-#ifdef COMPILER2
-#include "opto/optoreg.hpp"
-#endif // COMPILER2
-
-#ifdef COMPILER1
-class LIR_Assembler;
-class LIR_Opr;
-class StubAssembler;
-#endif // COMPILER1
-
-#ifdef COMPILER2
-class Node;
-#endif // COMPILER2
-
-#ifdef COMPILER1
-class XLoadBarrierStubC1;
-#endif // COMPILER1
-
-#ifdef COMPILER2
-class XLoadBarrierStubC2;
-#endif // COMPILER2
-
-class XBarrierSetAssembler : public XBarrierSetAssemblerBase {
-public:
- virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
- Register base, RegisterOrConstant ind_or_offs, Register dst,
- Register tmp1, Register tmp2,
- MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null = nullptr);
-
-#ifdef ASSERT
- virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
- Register base, RegisterOrConstant ind_or_offs, Register val,
- Register tmp1, Register tmp2, Register tmp3,
- MacroAssembler::PreservationLevel preservation_level);
-#endif // ASSERT
-
- virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
- Register src, Register dst, Register count,
- Register preserve1, Register preserve2);
-
- virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
- Register obj, Register tmp, Label& slowpath);
-
- virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_data_patch; }
-
-#ifdef COMPILER1
- void generate_c1_load_barrier_test(LIR_Assembler* ce,
- LIR_Opr ref) const;
-
- void generate_c1_load_barrier_stub(LIR_Assembler* ce,
- XLoadBarrierStubC1* stub) const;
-
- void generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
- DecoratorSet decorators) const;
-#endif // COMPILER1
-
-#ifdef COMPILER2
- OptoReg::Name refine_register(const Node* node, OptoReg::Name opto_reg) const;
-
- void generate_c2_load_barrier_stub(MacroAssembler* masm, XLoadBarrierStubC2* stub) const;
-#endif // COMPILER2
-};
-
-#endif // CPU_PPC_GC_X_XBARRIERSETASSEMBLER_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/gc/x/xGlobals_ppc.cpp b/src/hotspot/cpu/ppc/gc/x/xGlobals_ppc.cpp
deleted file mode 100644
index 3218a765fc703..0000000000000
--- a/src/hotspot/cpu/ppc/gc/x/xGlobals_ppc.cpp
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2021 SAP SE. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/shared/gcLogPrecious.hpp"
-#include "gc/shared/gc_globals.hpp"
-#include "gc/x/xGlobals.hpp"
-#include "runtime/globals.hpp"
-#include "runtime/os.hpp"
-#include "utilities/globalDefinitions.hpp"
-#include "utilities/powerOfTwo.hpp"
-#include
-
-#ifdef LINUX
-#include
-#endif // LINUX
-
-//
-// The overall memory layouts across different power platforms are similar and only differ with regards to
-// the position of the highest addressable bit; the position of the metadata bits and the size of the actual
-// addressable heap address space are adjusted accordingly.
-//
-// The following memory schema shows an exemplary layout in which bit '45' is the highest addressable bit.
-// It is assumed that this virtual memory address space layout is predominant on the power platform.
-//
-// Standard Address Space & Pointer Layout
-// ---------------------------------------
-//
-// +--------------------------------+ 0x00007FFFFFFFFFFF (127 TiB - 1)
-// . .
-// . .
-// . .
-// +--------------------------------+ 0x0000140000000000 (20 TiB)
-// | Remapped View |
-// +--------------------------------+ 0x0000100000000000 (16 TiB)
-// . .
-// +--------------------------------+ 0x00000c0000000000 (12 TiB)
-// | Marked1 View |
-// +--------------------------------+ 0x0000080000000000 (8 TiB)
-// | Marked0 View |
-// +--------------------------------+ 0x0000040000000000 (4 TiB)
-// . .
-// +--------------------------------+ 0x0000000000000000
-//
-// 6 4 4 4 4
-// 3 6 5 2 1 0
-// +--------------------+----+-----------------------------------------------+
-// |00000000 00000000 00|1111|11 11111111 11111111 11111111 11111111 11111111|
-// +--------------------+----+-----------------------------------------------+
-// | | |
-// | | * 41-0 Object Offset (42-bits, 4TB address space)
-// | |
-// | * 45-42 Metadata Bits (4-bits) 0001 = Marked0 (Address view 4-8TB)
-// | 0010 = Marked1 (Address view 8-12TB)
-// | 0100 = Remapped (Address view 16-20TB)
-// | 1000 = Finalizable (Address view N/A)
-// |
-// * 63-46 Fixed (18-bits, always zero)
-//
-
-// Maximum value as per spec (Power ISA v2.07): 2 ^ 60 bytes, i.e. 1 EiB (exbibyte)
-static const unsigned int MAXIMUM_MAX_ADDRESS_BIT = 60;
-
-// Most modern power processors provide an address space with not more than 45 bit addressable bit,
-// that is an address space of 32 TiB in size.
-static const unsigned int DEFAULT_MAX_ADDRESS_BIT = 45;
-
-// Minimum value returned, if probing fails: 64 GiB
-static const unsigned int MINIMUM_MAX_ADDRESS_BIT = 36;
-
-// Determines the highest addressable bit of the virtual address space (depends on platform)
-// by trying to interact with memory in that address range,
-// i.e. by syncing existing mappings (msync) or by temporarily mapping the memory area (mmap).
-// If one of those operations succeeds, it is proven that the targeted memory area is within the virtual address space.
-//
-// To reduce the number of required system calls to a bare minimum, the DEFAULT_MAX_ADDRESS_BIT is intentionally set
-// lower than what the ABI would theoretically permit.
-// Such an avoidance strategy, however, might impose unnecessary limits on processors that exceed this limit.
-// If DEFAULT_MAX_ADDRESS_BIT is addressable, the next higher bit will be tested as well to ensure that
-// the made assumption does not artificially restrict the memory availability.
-static unsigned int probe_valid_max_address_bit(size_t init_bit, size_t min_bit) {
- assert(init_bit >= min_bit, "Sanity");
- assert(init_bit <= MAXIMUM_MAX_ADDRESS_BIT, "Test bit is outside the assumed address space range");
-
-#ifdef LINUX
- unsigned int max_valid_address_bit = 0;
- void* last_allocatable_address = nullptr;
-
- const size_t page_size = os::vm_page_size();
-
- for (size_t i = init_bit; i >= min_bit; --i) {
- void* base_addr = (void*) (((unsigned long) 1U) << i);
-
- /* ==== Try msync-ing already mapped memory page ==== */
- if (msync(base_addr, page_size, MS_ASYNC) == 0) {
- // The page of the given address was synced by the linux kernel and must thus be both, mapped and valid.
- max_valid_address_bit = i;
- break;
- }
- if (errno != ENOMEM) {
- // An unexpected error occurred, i.e. an error not indicating that the targeted memory page is unmapped,
- // but pointing out another type of issue.
- // Even though this should never happen, those issues may come up due to undefined behavior.
-#ifdef ASSERT
- fatal("Received '%s' while probing the address space for the highest valid bit", os::errno_name(errno));
-#else // ASSERT
- log_warning_p(gc)("Received '%s' while probing the address space for the highest valid bit", os::errno_name(errno));
-#endif // ASSERT
- continue;
- }
-
- /* ==== Try mapping memory page on our own ==== */
- last_allocatable_address = mmap(base_addr, page_size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0);
- if (last_allocatable_address != MAP_FAILED) {
- munmap(last_allocatable_address, page_size);
- }
-
- if (last_allocatable_address == base_addr) {
- // As the linux kernel mapped exactly the page we have requested, the address must be valid.
- max_valid_address_bit = i;
- break;
- }
-
- log_info_p(gc, init)("Probe failed for bit '%zu'", i);
- }
-
- if (max_valid_address_bit == 0) {
- // Probing did not bring up any usable address bit.
- // As an alternative, the VM evaluates the address returned by mmap as it is expected that the reserved page
- // will be close to the probed address that was out-of-range.
- // As per mmap(2), "the kernel [will take] [the address] as a hint about where to
- // place the mapping; on Linux, the mapping will be created at a nearby page boundary".
- // It should thus be a "close enough" approximation to the real virtual memory address space limit.
- //
- // This recovery strategy is only applied in production builds.
- // In debug builds, an assertion in 'XPlatformAddressOffsetBits' will bail out the VM to indicate that
- // the assumed address space is no longer up-to-date.
- if (last_allocatable_address != MAP_FAILED) {
- const unsigned int bitpos = BitsPerSize_t - count_leading_zeros((size_t) last_allocatable_address) - 1;
- log_info_p(gc, init)("Did not find any valid addresses within the range, using address '%u' instead", bitpos);
- return bitpos;
- }
-
-#ifdef ASSERT
- fatal("Available address space can not be determined");
-#else // ASSERT
- log_warning_p(gc)("Cannot determine available address space. Falling back to default value.");
- return DEFAULT_MAX_ADDRESS_BIT;
-#endif // ASSERT
- } else {
- if (max_valid_address_bit == init_bit) {
- // An usable address bit has been found immediately.
- // To ensure that the entire virtual address space is exploited, the next highest bit will be tested as well.
- log_info_p(gc, init)("Hit valid address '%u' on first try, retrying with next higher bit", max_valid_address_bit);
- return MAX2(max_valid_address_bit, probe_valid_max_address_bit(init_bit + 1, init_bit + 1));
- }
- }
-
- log_info_p(gc, init)("Found valid address '%u'", max_valid_address_bit);
- return max_valid_address_bit;
-#else // LINUX
- return DEFAULT_MAX_ADDRESS_BIT;
-#endif // LINUX
-}
-
-size_t XPlatformAddressOffsetBits() {
- const static unsigned int valid_max_address_offset_bits =
- probe_valid_max_address_bit(DEFAULT_MAX_ADDRESS_BIT, MINIMUM_MAX_ADDRESS_BIT) + 1;
- assert(valid_max_address_offset_bits >= MINIMUM_MAX_ADDRESS_BIT,
- "Highest addressable bit is outside the assumed address space range");
-
- const size_t max_address_offset_bits = valid_max_address_offset_bits - 3;
- const size_t min_address_offset_bits = max_address_offset_bits - 2;
- const size_t address_offset = round_up_power_of_2(MaxHeapSize * XVirtualToPhysicalRatio);
- const size_t address_offset_bits = log2i_exact(address_offset);
-
- return clamp(address_offset_bits, min_address_offset_bits, max_address_offset_bits);
-}
-
-size_t XPlatformAddressMetadataShift() {
- return XPlatformAddressOffsetBits();
-}
diff --git a/src/hotspot/cpu/ppc/gc/x/xGlobals_ppc.hpp b/src/hotspot/cpu/ppc/gc/x/xGlobals_ppc.hpp
deleted file mode 100644
index be88b05b02a82..0000000000000
--- a/src/hotspot/cpu/ppc/gc/x/xGlobals_ppc.hpp
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2021 SAP SE. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#ifndef CPU_PPC_GC_X_XGLOBALS_PPC_HPP
-#define CPU_PPC_GC_X_XGLOBALS_PPC_HPP
-
-#include "globalDefinitions_ppc.hpp"
-
-const size_t XPlatformHeapViews = 3;
-const size_t XPlatformCacheLineSize = DEFAULT_CACHE_LINE_SIZE;
-
-size_t XPlatformAddressOffsetBits();
-size_t XPlatformAddressMetadataShift();
-
-#endif // CPU_PPC_GC_X_XGLOBALS_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/gc/x/x_ppc.ad b/src/hotspot/cpu/ppc/gc/x/x_ppc.ad
deleted file mode 100644
index b206b6593fb45..0000000000000
--- a/src/hotspot/cpu/ppc/gc/x/x_ppc.ad
+++ /dev/null
@@ -1,298 +0,0 @@
-//
-// Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
-// Copyright (c) 2021 SAP SE. All rights reserved.
-// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-//
-// This code is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License version 2 only, as
-// published by the Free Software Foundation.
-//
-// This code is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-// version 2 for more details (a copy is included in the LICENSE file that
-// accompanied this code).
-//
-// You should have received a copy of the GNU General Public License version
-// 2 along with this work; if not, write to the Free Software Foundation,
-// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-//
-// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-// or visit www.oracle.com if you need additional information or have any
-// questions.
-//
-
-source_hpp %{
-
-#include "gc/shared/gc_globals.hpp"
-#include "gc/x/c2/xBarrierSetC2.hpp"
-#include "gc/x/xThreadLocalData.hpp"
-
-%}
-
-source %{
-
-static void x_load_barrier(MacroAssembler* masm, const MachNode* node, Address ref_addr, Register ref,
- Register tmp, uint8_t barrier_data) {
- if (barrier_data == XLoadBarrierElided) {
- return;
- }
-
- XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data);
- __ ld(tmp, in_bytes(XThreadLocalData::address_bad_mask_offset()), R16_thread);
- __ and_(tmp, tmp, ref);
- __ bne_far(CCR0, *stub->entry(), MacroAssembler::bc_far_optimize_on_relocate);
- __ bind(*stub->continuation());
-}
-
-static void x_load_barrier_slow_path(MacroAssembler* masm, const MachNode* node, Address ref_addr, Register ref,
- Register tmp) {
- XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, XLoadBarrierStrong);
- __ b(*stub->entry());
- __ bind(*stub->continuation());
-}
-
-static void x_compare_and_swap(MacroAssembler* masm, const MachNode* node,
- Register res, Register mem, Register oldval, Register newval,
- Register tmp_xchg, Register tmp_mask,
- bool weak, bool acquire) {
- // z-specific load barrier requires strong CAS operations.
- // Weak CAS operations are thus only emitted if the barrier is elided.
- __ cmpxchgd(CCR0, tmp_xchg, oldval, newval, mem,
- MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), res, nullptr, true,
- weak && node->barrier_data() == XLoadBarrierElided);
-
- if (node->barrier_data() != XLoadBarrierElided) {
- Label skip_barrier;
-
- __ ld(tmp_mask, in_bytes(XThreadLocalData::address_bad_mask_offset()), R16_thread);
- __ and_(tmp_mask, tmp_mask, tmp_xchg);
- __ beq(CCR0, skip_barrier);
-
- // CAS must have failed because pointer in memory is bad.
- x_load_barrier_slow_path(masm, node, Address(mem), tmp_xchg, res /* used as tmp */);
-
- __ cmpxchgd(CCR0, tmp_xchg, oldval, newval, mem,
- MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), res, nullptr, true, weak);
-
- __ bind(skip_barrier);
- }
-
- if (acquire) {
- if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
- // Uses the isync instruction as an acquire barrier.
- // This exploits the compare and the branch in the z load barrier (load, compare and branch, isync).
- __ isync();
- } else {
- __ sync();
- }
- }
-}
-
-static void x_compare_and_exchange(MacroAssembler* masm, const MachNode* node,
- Register res, Register mem, Register oldval, Register newval, Register tmp,
- bool weak, bool acquire) {
- // z-specific load barrier requires strong CAS operations.
- // Weak CAS operations are thus only emitted if the barrier is elided.
- __ cmpxchgd(CCR0, res, oldval, newval, mem,
- MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), noreg, nullptr, true,
- weak && node->barrier_data() == XLoadBarrierElided);
-
- if (node->barrier_data() != XLoadBarrierElided) {
- Label skip_barrier;
- __ ld(tmp, in_bytes(XThreadLocalData::address_bad_mask_offset()), R16_thread);
- __ and_(tmp, tmp, res);
- __ beq(CCR0, skip_barrier);
-
- x_load_barrier_slow_path(masm, node, Address(mem), res, tmp);
-
- __ cmpxchgd(CCR0, res, oldval, newval, mem,
- MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), noreg, nullptr, true, weak);
-
- __ bind(skip_barrier);
- }
-
- if (acquire) {
- if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
- // Uses the isync instruction as an acquire barrier.
- // This exploits the compare and the branch in the z load barrier (load, compare and branch, isync).
- __ isync();
- } else {
- __ sync();
- }
- }
-}
-
-%}
-
-instruct xLoadP(iRegPdst dst, memoryAlg4 mem, iRegPdst tmp, flagsRegCR0 cr0)
-%{
- match(Set dst (LoadP mem));
- effect(TEMP_DEF dst, TEMP tmp, KILL cr0);
- ins_cost(MEMORY_REF_COST);
-
- predicate((UseZGC && !ZGenerational && n->as_Load()->barrier_data() != 0)
- && (n->as_Load()->is_unordered() || followed_by_acquire(n)));
-
- format %{ "LD $dst, $mem" %}
- ins_encode %{
- assert($mem$$index == 0, "sanity");
- __ ld($dst$$Register, $mem$$disp, $mem$$base$$Register);
- x_load_barrier(masm, this, Address($mem$$base$$Register, $mem$$disp), $dst$$Register, $tmp$$Register, barrier_data());
- %}
- ins_pipe(pipe_class_default);
-%}
-
-// Load Pointer Volatile
-instruct xLoadP_acq(iRegPdst dst, memoryAlg4 mem, iRegPdst tmp, flagsRegCR0 cr0)
-%{
- match(Set dst (LoadP mem));
- effect(TEMP_DEF dst, TEMP tmp, KILL cr0);
- ins_cost(3 * MEMORY_REF_COST);
-
- // Predicate on instruction order is implicitly present due to the predicate of the cheaper zLoadP operation
- predicate(UseZGC && !ZGenerational && n->as_Load()->barrier_data() != 0);
-
- format %{ "LD acq $dst, $mem" %}
- ins_encode %{
- __ ld($dst$$Register, $mem$$disp, $mem$$base$$Register);
- x_load_barrier(masm, this, Address($mem$$base$$Register, $mem$$disp), $dst$$Register, $tmp$$Register, barrier_data());
-
- // Uses the isync instruction as an acquire barrier.
- // This exploits the compare and the branch in the z load barrier (load, compare and branch, isync).
- __ isync();
- %}
- ins_pipe(pipe_class_default);
-%}
-
-instruct xCompareAndSwapP(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
- iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{
- match(Set res (CompareAndSwapP mem (Binary oldval newval)));
- effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0);
-
- predicate((UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong)
- && (((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*) n)->order() != MemNode::seqcst));
-
- format %{ "CMPXCHG $res, $mem, $oldval, $newval; as bool; ptr" %}
- ins_encode %{
- x_compare_and_swap(masm, this,
- $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register,
- $tmp_xchg$$Register, $tmp_mask$$Register,
- false /* weak */, false /* acquire */);
- %}
- ins_pipe(pipe_class_default);
-%}
-
-instruct xCompareAndSwapP_acq(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
- iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{
- match(Set res (CompareAndSwapP mem (Binary oldval newval)));
- effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0);
-
- predicate((UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong)
- && (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*) n)->order() == MemNode::seqcst));
-
- format %{ "CMPXCHG acq $res, $mem, $oldval, $newval; as bool; ptr" %}
- ins_encode %{
- x_compare_and_swap(masm, this,
- $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register,
- $tmp_xchg$$Register, $tmp_mask$$Register,
- false /* weak */, true /* acquire */);
- %}
- ins_pipe(pipe_class_default);
-%}
-
-instruct xCompareAndSwapPWeak(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
- iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{
- match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
- effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0);
-
- predicate((UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong)
- && ((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*) n)->order() != MemNode::seqcst);
-
- format %{ "weak CMPXCHG $res, $mem, $oldval, $newval; as bool; ptr" %}
- ins_encode %{
- x_compare_and_swap(masm, this,
- $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register,
- $tmp_xchg$$Register, $tmp_mask$$Register,
- true /* weak */, false /* acquire */);
- %}
- ins_pipe(pipe_class_default);
-%}
-
-instruct xCompareAndSwapPWeak_acq(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
- iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{
- match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
- effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0);
-
- predicate((UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong)
- && (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*) n)->order() == MemNode::seqcst));
-
- format %{ "weak CMPXCHG acq $res, $mem, $oldval, $newval; as bool; ptr" %}
- ins_encode %{
- x_compare_and_swap(masm, this,
- $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register,
- $tmp_xchg$$Register, $tmp_mask$$Register,
- true /* weak */, true /* acquire */);
- %}
- ins_pipe(pipe_class_default);
-%}
-
-instruct xCompareAndExchangeP(iRegPdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
- iRegPdst tmp, flagsRegCR0 cr0) %{
- match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
- effect(TEMP_DEF res, TEMP tmp, KILL cr0);
-
- predicate((UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong)
- && (
- ((CompareAndSwapNode*)n)->order() != MemNode::acquire
- && ((CompareAndSwapNode*)n)->order() != MemNode::seqcst
- ));
-
- format %{ "CMPXCHG $res, $mem, $oldval, $newval; as ptr; ptr" %}
- ins_encode %{
- x_compare_and_exchange(masm, this,
- $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, $tmp$$Register,
- false /* weak */, false /* acquire */);
- %}
- ins_pipe(pipe_class_default);
-%}
-
-instruct xCompareAndExchangeP_acq(iRegPdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
- iRegPdst tmp, flagsRegCR0 cr0) %{
- match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
- effect(TEMP_DEF res, TEMP tmp, KILL cr0);
-
- predicate((UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong)
- && (
- ((CompareAndSwapNode*)n)->order() == MemNode::acquire
- || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst
- ));
-
- format %{ "CMPXCHG acq $res, $mem, $oldval, $newval; as ptr; ptr" %}
- ins_encode %{
- x_compare_and_exchange(masm, this,
- $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, $tmp$$Register,
- false /* weak */, true /* acquire */);
- %}
- ins_pipe(pipe_class_default);
-%}
-
-instruct xGetAndSetP(iRegPdst res, iRegPdst mem, iRegPsrc newval, iRegPdst tmp, flagsRegCR0 cr0) %{
- match(Set res (GetAndSetP mem newval));
- effect(TEMP_DEF res, TEMP tmp, KILL cr0);
-
- predicate(UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() != 0);
-
- format %{ "GetAndSetP $res, $mem, $newval" %}
- ins_encode %{
- __ getandsetd($res$$Register, $newval$$Register, $mem$$Register, MacroAssembler::cmpxchgx_hint_atomic_update());
- x_load_barrier(masm, this, Address(noreg, (intptr_t) 0), $res$$Register, $tmp$$Register, barrier_data());
-
- if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
- __ isync();
- } else {
- __ sync();
- }
- %}
- ins_pipe(pipe_class_default);
-%}
diff --git a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
index 89ab1b1edeeb4..b9ea67dabe3d6 100644
--- a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
@@ -610,14 +610,14 @@ void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, R
// Resolve global handle
__ ld(dst, 0, dst);
- __ ld(tmp, load_bad_mask.disp(), load_bad_mask.base());
+ __ ld(tmp, load_bad_mask);
__ b(check_color);
__ bind(weak_tagged);
// Resolve weak handle
__ ld(dst, 0, dst);
- __ ld(tmp, mark_bad_mask.disp(), mark_bad_mask.base());
+ __ ld(tmp, mark_bad_mask);
__ bind(check_color);
__ and_(tmp, tmp, dst);
@@ -943,6 +943,8 @@ void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm,
__ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_native_oop_field_without_healing_addr(), R3_ARG1);
} else if (stub->is_atomic()) {
__ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr(), R3_ARG1);
+ } else if (stub->is_nokeepalive()) {
+ __ call_VM_leaf(ZBarrierSetRuntime::no_keepalive_store_barrier_on_oop_field_without_healing_addr(), R3_ARG1);
} else {
__ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), R3_ARG1);
}
diff --git a/src/hotspot/cpu/ppc/gc/z/z_ppc.ad b/src/hotspot/cpu/ppc/gc/z/z_ppc.ad
index 017574d40ff8b..97b49bc1b026e 100644
--- a/src/hotspot/cpu/ppc/gc/z/z_ppc.ad
+++ b/src/hotspot/cpu/ppc/gc/z/z_ppc.ad
@@ -83,7 +83,8 @@ static void z_store_barrier(MacroAssembler* masm, const MachNode* node, Register
z_color(masm, rnew_zpointer, rnew_zaddress);
} else {
bool is_native = (node->barrier_data() & ZBarrierNative) != 0;
- ZStoreBarrierStubC2* const stub = ZStoreBarrierStubC2::create(node, Address(ref_base, disp), rnew_zaddress, rnew_zpointer, is_native, is_atomic);
+ bool is_nokeepalive = (node->barrier_data() & ZBarrierNoKeepalive) != 0;
+ ZStoreBarrierStubC2* const stub = ZStoreBarrierStubC2::create(node, Address(ref_base, disp), rnew_zaddress, rnew_zpointer, is_native, is_atomic, is_nokeepalive);
ZBarrierSetAssembler* bs_asm = ZBarrierSet::assembler();
bs_asm->store_barrier_fast(masm, ref_base, disp, rnew_zaddress, rnew_zpointer, true /* in_nmethod */, is_atomic, *stub->entry(), *stub->continuation());
}
@@ -142,7 +143,7 @@ instruct zLoadP(iRegPdst dst, memoryAlg4 mem, flagsRegCR0 cr0)
effect(TEMP_DEF dst, KILL cr0);
ins_cost(MEMORY_REF_COST);
- predicate((UseZGC && ZGenerational && n->as_Load()->barrier_data() != 0)
+ predicate((UseZGC && n->as_Load()->barrier_data() != 0)
&& (n->as_Load()->is_unordered() || followed_by_acquire(n)));
format %{ "LD $dst, $mem" %}
@@ -162,7 +163,7 @@ instruct zLoadP_acq(iRegPdst dst, memoryAlg4 mem, flagsRegCR0 cr0)
ins_cost(3 * MEMORY_REF_COST);
// Predicate on instruction order is implicitly present due to the predicate of the cheaper zLoadP operation
- predicate(UseZGC && ZGenerational && n->as_Load()->barrier_data() != 0);
+ predicate(UseZGC && n->as_Load()->barrier_data() != 0);
format %{ "LD acq $dst, $mem" %}
ins_encode %{
@@ -180,7 +181,7 @@ instruct zLoadP_acq(iRegPdst dst, memoryAlg4 mem, flagsRegCR0 cr0)
// Store Pointer
instruct zStoreP(memoryAlg4 mem, iRegPsrc src, iRegPdst tmp, flagsRegCR0 cr0)
%{
- predicate(UseZGC && ZGenerational && n->as_Store()->barrier_data() != 0);
+ predicate(UseZGC && n->as_Store()->barrier_data() != 0);
match(Set mem (StoreP mem src));
effect(TEMP tmp, KILL cr0);
ins_cost(2 * MEMORY_REF_COST);
@@ -194,7 +195,7 @@ instruct zStoreP(memoryAlg4 mem, iRegPsrc src, iRegPdst tmp, flagsRegCR0 cr0)
instruct zStorePNull(memoryAlg4 mem, immP_0 zero, iRegPdst tmp, flagsRegCR0 cr0)
%{
- predicate(UseZGC && ZGenerational && n->as_Store()->barrier_data() != 0);
+ predicate(UseZGC && n->as_Store()->barrier_data() != 0);
match(Set mem (StoreP mem zero));
effect(TEMP tmp, KILL cr0);
ins_cost(MEMORY_REF_COST);
@@ -212,7 +213,7 @@ instruct zCompareAndSwapP(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc
match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr0);
- predicate((UseZGC && ZGenerational && n->as_LoadStore()->barrier_data() != 0)
+ predicate((UseZGC && n->as_LoadStore()->barrier_data() != 0)
&& (((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*) n)->order() != MemNode::seqcst));
format %{ "CMPXCHG $res, $mem, $oldval, $newval; as bool; ptr" %}
@@ -231,7 +232,7 @@ instruct zCompareAndSwapP_acq(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegP
match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr0);
- predicate((UseZGC && ZGenerational && n->as_LoadStore()->barrier_data() != 0)
+ predicate((UseZGC && n->as_LoadStore()->barrier_data() != 0)
&& (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*) n)->order() == MemNode::seqcst));
format %{ "CMPXCHG acq $res, $mem, $oldval, $newval; as bool; ptr" %}
@@ -249,7 +250,7 @@ instruct zCompareAndExchangeP(iRegPdst res, iRegPdst mem, iRegPsrc oldval, iRegP
match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
effect(TEMP_DEF res, TEMP tmp, KILL cr0);
- predicate((UseZGC && ZGenerational && n->as_LoadStore()->barrier_data() != 0)
+ predicate((UseZGC && n->as_LoadStore()->barrier_data() != 0)
&& (
((CompareAndSwapNode*)n)->order() != MemNode::acquire
&& ((CompareAndSwapNode*)n)->order() != MemNode::seqcst
@@ -269,7 +270,7 @@ instruct zCompareAndExchangeP_acq(iRegPdst res, iRegPdst mem, iRegPsrc oldval, i
match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
effect(TEMP_DEF res, TEMP tmp, KILL cr0);
- predicate((UseZGC && ZGenerational && n->as_LoadStore()->barrier_data() != 0)
+ predicate((UseZGC && n->as_LoadStore()->barrier_data() != 0)
&& (
((CompareAndSwapNode*)n)->order() == MemNode::acquire
|| ((CompareAndSwapNode*)n)->order() == MemNode::seqcst
@@ -288,7 +289,7 @@ instruct zGetAndSetP(iRegPdst res, iRegPdst mem, iRegPsrc newval, iRegPdst tmp,
match(Set res (GetAndSetP mem newval));
effect(TEMP_DEF res, TEMP tmp, KILL cr0);
- predicate(UseZGC && ZGenerational && n->as_LoadStore()->barrier_data() != 0);
+ predicate(UseZGC && n->as_LoadStore()->barrier_data() != 0);
format %{ "GetAndSetP $res, $mem, $newval" %}
ins_encode %{
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
index 8d8e39b8bbc00..c7adbfb52f068 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
@@ -2410,7 +2410,7 @@ void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass,
void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) {
assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required");
- Label L_fallthrough;
+ Label L_check_thread, L_fallthrough;
if (L_fast_path == nullptr) {
L_fast_path = &L_fallthrough;
} else if (L_slow_path == nullptr) {
@@ -2419,10 +2419,14 @@ void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fa
// Fast path check: class is fully initialized
lbz(R0, in_bytes(InstanceKlass::init_state_offset()), klass);
+ // acquire by cmp-branch-isync if fully_initialized
cmpwi(CCR0, R0, InstanceKlass::fully_initialized);
- beq(CCR0, *L_fast_path);
+ bne(CCR0, L_check_thread);
+ isync();
+ b(*L_fast_path);
// Fast path check: current thread is initializer thread
+ bind(L_check_thread);
ld(R0, in_bytes(InstanceKlass::init_thread_offset()), klass);
cmpd(CCR0, thread, R0);
if (L_slow_path == &L_fallthrough) {
@@ -2647,7 +2651,19 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
// flag == NE indicates failure
bind(success);
inc_held_monitor_count(temp);
+#ifdef ASSERT
+ // Check that unlocked label is reached with flag == EQ.
+ Label flag_correct;
+ beq(flag, flag_correct);
+ stop("compiler_fast_lock_object: Flag != EQ");
+#endif
bind(failure);
+#ifdef ASSERT
+ // Check that slow_path label is reached with flag == NE.
+ bne(flag, flag_correct);
+ stop("compiler_fast_lock_object: Flag != NE");
+ bind(flag_correct);
+#endif
}
void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Register oop, Register box,
@@ -2697,17 +2713,12 @@ void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Registe
bind(object_has_monitor);
STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
addi(current_header, current_header, -(int)markWord::monitor_value); // monitor
- ld(temp, in_bytes(ObjectMonitor::owner_offset()), current_header);
-
- // In case of LM_LIGHTWEIGHT, we may reach here with (temp & ObjectMonitor::ANONYMOUS_OWNER) != 0.
- // This is handled like owner thread mismatches: We take the slow path.
- cmpd(flag, temp, R16_thread);
- bne(flag, failure);
ld(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header);
-
addic_(displaced_header, displaced_header, -1);
blt(CCR0, notRecursive); // Not recursive if negative after decrement.
+
+ // Recursive unlock
std(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header);
if (flag == CCR0) { // Otherwise, flag is already EQ, here.
crorc(CCR0, Assembler::equal, CCR0, Assembler::equal); // Set CCR0 EQ
@@ -2725,7 +2736,7 @@ void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Registe
// StoreLoad achieves this.
membar(StoreLoad);
- // Check if the entry lists are empty.
+ // Check if the entry lists are empty (EntryList first - by convention).
ld(temp, in_bytes(ObjectMonitor::EntryList_offset()), current_header);
ld(displaced_header, in_bytes(ObjectMonitor::cxq_offset()), current_header);
orr(temp, temp, displaced_header); // Will be 0 if both are 0.
@@ -2735,20 +2746,32 @@ void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Registe
// Check if there is a successor.
ld(temp, in_bytes(ObjectMonitor::succ_offset()), current_header);
cmpdi(flag, temp, 0);
- bne(flag, success); // If so we are done.
+ // Invert equal bit
+ crnand(flag, Assembler::equal, flag, Assembler::equal);
+ beq(flag, success); // If there is a successor we are done.
// Save the monitor pointer in the current thread, so we can try
// to reacquire the lock in SharedRuntime::monitor_exit_helper().
std(current_header, in_bytes(JavaThread::unlocked_inflated_monitor_offset()), R16_thread);
-
- crxor(flag, Assembler::equal, flag, Assembler::equal); // Set flag = NE => slow path
- b(failure);
+ b(failure); // flag == NE
// flag == EQ indicates success, decrement held monitor count
// flag == NE indicates failure
bind(success);
dec_held_monitor_count(temp);
+#ifdef ASSERT
+ // Check that unlocked label is reached with flag == EQ.
+ Label flag_correct;
+ beq(flag, flag_correct);
+ stop("compiler_fast_unlock_object: Flag != EQ");
+#endif
bind(failure);
+#ifdef ASSERT
+ // Check that slow_path label is reached with flag == NE.
+ bne(flag, flag_correct);
+ stop("compiler_fast_unlock_object: Flag != NE");
+ bind(flag_correct);
+#endif
}
void MacroAssembler::compiler_fast_lock_lightweight_object(ConditionRegister flag, Register obj, Register box,
@@ -3049,7 +3072,6 @@ void MacroAssembler::compiler_fast_unlock_lightweight_object(ConditionRegister f
bind(not_recursive);
- Label set_eq_unlocked;
const Register t2 = tmp2;
// Set owner to null.
@@ -3061,7 +3083,7 @@ void MacroAssembler::compiler_fast_unlock_lightweight_object(ConditionRegister f
// StoreLoad achieves this.
membar(StoreLoad);
- // Check if the entry lists are empty.
+ // Check if the entry lists are empty (EntryList first - by convention).
ld(t, in_bytes(ObjectMonitor::EntryList_offset()), monitor);
ld(t2, in_bytes(ObjectMonitor::cxq_offset()), monitor);
orr(t, t, t2);
@@ -3071,17 +3093,14 @@ void MacroAssembler::compiler_fast_unlock_lightweight_object(ConditionRegister f
// Check if there is a successor.
ld(t, in_bytes(ObjectMonitor::succ_offset()), monitor);
cmpdi(CCR0, t, 0);
- bne(CCR0, set_eq_unlocked); // If so we are done.
+ // Invert equal bit
+ crnand(flag, Assembler::equal, flag, Assembler::equal);
+ beq(CCR0, unlocked); // If there is a successor we are done.
// Save the monitor pointer in the current thread, so we can try
// to reacquire the lock in SharedRuntime::monitor_exit_helper().
std(monitor, in_bytes(JavaThread::unlocked_inflated_monitor_offset()), R16_thread);
-
- crxor(CCR0, Assembler::equal, CCR0, Assembler::equal); // Set flag = NE => slow path
- b(slow_path);
-
- bind(set_eq_unlocked);
- crorc(CCR0, Assembler::equal, CCR0, Assembler::equal); // Set flag = EQ => fast path
+ b(slow_path); // flag == NE
}
bind(unlocked);
@@ -4600,23 +4619,6 @@ void MacroAssembler::zap_from_to(Register low, int before, Register high, int af
#endif // !PRODUCT
-void SkipIfEqualZero::skip_to_label_if_equal_zero(MacroAssembler* masm, Register temp,
- const bool* flag_addr, Label& label) {
- int simm16_offset = masm->load_const_optimized(temp, (address)flag_addr, R0, true);
- assert(sizeof(bool) == 1, "PowerPC ABI");
- masm->lbz(temp, simm16_offset, temp);
- masm->cmpwi(CCR0, temp, 0);
- masm->beq(CCR0, label);
-}
-
-SkipIfEqualZero::SkipIfEqualZero(MacroAssembler* masm, Register temp, const bool* flag_addr) : _masm(masm), _label() {
- skip_to_label_if_equal_zero(masm, temp, flag_addr, _label);
-}
-
-SkipIfEqualZero::~SkipIfEqualZero() {
- _masm->bind(_label);
-}
-
void MacroAssembler::cache_wb(Address line) {
assert(line.index() == noreg, "index should be noreg");
assert(line.disp() == 0, "displacement should be 0");
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
index 224e7bff99541..f0e7c6445354c 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
@@ -960,23 +960,4 @@ class MacroAssembler: public Assembler {
void zap_from_to(Register low, int before, Register high, int after, Register val, Register addr) PRODUCT_RETURN;
};
-// class SkipIfEqualZero:
-//
-// Instantiating this class will result in assembly code being output that will
-// jump around any code emitted between the creation of the instance and it's
-// automatic destruction at the end of a scope block, depending on the value of
-// the flag passed to the constructor, which will be checked at run-time.
-class SkipIfEqualZero : public StackObj {
- private:
- MacroAssembler* _masm;
- Label _label;
-
- public:
- // 'Temp' is a temp register that this object can use (and trash).
- explicit SkipIfEqualZero(MacroAssembler*, Register temp, const bool* flag_addr);
- static void skip_to_label_if_equal_zero(MacroAssembler*, Register temp,
- const bool* flag_addr, Label& label);
- ~SkipIfEqualZero();
-};
-
#endif // CPU_PPC_MACROASSEMBLER_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/nativeInst_ppc.cpp b/src/hotspot/cpu/ppc/nativeInst_ppc.cpp
index 9520f6a94f2a4..f61328fc7360c 100644
--- a/src/hotspot/cpu/ppc/nativeInst_ppc.cpp
+++ b/src/hotspot/cpu/ppc/nativeInst_ppc.cpp
@@ -92,10 +92,10 @@ address NativeCall::destination() const {
// Used in the runtime linkage of calls; see class CompiledIC.
//
// Add parameter assert_lock to switch off assertion
-// during code generation, where no patching lock is needed.
+// during code generation, where no lock is needed.
void NativeCall::set_destination_mt_safe(address dest, bool assert_lock) {
assert(!assert_lock ||
- (Patching_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) ||
+ (CodeCache_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) ||
CompiledICLocker::is_safe(addr_at(0)),
"concurrent code patching");
diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad
index ca9abfa3719b4..6d3daa025e8d2 100644
--- a/src/hotspot/cpu/ppc/ppc.ad
+++ b/src/hotspot/cpu/ppc/ppc.ad
@@ -1000,6 +1000,10 @@ int MachNode::compute_padding(int current_offset) const {
// Should the matcher clone input 'm' of node 'n'?
bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
+ if (is_encode_and_store_pattern(n, m)) {
+ mstack.push(m, Visit);
+ return true;
+ }
return false;
}
@@ -2150,10 +2154,6 @@ const RegMask* Matcher::predicate_reg_mask(void) {
return nullptr;
}
-const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
- return nullptr;
-}
-
// Vector calling convention not yet implemented.
bool Matcher::supports_vector_calling_convention(void) {
return false;
@@ -5407,7 +5407,7 @@ instruct loadRange(iRegIdst dst, memory mem) %{
// Load Compressed Pointer
instruct loadN(iRegNdst dst, memory mem) %{
match(Set dst (LoadN mem));
- predicate(n->as_Load()->is_unordered() || followed_by_acquire(n));
+ predicate((n->as_Load()->is_unordered() || followed_by_acquire(n)) && n->as_Load()->barrier_data() == 0);
ins_cost(MEMORY_REF_COST);
format %{ "LWZ $dst, $mem \t// load compressed ptr" %}
@@ -5419,6 +5419,7 @@ instruct loadN(iRegNdst dst, memory mem) %{
// Load Compressed Pointer acquire.
instruct loadN_ac(iRegNdst dst, memory mem) %{
match(Set dst (LoadN mem));
+ predicate(n->as_Load()->barrier_data() == 0);
ins_cost(3*MEMORY_REF_COST);
format %{ "LWZ $dst, $mem \t// load acquire compressed ptr\n\t"
@@ -5432,7 +5433,7 @@ instruct loadN_ac(iRegNdst dst, memory mem) %{
// Load Compressed Pointer and decode it if narrow_oop_shift == 0.
instruct loadN2P_unscaled(iRegPdst dst, memory mem) %{
match(Set dst (DecodeN (LoadN mem)));
- predicate(_kids[0]->_leaf->as_Load()->is_unordered() && CompressedOops::shift() == 0);
+ predicate(_kids[0]->_leaf->as_Load()->is_unordered() && CompressedOops::shift() == 0 && _kids[0]->_leaf->as_Load()->barrier_data() == 0);
ins_cost(MEMORY_REF_COST);
format %{ "LWZ $dst, $mem \t// DecodeN (unscaled)" %}
@@ -6423,6 +6424,7 @@ instruct reinterpretX(vecX dst) %{
// Store Compressed Oop
instruct storeN(memory dst, iRegN_P2N src) %{
match(Set dst (StoreN dst src));
+ predicate(n->as_Store()->barrier_data() == 0);
ins_cost(MEMORY_REF_COST);
format %{ "STW $src, $dst \t// compressed oop" %}
@@ -6476,23 +6478,6 @@ instruct storeD(memory mem, regD src) %{
ins_pipe(pipe_class_memory);
%}
-//----------Store Instructions With Zeros--------------------------------------
-
-instruct storeCM(memory mem, immI_0 zero) %{
- match(Set mem (StoreCM mem zero));
- ins_cost(MEMORY_REF_COST);
-
- format %{ "STB #0, $mem \t// CMS card-mark byte store" %}
- size(8);
- ins_encode %{
- __ li(R0, 0);
- // No release barrier: Oops are allowed to get visible after marking.
- guarantee($mem$$base$$Register != R1_SP, "use frame_slots_bias");
- __ stb(R0, $mem$$disp, $mem$$base$$Register);
- %}
- ins_pipe(pipe_class_memory);
-%}
-
// Convert oop pointer into compressed form.
// Nodes for postalloc expand.
@@ -7477,6 +7462,7 @@ instruct compareAndSwapI_regP_regI_regI(iRegIdst res, iRegPdst mem_ptr, iRegIsrc
instruct compareAndSwapN_regP_regN_regN(iRegIdst res, iRegPdst mem_ptr, iRegNsrc src1, iRegNsrc src2, flagsRegCR0 cr0) %{
match(Set res (CompareAndSwapN mem_ptr (Binary src1 src2)));
+ predicate(n->as_LoadStore()->barrier_data() == 0);
effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
format %{ "CMPXCHGW $res, $mem_ptr, $src1, $src2; as bool" %}
ins_encode %{
@@ -7676,7 +7662,7 @@ instruct weakCompareAndSwapI_acq_regP_regI_regI(iRegIdst res, iRegPdst mem_ptr,
instruct weakCompareAndSwapN_regP_regN_regN(iRegIdst res, iRegPdst mem_ptr, iRegNsrc src1, iRegNsrc src2, flagsRegCR0 cr0) %{
match(Set res (WeakCompareAndSwapN mem_ptr (Binary src1 src2)));
- predicate(((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*)n)->order() != MemNode::seqcst);
+ predicate(((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*)n)->order() != MemNode::seqcst && n->as_LoadStore()->barrier_data() == 0);
effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
format %{ "weak CMPXCHGW $res, $mem_ptr, $src1, $src2; as bool" %}
ins_encode %{
@@ -7690,7 +7676,7 @@ instruct weakCompareAndSwapN_regP_regN_regN(iRegIdst res, iRegPdst mem_ptr, iReg
instruct weakCompareAndSwapN_acq_regP_regN_regN(iRegIdst res, iRegPdst mem_ptr, iRegNsrc src1, iRegNsrc src2, flagsRegCR0 cr0) %{
match(Set res (WeakCompareAndSwapN mem_ptr (Binary src1 src2)));
- predicate(((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst);
+ predicate((((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst) && n->as_LoadStore()->barrier_data() == 0);
effect(TEMP_DEF res, TEMP cr0); // TEMP_DEF to avoid jump
format %{ "weak CMPXCHGW acq $res, $mem_ptr, $src1, $src2; as bool" %}
ins_encode %{
@@ -7939,7 +7925,7 @@ instruct compareAndExchangeI_acq_regP_regI_regI(iRegIdst res, iRegPdst mem_ptr,
instruct compareAndExchangeN_regP_regN_regN(iRegNdst res, iRegPdst mem_ptr, iRegNsrc src1, iRegNsrc src2, flagsRegCR0 cr0) %{
match(Set res (CompareAndExchangeN mem_ptr (Binary src1 src2)));
- predicate(((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*)n)->order() != MemNode::seqcst);
+ predicate(((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*)n)->order() != MemNode::seqcst && n->as_LoadStore()->barrier_data() == 0);
effect(TEMP_DEF res, TEMP cr0);
format %{ "CMPXCHGW $res, $mem_ptr, $src1, $src2; as narrow oop" %}
ins_encode %{
@@ -7953,7 +7939,7 @@ instruct compareAndExchangeN_regP_regN_regN(iRegNdst res, iRegPdst mem_ptr, iReg
instruct compareAndExchangeN_acq_regP_regN_regN(iRegNdst res, iRegPdst mem_ptr, iRegNsrc src1, iRegNsrc src2, flagsRegCR0 cr0) %{
match(Set res (CompareAndExchangeN mem_ptr (Binary src1 src2)));
- predicate(((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst);
+ predicate((((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst) && n->as_LoadStore()->barrier_data() == 0);
effect(TEMP_DEF res, TEMP cr0);
format %{ "CMPXCHGW acq $res, $mem_ptr, $src1, $src2; as narrow oop" %}
ins_encode %{
@@ -8262,6 +8248,7 @@ instruct getAndSetP(iRegPdst res, iRegPdst mem_ptr, iRegPsrc src, flagsRegCR0 cr
instruct getAndSetN(iRegNdst res, iRegPdst mem_ptr, iRegNsrc src, flagsRegCR0 cr0) %{
match(Set res (GetAndSetN mem_ptr src));
+ predicate(n->as_LoadStore()->barrier_data() == 0);
effect(TEMP_DEF res, TEMP cr0);
format %{ "GetAndSetN $res, $mem_ptr, $src" %}
ins_encode %{
@@ -12281,7 +12268,7 @@ instruct string_equalsL(rarg1RegP str1, rarg2RegP str2, rarg3RegI cnt, iRegIdst
%}
instruct array_equalsB(rarg1RegP ary1, rarg2RegP ary2, iRegIdst result,
- iRegIdst tmp1, iRegIdst tmp2, regCTR ctr, flagsRegCR0 cr0, flagsRegCR0 cr1) %{
+ iRegIdst tmp1, iRegIdst tmp2, regCTR ctr, flagsRegCR0 cr0, flagsRegCR1 cr1) %{
predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
match(Set result (AryEq ary1 ary2));
effect(TEMP_DEF result, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, KILL ctr, KILL cr0, KILL cr1);
@@ -12296,7 +12283,7 @@ instruct array_equalsB(rarg1RegP ary1, rarg2RegP ary2, iRegIdst result,
%}
instruct array_equalsC(rarg1RegP ary1, rarg2RegP ary2, iRegIdst result,
- iRegIdst tmp1, iRegIdst tmp2, regCTR ctr, flagsRegCR0 cr0, flagsRegCR0 cr1) %{
+ iRegIdst tmp1, iRegIdst tmp2, regCTR ctr, flagsRegCR0 cr0, flagsRegCR1 cr1) %{
predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
match(Set result (AryEq ary1 ary2));
effect(TEMP_DEF result, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, KILL ctr, KILL cr0, KILL cr1);
diff --git a/src/hotspot/cpu/ppc/register_ppc.hpp b/src/hotspot/cpu/ppc/register_ppc.hpp
index 302d49884fae3..b7ba4f053b5d6 100644
--- a/src/hotspot/cpu/ppc/register_ppc.hpp
+++ b/src/hotspot/cpu/ppc/register_ppc.hpp
@@ -27,6 +27,7 @@
#define CPU_PPC_REGISTER_PPC_HPP
#include "asm/register.hpp"
+#include "utilities/count_trailing_zeros.hpp"
// forward declaration
class VMRegImpl;
@@ -555,4 +556,12 @@ constexpr Register R29_TOC = R29;
constexpr Register R11_scratch1 = R11;
constexpr Register R12_scratch2 = R12;
+template <>
+inline Register AbstractRegSet::first() {
+ if (_bitset == 0) { return noreg; }
+ return as_Register(count_trailing_zeros(_bitset));
+}
+
+typedef AbstractRegSet RegSet;
+
#endif // CPU_PPC_REGISTER_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
index ee3f1911e2082..b3ace8898ad63 100644
--- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
+++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
@@ -49,7 +49,6 @@
#include "utilities/align.hpp"
#include "utilities/powerOfTwo.hpp"
#if INCLUDE_ZGC
-#include "gc/x/xBarrierSetAssembler.hpp"
#include "gc/z/zBarrierSetAssembler.hpp"
#endif
@@ -1976,7 +1975,7 @@ class StubGenerator: public StubCodeGenerator {
generate_conjoint_int_copy_core(aligned);
} else {
#if INCLUDE_ZGC
- if (UseZGC && ZGenerational) {
+ if (UseZGC) {
ZBarrierSetAssembler *zbs = (ZBarrierSetAssembler*)bs;
zbs->generate_conjoint_oop_copy(_masm, dest_uninitialized);
} else
@@ -2019,7 +2018,7 @@ class StubGenerator: public StubCodeGenerator {
generate_disjoint_int_copy_core(aligned);
} else {
#if INCLUDE_ZGC
- if (UseZGC && ZGenerational) {
+ if (UseZGC) {
ZBarrierSetAssembler *zbs = (ZBarrierSetAssembler*)bs;
zbs->generate_disjoint_oop_copy(_masm, dest_uninitialized);
} else
@@ -2137,7 +2136,7 @@ class StubGenerator: public StubCodeGenerator {
} else {
__ bind(store_null);
#if INCLUDE_ZGC
- if (UseZGC && ZGenerational) {
+ if (UseZGC) {
__ store_heap_oop(R10_oop, R8_offset, R4_to, R11_scratch1, R12_tmp, noreg,
MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS,
dest_uninitialized ? IS_DEST_UNINITIALIZED : 0);
@@ -2153,7 +2152,7 @@ class StubGenerator: public StubCodeGenerator {
// ======== loop entry is here ========
__ bind(load_element);
#if INCLUDE_ZGC
- if (UseZGC && ZGenerational) {
+ if (UseZGC) {
__ load_heap_oop(R10_oop, R8_offset, R3_from,
R11_scratch1, R12_tmp,
MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS,
@@ -4587,6 +4586,30 @@ address generate_lookup_secondary_supers_table_stub(u1 super_klass_index) {
return start;
}
+ // load Method* target of MethodHandle
+ // R3_ARG1 = jobject receiver
+ // R19_method = result Method*
+ address generate_upcall_stub_load_target() {
+
+ StubCodeMark mark(this, "StubRoutines", "upcall_stub_load_target");
+ address start = __ pc();
+
+ __ resolve_global_jobject(R3_ARG1, R22_tmp2, R23_tmp3, MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS);
+ // Load target method from receiver
+ __ load_heap_oop(R19_method, java_lang_invoke_MethodHandle::form_offset(), R3_ARG1,
+ R22_tmp2, R23_tmp3, MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS, IS_NOT_NULL);
+ __ load_heap_oop(R19_method, java_lang_invoke_LambdaForm::vmentry_offset(), R19_method,
+ R22_tmp2, R23_tmp3, MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS, IS_NOT_NULL);
+ __ load_heap_oop(R19_method, java_lang_invoke_MemberName::method_offset(), R19_method,
+ R22_tmp2, R23_tmp3, MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS, IS_NOT_NULL);
+ __ ld(R19_method, java_lang_invoke_ResolvedMethodName::vmtarget_offset(), R19_method);
+ __ std(R19_method, in_bytes(JavaThread::callee_target_offset()), R16_thread); // just in case callee is deoptimized
+
+ __ blr();
+
+ return start;
+ }
+
// Initialization
void generate_initial_stubs() {
// Generates all stubs and initializes the entry points
@@ -4651,6 +4674,7 @@ address generate_lookup_secondary_supers_table_stub(u1 super_klass_index) {
}
StubRoutines::_upcall_stub_exception_handler = generate_upcall_stub_exception_handler();
+ StubRoutines::_upcall_stub_load_target = generate_upcall_stub_load_target();
}
void generate_compiler_stubs() {
diff --git a/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp b/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp
index b60fd4f16d163..635bab900d157 100644
--- a/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp
+++ b/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp
@@ -24,6 +24,7 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "classfile/javaClasses.hpp"
#include "logging/logStream.hpp"
#include "memory/resourceArea.hpp"
#include "prims/upcallLinker.hpp"
@@ -118,7 +119,7 @@ static void restore_callee_saved_registers(MacroAssembler* _masm, const ABIDescr
static const int upcall_stub_code_base_size = 1024;
static const int upcall_stub_size_per_arg = 16; // arg save & restore + move
-address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
+address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature,
BasicType* out_sig_bt, int total_out_args,
BasicType ret_type,
jobject jabi, jobject jconv,
@@ -221,7 +222,6 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
__ block_comment("{ on_entry");
__ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, UpcallLinker::on_entry), R0);
__ addi(R3_ARG1, R1_SP, frame_data_offset);
- __ load_const_optimized(R4_ARG2, (intptr_t)receiver, R0);
__ call_c(call_target_address);
__ mr(R16_thread, R3_RET);
__ block_comment("} on_entry");
@@ -236,12 +236,12 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
arg_shuffle.generate(_masm, as_VMStorage(callerSP), frame::native_abi_minframe_size, frame::jit_out_preserve_size);
__ block_comment("} argument shuffle");
- __ block_comment("{ receiver ");
- __ get_vm_result(R3_ARG1);
- __ block_comment("} receiver ");
-
- __ load_const_optimized(R19_method, (intptr_t)entry);
- __ std(R19_method, in_bytes(JavaThread::callee_target_offset()), R16_thread);
+ __ block_comment("{ load target ");
+ __ load_const_optimized(call_target_address, StubRoutines::upcall_stub_load_target(), R0);
+ __ load_const_optimized(R3_ARG1, (intptr_t)receiver, R0);
+ __ mtctr(call_target_address);
+ __ bctrl(); // loads target Method* into R19_method
+ __ block_comment("} load target ");
__ push_cont_fastpath();
@@ -326,7 +326,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
#ifndef PRODUCT
stringStream ss;
- ss.print("upcall_stub_%s", entry->signature()->as_C_string());
+ ss.print("upcall_stub_%s", signature->as_C_string());
const char* name = _masm->code_string(ss.as_string());
#else // PRODUCT
const char* name = "upcall_stub";
diff --git a/src/hotspot/cpu/riscv/assembler_riscv.hpp b/src/hotspot/cpu/riscv/assembler_riscv.hpp
index d1021d9e283d2..7334ec675e31f 100644
--- a/src/hotspot/cpu/riscv/assembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/assembler_riscv.hpp
@@ -46,8 +46,10 @@
class Argument {
public:
enum {
- n_int_register_parameters_c = 8, // x10, x11, ... x17 (c_rarg0, c_rarg1, ...)
- n_float_register_parameters_c = 8, // f10, f11, ... f17 (c_farg0, c_farg1, ... )
+ // check more info at https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc
+ n_int_register_parameters_c = 8, // x10, x11, ... x17 (c_rarg0, c_rarg1, ...)
+ n_float_register_parameters_c = 8, // f10, f11, ... f17 (c_farg0, c_farg1, ... )
+ n_vector_register_parameters_c = 16, // v8, v9, ... v23
n_int_register_parameters_j = 8, // x11, ... x17, x10 (j_rarg0, j_rarg1, ...)
n_float_register_parameters_j = 8 // f10, f11, ... f17 (j_farg0, j_farg1, ...)
@@ -143,6 +145,10 @@ constexpr Register x19_sender_sp = x19; // Sender's SP while in interpreter
constexpr Register t0 = x5;
constexpr Register t1 = x6;
constexpr Register t2 = x7;
+constexpr Register t3 = x28;
+constexpr Register t4 = x29;
+constexpr Register t5 = x30;
+constexpr Register t6 = x31;
const Register g_INTArgReg[Argument::n_int_register_parameters_c] = {
c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5, c_rarg6, c_rarg7
@@ -1956,6 +1962,13 @@ enum Nf {
INSN(vbrev8_v, 0b1010111, 0b010, 0b01000, 0b010010); // reverse bits in every byte of element
INSN(vrev8_v, 0b1010111, 0b010, 0b01001, 0b010010); // reverse bytes in every elememt
+ // Vector AES instructions (Zvkned extension)
+ INSN(vaesem_vv, 0b1110111, 0b010, 0b00010, 0b101000);
+ INSN(vaesef_vv, 0b1110111, 0b010, 0b00011, 0b101000);
+
+ INSN(vaesdm_vv, 0b1110111, 0b010, 0b00000, 0b101000);
+ INSN(vaesdf_vv, 0b1110111, 0b010, 0b00001, 0b101000);
+
INSN(vclz_v, 0b1010111, 0b010, 0b01100, 0b010010); // count leading zeros
INSN(vctz_v, 0b1010111, 0b010, 0b01101, 0b010010); // count trailing zeros
@@ -2886,8 +2899,9 @@ enum Nf {
// Unconditional branch instructions
// --------------------------
protected:
- // All calls and jumps must go via MASM.
+ // All calls and jumps must go via MASM. Only use x1 (aka ra) as link register for now.
void jalr(Register Rd, Register Rs, const int32_t offset) {
+ assert(Rd != x5 && Rs != x5, "Register x5 must not be used for calls/jumps.");
/* jalr -> c.jr/c.jalr */
if (do_compress() && (offset == 0 && Rs != x0)) {
if (Rd == x1) {
@@ -2902,6 +2916,7 @@ enum Nf {
}
void jal(Register Rd, const int32_t offset) {
+ assert(Rd != x5, "Register x5 must not be used for calls/jumps.");
/* jal -> c.j, note c.jal is RV32C only */
if (do_compress() &&
Rd == x0 &&
@@ -2909,7 +2924,6 @@ enum Nf {
c_j(offset);
return;
}
-
_jal(Rd, offset);
}
diff --git a/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp b/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp
index fb81082072610..0be1252f57f8a 100644
--- a/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp
@@ -43,9 +43,7 @@ void C1SafepointPollStub::emit_code(LIR_Assembler* ce) {
__ bind(_entry);
InternalAddress safepoint_pc(__ pc() - __ offset() + safepoint_offset());
__ relocate(safepoint_pc.rspec(), [&] {
- int32_t offset;
- __ la(t0, safepoint_pc.target(), offset);
- __ addi(t0, t0, offset);
+ __ la(t0, safepoint_pc.target());
});
__ sd(t0, Address(xthread, JavaThread::saved_exception_pc_offset()));
@@ -93,7 +91,7 @@ void RangeCheckStub::emit_code(LIR_Assembler* ce) {
stub_id = C1StubId::throw_range_check_failed_id;
}
// t0 and t1 are used as args in generate_exception_throw,
- // so use ra as the tmp register for rt_call.
+ // so use x1/ra as the tmp register for rt_call.
__ rt_call(Runtime1::entry_for(stub_id), ra);
ce->add_call_info_here(_info);
ce->verify_oop_map(_info);
@@ -275,7 +273,7 @@ void SimpleExceptionStub::emit_code(LIR_Assembler* ce) {
if (_obj->is_cpu_register()) {
__ mv(t0, _obj->as_register());
}
- __ far_call(RuntimeAddress(Runtime1::entry_for(_stub)), t1);
+ __ far_call(RuntimeAddress(Runtime1::entry_for(_stub)));
ce->add_call_info_here(_info);
debug_only(__ should_not_reach_here());
}
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
index 940706b0a7376..21bf089118b45 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
@@ -838,10 +838,7 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch
__ decode_heap_oop(dest->as_register());
}
- if (!(UseZGC && !ZGenerational)) {
- // Load barrier has not yet been applied, so ZGC can't verify the oop here
- __ verify_oop(dest->as_register());
- }
+ __ verify_oop(dest->as_register());
}
}
@@ -980,6 +977,7 @@ void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
if (op->init_check()) {
__ lbu(t0, Address(op->klass()->as_register(),
InstanceKlass::init_state_offset()));
+ __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
__ mv(t1, (u1)InstanceKlass::fully_initialized);
add_debug_info_for_null_check_here(op->stub()->info());
__ bne(t0, t1, *op->stub()->entry(), /* is_far */ true);
@@ -1405,9 +1403,7 @@ void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmit
int pc_for_athrow_offset = __ offset();
InternalAddress pc_for_athrow(__ pc());
__ relocate(pc_for_athrow.rspec(), [&] {
- int32_t offset;
- __ la(exceptionPC->as_register(), pc_for_athrow.target(), offset);
- __ addi(exceptionPC->as_register(), exceptionPC->as_register(), offset);
+ __ la(exceptionPC->as_register(), pc_for_athrow.target());
});
add_call_info(pc_for_athrow_offset, info); // for exception handler
diff --git a/src/hotspot/cpu/riscv/c2_CodeStubs_riscv.cpp b/src/hotspot/cpu/riscv/c2_CodeStubs_riscv.cpp
index 7995750aba96b..320093ee0e332 100644
--- a/src/hotspot/cpu/riscv/c2_CodeStubs_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c2_CodeStubs_riscv.cpp
@@ -33,7 +33,7 @@
#define __ masm.
int C2SafepointPollStub::max_size() const {
- return 13 * 4;
+ return 5 * NativeInstruction::instruction_size;
}
void C2SafepointPollStub::emit(C2_MacroAssembler& masm) {
@@ -45,58 +45,31 @@ void C2SafepointPollStub::emit(C2_MacroAssembler& masm) {
__ bind(entry());
InternalAddress safepoint_pc(__ pc() - __ offset() + _safepoint_offset);
__ relocate(safepoint_pc.rspec(), [&] {
- int32_t offset;
- __ la(t0, safepoint_pc.target(), offset);
- __ addi(t0, t0, offset);
+ // emits auipc + addi for address inside code cache
+ __ la(t0, safepoint_pc.target());
});
__ sd(t0, Address(xthread, JavaThread::saved_exception_pc_offset()));
+ // emits auipc + jr for address inside code cache
__ far_jump(callback_addr);
}
int C2EntryBarrierStub::max_size() const {
// 4 bytes for alignment
- return 8 * 4 + 4;
+ return 3 * NativeInstruction::instruction_size + 4 + 4;
}
void C2EntryBarrierStub::emit(C2_MacroAssembler& masm) {
__ bind(entry());
- __ rt_call(StubRoutines::method_entry_barrier());
+ // emits auipc + jalr for address inside code cache
+ __ far_call(StubRoutines::method_entry_barrier());
__ j(continuation());
- // make guard value 4-byte aligned so that it can be accessed by atomic instructions on RISC-V
+ // make guard value 4-byte aligned so that it can be accessed atomically
__ align(4);
__ bind(guard());
__ relocate(entry_guard_Relocation::spec());
__ emit_int32(0); // nmethod guard value
}
-int C2HandleAnonOMOwnerStub::max_size() const {
- // Max size of stub has been determined by testing with 0 without using RISC-V compressed
- // instruction-set extension, in which case C2CodeStubList::emit() will throw an assertion
- // and report the actual size that is needed.
- return 20 DEBUG_ONLY(+8);
-}
-
-void C2HandleAnonOMOwnerStub::emit(C2_MacroAssembler& masm) {
- __ bind(entry());
- Register mon = monitor();
- Register t = tmp();
- assert(t != noreg, "need tmp register");
-
- // Fix owner to be the current thread.
- __ sd(xthread, Address(mon, ObjectMonitor::owner_offset()));
-
- // Pop owner object from lock-stack.
- __ lwu(t, Address(xthread, JavaThread::lock_stack_top_offset()));
- __ subw(t, t, oopSize);
-#ifdef ASSERT
- __ add(t0, xthread, t);
- __ sd(zr, Address(t0, 0));
-#endif
- __ sw(t, Address(xthread, JavaThread::lock_stack_top_offset()));
-
- __ j(continuation());
-}
-
#undef __
diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
index 75f87e35adf41..0ffdcbca72307 100644
--- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
@@ -234,7 +234,7 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg,
// StoreLoad achieves this.
membar(StoreLoad);
- // Check if the entry lists are empty.
+ // Check if the entry lists are empty (EntryList first - by convention).
ld(t0, Address(tmp, ObjectMonitor::EntryList_offset()));
ld(tmp1Reg, Address(tmp, ObjectMonitor::cxq_offset()));
orr(t0, t0, tmp1Reg);
@@ -566,7 +566,7 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box,
// StoreLoad achieves this.
membar(StoreLoad);
- // Check if the entry lists are empty.
+ // Check if the entry lists are empty (EntryList first - by convention).
ld(t0, Address(tmp1_monitor, ObjectMonitor::EntryList_offset()));
ld(tmp3_t, Address(tmp1_monitor, ObjectMonitor::cxq_offset()));
orr(t0, t0, tmp3_t);
diff --git a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp
index 062f80290626f..5493882be7264 100644
--- a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -39,7 +39,10 @@
#include "c1/c1_LIRAssembler.hpp"
#include "c1/c1_MacroAssembler.hpp"
#include "gc/g1/c1/g1BarrierSetC1.hpp"
-#endif
+#endif // COMPILER1
+#ifdef COMPILER2
+#include "gc/g1/c2/g1BarrierSetC2.hpp"
+#endif // COMPILER2
#define __ masm->
@@ -96,6 +99,55 @@ void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* mas
__ pop_reg(saved_regs, sp);
}
+static void generate_queue_test_and_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
+ const Register thread, const Register value, const Register tmp1, const Register tmp2) {
+ // Can we store a value in the given thread's buffer?
+ // (The index field is typed as size_t.)
+ __ ld(tmp1, Address(thread, in_bytes(index_offset))); // tmp1 := *(index address)
+ __ beqz(tmp1, runtime); // jump to runtime if index == 0 (full buffer)
+ // The buffer is not full, store value into it.
+ __ sub(tmp1, tmp1, wordSize); // tmp1 := next index
+ __ sd(tmp1, Address(thread, in_bytes(index_offset))); // *(index address) := next index
+ __ ld(tmp2, Address(thread, in_bytes(buffer_offset))); // tmp2 := buffer address
+ __ add(tmp2, tmp2, tmp1);
+ __ sd(value, Address(tmp2)); // *(buffer address + next index) := value
+}
+
+static void generate_pre_barrier_fast_path(MacroAssembler* masm,
+ const Register thread,
+ const Register tmp1) {
+ Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
+ // Is marking active?
+ if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
+ __ lwu(tmp1, in_progress);
+ } else {
+ assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
+ __ lbu(tmp1, in_progress);
+ }
+}
+
+static void generate_pre_barrier_slow_path(MacroAssembler* masm,
+ const Register obj,
+ const Register pre_val,
+ const Register thread,
+ const Register tmp1,
+ const Register tmp2,
+ Label& done,
+ Label& runtime) {
+ // Do we need to load the previous value?
+ if (obj != noreg) {
+ __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
+ }
+ // Is the previous value null?
+ __ beqz(pre_val, done, true);
+ generate_queue_test_and_insertion(masm,
+ G1ThreadLocalData::satb_mark_queue_index_offset(),
+ G1ThreadLocalData::satb_mark_queue_buffer_offset(),
+ runtime,
+ thread, pre_val, tmp1, tmp2);
+ __ j(done);
+}
+
void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
Register obj,
Register pre_val,
@@ -116,43 +168,10 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
assert_different_registers(obj, pre_val, tmp1, tmp2);
assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
- Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
- Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
- Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
-
- // Is marking active?
- if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { // 4-byte width
- __ lwu(tmp1, in_progress);
- } else {
- assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
- __ lbu(tmp1, in_progress);
- }
+ generate_pre_barrier_fast_path(masm, thread, tmp1);
+ // If marking is not active (*(mark queue active address) == 0), jump to done
__ beqz(tmp1, done);
-
- // Do we need to load the previous value?
- if (obj != noreg) {
- __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
- }
-
- // Is the previous value null?
- __ beqz(pre_val, done);
-
- // Can we store original value in the thread's buffer?
- // Is index == 0?
- // (The index field is typed as size_t.)
-
- __ ld(tmp1, index); // tmp := *index_adr
- __ beqz(tmp1, runtime); // tmp == 0?
- // If yes, goto runtime
-
- __ sub(tmp1, tmp1, wordSize); // tmp := tmp - wordSize
- __ sd(tmp1, index); // *index_adr := tmp
- __ ld(tmp2, buffer);
- __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr
-
- // Record the previous value
- __ sd(pre_val, Address(tmp1, 0));
- __ j(done);
+ generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp1, tmp2, done, runtime);
__ bind(runtime);
@@ -171,6 +190,49 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
}
+static void generate_post_barrier_fast_path(MacroAssembler* masm,
+ const Register store_addr,
+ const Register new_val,
+ const Register tmp1,
+ const Register tmp2,
+ Label& done,
+ bool new_val_may_be_null) {
+ // Does store cross heap regions?
+ __ xorr(tmp1, store_addr, new_val); // tmp1 := store address ^ new value
+ __ srli(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
+ __ beqz(tmp1, done);
+ // Crosses regions, storing null?
+ if (new_val_may_be_null) {
+ __ beqz(new_val, done);
+ }
+ // Storing region crossing non-null, is card young?
+ __ srli(tmp1, store_addr, CardTable::card_shift()); // tmp1 := card address relative to card table base
+ __ load_byte_map_base(tmp2); // tmp2 := card table base address
+ __ add(tmp1, tmp1, tmp2); // tmp1 := card address
+ __ lbu(tmp2, Address(tmp1)); // tmp2 := card
+}
+
+static void generate_post_barrier_slow_path(MacroAssembler* masm,
+ const Register thread,
+ const Register tmp1,
+ const Register tmp2,
+ Label& done,
+ Label& runtime) {
+ __ membar(MacroAssembler::StoreLoad); // StoreLoad membar
+ __ lbu(tmp2, Address(tmp1)); // tmp2 := card
+ __ beqz(tmp2, done, true);
+ // Storing a region crossing, non-null oop, card is clean.
+ // Dirty card and log.
+ STATIC_ASSERT(CardTable::dirty_card_val() == 0);
+ __ sb(zr, Address(tmp1)); // *(card address) := dirty_card_val
+ generate_queue_test_and_insertion(masm,
+ G1ThreadLocalData::dirty_card_queue_index_offset(),
+ G1ThreadLocalData::dirty_card_queue_buffer_offset(),
+ runtime,
+ thread, tmp1, tmp2, t0);
+ __ j(done);
+}
+
void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
Register store_addr,
Register new_val,
@@ -179,73 +241,119 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
Register tmp2) {
assert(thread == xthread, "must be");
assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, t0);
- assert(store_addr != noreg && new_val != noreg && tmp1 != noreg &&
- tmp2 != noreg, "expecting a register");
-
- Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
- Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
-
- BarrierSet* bs = BarrierSet::barrier_set();
- CardTableBarrierSet* ctbs = barrier_set_cast(bs);
+ assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg,
+ "expecting a register");
Label done;
Label runtime;
- // Does store cross heap regions?
+ generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, done, true /* new_val_may_be_null */);
+ // If card is young, jump to done (tmp2 holds the card value)
+ __ mv(t0, (int)G1CardTable::g1_young_card_val());
+ __ beq(tmp2, t0, done); // card == young_card_val?
+ generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, done, runtime);
- __ xorr(tmp1, store_addr, new_val);
- __ srli(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes);
- __ beqz(tmp1, done);
+ __ bind(runtime);
+ // save the live input values
+ RegSet saved = RegSet::of(store_addr);
+ __ push_reg(saved, sp);
+ __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp1, thread);
+ __ pop_reg(saved, sp);
- // crosses regions, storing null?
+ __ bind(done);
+}
- __ beqz(new_val, done);
+#if defined(COMPILER2)
- // storing region crossing non-null, is card already dirty?
+static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
+ SaveLiveRegisters save_registers(masm, stub);
+ if (c_rarg0 != arg) {
+ __ mv(c_rarg0, arg);
+ }
+ __ mv(c_rarg1, xthread);
+ __ mv(t1, runtime_path);
+ __ jalr(t1);
+}
- const Register card_addr = tmp1;
+void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2,
+ G1PreBarrierStubC2* stub) {
+ assert(thread == xthread, "must be");
+ assert_different_registers(obj, pre_val, tmp1, tmp2);
+ assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
- __ srli(card_addr, store_addr, CardTable::card_shift());
+ stub->initialize_registers(obj, pre_val, thread, tmp1, tmp2);
- // get the address of the card
- __ load_byte_map_base(tmp2);
- __ add(card_addr, card_addr, tmp2);
- __ lbu(tmp2, Address(card_addr));
- __ mv(t0, (int)G1CardTable::g1_young_card_val());
- __ beq(tmp2, t0, done);
+ generate_pre_barrier_fast_path(masm, thread, tmp1);
+ // If marking is active (*(mark queue active address) != 0), jump to stub (slow path)
+ __ bnez(tmp1, *stub->entry(), true);
- assert((int)CardTable::dirty_card_val() == 0, "must be 0");
+ __ bind(*stub->continuation());
+}
- __ membar(MacroAssembler::StoreLoad);
+void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm,
+ G1PreBarrierStubC2* stub) const {
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ Label runtime;
+ Register obj = stub->obj();
+ Register pre_val = stub->pre_val();
+ Register thread = stub->thread();
+ Register tmp1 = stub->tmp1();
+ Register tmp2 = stub->tmp2();
- __ lbu(tmp2, Address(card_addr));
- __ beqz(tmp2, done);
+ __ bind(*stub->entry());
+ generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp1, tmp2, *stub->continuation(), runtime);
- // storing a region crossing, non-null oop, card is clean.
- // dirty card and log.
+ __ bind(runtime);
+ generate_c2_barrier_runtime_call(masm, stub, pre_val, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry));
+ __ j(*stub->continuation());
+}
- __ sb(zr, Address(card_addr));
+void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2,
+ G1PostBarrierStubC2* stub) {
+ assert(thread == xthread, "must be");
+ assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, t0);
+ assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg,
+ "expecting a register");
- __ ld(t0, queue_index);
- __ beqz(t0, runtime);
- __ sub(t0, t0, wordSize);
- __ sd(t0, queue_index);
+ stub->initialize_registers(thread, tmp1, tmp2);
- __ ld(tmp2, buffer);
- __ add(t0, tmp2, t0);
- __ sd(card_addr, Address(t0, 0));
- __ j(done);
+ bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
+ generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, *stub->continuation(), new_val_may_be_null);
+ // If card is not young, jump to stub (slow path) (tmp2 holds the card value)
+ __ mv(t0, (int)G1CardTable::g1_young_card_val());
+ __ bne(tmp2, t0, *stub->entry(), true);
- __ bind(runtime);
- // save the live input values
- RegSet saved = RegSet::of(store_addr);
- __ push_reg(saved, sp);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
- __ pop_reg(saved, sp);
+ __ bind(*stub->continuation());
+}
- __ bind(done);
+void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
+ G1PostBarrierStubC2* stub) const {
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ Label runtime;
+ Register thread = stub->thread();
+ Register tmp1 = stub->tmp1(); // tmp1 holds the card address.
+ Register tmp2 = stub->tmp2();
+
+ __ bind(*stub->entry());
+ generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, *stub->continuation(), runtime);
+
+ __ bind(runtime);
+ generate_c2_barrier_runtime_call(masm, stub, tmp1, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
+ __ j(*stub->continuation());
}
+#endif // COMPILER2
+
void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp2) {
bool on_oop = is_reference_type(type);
diff --git a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp
index 96568994079dd..c7bee2ef6f3a8 100644
--- a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -36,6 +36,8 @@ class LIR_Assembler;
class StubAssembler;
class G1PreBarrierStub;
class G1PostBarrierStub;
+class G1PreBarrierStubC2;
+class G1PostBarrierStubC2;
class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
protected:
@@ -72,6 +74,27 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm);
#endif
+#ifdef COMPILER2
+ void g1_write_barrier_pre_c2(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2,
+ G1PreBarrierStubC2* c2_stub);
+ void generate_c2_pre_barrier_stub(MacroAssembler* masm,
+ G1PreBarrierStubC2* stub) const;
+ void g1_write_barrier_post_c2(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2,
+ G1PostBarrierStubC2* c2_stub);
+ void generate_c2_post_barrier_stub(MacroAssembler* masm,
+ G1PostBarrierStubC2* stub) const;
+#endif
+
void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp2);
};
diff --git a/src/hotspot/cpu/riscv/gc/g1/g1_riscv.ad b/src/hotspot/cpu/riscv/gc/g1/g1_riscv.ad
new file mode 100644
index 0000000000000..7a525323021dd
--- /dev/null
+++ b/src/hotspot/cpu/riscv/gc/g1/g1_riscv.ad
@@ -0,0 +1,564 @@
+//
+// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2024, Huawei Technologies Co., Ltd. All rights reserved.
+// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+
+source_hpp %{
+
+#include "gc/g1/c2/g1BarrierSetC2.hpp"
+#include "gc/shared/gc_globals.hpp"
+
+%}
+
+source %{
+
+#include "gc/g1/g1BarrierSetAssembler_riscv.hpp"
+#include "gc/g1/g1BarrierSetRuntime.hpp"
+
+static void write_barrier_pre(MacroAssembler* masm,
+ const MachNode* node,
+ Register obj,
+ Register pre_val,
+ Register tmp1,
+ Register tmp2,
+ RegSet preserve = RegSet(),
+ RegSet no_preserve = RegSet()) {
+ if (!G1PreBarrierStubC2::needs_barrier(node)) {
+ return;
+ }
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
+ G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
+ for (RegSetIterator reg = preserve.begin(); *reg != noreg; ++reg) {
+ stub->preserve(*reg);
+ }
+ for (RegSetIterator reg = no_preserve.begin(); *reg != noreg; ++reg) {
+ stub->dont_preserve(*reg);
+ }
+ g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, xthread, tmp1, tmp2, stub);
+}
+
+static void write_barrier_post(MacroAssembler* masm,
+ const MachNode* node,
+ Register store_addr,
+ Register new_val,
+ Register tmp1,
+ Register tmp2) {
+ if (!G1PostBarrierStubC2::needs_barrier(node)) {
+ return;
+ }
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
+ G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
+ g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, xthread, tmp1, tmp2, stub);
+}
+
+%}
+
+instruct g1StoreP(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreP mem src));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(STORE_COST);
+ format %{ "sd $src, $mem\t# ptr" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $src$$Register) /* preserve */);
+ __ sd($src$$Register, Address($mem$$Register));
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $src$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(istore_reg_mem);
+%}
+
+instruct g1StoreN(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreN mem src));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(STORE_COST);
+ format %{ "sw $src, $mem\t# compressed ptr" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $src$$Register) /* preserve */);
+ __ sw($src$$Register, Address($mem$$Register));
+ if ((barrier_data() & G1C2BarrierPost) != 0) {
+ if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
+ __ decode_heap_oop($tmp1$$Register, $src$$Register);
+ } else {
+ __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register);
+ }
+ }
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(istore_reg_mem);
+%}
+
+instruct g1EncodePAndStoreN(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreN mem (EncodeP src)));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(STORE_COST);
+ format %{ "encode_heap_oop $tmp1, $src\n\t"
+ "sw $tmp1, $mem\t# compressed ptr" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $src$$Register) /* preserve */);
+ if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
+ __ encode_heap_oop($tmp1$$Register, $src$$Register);
+ } else {
+ __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register);
+ }
+ __ sw($tmp1$$Register, Address($mem$$Register));
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $src$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(istore_reg_mem);
+%}
+
+instruct g1CompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(2 * VOLATILE_REF_COST);
+ format %{ "cmpxchg $res = $mem, $oldval, $newval\t# ptr" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ // Pass $oldval to the pre-barrier (instead of loading from $mem), because
+ // $oldval is the only value that can be overwritten.
+ // The same holds for g1CompareAndSwapP and its Acq variant.
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $oldval$$Register /* pre_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $newval$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct g1CompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
+%{
+ predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(VOLATILE_REF_COST);
+ format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# ptr" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ // Pass $oldval to the pre-barrier (instead of loading from $mem), because
+ // $oldval is the only value that can be overwritten.
+ // The same holds for g1CompareAndSwapP and its Acq variant.
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $oldval$$Register /* pre_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $newval$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct g1CompareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(2 * VOLATILE_REF_COST);
+ format %{ "cmpxchg $res = $mem, $oldval, $newval\t# narrow oop" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
+ __ decode_heap_oop($tmp1$$Register, $newval$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct g1CompareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(VOLATILE_REF_COST);
+ format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# narrow oop" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
+ __ decode_heap_oop($tmp1$$Register, $newval$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct g1CompareAndSwapP(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(2 * VOLATILE_REF_COST);
+ format %{ "cmpxchg $mem, $oldval, $newval\t# (ptr)\n\t"
+ "mv $res, $res == $oldval" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $oldval$$Register /* pre_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $newval$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct g1CompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(VOLATILE_REF_COST);
+ format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (ptr)\n\t"
+ "mv $res, $res == $oldval" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $oldval$$Register /* pre_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $newval$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct g1CompareAndSwapN(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndSwapN mem (Binary oldval newval)));
+ match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(2 * VOLATILE_REF_COST);
+ format %{ "cmpxchg $mem, $oldval, $newval\t# (narrow oop)\n\t"
+ "mv $res, $res == $oldval" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ __ decode_heap_oop($tmp1$$Register, $newval$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct g1CompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndSwapN mem (Binary oldval newval)));
+ match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(VOLATILE_REF_COST);
+ format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop)\n\t"
+ "mv $res, $res == $oldval" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ assert_different_registers($newval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ __ decode_heap_oop($tmp1$$Register, $newval$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct g1GetAndSetP(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set preval (GetAndSetP mem newval));
+ effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(2 * VOLATILE_REF_COST);
+ format %{ "atomic_xchg $preval, $newval, [$mem]" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ assert_different_registers($mem$$Register, $newval$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $preval$$Register /* pre_val (as a temporary register) */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
+ __ atomic_xchg($preval$$Register, $newval$$Register, $mem$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $newval$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_serial);
+%}
+
+instruct g1GetAndSetPAcq(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set preval (GetAndSetP mem newval));
+ effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(VOLATILE_REF_COST);
+ format %{ "atomic_xchg_acq $preval, $newval, [$mem]" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ assert_different_registers($mem$$Register, $newval$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $preval$$Register /* pre_val (as a temporary register) */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
+ __ atomic_xchgal($preval$$Register, $newval$$Register, $mem$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $newval$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_serial);
+%}
+
+instruct g1GetAndSetN(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set preval (GetAndSetN mem newval));
+ effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(2 * VOLATILE_REF_COST);
+ format %{ "atomic_xchgwu $preval, $newval, [$mem]" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ assert_different_registers($mem$$Register, $newval$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
+ __ atomic_xchgwu($preval$$Register, $newval$$Register, $mem$$Register);
+ __ decode_heap_oop($tmp1$$Register, $newval$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_serial);
+%}
+
+instruct g1GetAndSetNAcq(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
+ match(Set preval (GetAndSetN mem newval));
+ effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(VOLATILE_REF_COST);
+ format %{ "atomic_xchgwu_acq $preval, $newval, [$mem]" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ assert_different_registers($mem$$Register, $newval$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */,
+ RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
+ __ atomic_xchgalwu($preval$$Register, $newval$$Register, $mem$$Register);
+ __ decode_heap_oop($tmp1$$Register, $newval$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_serial);
+%}
+
+instruct g1LoadP(iRegPNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
+ match(Set dst (LoadP mem));
+ effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(LOAD_COST + BRANCH_COST);
+ format %{ "ld $dst, $mem\t# ptr" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ __ ld($dst$$Register, Address($mem$$Register));
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $dst$$Register /* pre_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(iload_reg_mem);
+%}
+
+instruct g1LoadN(iRegNNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
+ match(Set dst (LoadN mem));
+ effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(LOAD_COST + BRANCH_COST);
+ format %{ "lwu $dst, $mem\t# compressed ptr" %}
+ ins_encode %{
+ guarantee($mem$$disp == 0, "impossible encoding");
+ __ lwu($dst$$Register, Address($mem$$Register));
+ if ((barrier_data() & G1C2BarrierPre) != 0) {
+ __ decode_heap_oop($tmp1$$Register, $dst$$Register);
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ }
+ %}
+ ins_pipe(iload_reg_mem);
+%}
diff --git a/src/hotspot/cpu/riscv/gc/x/xBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/x/xBarrierSetAssembler_riscv.cpp
deleted file mode 100644
index 7306492970b81..0000000000000
--- a/src/hotspot/cpu/riscv/gc/x/xBarrierSetAssembler_riscv.cpp
+++ /dev/null
@@ -1,454 +0,0 @@
-/*
- * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "asm/macroAssembler.inline.hpp"
-#include "code/codeBlob.hpp"
-#include "code/vmreg.inline.hpp"
-#include "gc/x/xBarrier.inline.hpp"
-#include "gc/x/xBarrierSet.hpp"
-#include "gc/x/xBarrierSetAssembler.hpp"
-#include "gc/x/xBarrierSetRuntime.hpp"
-#include "gc/x/xThreadLocalData.hpp"
-#include "memory/resourceArea.hpp"
-#include "runtime/sharedRuntime.hpp"
-#include "utilities/macros.hpp"
-#ifdef COMPILER1
-#include "c1/c1_LIRAssembler.hpp"
-#include "c1/c1_MacroAssembler.hpp"
-#include "gc/x/c1/xBarrierSetC1.hpp"
-#endif // COMPILER1
-#ifdef COMPILER2
-#include "gc/x/c2/xBarrierSetC2.hpp"
-#endif // COMPILER2
-
-#ifdef PRODUCT
-#define BLOCK_COMMENT(str) /* nothing */
-#else
-#define BLOCK_COMMENT(str) __ block_comment(str)
-#endif
-
-#undef __
-#define __ masm->
-
-void XBarrierSetAssembler::load_at(MacroAssembler* masm,
- DecoratorSet decorators,
- BasicType type,
- Register dst,
- Address src,
- Register tmp1,
- Register tmp2) {
- if (!XBarrierSet::barrier_needed(decorators, type)) {
- // Barrier not needed
- BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
- return;
- }
-
- assert_different_registers(t1, src.base());
- assert_different_registers(t0, t1, dst);
-
- Label done;
-
- // Load bad mask into temp register.
- __ la(t0, src);
- __ ld(t1, address_bad_mask_from_thread(xthread));
- __ ld(dst, Address(t0));
-
- // Test reference against bad mask. If mask bad, then we need to fix it up.
- __ andr(t1, dst, t1);
- __ beqz(t1, done);
-
- __ enter();
-
- __ push_call_clobbered_registers_except(RegSet::of(dst));
-
- if (c_rarg0 != dst) {
- __ mv(c_rarg0, dst);
- }
-
- __ mv(c_rarg1, t0);
-
- __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
-
- // Make sure dst has the return value.
- if (dst != x10) {
- __ mv(dst, x10);
- }
-
- __ pop_call_clobbered_registers_except(RegSet::of(dst));
- __ leave();
-
- __ bind(done);
-}
-
-#ifdef ASSERT
-
-void XBarrierSetAssembler::store_at(MacroAssembler* masm,
- DecoratorSet decorators,
- BasicType type,
- Address dst,
- Register val,
- Register tmp1,
- Register tmp2,
- Register tmp3) {
- // Verify value
- if (is_reference_type(type)) {
- // Note that src could be noreg, which means we
- // are storing null and can skip verification.
- if (val != noreg) {
- Label done;
-
- // tmp1, tmp2 and tmp3 are often set to noreg.
- RegSet savedRegs = RegSet::of(t0);
- __ push_reg(savedRegs, sp);
-
- __ ld(t0, address_bad_mask_from_thread(xthread));
- __ andr(t0, val, t0);
- __ beqz(t0, done);
- __ stop("Verify oop store failed");
- __ should_not_reach_here();
- __ bind(done);
- __ pop_reg(savedRegs, sp);
- }
- }
-
- // Store value
- BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, noreg);
-}
-
-#endif // ASSERT
-
-void XBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
- DecoratorSet decorators,
- bool is_oop,
- Register src,
- Register dst,
- Register count,
- RegSet saved_regs) {
- if (!is_oop) {
- // Barrier not needed
- return;
- }
-
- BLOCK_COMMENT("XBarrierSetAssembler::arraycopy_prologue {");
-
- assert_different_registers(src, count, t0);
-
- __ push_reg(saved_regs, sp);
-
- if (count == c_rarg0 && src == c_rarg1) {
- // exactly backwards!!
- __ xorr(c_rarg0, c_rarg0, c_rarg1);
- __ xorr(c_rarg1, c_rarg0, c_rarg1);
- __ xorr(c_rarg0, c_rarg0, c_rarg1);
- } else {
- __ mv(c_rarg0, src);
- __ mv(c_rarg1, count);
- }
-
- __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_array_addr(), 2);
-
- __ pop_reg(saved_regs, sp);
-
- BLOCK_COMMENT("} XBarrierSetAssembler::arraycopy_prologue");
-}
-
-void XBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
- Register jni_env,
- Register robj,
- Register tmp,
- Label& slowpath) {
- BLOCK_COMMENT("XBarrierSetAssembler::try_resolve_jobject_in_native {");
-
- assert_different_registers(jni_env, robj, tmp);
-
- // Resolve jobject
- BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, robj, tmp, slowpath);
-
- // Compute the offset of address bad mask from the field of jni_environment
- long int bad_mask_relative_offset = (long int) (in_bytes(XThreadLocalData::address_bad_mask_offset()) -
- in_bytes(JavaThread::jni_environment_offset()));
-
- // Load the address bad mask
- __ ld(tmp, Address(jni_env, bad_mask_relative_offset));
-
- // Check address bad mask
- __ andr(tmp, robj, tmp);
- __ bnez(tmp, slowpath);
-
- BLOCK_COMMENT("} XBarrierSetAssembler::try_resolve_jobject_in_native");
-}
-
-#ifdef COMPILER2
-
-OptoReg::Name XBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
- if (!OptoReg::is_reg(opto_reg)) {
- return OptoReg::Bad;
- }
-
- const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
- if (vm_reg->is_FloatRegister()) {
- return opto_reg & ~1;
- }
-
- return opto_reg;
-}
-
-#undef __
-#define __ _masm->
-
-class XSaveLiveRegisters {
-private:
- MacroAssembler* const _masm;
- RegSet _gp_regs;
- FloatRegSet _fp_regs;
- VectorRegSet _vp_regs;
-
-public:
- void initialize(XLoadBarrierStubC2* stub) {
- // Record registers that needs to be saved/restored
- RegMaskIterator rmi(stub->live());
- while (rmi.has_next()) {
- const OptoReg::Name opto_reg = rmi.next();
- if (OptoReg::is_reg(opto_reg)) {
- const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
- if (vm_reg->is_Register()) {
- _gp_regs += RegSet::of(vm_reg->as_Register());
- } else if (vm_reg->is_FloatRegister()) {
- _fp_regs += FloatRegSet::of(vm_reg->as_FloatRegister());
- } else if (vm_reg->is_VectorRegister()) {
- const VMReg vm_reg_base = OptoReg::as_VMReg(opto_reg & ~(VectorRegister::max_slots_per_register - 1));
- _vp_regs += VectorRegSet::of(vm_reg_base->as_VectorRegister());
- } else {
- fatal("Unknown register type");
- }
- }
- }
-
- // Remove C-ABI SOE registers, tmp regs and _ref register that will be updated
- _gp_regs -= RegSet::range(x18, x27) + RegSet::of(x2) + RegSet::of(x8, x9) + RegSet::of(x5, stub->ref());
- }
-
- XSaveLiveRegisters(MacroAssembler* masm, XLoadBarrierStubC2* stub) :
- _masm(masm),
- _gp_regs(),
- _fp_regs(),
- _vp_regs() {
- // Figure out what registers to save/restore
- initialize(stub);
-
- // Save registers
- __ push_reg(_gp_regs, sp);
- __ push_fp(_fp_regs, sp);
- __ push_v(_vp_regs, sp);
- }
-
- ~XSaveLiveRegisters() {
- // Restore registers
- __ pop_v(_vp_regs, sp);
- __ pop_fp(_fp_regs, sp);
- __ pop_reg(_gp_regs, sp);
- }
-};
-
-class XSetupArguments {
-private:
- MacroAssembler* const _masm;
- const Register _ref;
- const Address _ref_addr;
-
-public:
- XSetupArguments(MacroAssembler* masm, XLoadBarrierStubC2* stub) :
- _masm(masm),
- _ref(stub->ref()),
- _ref_addr(stub->ref_addr()) {
-
- // Setup arguments
- if (_ref_addr.base() == noreg) {
- // No self healing
- if (_ref != c_rarg0) {
- __ mv(c_rarg0, _ref);
- }
- __ mv(c_rarg1, zr);
- } else {
- // Self healing
- if (_ref == c_rarg0) {
- // _ref is already at correct place
- __ la(c_rarg1, _ref_addr);
- } else if (_ref != c_rarg1) {
- // _ref is in wrong place, but not in c_rarg1, so fix it first
- __ la(c_rarg1, _ref_addr);
- __ mv(c_rarg0, _ref);
- } else if (_ref_addr.base() != c_rarg0) {
- assert(_ref == c_rarg1, "Mov ref first, vacating c_rarg0");
- __ mv(c_rarg0, _ref);
- __ la(c_rarg1, _ref_addr);
- } else {
- assert(_ref == c_rarg1, "Need to vacate c_rarg1 and _ref_addr is using c_rarg0");
- if (_ref_addr.base() == c_rarg0) {
- __ mv(t1, c_rarg1);
- __ la(c_rarg1, _ref_addr);
- __ mv(c_rarg0, t1);
- } else {
- ShouldNotReachHere();
- }
- }
- }
- }
-
- ~XSetupArguments() {
- // Transfer result
- if (_ref != x10) {
- __ mv(_ref, x10);
- }
- }
-};
-
-#undef __
-#define __ masm->
-
-void XBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, XLoadBarrierStubC2* stub) const {
- BLOCK_COMMENT("XLoadBarrierStubC2");
-
- // Stub entry
- __ bind(*stub->entry());
-
- {
- XSaveLiveRegisters save_live_registers(masm, stub);
- XSetupArguments setup_arguments(masm, stub);
-
- __ mv(t0, stub->slow_path());
- __ jalr(t0);
- }
-
- // Stub exit
- __ j(*stub->continuation());
-}
-
-#endif // COMPILER2
-
-#ifdef COMPILER1
-#undef __
-#define __ ce->masm()->
-
-void XBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
- LIR_Opr ref) const {
- assert_different_registers(xthread, ref->as_register(), t1);
- __ ld(t1, address_bad_mask_from_thread(xthread));
- __ andr(t1, t1, ref->as_register());
-}
-
-void XBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
- XLoadBarrierStubC1* stub) const {
- // Stub entry
- __ bind(*stub->entry());
-
- Register ref = stub->ref()->as_register();
- Register ref_addr = noreg;
- Register tmp = noreg;
-
- if (stub->tmp()->is_valid()) {
- // Load address into tmp register
- ce->leal(stub->ref_addr(), stub->tmp());
- ref_addr = tmp = stub->tmp()->as_pointer_register();
- } else {
- // Address already in register
- ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
- }
-
- assert_different_registers(ref, ref_addr, noreg);
-
- // Save x10 unless it is the result or tmp register
- // Set up SP to accommodate parameters and maybe x10.
- if (ref != x10 && tmp != x10) {
- __ sub(sp, sp, 32);
- __ sd(x10, Address(sp, 16));
- } else {
- __ sub(sp, sp, 16);
- }
-
- // Setup arguments and call runtime stub
- ce->store_parameter(ref_addr, 1);
- ce->store_parameter(ref, 0);
-
- __ far_call(stub->runtime_stub());
-
- // Verify result
- __ verify_oop(x10);
-
-
- // Move result into place
- if (ref != x10) {
- __ mv(ref, x10);
- }
-
- // Restore x10 unless it is the result or tmp register
- if (ref != x10 && tmp != x10) {
- __ ld(x10, Address(sp, 16));
- __ add(sp, sp, 32);
- } else {
- __ add(sp, sp, 16);
- }
-
- // Stub exit
- __ j(*stub->continuation());
-}
-
-#undef __
-#define __ sasm->
-
-void XBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
- DecoratorSet decorators) const {
- __ prologue("zgc_load_barrier stub", false);
-
- __ push_call_clobbered_registers_except(RegSet::of(x10));
-
- // Setup arguments
- __ load_parameter(0, c_rarg0);
- __ load_parameter(1, c_rarg1);
-
- __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
-
- __ pop_call_clobbered_registers_except(RegSet::of(x10));
-
- __ epilogue();
-}
-
-#endif // COMPILER1
-
-#undef __
-#define __ masm->
-
-void XBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
- // Check if mask is good.
- // verifies that XAddressBadMask & obj == 0
- __ ld(tmp2, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
- __ andr(tmp1, obj, tmp2);
- __ bnez(tmp1, error);
-
- BarrierSetAssembler::check_oop(masm, obj, tmp1, tmp2, error);
-}
-
-#undef __
diff --git a/src/hotspot/cpu/riscv/gc/x/xBarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/x/xBarrierSetAssembler_riscv.hpp
deleted file mode 100644
index cbf5077999bfb..0000000000000
--- a/src/hotspot/cpu/riscv/gc/x/xBarrierSetAssembler_riscv.hpp
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef CPU_RISCV_GC_X_XBARRIERSETASSEMBLER_RISCV_HPP
-#define CPU_RISCV_GC_X_XBARRIERSETASSEMBLER_RISCV_HPP
-
-#include "code/vmreg.hpp"
-#include "oops/accessDecorators.hpp"
-#ifdef COMPILER2
-#include "opto/optoreg.hpp"
-#endif // COMPILER2
-
-#ifdef COMPILER1
-class LIR_Assembler;
-class LIR_Opr;
-class StubAssembler;
-#endif // COMPILER1
-
-#ifdef COMPILER2
-class Node;
-#endif // COMPILER2
-
-#ifdef COMPILER1
-class XLoadBarrierStubC1;
-#endif // COMPILER1
-
-#ifdef COMPILER2
-class XLoadBarrierStubC2;
-#endif // COMPILER2
-
-class XBarrierSetAssembler : public XBarrierSetAssemblerBase {
-public:
- virtual void load_at(MacroAssembler* masm,
- DecoratorSet decorators,
- BasicType type,
- Register dst,
- Address src,
- Register tmp1,
- Register tmp2);
-
-#ifdef ASSERT
- virtual void store_at(MacroAssembler* masm,
- DecoratorSet decorators,
- BasicType type,
- Address dst,
- Register val,
- Register tmp1,
- Register tmp2,
- Register tmp3);
-#endif // ASSERT
-
- virtual void arraycopy_prologue(MacroAssembler* masm,
- DecoratorSet decorators,
- bool is_oop,
- Register src,
- Register dst,
- Register count,
- RegSet saved_regs);
-
- virtual void try_resolve_jobject_in_native(MacroAssembler* masm,
- Register jni_env,
- Register robj,
- Register tmp,
- Label& slowpath);
-
- virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_data_patch; }
-
-#ifdef COMPILER1
- void generate_c1_load_barrier_test(LIR_Assembler* ce,
- LIR_Opr ref) const;
-
- void generate_c1_load_barrier_stub(LIR_Assembler* ce,
- XLoadBarrierStubC1* stub) const;
-
- void generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
- DecoratorSet decorators) const;
-#endif // COMPILER1
-
-#ifdef COMPILER2
- OptoReg::Name refine_register(const Node* node,
- OptoReg::Name opto_reg);
-
- void generate_c2_load_barrier_stub(MacroAssembler* masm,
- XLoadBarrierStubC2* stub) const;
-#endif // COMPILER2
-
- void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);
-};
-
-#endif // CPU_RISCV_GC_X_XBARRIERSETASSEMBLER_RISCV_HPP
diff --git a/src/hotspot/cpu/riscv/gc/x/xGlobals_riscv.cpp b/src/hotspot/cpu/riscv/gc/x/xGlobals_riscv.cpp
deleted file mode 100644
index 602dab5674738..0000000000000
--- a/src/hotspot/cpu/riscv/gc/x/xGlobals_riscv.cpp
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "gc/shared/gcLogPrecious.hpp"
-#include "gc/shared/gc_globals.hpp"
-#include "gc/x/xGlobals.hpp"
-#include "runtime/globals.hpp"
-#include "runtime/os.hpp"
-#include "utilities/globalDefinitions.hpp"
-#include "utilities/powerOfTwo.hpp"
-
-#ifdef LINUX
-#include
-#endif // LINUX
-
-//
-// The heap can have three different layouts, depending on the max heap size.
-//
-// Address Space & Pointer Layout 1
-// --------------------------------
-//
-// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB)
-// . .
-// . .
-// . .
-// +--------------------------------+ 0x0000014000000000 (20TB)
-// | Remapped View |
-// +--------------------------------+ 0x0000010000000000 (16TB)
-// . .
-// +--------------------------------+ 0x00000c0000000000 (12TB)
-// | Marked1 View |
-// +--------------------------------+ 0x0000080000000000 (8TB)
-// | Marked0 View |
-// +--------------------------------+ 0x0000040000000000 (4TB)
-// . .
-// +--------------------------------+ 0x0000000000000000
-//
-// 6 4 4 4 4
-// 3 6 5 2 1 0
-// +--------------------+----+-----------------------------------------------+
-// |00000000 00000000 00|1111|11 11111111 11111111 11111111 11111111 11111111|
-// +--------------------+----+-----------------------------------------------+
-// | | |
-// | | * 41-0 Object Offset (42-bits, 4TB address space)
-// | |
-// | * 45-42 Metadata Bits (4-bits) 0001 = Marked0 (Address view 4-8TB)
-// | 0010 = Marked1 (Address view 8-12TB)
-// | 0100 = Remapped (Address view 16-20TB)
-// | 1000 = Finalizable (Address view N/A)
-// |
-// * 63-46 Fixed (18-bits, always zero)
-//
-//
-// Address Space & Pointer Layout 2
-// --------------------------------
-//
-// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB)
-// . .
-// . .
-// . .
-// +--------------------------------+ 0x0000280000000000 (40TB)
-// | Remapped View |
-// +--------------------------------+ 0x0000200000000000 (32TB)
-// . .
-// +--------------------------------+ 0x0000180000000000 (24TB)
-// | Marked1 View |
-// +--------------------------------+ 0x0000100000000000 (16TB)
-// | Marked0 View |
-// +--------------------------------+ 0x0000080000000000 (8TB)
-// . .
-// +--------------------------------+ 0x0000000000000000
-//
-// 6 4 4 4 4
-// 3 7 6 3 2 0
-// +------------------+-----+------------------------------------------------+
-// |00000000 00000000 0|1111|111 11111111 11111111 11111111 11111111 11111111|
-// +-------------------+----+------------------------------------------------+
-// | | |
-// | | * 42-0 Object Offset (43-bits, 8TB address space)
-// | |
-// | * 46-43 Metadata Bits (4-bits) 0001 = Marked0 (Address view 8-16TB)
-// | 0010 = Marked1 (Address view 16-24TB)
-// | 0100 = Remapped (Address view 32-40TB)
-// | 1000 = Finalizable (Address view N/A)
-// |
-// * 63-47 Fixed (17-bits, always zero)
-//
-//
-// Address Space & Pointer Layout 3
-// --------------------------------
-//
-// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB)
-// . .
-// . .
-// . .
-// +--------------------------------+ 0x0000500000000000 (80TB)
-// | Remapped View |
-// +--------------------------------+ 0x0000400000000000 (64TB)
-// . .
-// +--------------------------------+ 0x0000300000000000 (48TB)
-// | Marked1 View |
-// +--------------------------------+ 0x0000200000000000 (32TB)
-// | Marked0 View |
-// +--------------------------------+ 0x0000100000000000 (16TB)
-// . .
-// +--------------------------------+ 0x0000000000000000
-//
-// 6 4 4 4 4
-// 3 8 7 4 3 0
-// +------------------+----+-------------------------------------------------+
-// |00000000 00000000 |1111|1111 11111111 11111111 11111111 11111111 11111111|
-// +------------------+----+-------------------------------------------------+
-// | | |
-// | | * 43-0 Object Offset (44-bits, 16TB address space)
-// | |
-// | * 47-44 Metadata Bits (4-bits) 0001 = Marked0 (Address view 16-32TB)
-// | 0010 = Marked1 (Address view 32-48TB)
-// | 0100 = Remapped (Address view 64-80TB)
-// | 1000 = Finalizable (Address view N/A)
-// |
-// * 63-48 Fixed (16-bits, always zero)
-//
-
-// Default value if probing is not implemented for a certain platform: 128TB
-static const size_t DEFAULT_MAX_ADDRESS_BIT = 47;
-// Minimum value returned, if probing fails: 64GB
-static const size_t MINIMUM_MAX_ADDRESS_BIT = 36;
-
-static size_t probe_valid_max_address_bit() {
-#ifdef LINUX
- size_t max_address_bit = 0;
- const size_t page_size = os::vm_page_size();
- for (size_t i = DEFAULT_MAX_ADDRESS_BIT; i > MINIMUM_MAX_ADDRESS_BIT; --i) {
- const uintptr_t base_addr = ((uintptr_t) 1U) << i;
- if (msync((void*)base_addr, page_size, MS_ASYNC) == 0) {
- // msync succeeded, the address is valid, and maybe even already mapped.
- max_address_bit = i;
- break;
- }
- if (errno != ENOMEM) {
- // Some error occurred. This should never happen, but msync
- // has some undefined behavior, hence ignore this bit.
-#ifdef ASSERT
- fatal("Received '%s' while probing the address space for the highest valid bit", os::errno_name(errno));
-#else // ASSERT
- log_warning_p(gc)("Received '%s' while probing the address space for the highest valid bit", os::errno_name(errno));
-#endif // ASSERT
- continue;
- }
- // Since msync failed with ENOMEM, the page might not be mapped.
- // Try to map it, to see if the address is valid.
- void* const result_addr = mmap((void*) base_addr, page_size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0);
- if (result_addr != MAP_FAILED) {
- munmap(result_addr, page_size);
- }
- if ((uintptr_t) result_addr == base_addr) {
- // address is valid
- max_address_bit = i;
- break;
- }
- }
- if (max_address_bit == 0) {
- // probing failed, allocate a very high page and take that bit as the maximum
- const uintptr_t high_addr = ((uintptr_t) 1U) << DEFAULT_MAX_ADDRESS_BIT;
- void* const result_addr = mmap((void*) high_addr, page_size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0);
- if (result_addr != MAP_FAILED) {
- max_address_bit = BitsPerSize_t - count_leading_zeros((size_t) result_addr) - 1;
- munmap(result_addr, page_size);
- }
- }
- log_info_p(gc, init)("Probing address space for the highest valid bit: " SIZE_FORMAT, max_address_bit);
- return MAX2(max_address_bit, MINIMUM_MAX_ADDRESS_BIT);
-#else // LINUX
- return DEFAULT_MAX_ADDRESS_BIT;
-#endif // LINUX
-}
-
-size_t XPlatformAddressOffsetBits() {
- const static size_t valid_max_address_offset_bits = probe_valid_max_address_bit() + 1;
- const size_t max_address_offset_bits = valid_max_address_offset_bits - 3;
- const size_t min_address_offset_bits = max_address_offset_bits - 2;
- const size_t address_offset = round_up_power_of_2(MaxHeapSize * XVirtualToPhysicalRatio);
- const size_t address_offset_bits = log2i_exact(address_offset);
- return clamp(address_offset_bits, min_address_offset_bits, max_address_offset_bits);
-}
-
-size_t XPlatformAddressMetadataShift() {
- return XPlatformAddressOffsetBits();
-}
diff --git a/src/hotspot/cpu/riscv/gc/x/xGlobals_riscv.hpp b/src/hotspot/cpu/riscv/gc/x/xGlobals_riscv.hpp
deleted file mode 100644
index 836dc7aac0d1d..0000000000000
--- a/src/hotspot/cpu/riscv/gc/x/xGlobals_riscv.hpp
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef CPU_RISCV_GC_X_XGLOBALS_RISCV_HPP
-#define CPU_RISCV_GC_X_XGLOBALS_RISCV_HPP
-
-const size_t XPlatformHeapViews = 3;
-const size_t XPlatformCacheLineSize = 64;
-
-size_t XPlatformAddressOffsetBits();
-size_t XPlatformAddressMetadataShift();
-
-#endif // CPU_RISCV_GC_X_XGLOBALS_RISCV_HPP
diff --git a/src/hotspot/cpu/riscv/gc/x/x_riscv.ad b/src/hotspot/cpu/riscv/gc/x/x_riscv.ad
deleted file mode 100644
index ef02f301c6aeb..0000000000000
--- a/src/hotspot/cpu/riscv/gc/x/x_riscv.ad
+++ /dev/null
@@ -1,229 +0,0 @@
-//
-// Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
-// Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
-// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-//
-// This code is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License version 2 only, as
-// published by the Free Software Foundation.
-//
-// This code is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-// version 2 for more details (a copy is included in the LICENSE file that
-// accompanied this code).
-//
-// You should have received a copy of the GNU General Public License version
-// 2 along with this work; if not, write to the Free Software Foundation,
-// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-//
-// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-// or visit www.oracle.com if you need additional information or have any
-// questions.
-//
-
-source_hpp %{
-
-#include "gc/shared/gc_globals.hpp"
-#include "gc/x/c2/xBarrierSetC2.hpp"
-#include "gc/x/xThreadLocalData.hpp"
-
-%}
-
-source %{
-
-static void x_load_barrier(MacroAssembler* masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, int barrier_data) {
- if (barrier_data == XLoadBarrierElided) {
- return;
- }
- XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data);
- __ ld(tmp, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
- __ andr(tmp, tmp, ref);
- __ bnez(tmp, *stub->entry(), true /* far */);
- __ bind(*stub->continuation());
-}
-
-static void x_load_barrier_slow_path(MacroAssembler* masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {
- XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, XLoadBarrierStrong);
- __ j(*stub->entry());
- __ bind(*stub->continuation());
-}
-
-%}
-
-// Load Pointer
-instruct xLoadP(iRegPNoSp dst, memory mem, iRegPNoSp tmp)
-%{
- match(Set dst (LoadP mem));
- predicate(UseZGC && !ZGenerational && (n->as_Load()->barrier_data() != 0));
- effect(TEMP dst, TEMP tmp);
-
- ins_cost(4 * DEFAULT_COST);
-
- format %{ "ld $dst, $mem, #@zLoadP" %}
-
- ins_encode %{
- const Address ref_addr (as_Register($mem$$base), $mem$$disp);
- __ ld($dst$$Register, ref_addr);
- x_load_barrier(masm, this, ref_addr, $dst$$Register, $tmp$$Register /* tmp */, barrier_data());
- %}
-
- ins_pipe(iload_reg_mem);
-%}
-
-instruct xCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
- match(Set res (CompareAndSwapP mem (Binary oldval newval)));
- match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
- predicate(UseZGC && !ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
- effect(TEMP_DEF res, TEMP tmp);
-
- ins_cost(2 * VOLATILE_REF_COST);
-
- format %{ "cmpxchg $mem, $oldval, $newval, #@zCompareAndSwapP\n\t"
- "mv $res, $res == $oldval" %}
-
- ins_encode %{
- Label failed;
- guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
- Assembler::relaxed /* acquire */, Assembler::rl /* release */, $tmp$$Register);
- __ sub(t0, $tmp$$Register, $oldval$$Register);
- __ seqz($res$$Register, t0);
- if (barrier_data() != XLoadBarrierElided) {
- Label good;
- __ ld(t0, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
- __ andr(t0, t0, $tmp$$Register);
- __ beqz(t0, good);
- x_load_barrier_slow_path(masm, this, Address($mem$$Register), $tmp$$Register /* ref */, $res$$Register /* tmp */);
- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
- Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register,
- true /* result_as_bool */);
- __ bind(good);
- }
- %}
-
- ins_pipe(pipe_slow);
-%}
-
-instruct xCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
- match(Set res (CompareAndSwapP mem (Binary oldval newval)));
- match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
- predicate(UseZGC && !ZGenerational && needs_acquiring_load_reserved(n) && (n->as_LoadStore()->barrier_data() == XLoadBarrierStrong));
- effect(TEMP_DEF res, TEMP tmp);
-
- ins_cost(2 * VOLATILE_REF_COST);
-
- format %{ "cmpxchg $mem, $oldval, $newval, #@zCompareAndSwapPAcq\n\t"
- "mv $res, $res == $oldval" %}
-
- ins_encode %{
- Label failed;
- guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
- Assembler::aq /* acquire */, Assembler::rl /* release */, $tmp$$Register);
- __ sub(t0, $tmp$$Register, $oldval$$Register);
- __ seqz($res$$Register, t0);
- if (barrier_data() != XLoadBarrierElided) {
- Label good;
- __ ld(t0, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
- __ andr(t0, t0, $tmp$$Register);
- __ beqz(t0, good);
- x_load_barrier_slow_path(masm, this, Address($mem$$Register), $tmp$$Register /* ref */, $res$$Register /* tmp */);
- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
- Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register,
- true /* result_as_bool */);
- __ bind(good);
- }
- %}
-
- ins_pipe(pipe_slow);
-%}
-
-instruct xCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
- match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
- predicate(UseZGC && !ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
- effect(TEMP_DEF res, TEMP tmp);
-
- ins_cost(2 * VOLATILE_REF_COST);
-
- format %{ "cmpxchg $res = $mem, $oldval, $newval, #@zCompareAndExchangeP" %}
-
- ins_encode %{
- guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
- Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register);
- if (barrier_data() != XLoadBarrierElided) {
- Label good;
- __ ld(t0, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
- __ andr(t0, t0, $res$$Register);
- __ beqz(t0, good);
- x_load_barrier_slow_path(masm, this, Address($mem$$Register), $res$$Register /* ref */, $tmp$$Register /* tmp */);
- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
- Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register);
- __ bind(good);
- }
- %}
-
- ins_pipe(pipe_slow);
-%}
-
-instruct xCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
- match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
- predicate(UseZGC && !ZGenerational && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
- effect(TEMP_DEF res, TEMP tmp);
-
- ins_cost(2 * VOLATILE_REF_COST);
-
- format %{ "cmpxchg $res = $mem, $oldval, $newval, #@zCompareAndExchangePAcq" %}
-
- ins_encode %{
- guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
- Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register);
- if (barrier_data() != XLoadBarrierElided) {
- Label good;
- __ ld(t0, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
- __ andr(t0, t0, $res$$Register);
- __ beqz(t0, good);
- x_load_barrier_slow_path(masm, this, Address($mem$$Register), $res$$Register /* ref */, $tmp$$Register /* tmp */);
- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
- Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register);
- __ bind(good);
- }
- %}
-
- ins_pipe(pipe_slow);
-%}
-
-instruct xGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp) %{
- match(Set prev (GetAndSetP mem newv));
- predicate(UseZGC && !ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
- effect(TEMP_DEF prev, TEMP tmp);
-
- ins_cost(2 * VOLATILE_REF_COST);
-
- format %{ "atomic_xchg $prev, $newv, [$mem], #@zGetAndSetP" %}
-
- ins_encode %{
- __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
- x_load_barrier(masm, this, Address(noreg, 0), $prev$$Register, $tmp$$Register /* tmp */, barrier_data());
- %}
-
- ins_pipe(pipe_serial);
-%}
-
-instruct xGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp) %{
- match(Set prev (GetAndSetP mem newv));
- predicate(UseZGC && !ZGenerational && needs_acquiring_load_reserved(n) && (n->as_LoadStore()->barrier_data() != 0));
- effect(TEMP_DEF prev, TEMP tmp);
-
- ins_cost(VOLATILE_REF_COST);
-
- format %{ "atomic_xchg_acq $prev, $newv, [$mem], #@zGetAndSetPAcq" %}
-
- ins_encode %{
- __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
- x_load_barrier(masm, this, Address(noreg, 0), $prev$$Register, $tmp$$Register /* tmp */, barrier_data());
- %}
- ins_pipe(pipe_serial);
-%}
diff --git a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp
index cbb918ade00fe..cd83eafcaeba5 100644
--- a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp
@@ -724,8 +724,8 @@ void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, Z
{
SaveLiveRegisters save_live_registers(masm, stub);
ZSetupArguments setup_arguments(masm, stub);
- __ mv(t0, stub->slow_path());
- __ jalr(t0);
+ __ mv(t1, stub->slow_path());
+ __ jalr(t1);
}
// Stub exit
@@ -758,13 +758,14 @@ void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm,
__ la(c_rarg0, stub->ref_addr());
if (stub->is_native()) {
- __ la(t0, RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_native_oop_field_without_healing_addr()));
+ __ rt_call(ZBarrierSetRuntime::store_barrier_on_native_oop_field_without_healing_addr());
} else if (stub->is_atomic()) {
- __ la(t0, RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr()));
+ __ rt_call(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr());
+ } else if (stub->is_nokeepalive()) {
+ __ rt_call(ZBarrierSetRuntime::no_keepalive_store_barrier_on_oop_field_without_healing_addr());
} else {
- __ la(t0, RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr()));
+ __ rt_call(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr());
}
- __ jalr(t0);
}
// Stub exit
diff --git a/src/hotspot/cpu/riscv/gc/z/z_riscv.ad b/src/hotspot/cpu/riscv/gc/z/z_riscv.ad
index 4c94e504475ee..8e33d514f46cb 100644
--- a/src/hotspot/cpu/riscv/gc/z/z_riscv.ad
+++ b/src/hotspot/cpu/riscv/gc/z/z_riscv.ad
@@ -82,7 +82,8 @@ static void z_store_barrier(MacroAssembler* masm, const MachNode* node, Address
z_color(masm, node, rnew_zpointer, rnew_zaddress, tmp);
} else {
bool is_native = (node->barrier_data() & ZBarrierNative) != 0;
- ZStoreBarrierStubC2* const stub = ZStoreBarrierStubC2::create(node, ref_addr, rnew_zaddress, rnew_zpointer, is_native, is_atomic);
+ bool is_nokeepalive = (node->barrier_data() & ZBarrierNoKeepalive) != 0;
+ ZStoreBarrierStubC2* const stub = ZStoreBarrierStubC2::create(node, ref_addr, rnew_zaddress, rnew_zpointer, is_native, is_atomic, is_nokeepalive);
ZBarrierSetAssembler* bs_asm = ZBarrierSet::assembler();
bs_asm->store_barrier_fast(masm, ref_addr, rnew_zaddress, rnew_zpointer, tmp, true /* in_nmethod */, is_atomic, *stub->entry(), *stub->continuation());
}
@@ -90,11 +91,11 @@ static void z_store_barrier(MacroAssembler* masm, const MachNode* node, Address
%}
// Load Pointer
-instruct zLoadP(iRegPNoSp dst, memory mem, iRegPNoSp tmp)
+instruct zLoadP(iRegPNoSp dst, memory mem, iRegPNoSp tmp, rFlagsReg cr)
%{
match(Set dst (LoadP mem));
- predicate(UseZGC && ZGenerational && n->as_Load()->barrier_data() != 0);
- effect(TEMP dst, TEMP tmp);
+ predicate(UseZGC && n->as_Load()->barrier_data() != 0);
+ effect(TEMP dst, TEMP tmp, KILL cr);
ins_cost(4 * DEFAULT_COST);
@@ -110,11 +111,11 @@ instruct zLoadP(iRegPNoSp dst, memory mem, iRegPNoSp tmp)
%}
// Store Pointer
-instruct zStoreP(memory mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2)
+instruct zStoreP(memory mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
%{
- predicate(UseZGC && ZGenerational && n->as_Store()->barrier_data() != 0);
+ predicate(UseZGC && n->as_Store()->barrier_data() != 0);
match(Set mem (StoreP mem src));
- effect(TEMP tmp1, TEMP tmp2);
+ effect(TEMP tmp1, TEMP tmp2, KILL cr);
ins_cost(125); // XXX
format %{ "sd $mem, $src\t# ptr" %}
@@ -127,11 +128,11 @@ instruct zStoreP(memory mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2)
%}
instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval,
- iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, iRegPNoSp tmp1) %{
+ iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, iRegPNoSp tmp1, rFlagsReg cr) %{
match(Set res (CompareAndSwapP mem (Binary oldval newval)));
match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
- predicate(UseZGC && ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
- effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP tmp1, TEMP_DEF res);
+ predicate(UseZGC && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
+ effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP tmp1, TEMP_DEF res, KILL cr);
ins_cost(2 * VOLATILE_REF_COST);
@@ -150,11 +151,11 @@ instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newva
%}
instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval,
- iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, iRegPNoSp tmp1) %{
+ iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, iRegPNoSp tmp1, rFlagsReg cr) %{
match(Set res (CompareAndSwapP mem (Binary oldval newval)));
match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
- predicate(UseZGC && ZGenerational && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
- effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP tmp1, TEMP_DEF res);
+ predicate(UseZGC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
+ effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP tmp1, TEMP_DEF res, KILL cr);
ins_cost(2 * VOLATILE_REF_COST);
@@ -173,10 +174,10 @@ instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
%}
instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval,
- iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, iRegPNoSp tmp1) %{
+ iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, iRegPNoSp tmp1, rFlagsReg cr) %{
match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
- predicate(UseZGC && ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
- effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP tmp1, TEMP_DEF res);
+ predicate(UseZGC && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
+ effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP tmp1, TEMP_DEF res, KILL cr);
ins_cost(2 * VOLATILE_REF_COST);
@@ -195,10 +196,10 @@ instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP n
%}
instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval,
- iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, iRegPNoSp tmp1) %{
+ iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, iRegPNoSp tmp1, rFlagsReg cr) %{
match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
- predicate(UseZGC && ZGenerational && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
- effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP tmp1, TEMP_DEF res);
+ predicate(UseZGC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
+ effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP tmp1, TEMP_DEF res, KILL cr);
ins_cost(2 * VOLATILE_REF_COST);
@@ -216,10 +217,10 @@ instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
ins_pipe(pipe_slow);
%}
-instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp) %{
+instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp, rFlagsReg cr) %{
match(Set prev (GetAndSetP mem newv));
- predicate(UseZGC && ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
- effect(TEMP_DEF prev, TEMP tmp);
+ predicate(UseZGC && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
+ effect(TEMP_DEF prev, TEMP tmp, KILL cr);
ins_cost(2 * VOLATILE_REF_COST);
@@ -234,10 +235,10 @@ instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp) %{
ins_pipe(pipe_serial);
%}
-instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp) %{
+instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp, rFlagsReg cr) %{
match(Set prev (GetAndSetP mem newv));
- predicate(UseZGC && ZGenerational && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
- effect(TEMP_DEF prev, TEMP tmp);
+ predicate(UseZGC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
+ effect(TEMP_DEF prev, TEMP tmp, KILL cr);
ins_cost(2 * VOLATILE_REF_COST);
diff --git a/src/hotspot/cpu/riscv/globals_riscv.hpp b/src/hotspot/cpu/riscv/globals_riscv.hpp
index dd31de14704ab..7bd0200f11803 100644
--- a/src/hotspot/cpu/riscv/globals_riscv.hpp
+++ b/src/hotspot/cpu/riscv/globals_riscv.hpp
@@ -112,11 +112,11 @@ define_pd_global(intx, InlineSmallCode, 1000);
product(bool, UseZicbom, false, EXPERIMENTAL, "Use Zicbom instructions") \
product(bool, UseZicbop, false, EXPERIMENTAL, "Use Zicbop instructions") \
product(bool, UseZicboz, false, EXPERIMENTAL, "Use Zicboz instructions") \
- product(bool, UseZtso, false, EXPERIMENTAL, "Assume Ztso memory model") \
product(bool, UseZihintpause, false, EXPERIMENTAL, \
"Use Zihintpause instructions") \
+ product(bool, UseZtso, false, EXPERIMENTAL, "Assume Ztso memory model") \
product(bool, UseZvbb, false, EXPERIMENTAL, "Use Zvbb instructions") \
- product(bool, UseZvfh, false, EXPERIMENTAL, "Use Zvfh instructions") \
+ product(bool, UseZvfh, false, "Use Zvfh instructions") \
product(bool, UseZvkn, false, EXPERIMENTAL, \
"Use Zvkn group extension, Zvkned, Zvknhb, Zvkb, Zvkt") \
product(bool, UseRVVForBigIntegerShiftIntrinsics, true, \
diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
index 3eb7abb5ee3b9..f383557e43fc5 100644
--- a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
+++ b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
@@ -193,9 +193,7 @@ void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(Register reg, i
void InterpreterMacroAssembler::get_dispatch() {
ExternalAddress target((address)Interpreter::dispatch_table());
relocate(target.rspec(), [&] {
- int32_t offset;
- la(xdispatch, target.target(), offset);
- addi(xdispatch, xdispatch, offset);
+ la(xdispatch, target.target());
});
}
@@ -421,13 +419,13 @@ void InterpreterMacroAssembler::jump_from_interpreted(Register method) {
// interp_only_mode if these events CAN be enabled.
lwu(t0, Address(xthread, JavaThread::interp_only_mode_offset()));
beqz(t0, run_compiled_code);
- ld(t0, Address(method, Method::interpreter_entry_offset()));
- jr(t0);
+ ld(t1, Address(method, Method::interpreter_entry_offset()));
+ jr(t1);
bind(run_compiled_code);
}
- ld(t0, Address(method, Method::from_interpreted_offset()));
- jr(t0);
+ ld(t1, Address(method, Method::from_interpreted_offset()));
+ jr(t1);
}
// The following two routines provide a hook so that an implementation
diff --git a/src/hotspot/cpu/riscv/jniFastGetField_riscv.cpp b/src/hotspot/cpu/riscv/jniFastGetField_riscv.cpp
index f7d702c631069..2b4a8b87e5404 100644
--- a/src/hotspot/cpu/riscv/jniFastGetField_riscv.cpp
+++ b/src/hotspot/cpu/riscv/jniFastGetField_riscv.cpp
@@ -75,9 +75,7 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
Address target(SafepointSynchronize::safepoint_counter_addr());
__ relocate(target.rspec(), [&] {
- int32_t offset;
- __ la(rcounter_addr, target.target(), offset);
- __ addi(rcounter_addr, rcounter_addr, offset);
+ __ la(rcounter_addr, target.target());
});
Label slow;
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
index b99ba542423a1..b58590e790fac 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
@@ -454,12 +454,7 @@ void MacroAssembler::call_VM_base(Register oop_result,
ld(t0, Address(java_thread, in_bytes(Thread::pending_exception_offset())));
Label ok;
beqz(t0, ok);
- RuntimeAddress target(StubRoutines::forward_exception_entry());
- relocate(target.rspec(), [&] {
- int32_t offset;
- la(t0, target.target(), offset);
- jr(t0, offset);
- });
+ j(RuntimeAddress(StubRoutines::forward_exception_entry()));
bind(ok);
}
@@ -493,6 +488,7 @@ void MacroAssembler::clinit_barrier(Register klass, Register tmp, Label* L_fast_
// Fast path check: class is fully initialized
lbu(tmp, Address(klass, InstanceKlass::init_state_offset()));
+ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
sub(tmp, tmp, InstanceKlass::fully_initialized);
beqz(tmp, *L_fast_path);
@@ -534,7 +530,7 @@ void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file,
movptr(t0, (address) b);
}
- // call indirectly to solve generation ordering problem
+ // Call indirectly to solve generation ordering problem
RuntimeAddress target(StubRoutines::verify_oop_subroutine_entry_address());
relocate(target.rspec(), [&] {
int32_t offset;
@@ -579,7 +575,7 @@ void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* f
movptr(t0, (address) b);
}
- // call indirectly to solve generation ordering problem
+ // Call indirectly to solve generation ordering problem
RuntimeAddress target(StubRoutines::verify_oop_subroutine_entry_address());
relocate(target.rspec(), [&] {
int32_t offset;
@@ -759,21 +755,21 @@ void MacroAssembler::emit_static_call_stub() {
// Jump to the entry point of the c2i stub.
int32_t offset = 0;
- movptr(t0, 0, offset, t1); // lui + lui + slli + add
- jr(t0, offset);
+ movptr(t1, 0, offset, t0); // lui + lui + slli + add
+ jr(t1, offset);
}
void MacroAssembler::call_VM_leaf_base(address entry_point,
int number_of_arguments,
Label *retaddr) {
int32_t offset = 0;
- push_reg(RegSet::of(t0, xmethod), sp); // push << t0 & xmethod >> to sp
- mv(t0, entry_point, offset);
- jalr(t0, offset);
+ push_reg(RegSet::of(t1, xmethod), sp); // push << t1 & xmethod >> to sp
+ mv(t1, entry_point, offset);
+ jalr(t1, offset);
if (retaddr != nullptr) {
bind(*retaddr);
}
- pop_reg(RegSet::of(t0, xmethod), sp); // pop << t0 & xmethod >> from sp
+ pop_reg(RegSet::of(t1, xmethod), sp); // pop << t1 & xmethod >> from sp
}
void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
@@ -940,6 +936,7 @@ void MacroAssembler::li(Register Rd, int64_t imm) {
void MacroAssembler::load_link_jump(const address source, Register temp) {
assert(temp != noreg && temp != x0, "expecting a register");
+ assert(temp != x5, "temp register must not be x5.");
assert_cond(source != nullptr);
int64_t distance = source - pc();
assert(is_simm32(distance), "Must be");
@@ -967,24 +964,27 @@ void MacroAssembler::j(const address dest, Register temp) {
if (is_simm21(distance) && ((distance % 2) == 0)) {
Assembler::jal(x0, distance);
} else {
- assert(temp != noreg && temp != x0, "expecting a register");
+ assert(temp != noreg && temp != x0, "Expecting a register");
+ assert(temp != x1 && temp != x5, "temp register must not be x1/x5.");
int32_t offset = 0;
la(temp, dest, offset);
jr(temp, offset);
}
}
-void MacroAssembler::j(const Address &adr, Register temp) {
- switch (adr.getMode()) {
+void MacroAssembler::j(const Address &dest, Register temp) {
+ switch (dest.getMode()) {
case Address::literal: {
- relocate(adr.rspec(), [&] {
- j(adr.target(), temp);
+ relocate(dest.rspec(), [&] {
+ int32_t offset;
+ la(temp, dest.target(), offset);
+ jr(temp, offset);
});
break;
}
case Address::base_plus_offset: {
- int32_t offset = ((int32_t)adr.offset() << 20) >> 20;
- la(temp, Address(adr.base(), adr.offset() - offset));
+ int32_t offset = ((int32_t)dest.offset() << 20) >> 20;
+ la(temp, Address(dest.base(), dest.offset() - offset));
jr(temp, offset);
break;
}
@@ -1005,12 +1005,14 @@ void MacroAssembler::j(Label &lab, Register temp) {
void MacroAssembler::jr(Register Rd, int32_t offset) {
assert(Rd != noreg, "expecting a register");
+ assert(Rd != x1 && Rd != x5, "Rd register must not be x1/x5.");
Assembler::jalr(x0, Rd, offset);
}
void MacroAssembler::call(const address dest, Register temp) {
assert_cond(dest != nullptr);
assert(temp != noreg, "expecting a register");
+ assert(temp != x5, "temp register must not be x5.");
int32_t offset = 0;
la(temp, dest, offset);
jalr(temp, offset);
@@ -1018,10 +1020,12 @@ void MacroAssembler::call(const address dest, Register temp) {
void MacroAssembler::jalr(Register Rs, int32_t offset) {
assert(Rs != noreg, "expecting a register");
+ assert(Rs != x5, "Rs register must not be x5.");
Assembler::jalr(x1, Rs, offset);
}
void MacroAssembler::rt_call(address dest, Register tmp) {
+ assert(tmp != x5, "tmp register must not be x5.");
CodeBlob *cb = CodeCache::find_blob(dest);
RuntimeAddress target(dest);
if (cb) {
@@ -1761,7 +1765,7 @@ void MacroAssembler::pop_CPU_state(bool restore_vectors, int vector_size_in_byte
static int patch_offset_in_jal(address branch, int64_t offset) {
assert(Assembler::is_simm21(offset) && ((offset % 2) == 0),
- "offset is too large to be patched in one jal instruction!\n");
+ "offset (%ld) is too large to be patched in one jal instruction!\n", offset);
Assembler::patch(branch, 31, 31, (offset >> 20) & 0x1); // offset[20] ==> branch[31]
Assembler::patch(branch, 30, 21, (offset >> 1) & 0x3ff); // offset[10:1] ==> branch[30:21]
Assembler::patch(branch, 20, 20, (offset >> 11) & 0x1); // offset[11] ==> branch[20]
@@ -2565,27 +2569,6 @@ void MacroAssembler::bang_stack_size(Register size, Register tmp) {
}
}
-SkipIfEqual::SkipIfEqual(MacroAssembler* masm, const bool* flag_addr, bool value) {
- int32_t offset = 0;
- _masm = masm;
- ExternalAddress target((address)flag_addr);
- _masm->relocate(target.rspec(), [&] {
- int32_t offset;
- _masm->la(t0, target.target(), offset);
- _masm->lbu(t0, Address(t0, offset));
- });
- if (value) {
- _masm->bnez(t0, _label);
- } else {
- _masm->beqz(t0, _label);
- }
-}
-
-SkipIfEqual::~SkipIfEqual() {
- _masm->bind(_label);
- _masm = nullptr;
-}
-
void MacroAssembler::load_mirror(Register dst, Register method, Register tmp1, Register tmp2) {
const int mirror_offset = in_bytes(Klass::java_mirror_offset());
ld(dst, Address(xmethod, Method::const_offset()));
@@ -2947,7 +2930,7 @@ int MacroAssembler::corrected_idivq(Register result, Register rs1, Register rs2,
return idivq_offset;
}
-// Look up the method for a megamorpic invkkeinterface call.
+// Look up the method for a megamorphic invokeinterface call.
// The target method is determined by .
// The receiver klass is in recv_klass.
// On success, the result will be in method_result, and execution falls through.
@@ -2962,9 +2945,9 @@ void MacroAssembler::lookup_interface_method(Register recv_klass,
assert_different_registers(recv_klass, intf_klass, scan_tmp);
assert_different_registers(method_result, intf_klass, scan_tmp);
assert(recv_klass != method_result || !return_method,
- "recv_klass can be destroyed when mehtid isn't needed");
+ "recv_klass can be destroyed when method isn't needed");
assert(itable_index.is_constant() || itable_index.as_register() == method_result,
- "caller must be same register for non-constant itable index as for method");
+ "caller must use same register for non-constant itable index as for method");
// Compute start of first itableOffsetEntry (which is at the end of the vtable).
int vtable_base = in_bytes(Klass::vtable_start_offset());
@@ -3657,6 +3640,7 @@ void MacroAssembler::far_jump(const Address &entry, Register tmp) {
}
void MacroAssembler::far_call(const Address &entry, Register tmp) {
+ assert(tmp != x5, "tmp register must not be x5.");
assert(CodeCache::find_blob(entry.target()) != nullptr,
"destination of far call not found in code cache");
assert(entry.rspec().type() == relocInfo::external_word_type
@@ -4071,7 +4055,7 @@ void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass,
Register tmp1,
Register tmp2,
Register tmp3) {
- assert_different_registers(r_sub_klass, r_super_klass, tmp1, tmp2, tmp3, result, t0);
+ assert_different_registers(r_sub_klass, r_super_klass, tmp1, tmp2, tmp3, result, t0, t1);
const Register
r_array_base = tmp1, // X11
@@ -4138,8 +4122,8 @@ void MacroAssembler::get_thread(Register thread) {
RegSet::range(x28, x31) + ra - thread;
push_reg(saved_regs, sp);
- mv(ra, CAST_FROM_FN_PTR(address, Thread::current));
- jalr(ra);
+ mv(t1, CAST_FROM_FN_PTR(address, Thread::current));
+ jalr(t1);
if (thread != c_rarg0) {
mv(thread, c_rarg0);
}
@@ -4186,8 +4170,7 @@ void MacroAssembler::reserved_stack_check() {
// We have already removed our own frame.
// throw_delayed_StackOverflowError will think that it's been
// called by our caller.
- la(t0, RuntimeAddress(SharedRuntime::throw_delayed_StackOverflowError_entry()));
- jr(t0);
+ j(RuntimeAddress(SharedRuntime::throw_delayed_StackOverflowError_entry()));
should_not_reach_here();
bind(no_reserved_zone_enabling);
@@ -4206,7 +4189,7 @@ void MacroAssembler::read_polling_page(Register r, int32_t offset, relocInfo::re
});
}
-void MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
+void MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
#ifdef ASSERT
{
ThreadInVMfromUnknown tiv;
@@ -4298,7 +4281,7 @@ address MacroAssembler::load_and_call(Address entry) {
}
#endif
relocate(entry.rspec(), [&] {
- load_link_jump(target);
+ load_link_jump(target, t1);
});
postcond(pc() != badAddress);
@@ -4308,7 +4291,7 @@ address MacroAssembler::load_and_call(Address entry) {
address MacroAssembler::ic_call(address entry, jint method_index) {
RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
IncompressibleRegion ir(this); // relocations
- movptr(t1, (address)Universe::non_oop_word(), t0);
+ movptr(t0, (address)Universe::non_oop_word(), t1);
assert_cond(entry != nullptr);
return reloc_call(Address(entry, rh));
}
@@ -4322,9 +4305,9 @@ int MacroAssembler::ic_check_size() {
int MacroAssembler::ic_check(int end_alignment) {
IncompressibleRegion ir(this);
Register receiver = j_rarg0;
- Register data = t1;
+ Register data = t0;
- Register tmp1 = t0; // t0 always scratch
+ Register tmp1 = t1; // scratch
// t2 is saved on call, thus should have been saved before this check.
// Hence we can clobber it.
Register tmp2 = t2;
@@ -4422,8 +4405,8 @@ address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset,
// - load the call
// - call
Label target;
- ld(t0, target); // auipc + ld
- jr(t0); // jalr
+ ld(t1, target); // auipc + ld
+ jr(t1); // jalr
bind(target);
assert(offset() - stub_start_offset == MacroAssembler::NativeShortCall::trampoline_data_offset,
"should be");
@@ -4507,14 +4490,15 @@ void MacroAssembler::decrementw(const Address dst, int32_t value, Register tmp1,
sw(tmp1, adr);
}
-void MacroAssembler::cmpptr(Register src1, Address src2, Label& equal) {
- assert_different_registers(src1, t0);
+void MacroAssembler::cmpptr(Register src1, const Address &src2, Label& equal, Register tmp) {
+ assert_different_registers(src1, tmp);
+ assert(src2.getMode() == Address::literal, "must be applied to a literal address");
relocate(src2.rspec(), [&] {
int32_t offset;
- la(t0, src2.target(), offset);
- ld(t0, Address(t0, offset));
+ la(tmp, src2.target(), offset);
+ ld(tmp, Address(tmp, offset));
});
- beq(src1, t0, equal);
+ beq(src1, tmp, equal);
}
void MacroAssembler::load_method_holder_cld(Register result, Register method) {
@@ -5147,11 +5131,11 @@ const int MacroAssembler::zero_words_block_size = 8;
// ptr: Address of a buffer to be zeroed.
// cnt: Count in HeapWords.
//
-// ptr, cnt, and t0 are clobbered.
+// ptr, cnt, t1, and t0 are clobbered.
address MacroAssembler::zero_words(Register ptr, Register cnt) {
assert(is_power_of_2(zero_words_block_size), "adjust this");
assert(ptr == x28 && cnt == x29, "mismatch in register usage");
- assert_different_registers(cnt, t0);
+ assert_different_registers(cnt, t0, t1);
BLOCK_COMMENT("zero_words {");
@@ -5169,6 +5153,7 @@ address MacroAssembler::zero_words(Register ptr, Register cnt) {
return nullptr;
}
} else {
+ // Clobbers t1
rt_call(zero_blocks.target());
}
}
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
index fd174f241eb0b..b248db3993379 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
@@ -627,7 +627,7 @@ class MacroAssembler: public Assembler {
void bgtz(Register Rs, const address dest);
private:
- void load_link_jump(const address source, Register temp = t0);
+ void load_link_jump(const address source, Register temp);
void jump_link(const address dest, Register temp);
public:
// We try to follow risc-v asm menomics.
@@ -635,18 +635,42 @@ class MacroAssembler: public Assembler {
// we often need to resort to movptr, li <48imm>.
// https://github.com/riscv-non-isa/riscv-asm-manual/blob/master/riscv-asm.md
+ // Hotspot only use the standard calling convention using x1/ra.
+ // The alternative calling convection using x5/t0 is not used.
+ // Using x5 as a temp causes the CPU to mispredict returns.
+
+ // JALR, return address stack updates:
+ // | rd is x1/x5 | rs1 is x1/x5 | rd=rs1 | RAS action
+ // | ----------- | ------------ | ------ |-------------
+ // | No | No | — | None
+ // | No | Yes | — | Pop
+ // | Yes | No | — | Push
+ // | Yes | Yes | No | Pop, then push
+ // | Yes | Yes | Yes | Push
+ //
+ // JAL, return address stack updates:
+ // | rd is x1/x5 | RAS action
+ // | ----------- | ----------
+ // | Yes | Push
+ // | No | None
+ //
+ // JUMPs uses Rd = x0/zero and Rs = x6/t1 or imm
+ // CALLS uses Rd = x1/ra and Rs = x6/t1 or imm (or x1/ra*)
+ // RETURNS uses Rd = x0/zero and Rs = x1/ra
+ // *use of x1/ra should not normally be used, special case only.
+
// jump: jal x0, offset
// For long reach uses temp register for:
// la + jr
- void j(const address dest, Register temp = t0);
- void j(const Address &adr, Register temp = t0);
- void j(Label &l, Register temp = t0);
+ void j(const address dest, Register temp = t1);
+ void j(const Address &dest, Register temp = t1);
+ void j(Label &l, Register temp = noreg);
// jump register: jalr x0, offset(rs)
void jr(Register Rd, int32_t offset = 0);
// call: la + jalr x1
- void call(const address dest, Register temp = t0);
+ void call(const address dest, Register temp = t1);
// jalr: jalr x1, offset(rs)
void jalr(Register Rs, int32_t offset = 0);
@@ -654,7 +678,8 @@ class MacroAssembler: public Assembler {
// Emit a runtime call. Only invalidates the tmp register which
// is used to keep the entry address for jalr/movptr.
// Uses call() for intra code cache, else movptr + jalr.
- void rt_call(address dest, Register tmp = t0);
+ // Clobebrs t1
+ void rt_call(address dest, Register tmp = t1);
// ret: jalr x0, 0(x1)
inline void ret() {
@@ -1165,8 +1190,9 @@ class MacroAssembler: public Assembler {
// - relocInfo::external_word_type
// - relocInfo::runtime_call_type
// - relocInfo::none
- void far_call(const Address &entry, Register tmp = t0);
- void far_jump(const Address &entry, Register tmp = t0);
+ // Clobbers t1 default.
+ void far_call(const Address &entry, Register tmp = t1);
+ void far_jump(const Address &entry, Register tmp = t1);
static int far_branch_size() {
return 2 * 4; // auipc + jalr, see far_call() & far_jump()
@@ -1301,7 +1327,7 @@ class MacroAssembler: public Assembler {
void decrement(const Address dst, int64_t value = 1, Register tmp1 = t0, Register tmp2 = t1);
void decrementw(const Address dst, int32_t value = 1, Register tmp1 = t0, Register tmp2 = t1);
- void cmpptr(Register src1, Address src2, Label& equal);
+ void cmpptr(Register src1, const Address &src2, Label& equal, Register tmp = t0);
void clinit_barrier(Register klass, Register tmp, Label* L_fast_path = nullptr, Label* L_slow_path = nullptr);
void load_method_holder_cld(Register result, Register method);
@@ -1768,22 +1794,4 @@ class MacroAssembler: public Assembler {
inline bool AbstractAssembler::pd_check_instruction_mark() { return false; }
#endif
-/**
- * class SkipIfEqual:
- *
- * Instantiating this class will result in assembly code being output that will
- * jump around any code emitted between the creation of the instance and it's
- * automatic destruction at the end of a scope block, depending on the value of
- * the flag passed to the constructor, which will be checked at run-time.
- */
-class SkipIfEqual {
- private:
- MacroAssembler* _masm;
- Label _label;
-
- public:
- SkipIfEqual(MacroAssembler*, const bool* flag_addr, bool value);
- ~SkipIfEqual();
-};
-
#endif // CPU_RISCV_MACROASSEMBLER_RISCV_HPP
diff --git a/src/hotspot/cpu/riscv/methodHandles_riscv.cpp b/src/hotspot/cpu/riscv/methodHandles_riscv.cpp
index deeb771d83bb8..8ed4b93ad4de9 100644
--- a/src/hotspot/cpu/riscv/methodHandles_riscv.cpp
+++ b/src/hotspot/cpu/riscv/methodHandles_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -28,6 +28,7 @@
#include "asm/macroAssembler.hpp"
#include "classfile/javaClasses.inline.hpp"
#include "classfile/vmClasses.hpp"
+#include "compiler/disassembler.hpp"
#include "interpreter/interpreter.hpp"
#include "interpreter/interpreterRuntime.hpp"
#include "memory/allocation.inline.hpp"
@@ -37,7 +38,7 @@
#include "runtime/frame.inline.hpp"
#include "runtime/stubRoutines.hpp"
-#define __ _masm->
+#define __ Disassembler::hook(__FILE__, __LINE__, _masm)->
#ifdef PRODUCT
#define BLOCK_COMMENT(str) /* nothing */
@@ -108,17 +109,17 @@ void MethodHandles::jump_from_method_handle(MacroAssembler* _masm, Register meth
// compiled code in threads for which the event is enabled. Check here for
// interp_only_mode if these events CAN be enabled.
- __ lwu(t0, Address(xthread, JavaThread::interp_only_mode_offset()));
- __ beqz(t0, run_compiled_code);
- __ ld(t0, Address(method, Method::interpreter_entry_offset()));
- __ jr(t0);
+ __ lwu(t1, Address(xthread, JavaThread::interp_only_mode_offset()));
+ __ beqz(t1, run_compiled_code);
+ __ ld(t1, Address(method, Method::interpreter_entry_offset()));
+ __ jr(t1);
__ BIND(run_compiled_code);
}
const ByteSize entry_offset = for_compiler_entry ? Method::from_compiled_offset() :
Method::from_interpreted_offset();
- __ ld(t0,Address(method, entry_offset));
- __ jr(t0);
+ __ ld(t1, Address(method, entry_offset));
+ __ jr(t1);
__ bind(L_no_such_method);
__ far_jump(RuntimeAddress(SharedRuntime::throw_AbstractMethodError_entry()));
}
@@ -444,7 +445,6 @@ void MethodHandles::generate_method_handle_dispatch(MacroAssembler* _masm,
__ far_jump(RuntimeAddress(SharedRuntime::throw_IncompatibleClassChangeError_entry()));
}
}
-
}
#ifndef PRODUCT
diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp
index 6c9e0986869b6..988a5b7933224 100644
--- a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp
+++ b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp
@@ -91,10 +91,10 @@ bool NativeShortCallTrampolineStub::is_at(address addr) {
if (MacroAssembler::is_auipc_at(addr) &&
MacroAssembler::is_ld_at(addr + instr_size) &&
MacroAssembler::is_jalr_at(addr + 2 * instr_size) &&
- (MacroAssembler::extract_rd(addr) == x5) &&
- (MacroAssembler::extract_rd(addr + instr_size) == x5) &&
- (MacroAssembler::extract_rs1(addr + instr_size) == x5) &&
- (MacroAssembler::extract_rs1(addr + 2 * instr_size) == x5) &&
+ (MacroAssembler::extract_rd(addr) == x6) &&
+ (MacroAssembler::extract_rd(addr + instr_size) == x6) &&
+ (MacroAssembler::extract_rs1(addr + instr_size) == x6) &&
+ (MacroAssembler::extract_rs1(addr + 2 * instr_size) == x6) &&
(Assembler::extract(Assembler::ld_instr(addr + 4), 31, 20) == trampoline_data_offset)) {
return true;
}
@@ -215,10 +215,10 @@ void NativeShortCall::print() {
// Used in the runtime linkage of calls; see class CompiledIC.
//
// Add parameter assert_lock to switch off assertion
-// during code generation, where no patching lock is needed.
+// during code generation, where no lock is needed.
bool NativeShortCall::set_destination_mt_safe(address dest, bool assert_lock) {
assert(!assert_lock ||
- (Patching_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) ||
+ (CodeCache_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) ||
CompiledICLocker::is_safe(instruction_address()),
"concurrent code patching");
@@ -386,7 +386,7 @@ void NativeFarCall::print() {
bool NativeFarCall::set_destination_mt_safe(address dest, bool assert_lock) {
assert(NativeFarCall::is_at(addr_at(0)), "unexpected code at call site");
assert(!assert_lock ||
- (Patching_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) ||
+ (CodeCache_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) ||
CompiledICLocker::is_safe(addr_at(0)),
"concurrent code patching");
@@ -460,10 +460,10 @@ bool NativeFarCall::is_at(address addr) {
if (MacroAssembler::is_auipc_at(addr) &&
MacroAssembler::is_ld_at(addr + instr_size) &&
MacroAssembler::is_jalr_at(addr + 2 * instr_size) &&
- (MacroAssembler::extract_rd(addr) == x5) &&
- (MacroAssembler::extract_rd(addr + instr_size) == x5) &&
- (MacroAssembler::extract_rs1(addr + instr_size) == x5) &&
- (MacroAssembler::extract_rs1(addr + 2 * instr_size) == x5) &&
+ (MacroAssembler::extract_rd(addr) == x6) &&
+ (MacroAssembler::extract_rd(addr + instr_size) == x6) &&
+ (MacroAssembler::extract_rs1(addr + instr_size) == x6) &&
+ (MacroAssembler::extract_rs1(addr + 2 * instr_size) == x6) &&
(MacroAssembler::extract_rd(addr + 2 * instr_size) == x1)) {
return true;
}
@@ -789,8 +789,8 @@ void NativeGeneralJump::insert_unconditional(address code_pos, address entry) {
Assembler::IncompressibleRegion ir(&a); // Fixed length: see NativeGeneralJump::get_instruction_size()
int32_t offset = 0;
- a.movptr(t0, entry, offset, t1); // lui, lui, slli, add
- a.jr(t0, offset); // jalr
+ a.movptr(t1, entry, offset, t0); // lui, lui, slli, add
+ a.jr(t1, offset); // jalr
ICache::invalidate_range(code_pos, instruction_size);
}
diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad
index 05f55fd0da7af..864bfddef4caa 100644
--- a/src/hotspot/cpu/riscv/riscv.ad
+++ b/src/hotspot/cpu/riscv/riscv.ad
@@ -1259,13 +1259,13 @@ int MachCallRuntimeNode::ret_addr_offset() {
// jal(addr)
// or with far branches
// jal(trampoline_stub)
- // for real runtime callouts it will be 11 instructions
+ // for real runtime callouts it will be 9 instructions
// see riscv_enc_java_to_runtime
- // la(t1, retaddr) -> auipc + addi
- // la(t0, RuntimeAddress(addr)) -> lui + addi + slli + addi + slli + addi
+ // la(t0, retaddr) -> auipc + addi
// addi(sp, sp, -2 * wordSize) -> addi
- // sd(t1, Address(sp, wordSize)) -> sd
- // jalr(t0) -> jalr
+ // sd(t0, Address(sp, wordSize)) -> sd
+ // movptr(t1, addr, offset, t0) -> lui + lui + slli + add
+ // jalr(t1, offset) -> jalr
CodeBlob *cb = CodeCache::find_blob(_entry_point);
if (cb != nullptr) {
if (UseTrampolines) {
@@ -1273,7 +1273,7 @@ int MachCallRuntimeNode::ret_addr_offset() {
}
return 3 * NativeInstruction::instruction_size;
} else {
- return 11 * NativeInstruction::instruction_size;
+ return 9 * NativeInstruction::instruction_size;
}
}
@@ -1822,13 +1822,13 @@ void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
assert_cond(st != nullptr);
st->print_cr("# MachUEPNode");
if (UseCompressedClassPointers) {
- st->print_cr("\tlwu t0, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tlwu t2, [t1 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
+ st->print_cr("\tlwu t1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tlwu t2, [t0 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
} else {
- st->print_cr("\tld t0, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tld t2, [t1 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
+ st->print_cr("\tld t1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tld t2, [t0 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
}
- st->print_cr("\tbeq t0, t2, ic_hit");
+ st->print_cr("\tbeq t1, t2, ic_hit");
st->print_cr("\tj, SharedRuntime::_ic_miss_stub\t # Inline cache check");
st->print_cr("\tic_hit:");
}
@@ -1857,8 +1857,8 @@ uint MachUEPNode::size(PhaseRegAlloc* ra_) const
// Emit exception handler code.
int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
{
- // auipc t0, #exception_blob_entry_point
- // jr (offset)t0
+ // auipc t1, #exception_blob_entry_point
+ // jr (offset)t1
// Note that the code buffer's insts_mark is always relative to insts.
// That's why we must use the macroassembler to generate a handler.
address base = __ start_a_stub(size_exception_handler());
@@ -1966,18 +1966,18 @@ const RegMask* Matcher::predicate_reg_mask(void) {
return &_VMASK_REG_mask;
}
-const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
- return new TypeVectMask(elemTy, length);
-}
-
// Vector calling convention not yet implemented.
bool Matcher::supports_vector_calling_convention(void) {
- return false;
+ return EnableVectorSupport && UseVectorStubs;
}
OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
- Unimplemented();
- return OptoRegPair(0, 0);
+ assert(EnableVectorSupport && UseVectorStubs, "sanity");
+ assert(ideal_reg == Op_VecA, "sanity");
+ // check more info at https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc
+ int lo = V8_num;
+ int hi = V8_K_num;
+ return OptoRegPair(hi, lo);
}
// Is this branch offset short enough that a short branch can be used?
@@ -2224,7 +2224,8 @@ bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
assert_cond(m != nullptr);
if (is_vshift_con_pattern(n, m) || // ShiftV src (ShiftCntV con)
is_vector_bitwise_not_pattern(n, m) ||
- is_vector_scalar_bitwise_pattern(n, m)) {
+ is_vector_scalar_bitwise_pattern(n, m) ||
+ is_encode_and_store_pattern(n, m)) {
mstack.push(m, Visit);
return true;
}
@@ -2503,12 +2504,14 @@ encode %{
__ post_call_nop();
} else {
Label retaddr;
- __ la(t1, retaddr);
- __ la(t0, RuntimeAddress(entry));
+ __ la(t0, retaddr);
// Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
__ addi(sp, sp, -2 * wordSize);
- __ sd(t1, Address(sp, wordSize));
- __ jalr(t0);
+ __ sd(t0, Address(sp, wordSize));
+ int32_t offset = 0;
+ // No relocation needed
+ __ movptr(t1, entry, offset, t0); // lui + lui + slli + add
+ __ jalr(t1, offset);
__ bind(retaddr);
__ post_call_nop();
__ addi(sp, sp, 2 * wordSize);
@@ -4785,6 +4788,7 @@ instruct loadP(iRegPNoSp dst, memory mem)
// Load Compressed Pointer
instruct loadN(iRegNNoSp dst, memory mem)
%{
+ predicate(n->as_Load()->barrier_data() == 0);
match(Set dst (LoadN mem));
ins_cost(LOAD_COST);
@@ -5033,41 +5037,6 @@ instruct loadConD0(fRegD dst, immD0 con) %{
ins_pipe(fp_load_constant_d);
%}
-// Store Instructions
-// Store CMS card-mark Immediate
-instruct storeimmCM0(immI0 zero, memory mem)
-%{
- match(Set mem (StoreCM mem zero));
-
- ins_cost(STORE_COST);
- format %{ "storestore (elided)\n\t"
- "sb zr, $mem\t# byte, #@storeimmCM0" %}
-
- ins_encode %{
- __ sb(zr, Address(as_Register($mem$$base), $mem$$disp));
- %}
-
- ins_pipe(istore_mem);
-%}
-
-// Store CMS card-mark Immediate with intervening StoreStore
-// needed when using CMS with no conditional card marking
-instruct storeimmCM0_ordered(immI0 zero, memory mem)
-%{
- match(Set mem (StoreCM mem zero));
-
- ins_cost(ALU_COST + STORE_COST);
- format %{ "membar(StoreStore)\n\t"
- "sb zr, $mem\t# byte, #@storeimmCM0_ordered" %}
-
- ins_encode %{
- __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
- __ sb(zr, Address(as_Register($mem$$base), $mem$$disp));
- %}
-
- ins_pipe(istore_mem);
-%}
-
// Store Byte
instruct storeB(iRegIorL2I src, memory mem)
%{
@@ -5220,6 +5189,7 @@ instruct storeimmP0(immP0 zero, memory mem)
// Store Compressed Pointer
instruct storeN(iRegN src, memory mem)
%{
+ predicate(n->as_Store()->barrier_data() == 0);
match(Set mem (StoreN mem src));
ins_cost(STORE_COST);
@@ -5234,6 +5204,7 @@ instruct storeN(iRegN src, memory mem)
instruct storeImmN0(immN0 zero, memory mem)
%{
+ predicate(n->as_Store()->barrier_data() == 0);
match(Set mem (StoreN mem zero));
ins_cost(STORE_COST);
@@ -5424,6 +5395,7 @@ instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval
instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval)
%{
+ predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set res (CompareAndSwapN mem (Binary oldval newval)));
ins_cost(LOAD_COST + STORE_COST + ALU_COST * 8 + BRANCH_COST * 4);
@@ -5545,7 +5517,7 @@ instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP new
instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval)
%{
- predicate(needs_acquiring_load_reserved(n));
+ predicate(needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == 0);
match(Set res (CompareAndSwapN mem (Binary oldval newval)));
@@ -5653,6 +5625,7 @@ instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL ne
instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval)
%{
+ predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 3);
@@ -5786,7 +5759,7 @@ instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL
instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval)
%{
- predicate(needs_acquiring_load_reserved(n));
+ predicate(needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == 0);
match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
@@ -5914,6 +5887,7 @@ instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL ne
instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval)
%{
+ predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 4);
@@ -6045,7 +6019,7 @@ instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL
instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval)
%{
- predicate(needs_acquiring_load_reserved(n));
+ predicate(needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == 0);
match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
@@ -6117,6 +6091,8 @@ instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev)
instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev)
%{
+ predicate(n->as_LoadStore()->barrier_data() == 0);
+
match(Set prev (GetAndSetN mem newv));
ins_cost(ALU_COST);
@@ -6182,7 +6158,7 @@ instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev)
instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev)
%{
- predicate(needs_acquiring_load_reserved(n));
+ predicate(needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == 0);
match(Set prev (GetAndSetN mem newv));
@@ -10015,11 +9991,11 @@ instruct CallStaticJavaDirect(method meth)
// Call Java Dynamic Instruction
// Note: If this code changes, the corresponding ret_addr_offset() and
// compute_padding() functions will have to be adjusted.
-instruct CallDynamicJavaDirect(method meth, rFlagsReg cr)
+instruct CallDynamicJavaDirect(method meth)
%{
match(CallDynamicJava);
- effect(USE meth, KILL cr);
+ effect(USE meth);
ins_cost(BRANCH_COST + ALU_COST * 5);
@@ -10034,11 +10010,11 @@ instruct CallDynamicJavaDirect(method meth, rFlagsReg cr)
// Call Runtime Instruction
-instruct CallRuntimeDirect(method meth, rFlagsReg cr)
+instruct CallRuntimeDirect(method meth)
%{
match(CallRuntime);
- effect(USE meth, KILL cr);
+ effect(USE meth);
ins_cost(BRANCH_COST);
@@ -10051,11 +10027,11 @@ instruct CallRuntimeDirect(method meth, rFlagsReg cr)
// Call Runtime Instruction
-instruct CallLeafDirect(method meth, rFlagsReg cr)
+instruct CallLeafDirect(method meth)
%{
match(CallLeaf);
- effect(USE meth, KILL cr);
+ effect(USE meth);
ins_cost(BRANCH_COST);
@@ -10066,13 +10042,30 @@ instruct CallLeafDirect(method meth, rFlagsReg cr)
ins_pipe(pipe_class_call);
%}
+// Call Runtime Instruction without safepoint and with vector arguments
+
+instruct CallLeafDirectVector(method meth)
+%{
+ match(CallLeafVector);
+
+ effect(USE meth);
+
+ ins_cost(BRANCH_COST);
+
+ format %{ "CALL, runtime leaf vector $meth" %}
+
+ ins_encode(riscv_enc_java_to_runtime(meth));
+
+ ins_pipe(pipe_class_call);
+%}
+
// Call Runtime Instruction
-instruct CallLeafNoFPDirect(method meth, rFlagsReg cr)
+instruct CallLeafNoFPDirect(method meth)
%{
match(CallLeafNoFP);
- effect(USE meth, KILL cr);
+ effect(USE meth);
ins_cost(BRANCH_COST);
@@ -10107,11 +10100,11 @@ instruct partialSubtypeCheck(iRegP_R15 result, iRegP_R14 sub, iRegP_R10 super, i
%}
instruct partialSubtypeCheckConstSuper(iRegP_R14 sub, iRegP_R10 super_reg, immP super_con, iRegP_R15 result,
- iRegP_R11 tmpR11, iRegP_R12 tmpR12, iRegP_R13 tmpR13, iRegP_R16 tmpR16)
+ iRegP_R11 tmpR11, iRegP_R12 tmpR12, iRegP_R13 tmpR13, iRegP_R16 tmpR16, rFlagsReg cr)
%{
predicate(UseSecondarySupersTable);
match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
- effect(TEMP tmpR11, TEMP tmpR12, TEMP tmpR13, TEMP tmpR16);
+ effect(TEMP tmpR11, TEMP tmpR12, TEMP tmpR13, TEMP tmpR16, KILL cr);
ins_cost(7 * DEFAULT_COST); // needs to be less than competing nodes
format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
@@ -10388,12 +10381,12 @@ instruct stringL_indexof_char(iRegP_R11 str1, iRegI_R12 cnt1, iRegI_R13 ch,
// clearing of an array
instruct clearArray_reg_reg(iRegL_R29 cnt, iRegP_R28 base, iRegP_R30 tmp1,
- iRegP_R31 tmp2, Universe dummy)
+ iRegP_R31 tmp2, rFlagsReg cr, Universe dummy)
%{
// temp registers must match the one used in StubGenerator::generate_zero_blocks()
predicate(UseBlockZeroing || !UseRVV);
match(Set dummy (ClearArray cnt base));
- effect(USE_KILL cnt, USE_KILL base, TEMP tmp1, TEMP tmp2);
+ effect(USE_KILL cnt, USE_KILL base, TEMP tmp1, TEMP tmp2, KILL cr);
ins_cost(4 * DEFAULT_COST);
format %{ "ClearArray $cnt, $base\t#@clearArray_reg_reg" %}
diff --git a/src/hotspot/cpu/riscv/runtime_riscv.cpp b/src/hotspot/cpu/riscv/runtime_riscv.cpp
index 9e16278c3b547..2f879b07e2694 100644
--- a/src/hotspot/cpu/riscv/runtime_riscv.cpp
+++ b/src/hotspot/cpu/riscv/runtime_riscv.cpp
@@ -351,7 +351,7 @@ void OptoRuntime::generate_exception_blob() {
// x10: exception handler
// We have a handler in x10 (could be deopt blob).
- __ mv(t0, x10);
+ __ mv(t1, x10);
// Get the exception oop
__ ld(x10, Address(xthread, JavaThread::exception_oop_offset()));
@@ -365,11 +365,11 @@ void OptoRuntime::generate_exception_blob() {
__ sd(zr, Address(xthread, JavaThread::exception_oop_offset()));
// x10: exception oop
- // t0: exception handler
+ // t1: exception handler
// x14: exception pc
// Jump to handler
- __ jr(t0);
+ __ jr(t1);
// Make sure all code is generated
masm->flush();
diff --git a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
index 27da26d404cc0..29c96112eada8 100644
--- a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
+++ b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
@@ -468,8 +468,8 @@ static void gen_c2i_adapter(MacroAssembler *masm,
__ mv(esp, sp); // Interp expects args on caller's expression stack
- __ ld(t0, Address(xmethod, in_bytes(Method::interpreter_entry_offset())));
- __ jr(t0);
+ __ ld(t1, Address(xmethod, in_bytes(Method::interpreter_entry_offset())));
+ __ jr(t1);
}
void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,
@@ -610,8 +610,7 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
Label skip_fixup;
const Register receiver = j_rarg0;
- const Register data = t1;
- const Register tmp = t2; // A call-clobbered register not used for arg passing
+ const Register data = t0;
// -------------------------------------------------------------------------
// Generate a C2I adapter. On entry we know xmethod holds the Method* during calls
@@ -666,7 +665,20 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
int SharedRuntime::vector_calling_convention(VMRegPair *regs,
uint num_bits,
uint total_args_passed) {
- Unimplemented();
+ assert(total_args_passed <= Argument::n_vector_register_parameters_c, "unsupported");
+ assert(num_bits >= 64 && num_bits <= 2048 && is_power_of_2(num_bits), "unsupported");
+
+ // check more info at https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc
+ static const VectorRegister VEC_ArgReg[Argument::n_vector_register_parameters_c] = {
+ v8, v9, v10, v11, v12, v13, v14, v15,
+ v16, v17, v18, v19, v20, v21, v22, v23
+ };
+
+ const int next_reg_val = 3;
+ for (uint i = 0; i < total_args_passed; i++) {
+ VMReg vmreg = VEC_ArgReg[i]->as_VMReg();
+ regs[i].set_pair(vmreg->next(next_reg_val), vmreg);
+ }
return 0;
}
@@ -1127,8 +1139,7 @@ static void gen_continuation_yield(MacroAssembler* masm,
Label ok;
__ beqz(t0, ok);
__ leave();
- __ la(t0, RuntimeAddress(StubRoutines::forward_exception_entry()));
- __ jr(t0);
+ __ j(RuntimeAddress(StubRoutines::forward_exception_entry()));
__ bind(ok);
__ leave();
@@ -1439,8 +1450,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
// restoring them except fp. fp is the only callee save register
// as far as the interpreter and the compiler(s) are concerned.
-
- const Register ic_reg = t1;
const Register receiver = j_rarg0;
__ verify_oop(receiver);
@@ -1724,6 +1733,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
__ sw(t0, Address(t1));
+ // Clobbers t1
__ rt_call(native_func);
__ bind(native_return);
@@ -2606,20 +2616,19 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(SharedStubId id, address desti
__ reset_last_Java_frame(false);
// check for pending exceptions
Label pending;
- __ ld(t0, Address(xthread, Thread::pending_exception_offset()));
- __ bnez(t0, pending);
+ __ ld(t1, Address(xthread, Thread::pending_exception_offset()));
+ __ bnez(t1, pending);
// get the returned Method*
__ get_vm_result_2(xmethod, xthread);
__ sd(xmethod, Address(sp, reg_saver.reg_offset_in_bytes(xmethod)));
- // x10 is where we want to jump, overwrite t0 which is saved and temporary
- __ sd(x10, Address(sp, reg_saver.reg_offset_in_bytes(t0)));
+ // x10 is where we want to jump, overwrite t1 which is saved and temporary
+ __ sd(x10, Address(sp, reg_saver.reg_offset_in_bytes(t1)));
reg_saver.restore_live_registers(masm);
// We are back to the original state on entry and ready to go.
-
- __ jr(t0);
+ __ jr(t1);
// Pending exception after the safepoint
diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
index 5970111088fd1..bce0c8f1f3de7 100644
--- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
+++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
@@ -508,7 +508,7 @@ class StubGenerator: public StubCodeGenerator {
// complete return to VM
assert(StubRoutines::_call_stub_return_address != nullptr,
"_call_stub_return_address must have been generated before");
- __ j(StubRoutines::_call_stub_return_address);
+ __ j(RuntimeAddress(StubRoutines::_call_stub_return_address));
return start;
}
@@ -946,7 +946,7 @@ class StubGenerator: public StubCodeGenerator {
// The size of copy32_loop body increases significantly with ZGC GC barriers.
// Need conditional far branches to reach a point beyond the loop in this case.
- bool is_far = UseZGC && ZGenerational;
+ bool is_far = UseZGC;
__ beqz(count, done, is_far);
__ slli(cnt, count, exact_log2(granularity));
@@ -2276,6 +2276,174 @@ class StubGenerator: public StubCodeGenerator {
StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
}
+ void generate_aes_loadkeys(const Register &key, VectorRegister *working_vregs, int rounds) {
+ const int step = 16;
+ for (int i = 0; i < rounds; i++) {
+ __ vle32_v(working_vregs[i], key);
+ // The keys are stored in little-endian array, while we need
+ // to operate in big-endian.
+ // So performing an endian-swap here with vrev8.v instruction
+ __ vrev8_v(working_vregs[i], working_vregs[i]);
+ __ addi(key, key, step);
+ }
+ }
+
+ void generate_aes_encrypt(const VectorRegister &res, VectorRegister *working_vregs, int rounds) {
+ assert(rounds <= 15, "rounds should be less than or equal to working_vregs size");
+
+ __ vxor_vv(res, res, working_vregs[0]);
+ for (int i = 1; i < rounds - 1; i++) {
+ __ vaesem_vv(res, working_vregs[i]);
+ }
+ __ vaesef_vv(res, working_vregs[rounds - 1]);
+ }
+
+ // Arguments:
+ //
+ // Inputs:
+ // c_rarg0 - source byte array address
+ // c_rarg1 - destination byte array address
+ // c_rarg2 - K (key) in little endian int array
+ //
+ address generate_aescrypt_encryptBlock() {
+ assert(UseAESIntrinsics, "need AES instructions (Zvkned extension) support");
+
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
+
+ Label L_aes128, L_aes192;
+
+ const Register from = c_rarg0; // source array address
+ const Register to = c_rarg1; // destination array address
+ const Register key = c_rarg2; // key array address
+ const Register keylen = c_rarg3;
+
+ VectorRegister working_vregs[] = {
+ v4, v5, v6, v7, v8, v9, v10, v11,
+ v12, v13, v14, v15, v16, v17, v18
+ };
+ const VectorRegister res = v19;
+
+ address start = __ pc();
+ __ enter();
+
+ __ lwu(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
+
+ __ vsetivli(x0, 4, Assembler::e32, Assembler::m1);
+ __ vle32_v(res, from);
+
+ __ mv(t2, 52);
+ __ blt(keylen, t2, L_aes128);
+ __ beq(keylen, t2, L_aes192);
+ // Else we fallthrough to the biggest case (256-bit key size)
+
+ // Note: the following function performs key += 15*16
+ generate_aes_loadkeys(key, working_vregs, 15);
+ generate_aes_encrypt(res, working_vregs, 15);
+ __ vse32_v(res, to);
+ __ mv(c_rarg0, 0);
+ __ leave();
+ __ ret();
+
+ __ bind(L_aes192);
+ // Note: the following function performs key += 13*16
+ generate_aes_loadkeys(key, working_vregs, 13);
+ generate_aes_encrypt(res, working_vregs, 13);
+ __ vse32_v(res, to);
+ __ mv(c_rarg0, 0);
+ __ leave();
+ __ ret();
+
+ __ bind(L_aes128);
+ // Note: the following function performs key += 11*16
+ generate_aes_loadkeys(key, working_vregs, 11);
+ generate_aes_encrypt(res, working_vregs, 11);
+ __ vse32_v(res, to);
+ __ mv(c_rarg0, 0);
+ __ leave();
+ __ ret();
+
+ return start;
+ }
+
+ void generate_aes_decrypt(const VectorRegister &res, VectorRegister *working_vregs, int rounds) {
+ assert(rounds <= 15, "rounds should be less than or equal to working_vregs size");
+
+ __ vxor_vv(res, res, working_vregs[rounds - 1]);
+ for (int i = rounds - 2; i > 0; i--) {
+ __ vaesdm_vv(res, working_vregs[i]);
+ }
+ __ vaesdf_vv(res, working_vregs[0]);
+ }
+
+ // Arguments:
+ //
+ // Inputs:
+ // c_rarg0 - source byte array address
+ // c_rarg1 - destination byte array address
+ // c_rarg2 - K (key) in little endian int array
+ //
+ address generate_aescrypt_decryptBlock() {
+ assert(UseAESIntrinsics, "need AES instructions (Zvkned extension) support");
+
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
+
+ Label L_aes128, L_aes192;
+
+ const Register from = c_rarg0; // source array address
+ const Register to = c_rarg1; // destination array address
+ const Register key = c_rarg2; // key array address
+ const Register keylen = c_rarg3;
+
+ VectorRegister working_vregs[] = {
+ v4, v5, v6, v7, v8, v9, v10, v11,
+ v12, v13, v14, v15, v16, v17, v18
+ };
+ const VectorRegister res = v19;
+
+ address start = __ pc();
+ __ enter(); // required for proper stackwalking of RuntimeStub frame
+
+ __ lwu(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
+
+ __ vsetivli(x0, 4, Assembler::e32, Assembler::m1);
+ __ vle32_v(res, from);
+
+ __ mv(t2, 52);
+ __ blt(keylen, t2, L_aes128);
+ __ beq(keylen, t2, L_aes192);
+ // Else we fallthrough to the biggest case (256-bit key size)
+
+ // Note: the following function performs key += 15*16
+ generate_aes_loadkeys(key, working_vregs, 15);
+ generate_aes_decrypt(res, working_vregs, 15);
+ __ vse32_v(res, to);
+ __ mv(c_rarg0, 0);
+ __ leave();
+ __ ret();
+
+ __ bind(L_aes192);
+ // Note: the following function performs key += 13*16
+ generate_aes_loadkeys(key, working_vregs, 13);
+ generate_aes_decrypt(res, working_vregs, 13);
+ __ vse32_v(res, to);
+ __ mv(c_rarg0, 0);
+ __ leave();
+ __ ret();
+
+ __ bind(L_aes128);
+ // Note: the following function performs key += 11*16
+ generate_aes_loadkeys(key, working_vregs, 11);
+ generate_aes_decrypt(res, working_vregs, 11);
+ __ vse32_v(res, to);
+ __ mv(c_rarg0, 0);
+ __ leave();
+ __ ret();
+
+ return start;
+ }
+
// code for comparing 16 bytes of strings with same encoding
void compare_string_16_bytes_same(Label &DIFF1, Label &DIFF2) {
const Register result = x10, str1 = x11, cnt1 = x12, str2 = x13, tmp1 = x28, tmp2 = x29, tmp4 = x7, tmp5 = x31;
@@ -3782,8 +3950,7 @@ class StubGenerator: public StubCodeGenerator {
Label thaw_success;
// t1 contains the size of the frames to thaw, 0 if overflow or no more frames
__ bnez(t1, thaw_success);
- __ la(t0, RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
- __ jr(t0);
+ __ j(RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
__ bind(thaw_success);
// make room for the thawed frames
@@ -4482,7 +4649,7 @@ class StubGenerator: public StubCodeGenerator {
RegSet reg_cache_saved_regs = RegSet::of(x24, x25, x26, x27); // s8, s9, s10, s11
RegSet reg_cache_regs;
reg_cache_regs += reg_cache_saved_regs;
- reg_cache_regs += RegSet::of(x28, x29, x30, x31); // t3, t4, t5, t6
+ reg_cache_regs += RegSet::of(t3, t4, t5, t6);
BufRegCache reg_cache(_masm, reg_cache_regs);
RegSet saved_regs;
@@ -5462,8 +5629,8 @@ class StubGenerator: public StubCodeGenerator {
Register isMIME = c_rarg6;
Register codec = c_rarg7;
- Register dstBackup = x31;
- Register length = x28; // t3, total length of src data in bytes
+ Register dstBackup = t6;
+ Register length = t3; // total length of src data in bytes
Label ProcessData, Exit;
Label ProcessScalar, ScalarLoop;
@@ -5498,7 +5665,7 @@ class StubGenerator: public StubCodeGenerator {
Register stepSrcM1 = send;
Register stepSrcM2 = doff;
Register stepDst = isURL;
- Register size = x29; // t4
+ Register size = t4;
__ mv(size, MaxVectorSize * 2);
__ mv(stepSrcM1, MaxVectorSize * 4);
@@ -5550,7 +5717,7 @@ class StubGenerator: public StubCodeGenerator {
// scalar version
{
Register byte0 = soff, byte1 = send, byte2 = doff, byte3 = isURL;
- Register combined32Bits = x29; // t5
+ Register combined32Bits = t4;
// encoded: [byte0[5:0] : byte1[5:0] : byte2[5:0]] : byte3[5:0]] =>
// plain: [byte0[5:0]+byte1[5:4] : byte1[3:0]+byte2[5:2] : byte2[1:0]+byte3[5:0]]
@@ -5708,10 +5875,10 @@ class StubGenerator: public StubCodeGenerator {
Register nmax = c_rarg4;
Register base = c_rarg5;
Register count = c_rarg6;
- Register temp0 = x28; // t3
- Register temp1 = x29; // t4
- Register temp2 = x30; // t5
- Register temp3 = x31; // t6
+ Register temp0 = t3;
+ Register temp1 = t4;
+ Register temp2 = t5;
+ Register temp3 = t6;
VectorRegister vzero = v31;
VectorRegister vbytes = v8; // group: v8, v9, v10, v11
@@ -6071,6 +6238,58 @@ static const int64_t right_3_bits = right_n_bits(3);
return start;
}
+ void generate_vector_math_stubs() {
+ if (!UseRVV) {
+ log_info(library)("vector is not supported, skip loading vector math (sleef) library!");
+ return;
+ }
+
+ // Get native vector math stub routine addresses
+ void* libsleef = nullptr;
+ char ebuf[1024];
+ char dll_name[JVM_MAXPATHLEN];
+ if (os::dll_locate_lib(dll_name, sizeof(dll_name), Arguments::get_dll_dir(), "sleef")) {
+ libsleef = os::dll_load(dll_name, ebuf, sizeof ebuf);
+ }
+ if (libsleef == nullptr) {
+ log_info(library)("Failed to load native vector math (sleef) library, %s!", ebuf);
+ return;
+ }
+
+ // Method naming convention
+ // All the methods are named as _
+ //
+ // Where:
+ // is the operation name, e.g. sin, cos
+ // is to indicate float/double
+ // "fx/dx" for vector float/double operation
+ // is the precision level
+ // "u10/u05" represents 1.0/0.5 ULP error bounds
+ // We use "u10" for all operations by default
+ // But for those functions do not have u10 support, we use "u05" instead
+ // rvv, indicates riscv vector extension
+ //
+ // e.g. sinfx_u10rvv is the method for computing vector float sin using rvv instructions
+ //
+ log_info(library)("Loaded library %s, handle " INTPTR_FORMAT, JNI_LIB_PREFIX "sleef" JNI_LIB_SUFFIX, p2i(libsleef));
+
+ for (int op = 0; op < VectorSupport::NUM_VECTOR_OP_MATH; op++) {
+ int vop = VectorSupport::VECTOR_OP_MATH_START + op;
+ if (vop == VectorSupport::VECTOR_OP_TANH) { // skip tanh because of performance regression
+ continue;
+ }
+
+ // The native library does not support u10 level of "hypot".
+ const char* ulf = (vop == VectorSupport::VECTOR_OP_HYPOT) ? "u05" : "u10";
+
+ snprintf(ebuf, sizeof(ebuf), "%sfx_%srvv", VectorSupport::mathname[op], ulf);
+ StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_SCALABLE][op] = (address)os::dll_lookup(libsleef, ebuf);
+
+ snprintf(ebuf, sizeof(ebuf), "%sdx_%srvv", VectorSupport::mathname[op], ulf);
+ StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_SCALABLE][op] = (address)os::dll_lookup(libsleef, ebuf);
+ }
+ }
+
#endif // COMPILER2
/**
@@ -6102,7 +6321,7 @@ static const int64_t right_3_bits = right_n_bits(3);
__ kernel_crc32(crc, buf, len,
c_rarg3, c_rarg4, c_rarg5, c_rarg6, // tmp's for tables
- c_rarg7, t2, x28, x29, x30, x31); // misc tmps
+ c_rarg7, t2, t3, t4, t5, t6); // misc tmps
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret();
@@ -6124,6 +6343,29 @@ static const int64_t right_3_bits = right_n_bits(3);
return start;
}
+ // load Method* target of MethodHandle
+ // j_rarg0 = jobject receiver
+ // xmethod = Method* result
+ address generate_upcall_stub_load_target() {
+
+ StubCodeMark mark(this, "StubRoutines", "upcall_stub_load_target");
+ address start = __ pc();
+
+ __ resolve_global_jobject(j_rarg0, t0, t1);
+ // Load target method from receiver
+ __ load_heap_oop(xmethod, Address(j_rarg0, java_lang_invoke_MethodHandle::form_offset()), t0, t1);
+ __ load_heap_oop(xmethod, Address(xmethod, java_lang_invoke_LambdaForm::vmentry_offset()), t0, t1);
+ __ load_heap_oop(xmethod, Address(xmethod, java_lang_invoke_MemberName::method_offset()), t0, t1);
+ __ access_load_at(T_ADDRESS, IN_HEAP, xmethod,
+ Address(xmethod, java_lang_invoke_ResolvedMethodName::vmtarget_offset()),
+ noreg, noreg);
+ __ sd(xmethod, Address(xthread, JavaThread::callee_target_offset())); // just in case callee is deoptimized
+
+ __ ret();
+
+ return start;
+ }
+
#undef __
// Initialization
@@ -6189,6 +6431,7 @@ static const int64_t right_3_bits = right_n_bits(3);
#endif // COMPILER2
StubRoutines::_upcall_stub_exception_handler = generate_upcall_stub_exception_handler();
+ StubRoutines::_upcall_stub_load_target = generate_upcall_stub_load_target();
StubRoutines::riscv::set_completed();
}
@@ -6219,6 +6462,11 @@ static const int64_t right_3_bits = right_n_bits(3);
StubRoutines::_montgomerySquare = g.generate_square();
}
+ if (UseAESIntrinsics) {
+ StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
+ StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
+ }
+
if (UsePoly1305Intrinsics) {
StubRoutines::_poly1305_processBlocks = generate_poly1305_processBlocks();
}
@@ -6267,6 +6515,8 @@ static const int64_t right_3_bits = right_n_bits(3);
generate_string_indexof_stubs();
+ generate_vector_math_stubs();
+
#endif // COMPILER2
}
diff --git a/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp b/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp
index 1f32488777d57..0281c66b97db1 100644
--- a/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp
+++ b/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -27,6 +27,7 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "classfile/javaClasses.hpp"
+#include "compiler/disassembler.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
#include "interpreter/bytecodeHistogram.hpp"
#include "interpreter/bytecodeTracer.hpp"
@@ -70,7 +71,7 @@
// Max size with JVMTI
int TemplateInterpreter::InterpreterCodeSize = 256 * 1024;
-#define __ _masm->
+#define __ Disassembler::hook(__FILE__, __LINE__, _masm)->
//-----------------------------------------------------------------------------
@@ -165,7 +166,6 @@ address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::M
address fn = nullptr;
address entry_point = nullptr;
- Register continuation = ra;
switch (kind) {
case Interpreter::java_lang_math_abs:
entry_point = __ pc();
@@ -184,83 +184,82 @@ address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::M
__ fld(f10, Address(esp));
__ mv(sp, x19_sender_sp);
__ mv(x9, ra);
- continuation = x9; // The first callee-saved register
if (StubRoutines::dsin() == nullptr) {
fn = CAST_FROM_FN_PTR(address, SharedRuntime::dsin);
} else {
fn = CAST_FROM_FN_PTR(address, StubRoutines::dsin());
}
__ call(fn);
+ __ mv(ra, x9);
break;
case Interpreter::java_lang_math_cos :
entry_point = __ pc();
__ fld(f10, Address(esp));
__ mv(sp, x19_sender_sp);
__ mv(x9, ra);
- continuation = x9; // The first callee-saved register
if (StubRoutines::dcos() == nullptr) {
fn = CAST_FROM_FN_PTR(address, SharedRuntime::dcos);
} else {
fn = CAST_FROM_FN_PTR(address, StubRoutines::dcos());
}
__ call(fn);
+ __ mv(ra, x9);
break;
case Interpreter::java_lang_math_tan :
entry_point = __ pc();
__ fld(f10, Address(esp));
__ mv(sp, x19_sender_sp);
__ mv(x9, ra);
- continuation = x9; // The first callee-saved register
if (StubRoutines::dtan() == nullptr) {
fn = CAST_FROM_FN_PTR(address, SharedRuntime::dtan);
} else {
fn = CAST_FROM_FN_PTR(address, StubRoutines::dtan());
}
__ call(fn);
+ __ mv(ra, x9);
break;
case Interpreter::java_lang_math_log :
entry_point = __ pc();
__ fld(f10, Address(esp));
__ mv(sp, x19_sender_sp);
__ mv(x9, ra);
- continuation = x9; // The first callee-saved register
if (StubRoutines::dlog() == nullptr) {
fn = CAST_FROM_FN_PTR(address, SharedRuntime::dlog);
} else {
fn = CAST_FROM_FN_PTR(address, StubRoutines::dlog());
}
__ call(fn);
+ __ mv(ra, x9);
break;
case Interpreter::java_lang_math_log10 :
entry_point = __ pc();
__ fld(f10, Address(esp));
__ mv(sp, x19_sender_sp);
__ mv(x9, ra);
- continuation = x9; // The first callee-saved register
if (StubRoutines::dlog10() == nullptr) {
fn = CAST_FROM_FN_PTR(address, SharedRuntime::dlog10);
} else {
fn = CAST_FROM_FN_PTR(address, StubRoutines::dlog10());
}
__ call(fn);
+ __ mv(ra, x9);
break;
case Interpreter::java_lang_math_exp :
entry_point = __ pc();
__ fld(f10, Address(esp));
__ mv(sp, x19_sender_sp);
__ mv(x9, ra);
- continuation = x9; // The first callee-saved register
if (StubRoutines::dexp() == nullptr) {
fn = CAST_FROM_FN_PTR(address, SharedRuntime::dexp);
} else {
fn = CAST_FROM_FN_PTR(address, StubRoutines::dexp());
}
__ call(fn);
+ __ mv(ra, x9);
break;
case Interpreter::java_lang_math_pow :
entry_point = __ pc();
__ mv(x9, ra);
- continuation = x9;
__ fld(f10, Address(esp, 2 * Interpreter::stackElementSize));
__ fld(f11, Address(esp));
__ mv(sp, x19_sender_sp);
@@ -270,6 +269,7 @@ address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::M
fn = CAST_FROM_FN_PTR(address, StubRoutines::dpow());
}
__ call(fn);
+ __ mv(ra, x9);
break;
case Interpreter::java_lang_math_fmaD :
if (UseFMA) {
@@ -295,7 +295,7 @@ address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::M
;
}
if (entry_point != nullptr) {
- __ jr(continuation);
+ __ ret();
}
return entry_point;
@@ -421,7 +421,7 @@ address TemplateInterpreterGenerator::generate_exception_handler_common(
c_rarg1, c_rarg2);
}
// throw exception
- __ j(address(Interpreter::throw_exception_entry()));
+ __ j(RuntimeAddress(Interpreter::throw_exception_entry()));
return entry;
}
@@ -1748,13 +1748,21 @@ void TemplateInterpreterGenerator::set_vtos_entry_points(Template* t,
address& vep) {
assert(t != nullptr && t->is_valid() && t->tos_in() == vtos, "illegal template");
Label L;
- aep = __ pc(); __ push_ptr(); __ j(L);
- fep = __ pc(); __ push_f(); __ j(L);
- dep = __ pc(); __ push_d(); __ j(L);
- lep = __ pc(); __ push_l(); __ j(L);
- bep = cep = sep =
- iep = __ pc(); __ push_i();
- vep = __ pc();
+ aep = __ pc(); // atos entry point
+ __ push_ptr();
+ __ j(L);
+ fep = __ pc(); // ftos entry point
+ __ push_f();
+ __ j(L);
+ dep = __ pc(); // dtos entry point
+ __ push_d();
+ __ j(L);
+ lep = __ pc(); // ltos entry point
+ __ push_l();
+ __ j(L);
+ bep = cep = sep = iep = __ pc(); // [bcsi]tos entry point
+ __ push_i();
+ vep = __ pc(); // vtos entry point
__ bind(L);
generate_and_dispatch(t);
}
diff --git a/src/hotspot/cpu/riscv/templateTable_riscv.cpp b/src/hotspot/cpu/riscv/templateTable_riscv.cpp
index 2e6902180a892..9f37774e29743 100644
--- a/src/hotspot/cpu/riscv/templateTable_riscv.cpp
+++ b/src/hotspot/cpu/riscv/templateTable_riscv.cpp
@@ -26,6 +26,7 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "compiler/disassembler.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
#include "gc/shared/collectedHeap.hpp"
#include "gc/shared/tlab_globals.hpp"
@@ -49,7 +50,7 @@
#include "runtime/synchronizer.hpp"
#include "utilities/powerOfTwo.hpp"
-#define __ _masm->
+#define __ Disassembler::hook(__FILE__, __LINE__, _masm)->
// Address computation: local variables
@@ -704,7 +705,7 @@ void TemplateTable::wide_aload() {
}
void TemplateTable::index_check(Register array, Register index) {
- // destroys x11, t0
+ // destroys x11, t0, t1
// sign extend index for use by indexed load
// check index
const Register length = t0;
@@ -717,8 +718,8 @@ void TemplateTable::index_check(Register array, Register index) {
__ sign_extend(index, index, 32);
__ bltu(index, length, ok);
__ mv(x13, array);
- __ mv(t0, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
- __ jr(t0);
+ __ mv(t1, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
+ __ jr(t1);
__ bind(ok);
}
@@ -1084,7 +1085,7 @@ void TemplateTable::aastore() {
// Come here on failure
// object is at TOS
- __ j(Interpreter::_throw_ArrayStoreException_entry);
+ __ j(RuntimeAddress(Interpreter::_throw_ArrayStoreException_entry));
// Come here on success
__ bind(ok_is_subtype);
@@ -1312,8 +1313,8 @@ void TemplateTable::idiv() {
// explicitly check for div0
Label no_div0;
__ bnez(x10, no_div0);
- __ mv(t0, Interpreter::_throw_ArithmeticException_entry);
- __ jr(t0);
+ __ mv(t1, Interpreter::_throw_ArithmeticException_entry);
+ __ jr(t1);
__ bind(no_div0);
__ pop_i(x11);
// x10 <== x11 idiv x10
@@ -1325,8 +1326,8 @@ void TemplateTable::irem() {
// explicitly check for div0
Label no_div0;
__ bnez(x10, no_div0);
- __ mv(t0, Interpreter::_throw_ArithmeticException_entry);
- __ jr(t0);
+ __ mv(t1, Interpreter::_throw_ArithmeticException_entry);
+ __ jr(t1);
__ bind(no_div0);
__ pop_i(x11);
// x10 <== x11 irem x10
@@ -1344,8 +1345,8 @@ void TemplateTable::ldiv() {
// explicitly check for div0
Label no_div0;
__ bnez(x10, no_div0);
- __ mv(t0, Interpreter::_throw_ArithmeticException_entry);
- __ jr(t0);
+ __ mv(t1, Interpreter::_throw_ArithmeticException_entry);
+ __ jr(t1);
__ bind(no_div0);
__ pop_l(x11);
// x10 <== x11 ldiv x10
@@ -1357,8 +1358,8 @@ void TemplateTable::lrem() {
// explicitly check for div0
Label no_div0;
__ bnez(x10, no_div0);
- __ mv(t0, Interpreter::_throw_ArithmeticException_entry);
- __ jr(t0);
+ __ mv(t1, Interpreter::_throw_ArithmeticException_entry);
+ __ jr(t1);
__ bind(no_div0);
__ pop_l(x11);
// x10 <== x11 lrem x10
@@ -1767,8 +1768,8 @@ void TemplateTable::branch(bool is_jsr, bool is_wide) {
__ andi(sp, esp, -16);
// and begin the OSR nmethod
- __ ld(t0, Address(x9, nmethod::osr_entry_point_offset()));
- __ jr(t0);
+ __ ld(t1, Address(x9, nmethod::osr_entry_point_offset()));
+ __ jr(t1);
}
}
@@ -2170,7 +2171,7 @@ void TemplateTable::_return(TosState state) {
void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
Register Rcache,
Register index) {
- const Register temp = x9;
+ const Register temp = x9; // s1
assert_different_registers(Rcache, index, temp);
assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
@@ -2464,7 +2465,7 @@ void TemplateTable::jvmti_post_field_access(Register cache, Register index,
// take the time to call into the VM.
Label L1;
assert_different_registers(cache, index, x10);
- ExternalAddress target((address) JvmtiExport::get_field_access_count_addr());
+ ExternalAddress target(JvmtiExport::get_field_access_count_addr());
__ relocate(target.rspec(), [&] {
int32_t offset;
__ la(t0, target.target(), offset);
@@ -2675,7 +2676,7 @@ void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is
// we take the time to call into the VM.
Label L1;
assert_different_registers(cache, index, x10);
- ExternalAddress target((address)JvmtiExport::get_field_modification_count_addr());
+ ExternalAddress target(JvmtiExport::get_field_modification_count_addr());
__ relocate(target.rspec(), [&] {
int32_t offset;
__ la(t0, target.target(), offset);
@@ -2968,7 +2969,7 @@ void TemplateTable::jvmti_post_fast_field_mod() {
// Check to see if a field modification watch has been set before
// we take the time to call into the VM.
Label L2;
- ExternalAddress target((address)JvmtiExport::get_field_modification_count_addr());
+ ExternalAddress target(JvmtiExport::get_field_modification_count_addr());
__ relocate(target.rspec(), [&] {
int32_t offset;
__ la(t0, target.target(), offset);
@@ -3100,7 +3101,7 @@ void TemplateTable::fast_accessfield(TosState state) {
// Check to see if a field access watch has been set before we
// take the time to call into the VM.
Label L1;
- ExternalAddress target((address)JvmtiExport::get_field_access_count_addr());
+ ExternalAddress target(JvmtiExport::get_field_access_count_addr());
__ relocate(target.rspec(), [&] {
int32_t offset;
__ la(t0, target.target(), offset);
@@ -3671,7 +3672,7 @@ void TemplateTable::checkcast() {
// Come here on failure
__ push_reg(x13);
// object is at TOS
- __ j(Interpreter::_throw_ClassCastException_entry);
+ __ j(RuntimeAddress(Interpreter::_throw_ClassCastException_entry));
// Come here on success
__ bind(ok_is_subtype);
@@ -3778,7 +3779,7 @@ void TemplateTable::_breakpoint() {
void TemplateTable::athrow() {
transition(atos, vtos);
__ null_check(x10);
- __ j(Interpreter::throw_exception_entry());
+ __ j(RuntimeAddress(Interpreter::throw_exception_entry()));
}
//-----------------------------------------------------------------------------
@@ -3961,8 +3962,8 @@ void TemplateTable::wide() {
__ load_unsigned_byte(x9, at_bcp(1));
__ mv(t0, (address)Interpreter::_wentry_point);
__ shadd(t0, x9, t0, t1, 3);
- __ ld(t0, Address(t0));
- __ jr(t0);
+ __ ld(t1, Address(t0));
+ __ jr(t1);
}
// Multi arrays
diff --git a/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp b/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp
index 383f332f8fd94..3b4f26b6dc333 100644
--- a/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp
+++ b/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp
@@ -25,6 +25,7 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
+#include "classfile/javaClasses.hpp"
#include "logging/logStream.hpp"
#include "memory/resourceArea.hpp"
#include "prims/upcallLinker.hpp"
@@ -117,7 +118,7 @@ static void restore_callee_saved_registers(MacroAssembler* _masm, const ABIDescr
static const int upcall_stub_code_base_size = 1024;
static const int upcall_stub_size_per_arg = 16;
-address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
+address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature,
BasicType* out_sig_bt, int total_out_args,
BasicType ret_type,
jobject jabi, jobject jconv,
@@ -223,7 +224,6 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
__ block_comment("{ on_entry");
__ la(c_rarg0, Address(sp, frame_data_offset));
- __ movptr(c_rarg1, (address) receiver);
__ rt_call(CAST_FROM_FN_PTR(address, UpcallLinker::on_entry));
__ mv(xthread, x10);
__ reinit_heapbase();
@@ -260,17 +260,15 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
arg_shuffle.generate(_masm, as_VMStorage(shuffle_reg), abi._shadow_space_bytes, 0);
__ block_comment("} argument shuffle");
- __ block_comment("{ receiver ");
- __ get_vm_result(j_rarg0, xthread);
- __ block_comment("} receiver ");
-
- __ mov_metadata(xmethod, entry);
- __ sd(xmethod, Address(xthread, JavaThread::callee_target_offset())); // just in case callee is deoptimized
+ __ block_comment("{ load target ");
+ __ movptr(j_rarg0, (address) receiver);
+ __ far_call(RuntimeAddress(StubRoutines::upcall_stub_load_target())); // loads Method* into xmethod
+ __ block_comment("} load target ");
__ push_cont_fastpath(xthread);
- __ ld(t0, Address(xmethod, Method::from_compiled_offset()));
- __ jalr(t0);
+ __ ld(t1, Address(xmethod, Method::from_compiled_offset()));
+ __ jalr(t1);
__ pop_cont_fastpath(xthread);
@@ -338,7 +336,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
#ifndef PRODUCT
stringStream ss;
- ss.print("upcall_stub_%s", entry->signature()->as_C_string());
+ ss.print("upcall_stub_%s", signature->as_C_string());
const char *name = _masm->code_string(ss.as_string());
#else // PRODUCT
const char* name = "upcall_stub";
diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.cpp b/src/hotspot/cpu/riscv/vm_version_riscv.cpp
index e9c6226f44639..c32d2af993937 100644
--- a/src/hotspot/cpu/riscv/vm_version_riscv.cpp
+++ b/src/hotspot/cpu/riscv/vm_version_riscv.cpp
@@ -122,17 +122,6 @@ void VM_Version::common_initialize() {
FLAG_SET_DEFAULT(AllocatePrefetchDistance, 0);
}
- if (UseAES || UseAESIntrinsics) {
- if (UseAES && !FLAG_IS_DEFAULT(UseAES)) {
- warning("AES instructions are not available on this CPU");
- FLAG_SET_DEFAULT(UseAES, false);
- }
- if (UseAESIntrinsics && !FLAG_IS_DEFAULT(UseAESIntrinsics)) {
- warning("AES intrinsics are not available on this CPU");
- FLAG_SET_DEFAULT(UseAESIntrinsics, false);
- }
- }
-
if (UseAESCTRIntrinsics) {
warning("AES/CTR intrinsics are not available on this CPU");
FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
@@ -429,6 +418,23 @@ void VM_Version::c2_initialize() {
if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA3Intrinsics || UseSHA512Intrinsics)) {
FLAG_SET_DEFAULT(UseSHA, false);
}
+
+ // AES
+ if (UseZvkn) {
+ UseAES = UseAES || FLAG_IS_DEFAULT(UseAES);
+ UseAESIntrinsics =
+ UseAESIntrinsics || (UseAES && FLAG_IS_DEFAULT(UseAESIntrinsics));
+ if (UseAESIntrinsics && !UseAES) {
+ warning("UseAESIntrinsics enabled, but UseAES not, enabling");
+ UseAES = true;
+ }
+ } else if (UseAESIntrinsics || UseAES) {
+ if (!FLAG_IS_DEFAULT(UseAESIntrinsics) || !FLAG_IS_DEFAULT(UseAES)) {
+ warning("AES intrinsics require Zvkn extension (not available on this CPU).");
+ }
+ FLAG_SET_DEFAULT(UseAES, false);
+ FLAG_SET_DEFAULT(UseAESIntrinsics, false);
+ }
}
#endif // COMPILER2
diff --git a/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp b/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp
index 5d945dbc32309..ce6d9c2f38f5c 100644
--- a/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp
+++ b/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp
@@ -131,8 +131,8 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index) {
// xmethod: Method*
// x12: receiver
address ame_addr = __ pc();
- __ ld(t0, Address(xmethod, Method::from_compiled_offset()));
- __ jr(t0);
+ __ ld(t1, Address(xmethod, Method::from_compiled_offset()));
+ __ jr(t1);
masm->flush();
bookkeeping(masm, tty, s, npe_addr, ame_addr, true, vtable_index, slop_bytes, 0);
@@ -160,6 +160,13 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
MacroAssembler* masm = new MacroAssembler(&cb);
assert_cond(masm != nullptr);
+ // Real entry arguments:
+ // t0: CompiledICData
+ // j_rarg0: Receiver
+ // Make sure the move of CompiledICData from t0 to t1 is the frist thing that happens.
+ // Otherwise we risk clobber t0 as it is used as scratch.
+ __ mv(t1, t0);
+
#if (!defined(PRODUCT) && defined(COMPILER2))
if (CountCompiledCalls) {
__ la(x18, ExternalAddress((address) SharedRuntime::nof_megamorphic_calls_addr()));
@@ -170,8 +177,8 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
// get receiver (need to skip return address on top of stack)
assert(VtableStub::receiver_location() == j_rarg0->as_VMReg(), "receiver expected in j_rarg0");
- // Entry arguments:
- // t1: CompiledICData
+ // Arguments from this point:
+ // t1 (moved from t0): CompiledICData
// j_rarg0: Receiver
// This stub is called from compiled code which has no callee-saved registers,
@@ -220,8 +227,8 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
// xmethod: Method*
// j_rarg0: receiver
address ame_addr = __ pc();
- __ ld(t0, Address(xmethod, Method::from_compiled_offset()));
- __ jr(t0);
+ __ ld(t1, Address(xmethod, Method::from_compiled_offset()));
+ __ jr(t1);
__ bind(L_no_such_interface);
// Handle IncompatibleClassChangeError in itable stubs.
diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
index d288f4a893d0a..d2e860aa32032 100644
--- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
@@ -131,9 +131,19 @@ void LIR_Assembler::osr_entry() {
// copied into place by code emitted in the IR.
Register OSR_buf = osrBufferPointer()->as_register();
- { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
- int monitor_offset = BytesPerWord * method()->max_locals() +
- (2 * BytesPerWord) * (number_of_locks - 1);
+ {
+ assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
+
+ const int locals_space = BytesPerWord * method() -> max_locals();
+ int monitor_offset = locals_space + (2 * BytesPerWord) * (number_of_locks - 1);
+ bool large_offset = !Immediate::is_simm20(monitor_offset + BytesPerWord) && number_of_locks > 0;
+
+ if (large_offset) {
+ // z_lg can only handle displacement upto 20bit signed binary integer
+ __ z_algfi(OSR_buf, locals_space);
+ monitor_offset -= locals_space;
+ }
+
// SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
// the OSR buffer using 2 word entries: first the lock and then
// the oop.
@@ -147,6 +157,10 @@ void LIR_Assembler::osr_entry() {
__ z_lg(Z_R1_scratch, slot_offset + 1*BytesPerWord, OSR_buf);
__ z_stg(Z_R1_scratch, frame_map()->address_for_monitor_object(i));
}
+
+ if (large_offset) {
+ __ z_slgfi(OSR_buf, locals_space);
+ }
}
}
@@ -216,7 +230,11 @@ int LIR_Assembler::emit_unwind_handler() {
LIR_Opr lock = FrameMap::as_opr(Z_R1_scratch);
monitor_address(0, lock);
stub = new MonitorExitStub(lock, true, 0);
- __ unlock_object(Rtmp1, Rtmp2, lock->as_register(), *stub->entry());
+ if (LockingMode == LM_MONITOR) {
+ __ branch_optimized(Assembler::bcondAlways, *stub->entry());
+ } else {
+ __ unlock_object(Rtmp1, Rtmp2, lock->as_register(), *stub->entry());
+ }
__ bind(*stub->continuation());
}
@@ -2350,6 +2368,7 @@ void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr de
void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
if (op->init_check()) {
// Make sure klass is initialized & doesn't have finalizer.
+ // init_state needs acquire, but S390 is TSO, and so we are already good.
const int state_offset = in_bytes(InstanceKlass::init_state_offset());
Register iklass = op->klass()->as_register();
add_debug_info_for_null_check_here(op->stub()->info());
diff --git a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
index f6dd20db3d67f..41c2ae260a693 100644
--- a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
@@ -120,6 +120,8 @@ void C1_MacroAssembler::lock_object(Register Rmark, Register Roop, Register Rbox
branch_optimized(Assembler::bcondNotZero, slow_case);
// done
bind(done);
+ } else {
+ assert(false, "Unhandled LockingMode:%d", LockingMode);
}
}
@@ -151,6 +153,8 @@ void C1_MacroAssembler::unlock_object(Register Rmark, Register Roop, Register Rb
// If the object header was not pointing to the displaced header,
// we do unlocking via runtime call.
branch_optimized(Assembler::bcondNotEqual, slow_case);
+ } else {
+ assert(false, "Unhandled LockingMode:%d", LockingMode);
}
// done
bind(done);
diff --git a/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp b/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp
index 37631298920ca..544c82d34a769 100644
--- a/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2018, 2023 SAP SE. All rights reserved.
+ * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -42,11 +42,47 @@
#include "c1/c1_LIRAssembler.hpp"
#include "c1/c1_MacroAssembler.hpp"
#include "gc/g1/c1/g1BarrierSetC1.hpp"
-#endif
+#endif // COMPILER1
+#ifdef COMPILER2
+#include "gc/g1/c2/g1BarrierSetC2.hpp"
+#endif // COMPILER2
#define __ masm->
-#define BLOCK_COMMENT(str) if (PrintAssembly) __ block_comment(str)
+#define BLOCK_COMMENT(str) __ block_comment(str)
+
+static void generate_pre_barrier_fast_path(MacroAssembler* masm,
+ const Register thread,
+ const Register tmp1) {
+ Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
+ // Is marking active?
+ if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
+ __ load_and_test_int(tmp1, in_progress);
+ } else {
+ assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
+ __ load_and_test_byte(tmp1, in_progress);
+ }
+}
+
+static void generate_queue_test_and_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
+ const Register Z_thread, const Register value, const Register temp) {
+ BLOCK_COMMENT("generate_queue_test_and_insertion {");
+
+ assert_different_registers(temp, value);
+ // Can we store a value in the given thread's buffer?
+ // (The index field is typed as size_t.)
+
+ __ load_and_test_long(temp, Address(Z_thread, in_bytes(index_offset))); // temp := *(index address)
+ __ branch_optimized(Assembler::bcondEqual, runtime); // jump to runtime if index == 0 (full buffer)
+
+ // The buffer is not full, store value into it.
+ __ add2reg(temp, -wordSize); // temp := next index
+ __ z_stg(temp, in_bytes(index_offset), Z_thread); // *(index address) := next index
+
+ __ z_ag(temp, Address(Z_thread, in_bytes(buffer_offset))); // temp := buffer address + next index
+ __ z_stg(value, 0, temp); // *(buffer address + next index) := value
+ BLOCK_COMMENT("} generate_queue_test_and_insertion");
+}
void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
Register addr, Register count) {
@@ -59,13 +95,8 @@ void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm
assert_different_registers(addr, Z_R0_scratch); // would be destroyed by push_frame()
assert_different_registers(count, Z_R0_scratch); // would be destroyed by push_frame()
Register Rtmp1 = Z_R0_scratch;
- const int active_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
- if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
- __ load_and_test_int(Rtmp1, Address(Z_thread, active_offset));
- } else {
- guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
- __ load_and_test_byte(Rtmp1, Address(Z_thread, active_offset));
- }
+
+ generate_pre_barrier_fast_path(masm, Z_thread, Rtmp1);
__ z_bre(filtered); // Activity indicator is zero, so there is no marking going on currently.
RegisterSaver::save_live_registers(masm, RegisterSaver::arg_registers); // Creates frame.
@@ -100,6 +131,181 @@ void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* mas
}
}
+#if defined(COMPILER2)
+
+#undef __
+#define __ masm->
+
+static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register pre_val, const address runtime_path) {
+ BLOCK_COMMENT("generate_c2_barrier_runtime_call {");
+ SaveLiveRegisters save_registers(masm, stub);
+ __ call_VM_leaf(runtime_path, pre_val, Z_thread);
+ BLOCK_COMMENT("} generate_c2_barrier_runtime_call");
+}
+
+void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register thread,
+ Register tmp1,
+ G1PreBarrierStubC2* stub) {
+
+ BLOCK_COMMENT("g1_write_barrier_pre_c2 {");
+
+ assert(thread == Z_thread, "must be");
+ assert_different_registers(obj, pre_val, tmp1);
+ assert(pre_val != noreg && tmp1 != noreg, "expecting a register");
+
+ stub->initialize_registers(obj, pre_val, thread, tmp1, noreg);
+
+ generate_pre_barrier_fast_path(masm, thread, tmp1);
+ __ branch_optimized(Assembler::bcondNotEqual, *stub->entry()); // Activity indicator is zero, so there is no marking going on currently.
+
+ __ bind(*stub->continuation());
+
+ BLOCK_COMMENT("} g1_write_barrier_pre_c2");
+}
+
+void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm,
+ G1PreBarrierStubC2* stub) const {
+
+ BLOCK_COMMENT("generate_c2_pre_barrier_stub {");
+
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+
+ Label runtime;
+ Register obj = stub->obj();
+ Register pre_val = stub->pre_val();
+ Register thread = stub->thread();
+ Register tmp1 = stub->tmp1();
+
+ __ bind(*stub->entry());
+
+ BLOCK_COMMENT("generate_pre_val_not_null_test {");
+ if (obj != noreg) {
+ __ load_heap_oop(pre_val, Address(obj), noreg, noreg, AS_RAW);
+ }
+ __ z_ltgr(pre_val, pre_val);
+ __ branch_optimized(Assembler::bcondEqual, *stub->continuation());
+ BLOCK_COMMENT("} generate_pre_val_not_null_test");
+
+ generate_queue_test_and_insertion(masm,
+ G1ThreadLocalData::satb_mark_queue_index_offset(),
+ G1ThreadLocalData::satb_mark_queue_buffer_offset(),
+ runtime,
+ Z_thread, pre_val, tmp1);
+
+ __ branch_optimized(Assembler::bcondAlways, *stub->continuation());
+
+ __ bind(runtime);
+
+ generate_c2_barrier_runtime_call(masm, stub, pre_val, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry));
+
+ __ branch_optimized(Assembler::bcondAlways, *stub->continuation());
+
+ BLOCK_COMMENT("} generate_c2_pre_barrier_stub");
+}
+
+void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2,
+ G1PostBarrierStubC2* stub) {
+ BLOCK_COMMENT("g1_write_barrier_post_c2 {");
+
+ assert(thread == Z_thread, "must be");
+ assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, Z_R1_scratch);
+
+ assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
+
+ stub->initialize_registers(thread, tmp1, tmp2);
+
+ BLOCK_COMMENT("generate_region_crossing_test {");
+ if (VM_Version::has_DistinctOpnds()) {
+ __ z_xgrk(tmp1, store_addr, new_val);
+ } else {
+ __ z_lgr(tmp1, store_addr);
+ __ z_xgr(tmp1, new_val);
+ }
+ __ z_srag(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes);
+ __ branch_optimized(Assembler::bcondEqual, *stub->continuation());
+ BLOCK_COMMENT("} generate_region_crossing_test");
+
+ // crosses regions, storing null?
+ if ((stub->barrier_data() & G1C2BarrierPostNotNull) == 0) {
+ __ z_ltgr(new_val, new_val);
+ __ branch_optimized(Assembler::bcondEqual, *stub->continuation());
+ }
+
+ BLOCK_COMMENT("generate_card_young_test {");
+ CardTableBarrierSet* ct = barrier_set_cast(BarrierSet::barrier_set());
+ // calculate address of card
+ __ load_const_optimized(tmp2, (address)ct->card_table()->byte_map_base()); // Card table base.
+ __ z_srlg(tmp1, store_addr, CardTable::card_shift()); // Index into card table.
+ __ z_algr(tmp1, tmp2); // Explicit calculation needed for cli.
+
+ // Filter young.
+ __ z_cli(0, tmp1, G1CardTable::g1_young_card_val());
+
+ BLOCK_COMMENT("} generate_card_young_test");
+
+ // From here on, tmp1 holds the card address.
+ __ branch_optimized(Assembler::bcondNotEqual, *stub->entry());
+
+ __ bind(*stub->continuation());
+
+ BLOCK_COMMENT("} g1_write_barrier_post_c2");
+}
+
+void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
+ G1PostBarrierStubC2* stub) const {
+
+ BLOCK_COMMENT("generate_c2_post_barrier_stub {");
+
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ Label runtime;
+
+ Register thread = stub->thread();
+ Register tmp1 = stub->tmp1(); // tmp1 holds the card address.
+ Register tmp2 = stub->tmp2();
+ Register Rcard_addr = tmp1;
+
+ __ bind(*stub->entry());
+
+ BLOCK_COMMENT("generate_card_clean_test {");
+ __ z_sync(); // Required to support concurrent cleaning.
+ __ z_cli(0, Rcard_addr, 0); // Reload after membar.
+ __ branch_optimized(Assembler::bcondEqual, *stub->continuation());
+ BLOCK_COMMENT("} generate_card_clean_test");
+
+ BLOCK_COMMENT("generate_dirty_card {");
+ // Storing a region crossing, non-null oop, card is clean.
+ // Dirty card and log.
+ STATIC_ASSERT(CardTable::dirty_card_val() == 0);
+ __ z_mvi(0, Rcard_addr, CardTable::dirty_card_val());
+ BLOCK_COMMENT("} generate_dirty_card");
+
+ generate_queue_test_and_insertion(masm,
+ G1ThreadLocalData::dirty_card_queue_index_offset(),
+ G1ThreadLocalData::dirty_card_queue_buffer_offset(),
+ runtime,
+ Z_thread, tmp1, tmp2);
+
+ __ branch_optimized(Assembler::bcondAlways, *stub->continuation());
+
+ __ bind(runtime);
+
+ generate_c2_barrier_runtime_call(masm, stub, tmp1, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
+
+ __ branch_optimized(Assembler::bcondAlways, *stub->continuation());
+
+ BLOCK_COMMENT("} generate_c2_post_barrier_stub");
+}
+
+#endif //COMPILER2
+
void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
const Address& src, Register dst, Register tmp1, Register tmp2, Label *L_handle_null) {
bool on_oop = is_reference_type(type);
@@ -136,9 +342,6 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, Decorator
const Register Robj = obj ? obj->base() : noreg,
Roff = obj ? obj->index() : noreg;
- const int active_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
- const int buffer_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset());
- const int index_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset());
assert_different_registers(Rtmp1, Rtmp2, Z_R0_scratch); // None of the Rtmp must be Z_R0!!
assert_different_registers(Robj, Z_R0_scratch); // Used for addressing. Furthermore, push_frame destroys Z_R0!!
assert_different_registers(Rval, Z_R0_scratch); // push_frame destroys Z_R0!!
@@ -147,14 +350,7 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, Decorator
BLOCK_COMMENT("g1_write_barrier_pre {");
- // Is marking active?
- // Note: value is loaded for test purposes only. No further use here.
- if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
- __ load_and_test_int(Rtmp1, Address(Z_thread, active_offset));
- } else {
- guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
- __ load_and_test_byte(Rtmp1, Address(Z_thread, active_offset));
- }
+ generate_pre_barrier_fast_path(masm, Z_thread, Rtmp1);
__ z_bre(filtered); // Activity indicator is zero, so there is no marking going on currently.
assert(Rpre_val != noreg, "must have a real register");
@@ -194,24 +390,14 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, Decorator
// We can store the original value in the thread's buffer
// only if index > 0. Otherwise, we need runtime to handle.
// (The index field is typed as size_t.)
- Register Rbuffer = Rtmp1, Rindex = Rtmp2;
- assert_different_registers(Rbuffer, Rindex, Rpre_val);
-
- __ z_lg(Rbuffer, buffer_offset, Z_thread);
- __ load_and_test_long(Rindex, Address(Z_thread, index_offset));
- __ z_bre(callRuntime); // If index == 0, goto runtime.
-
- __ add2reg(Rindex, -wordSize); // Decrement index.
- __ z_stg(Rindex, index_offset, Z_thread);
-
- // Record the previous value.
- __ z_stg(Rpre_val, 0, Rbuffer, Rindex);
+ generate_queue_test_and_insertion(masm,
+ G1ThreadLocalData::satb_mark_queue_index_offset(),
+ G1ThreadLocalData::satb_mark_queue_buffer_offset(),
+ callRuntime,
+ Z_thread, Rpre_val, Rtmp2);
__ z_bru(filtered); // We are done.
- Rbuffer = noreg; // end of life
- Rindex = noreg; // end of life
-
__ bind(callRuntime);
// Save some registers (inputs and result) over runtime call
@@ -326,23 +512,16 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, Decorato
Register Rcard_addr_x = Rcard_addr;
Register Rqueue_index = (Rtmp2 != Z_R0_scratch) ? Rtmp2 : Rtmp1;
- Register Rqueue_buf = (Rtmp3 != Z_R0_scratch) ? Rtmp3 : Rtmp1;
- const int qidx_off = in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset());
- const int qbuf_off = in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset());
- if ((Rcard_addr == Rqueue_buf) || (Rcard_addr == Rqueue_index)) {
+ if (Rcard_addr == Rqueue_index) {
Rcard_addr_x = Z_R0_scratch; // Register shortage. We have to use Z_R0.
}
__ lgr_if_needed(Rcard_addr_x, Rcard_addr);
- __ load_and_test_long(Rqueue_index, Address(Z_thread, qidx_off));
- __ z_bre(callRuntime); // Index == 0 then jump to runtime.
-
- __ z_lg(Rqueue_buf, qbuf_off, Z_thread);
-
- __ add2reg(Rqueue_index, -wordSize); // Decrement index.
- __ z_stg(Rqueue_index, qidx_off, Z_thread);
-
- __ z_stg(Rcard_addr_x, 0, Rqueue_index, Rqueue_buf); // Store card.
+ generate_queue_test_and_insertion(masm,
+ G1ThreadLocalData::dirty_card_queue_index_offset(),
+ G1ThreadLocalData::dirty_card_queue_buffer_offset(),
+ callRuntime,
+ Z_thread, Rcard_addr_x, Rqueue_index);
__ z_bru(filtered);
__ bind(callRuntime);
diff --git a/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.hpp b/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.hpp
index cc1d51d2fa13e..0f0bdd8b83cfd 100644
--- a/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2018 SAP SE. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -34,6 +34,8 @@ class LIR_Assembler;
class StubAssembler;
class G1PreBarrierStub;
class G1PostBarrierStub;
+class G1PreBarrierStubC2;
+class G1PostBarrierStubC2;
class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
protected:
@@ -62,7 +64,27 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm);
-#endif
+#endif // COMPILER1
+
+#ifdef COMPILER2
+ void g1_write_barrier_pre_c2(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register thread,
+ Register tmp1,
+ G1PreBarrierStubC2* c2_stub);
+ void generate_c2_pre_barrier_stub(MacroAssembler* masm,
+ G1PreBarrierStubC2* stub) const;
+ void g1_write_barrier_post_c2(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2,
+ G1PostBarrierStubC2* c2_stub);
+ void generate_c2_post_barrier_stub(MacroAssembler* masm,
+ G1PostBarrierStubC2* stub) const;
+#endif // COMPILER2
virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
const Address& src, Register dst, Register tmp1, Register tmp2, Label *L_handle_null = nullptr);
diff --git a/src/hotspot/cpu/s390/gc/g1/g1_s390.ad b/src/hotspot/cpu/s390/gc/g1/g1_s390.ad
new file mode 100644
index 0000000000000..31f60c4aeff0b
--- /dev/null
+++ b/src/hotspot/cpu/s390/gc/g1/g1_s390.ad
@@ -0,0 +1,457 @@
+//
+// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+// Copyright 2024 IBM Corporation. All rights reserved.
+// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+
+source_hpp %{
+
+#include "gc/g1/c2/g1BarrierSetC2.hpp"
+#include "gc/shared/gc_globals.hpp"
+
+%}
+
+source %{
+
+#include "gc/g1/g1BarrierSetAssembler_s390.hpp"
+#include "gc/g1/g1BarrierSetRuntime.hpp"
+
+static void write_barrier_pre(MacroAssembler* masm,
+ const MachNode* node,
+ Register obj,
+ Register pre_val,
+ Register tmp1,
+ RegSet preserve = RegSet(),
+ RegSet no_preserve = RegSet()) {
+ if (!G1PreBarrierStubC2::needs_barrier(node)) {
+ return;
+ }
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
+ G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
+ for (RegSetIterator reg = preserve.begin(); *reg != noreg; ++reg) {
+ stub->preserve(*reg);
+ }
+ for (RegSetIterator reg = no_preserve.begin(); *reg != noreg; ++reg) {
+ stub->dont_preserve(*reg);
+ }
+ g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, Z_thread, tmp1, stub);
+}
+
+static void write_barrier_post(MacroAssembler* masm,
+ const MachNode* node,
+ Register store_addr,
+ Register new_val,
+ Register tmp1,
+ Register tmp2) {
+ if (!G1PostBarrierStubC2::needs_barrier(node)) {
+ return;
+ }
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
+ G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
+ g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, Z_thread, tmp1, tmp2, stub);
+}
+
+%} // source
+
+// store pointer
+instruct g1StoreP(indirect dst, memoryRegP src, iRegL tmp1, iRegL tmp2, flagsReg cr) %{
+ predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
+ match(Set dst (StoreP dst src));
+ effect(TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(MEMORY_REF_COST);
+ format %{ "STG $src,$dst\t # ptr" %}
+ ins_encode %{
+ __ block_comment("g1StoreP {");
+ write_barrier_pre(masm, this,
+ $dst$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ RegSet::of($dst$$Register, $src$$Register) /* preserve */);
+
+ __ z_stg($src$$Register, Address($dst$$Register));
+
+ write_barrier_post(masm, this,
+ $dst$$Register, /* store_addr */
+ $src$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ __ block_comment("} g1StoreP");
+ %}
+ ins_pipe(pipe_class_dummy);
+%}
+
+// Store Compressed Pointer
+instruct g1StoreN(indirect mem, iRegN_P2N src, iRegL tmp1, iRegL tmp2, iRegL tmp3, flagsReg cr) %{
+ predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreN mem src));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(MEMORY_REF_COST);
+ format %{ "STY $src,$mem\t # (cOop)" %}
+ ins_encode %{
+ __ block_comment("g1StoreN {");
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ RegSet::of($mem$$Register, $src$$Register) /* preserve */);
+
+ __ z_sty($src$$Register, Address($mem$$Register));
+
+ if ((barrier_data() & G1C2BarrierPost) != 0) {
+ if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
+ __ oop_decoder($tmp1$$Register, $src$$Register, true /* maybe_null */);
+ } else {
+ __ oop_decoder($tmp1$$Register, $src$$Register, false /* maybe_null */);
+ }
+ }
+
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ __ block_comment("} g1StoreN");
+ %}
+
+ ins_pipe(pipe_class_dummy);
+%}
+
+instruct g1CompareAndSwapN(indirect mem_ptr, rarg5RegN oldval, iRegN_P2N newval, iRegI res, iRegL tmp1, iRegL tmp2, iRegL tmp3, flagsReg cr) %{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
+ match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
+ effect(USE mem_ptr, TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL oldval, KILL cr);
+ format %{ "$res = CompareAndSwapN $oldval,$newval,$mem_ptr" %}
+ ins_encode %{
+ assert_different_registers($oldval$$Register, $mem_ptr$$Register);
+ assert_different_registers($newval$$Register, $mem_ptr$$Register);
+ __ block_comment("g1compareAndSwapN {");
+
+ Register Rcomp = reg_to_register_object($oldval$$reg);
+ Register Rnew = reg_to_register_object($newval$$reg);
+ Register Raddr = reg_to_register_object($mem_ptr$$reg);
+ Register Rres = reg_to_register_object($res$$reg);
+
+ write_barrier_pre(masm, this,
+ Raddr /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ RegSet::of(Raddr, Rcomp, Rnew) /* preserve */,
+ RegSet::of(Rres) /* no_preserve */);
+
+ __ z_cs(Rcomp, Rnew, 0, Raddr);
+
+ assert_different_registers(Rres, Raddr);
+ if (VM_Version::has_LoadStoreConditional()) {
+ __ load_const_optimized(Z_R0_scratch, 0L); // false (failed)
+ __ load_const_optimized(Rres, 1L); // true (succeed)
+ __ z_locgr(Rres, Z_R0_scratch, Assembler::bcondNotEqual);
+ } else {
+ Label done;
+ __ load_const_optimized(Rres, 0L); // false (failed)
+ __ z_brne(done); // Assume true to be the common case.
+ __ load_const_optimized(Rres, 1L); // true (succeed)
+ __ bind(done);
+ }
+
+ __ oop_decoder($tmp3$$Register, Rnew, true /* maybe_null */);
+
+ write_barrier_post(masm, this,
+ Raddr /* store_addr */,
+ $tmp3$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ __ block_comment("} g1compareAndSwapN");
+ %}
+ ins_pipe(pipe_class_dummy);
+%}
+
+instruct g1CompareAndExchangeN(iRegP mem_ptr, rarg5RegN oldval, iRegN_P2N newval, iRegN res, iRegL tmp1, iRegL tmp2, iRegL tmp3, flagsReg cr) %{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndExchangeN mem_ptr (Binary oldval newval)));
+ effect(USE mem_ptr, TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL oldval, KILL cr);
+ format %{ "$res = CompareAndExchangeN $oldval,$newval,$mem_ptr" %}
+ ins_encode %{
+ assert_different_registers($oldval$$Register, $mem_ptr$$Register);
+ assert_different_registers($newval$$Register, $mem_ptr$$Register);
+ __ block_comment("g1CompareAndExchangeN {");
+ write_barrier_pre(masm, this,
+ $mem_ptr$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ RegSet::of($mem_ptr$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+
+ Register Rcomp = reg_to_register_object($oldval$$reg);
+ Register Rnew = reg_to_register_object($newval$$reg);
+ Register Raddr = reg_to_register_object($mem_ptr$$reg);
+
+ Register Rres = reg_to_register_object($res$$reg);
+ assert_different_registers(Rres, Raddr);
+
+ __ z_lgr(Rres, Rcomp); // previous contents
+ __ z_csy(Rres, Rnew, 0, Raddr); // Try to store new value.
+
+ __ oop_decoder($tmp1$$Register, Rnew, true /* maybe_null */);
+
+ write_barrier_post(masm, this,
+ Raddr /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ __ block_comment("} g1CompareAndExchangeN");
+ %}
+ ins_pipe(pipe_class_dummy);
+%}
+
+// Load narrow oop
+instruct g1LoadN(iRegN dst, indirect mem, iRegP tmp1, iRegP tmp2, flagsReg cr) %{
+ predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
+ match(Set dst (LoadN mem));
+ effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(MEMORY_REF_COST);
+ format %{ "LoadN $dst,$mem\t # (cOop)" %}
+ ins_encode %{
+ __ block_comment("g1LoadN {");
+ __ z_llgf($dst$$Register, Address($mem$$Register));
+ if ((barrier_data() & G1C2BarrierPre) != 0) {
+ __ oop_decoder($tmp1$$Register, $dst$$Register, true);
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register );
+ }
+ __ block_comment("} g1LoadN");
+ %}
+ ins_pipe(pipe_class_dummy);
+%}
+
+instruct g1GetAndSetN(indirect mem, iRegN dst, iRegI tmp, iRegL tmp1, iRegL tmp2, iRegL tmp3, flagsReg cr) %{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set dst (GetAndSetN mem dst));
+ effect(KILL cr, TEMP tmp, TEMP tmp1, TEMP tmp2, TEMP tmp3); // USE_DEF dst by match rule.
+ format %{ "XCHGN $dst,[$mem]\t # EXCHANGE (coop, atomic), temp $tmp" %}
+ ins_encode %{
+ __ block_comment("g1GetAndSetN {");
+ assert_different_registers($mem$$Register, $dst$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ RegSet::of($mem$$Register, $dst$$Register) /* preserve */);
+
+ Register Rdst = reg_to_register_object($dst$$reg);
+ Register Rtmp = reg_to_register_object($tmp$$reg);
+ guarantee(Rdst != Rtmp, "Fix match rule to use TEMP_DEF");
+ Label retry;
+
+ // Iterate until swap succeeds.
+ __ z_llgf(Rtmp, Address($mem$$Register)); // current contents
+ __ bind(retry);
+ // Calculate incremented value.
+ __ z_csy(Rtmp, Rdst, Address($mem$$Register)); // Try to store new value.
+ __ z_brne(retry); // Yikes, concurrent update, need to retry.
+
+ __ oop_decoder($tmp1$$Register, $dst$$Register, true /* maybe_null */);
+
+ __ z_lgr(Rdst, Rtmp); // Exchanged value from memory is return value.
+
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+
+ __ block_comment("} g1GetAndSetN");
+ %}
+ ins_pipe(pipe_class_dummy);
+%}
+
+instruct g1CompareAndSwapP(iRegP mem_ptr, rarg5RegP oldval, iRegP_N2P newval, iRegI res, iRegL tmp1, iRegL tmp2, flagsReg cr) %{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
+ match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, USE mem_ptr, USE_KILL oldval, KILL cr);
+ format %{ "$res = CompareAndSwapP $oldval,$newval,$mem_ptr" %}
+ ins_encode %{
+ __ block_comment("g1CompareAndSwapP {");
+ assert_different_registers($oldval$$Register, $mem_ptr$$Register);
+ assert_different_registers($newval$$Register, $mem_ptr$$Register);
+
+ Register Rcomp = reg_to_register_object($oldval$$reg);
+ Register Rnew = reg_to_register_object($newval$$reg);
+ Register Raddr = reg_to_register_object($mem_ptr$$reg);
+ Register Rres = reg_to_register_object($res$$reg);
+
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ Rcomp /* pre_val */,
+ $tmp1$$Register /* tmp1 */,
+ RegSet::of(Raddr, Rcomp, Rnew) /* preserve */,
+ RegSet::of(Rres) /* no_preserve */);
+
+ __ z_csg(Rcomp, Rnew, 0, Raddr);
+
+ if (VM_Version::has_LoadStoreConditional()) {
+ __ load_const_optimized(Z_R0_scratch, 0L); // false (failed)
+ __ load_const_optimized(Rres, 1L); // true (succeed)
+ __ z_locgr(Rres, Z_R0_scratch, Assembler::bcondNotEqual);
+ } else {
+ Label done;
+ __ load_const_optimized(Rres, 0L); // false (failed)
+ __ z_brne(done); // Assume true to be the common case.
+ __ load_const_optimized(Rres, 1L); // true (succeed)
+ __ bind(done);
+ }
+
+ write_barrier_post(masm, this,
+ Raddr /* store_addr */,
+ Rnew /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ __ block_comment("} g1CompareAndSwapP");
+ %}
+ ins_pipe(pipe_class_dummy);
+%}
+
+instruct g1CompareAndExchangeP(iRegP mem_ptr, rarg5RegP oldval, iRegP_N2P newval, iRegP res, iRegL tmp1, iRegL tmp2, flagsReg cr) %{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndExchangeP mem_ptr (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, USE mem_ptr, USE_KILL oldval, KILL cr);
+ format %{ "$res = CompareAndExchangeP $oldval,$newval,$mem_ptr" %}
+ ins_encode %{
+ __ block_comment("g1CompareAndExchangeP {");
+ assert_different_registers($oldval$$Register, $mem_ptr$$Register);
+ assert_different_registers($newval$$Register, $mem_ptr$$Register);
+
+ // Pass $oldval to the pre-barrier (instead of loading from $mem), because
+ // $oldval is the only value that can be overwritten.
+ // The same holds for g1CompareAndSwapP.
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $oldval$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ RegSet::of($mem_ptr$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+
+ __ z_lgr($res$$Register, $oldval$$Register); // previous content
+
+ __ z_csg($oldval$$Register, $newval$$Register, 0, $mem_ptr$$reg);
+
+ write_barrier_post(masm, this,
+ $mem_ptr$$Register /* store_addr */,
+ $newval$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ __ block_comment("} g1CompareAndExchangeP");
+ %}
+ ins_pipe(pipe_class_dummy);
+%}
+
+// Load Pointer
+instruct g1LoadP(iRegP dst, memory mem, iRegL tmp1, flagsReg cr) %{
+ predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
+ match(Set dst (LoadP mem));
+ effect(TEMP dst, TEMP tmp1, KILL cr);
+ ins_cost(MEMORY_REF_COST);
+ format %{ "LG $dst,$mem\t # ptr" %}
+ ins_encode %{
+ __ block_comment("g1LoadP {");
+ __ z_lg($dst$$Register, $mem$$Address);
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $dst$$Register /* pre_val */,
+ $tmp1$$Register );
+ __ block_comment("} g1LoadP");
+ %}
+ ins_pipe(pipe_class_dummy);
+%}
+
+instruct g1GetAndSetP(indirect mem, iRegP dst, iRegL tmp, iRegL tmp1, iRegL tmp2, flagsReg cr) %{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set dst (GetAndSetP mem dst));
+ effect(KILL cr, TEMP tmp, TEMP tmp1, TEMP tmp2); // USE_DEF dst by match rule.
+ format %{ "XCHGP $dst,[$mem]\t # EXCHANGE (oop, atomic), temp $tmp" %}
+ ins_encode %{
+ __ block_comment("g1GetAndSetP {");
+
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp$$Register /* pre_val (as a temporary register) */,
+ $tmp1$$Register /* tmp1 */,
+ RegSet::of($mem$$Register, $dst$$Register) /* preserve */);
+
+ __ z_lgr($tmp1$$Register, $dst$$Register);
+ Register Rdst = reg_to_register_object($dst$$reg);
+ Register Rtmp = reg_to_register_object($tmp$$reg);
+ guarantee(Rdst != Rtmp, "Fix match rule to use TEMP_DEF");
+ Label retry;
+
+ // Iterate until swap succeeds.
+ __ z_lg(Rtmp, Address($mem$$Register)); // current contents
+ __ bind(retry);
+ // Calculate incremented value.
+ __ z_csg(Rtmp, Rdst, Address($mem$$Register)); // Try to store new value.
+ __ z_brne(retry); // Yikes, concurrent update, need to retry.
+ __ z_lgr(Rdst, Rtmp); // Exchanged value from memory is return value.
+
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp$$Register /* tmp2 */);
+ __ block_comment("} g1GetAndSetP");
+ %}
+ ins_pipe(pipe_class_dummy);
+%}
+
+instruct g1EncodePAndStoreN(indirect mem, iRegP src, iRegL tmp1, iRegL tmp2, flagsReg cr)
+%{
+ predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreN mem (EncodeP src)));
+ effect(TEMP tmp1, TEMP tmp2, KILL cr);
+ // ins_cost(INSN_COST);
+ format %{ "encode_heap_oop $tmp1, $src\n\t"
+ "st $tmp1, $mem\t# compressed ptr" %}
+ ins_encode %{
+ __ block_comment("g1EncodePAndStoreN {");
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp1 */,
+ RegSet::of($mem$$Register, $src$$Register) /* preserve */);
+ if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
+ __ oop_encoder($tmp1$$Register, $src$$Register, true /* maybe_null */);
+ } else {
+ __ oop_encoder($tmp1$$Register, $src$$Register, false /* maybe_null */);
+ }
+ __ z_st($tmp1$$Register, Address($mem$$Register));
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $src$$Register /* new_val */,
+ $tmp1$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ __ block_comment("} g1EncodePAndStoreN");
+ %}
+ ins_pipe(pipe_class_dummy);
+%}
diff --git a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp
index 28892da6ca4c1..d826b4a06f336 100644
--- a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp
@@ -33,6 +33,9 @@
#include "runtime/jniHandles.hpp"
#include "runtime/stubRoutines.hpp"
#include "utilities/macros.hpp"
+#ifdef COMPILER2
+#include "gc/shared/c2/barrierSetC2.hpp"
+#endif // COMPILER2
#define __ masm->
@@ -194,8 +197,93 @@ void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) {
#ifdef COMPILER2
-OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
- Unimplemented(); // This must be implemented to support late barrier expansion.
+OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) const {
+ if (!OptoReg::is_reg(opto_reg)) {
+ return OptoReg::Bad;
+ }
+
+ VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+ if ((vm_reg->is_Register() || vm_reg ->is_FloatRegister()) && (opto_reg & 1) != 0) {
+ return OptoReg::Bad;
+ }
+
+ return opto_reg;
+}
+
+#undef __
+#define __ _masm->
+
+SaveLiveRegisters::SaveLiveRegisters(MacroAssembler *masm, BarrierStubC2 *stub)
+ : _masm(masm), _reg_mask(stub->preserve_set()) {
+
+ const int register_save_size = iterate_over_register_mask(ACTION_COUNT_ONLY) * BytesPerWord;
+
+ _frame_size = align_up(register_save_size, frame::alignment_in_bytes) + frame::z_abi_160_size; // FIXME: this could be restricted to argument only
+
+ __ save_return_pc();
+ __ push_frame(_frame_size, Z_R14); // FIXME: check if Z_R1_scaratch can do a job here;
+
+ __ z_lg(Z_R14, _z_common_abi(return_pc) + _frame_size, Z_SP);
+
+ iterate_over_register_mask(ACTION_SAVE, _frame_size);
+}
+
+SaveLiveRegisters::~SaveLiveRegisters() {
+ iterate_over_register_mask(ACTION_RESTORE, _frame_size);
+
+ __ pop_frame();
+
+ __ restore_return_pc();
+}
+
+int SaveLiveRegisters::iterate_over_register_mask(IterationAction action, int offset) {
+ int reg_save_index = 0;
+ RegMaskIterator live_regs_iterator(_reg_mask);
+
+ while(live_regs_iterator.has_next()) {
+ const OptoReg::Name opto_reg = live_regs_iterator.next();
+
+ // Filter out stack slots (spilled registers, i.e., stack-allocated registers).
+ if (!OptoReg::is_reg(opto_reg)) {
+ continue;
+ }
+
+ const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+ if (vm_reg->is_Register()) {
+ Register std_reg = vm_reg->as_Register();
+
+ if (std_reg->encoding() >= Z_R2->encoding() && std_reg->encoding() <= Z_R15->encoding()) {
+ reg_save_index++;
+
+ if (action == ACTION_SAVE) {
+ __ z_stg(std_reg, offset - reg_save_index * BytesPerWord, Z_SP);
+ } else if (action == ACTION_RESTORE) {
+ __ z_lg(std_reg, offset - reg_save_index * BytesPerWord, Z_SP);
+ } else {
+ assert(action == ACTION_COUNT_ONLY, "Sanity");
+ }
+ }
+ } else if (vm_reg->is_FloatRegister()) {
+ FloatRegister fp_reg = vm_reg->as_FloatRegister();
+ if (fp_reg->encoding() >= Z_F0->encoding() && fp_reg->encoding() <= Z_F15->encoding()
+ && fp_reg->encoding() != Z_F1->encoding()) {
+ reg_save_index++;
+
+ if (action == ACTION_SAVE) {
+ __ z_std(fp_reg, offset - reg_save_index * BytesPerWord, Z_SP);
+ } else if (action == ACTION_RESTORE) {
+ __ z_ld(fp_reg, offset - reg_save_index * BytesPerWord, Z_SP);
+ } else {
+ assert(action == ACTION_COUNT_ONLY, "Sanity");
+ }
+ }
+ } else if (false /* vm_reg->is_VectorRegister() */){
+ fatal("Vector register support is not there yet!");
+ } else {
+ fatal("Register type is not known");
+ }
+ }
+ return reg_save_index;
}
#endif // COMPILER2
diff --git a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp
index de1de8a51a7f1..fb61adc55b500 100644
--- a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp
@@ -32,7 +32,9 @@
#ifdef COMPILER2
#include "code/vmreg.hpp"
#include "opto/optoreg.hpp"
+#include "opto/regmask.hpp"
+class BarrierStubC2;
class Node;
#endif // COMPILER2
@@ -62,8 +64,42 @@ class BarrierSetAssembler: public CHeapObj {
#ifdef COMPILER2
OptoReg::Name refine_register(const Node* node,
- OptoReg::Name opto_reg);
+ OptoReg::Name opto_reg) const;
#endif // COMPILER2
};
+#ifdef COMPILER2
+
+// This class saves and restores the registers that need to be preserved across
+// the runtime call represented by a given C2 barrier stub. Use as follows:
+// {
+// SaveLiveRegisters save(masm, stub);
+// ..
+// __ call_VM_leaf(...);
+// ..
+// }
+
+class SaveLiveRegisters {
+ MacroAssembler* _masm;
+ RegMask _reg_mask;
+ Register _result_reg;
+ int _frame_size;
+
+ public:
+ SaveLiveRegisters(MacroAssembler *masm, BarrierStubC2 *stub);
+
+ ~SaveLiveRegisters();
+
+ private:
+ enum IterationAction : int {
+ ACTION_SAVE,
+ ACTION_RESTORE,
+ ACTION_COUNT_ONLY
+ };
+
+ int iterate_over_register_mask(IterationAction action, int offset = 0);
+};
+
+#endif // COMPILER2
+
#endif // CPU_S390_GC_SHARED_BARRIERSETASSEMBLER_S390_HPP
diff --git a/src/hotspot/cpu/s390/interp_masm_s390.cpp b/src/hotspot/cpu/s390/interp_masm_s390.cpp
index d00b6c3e2cc2e..5e80817aaba7b 100644
--- a/src/hotspot/cpu/s390/interp_masm_s390.cpp
+++ b/src/hotspot/cpu/s390/interp_masm_s390.cpp
@@ -2131,18 +2131,6 @@ void InterpreterMacroAssembler::notify_method_exit(bool native_method,
if (!native_method) pop(state);
bind(jvmti_post_done);
}
-
-#if 0
- // Dtrace currently not supported on z/Architecture.
- {
- SkipIfEqual skip(this, &DTraceMethodProbes, false);
- push(state);
- get_method(c_rarg1);
- call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
- r15_thread, c_rarg1);
- pop(state);
- }
-#endif
}
void InterpreterMacroAssembler::skip_if_jvmti_mode(Label &Lskip, Register Rscratch) {
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.cpp b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
index af281345b1477..6b739553b1580 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
@@ -2127,8 +2127,9 @@ unsigned int MacroAssembler::push_frame_abi160(unsigned int bytes) {
// Pop current C frame.
void MacroAssembler::pop_frame() {
- BLOCK_COMMENT("pop_frame:");
+ BLOCK_COMMENT("pop_frame {");
Assembler::z_lg(Z_SP, _z_abi(callers_sp), Z_SP);
+ BLOCK_COMMENT("} pop_frame");
}
// Pop current C frame and restore return PC register (Z_R14).
@@ -3458,7 +3459,8 @@ void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fa
L_slow_path = &L_fallthrough;
}
- // Fast path check: class is fully initialized
+ // Fast path check: class is fully initialized.
+ // init_state needs acquire, but S390 is TSO, and so we are already good.
z_cli(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
z_bre(*L_fast_path);
@@ -3665,7 +3667,7 @@ void MacroAssembler::compiler_fast_unlock_object(Register oop, Register box, Reg
// We need a full fence after clearing owner to avoid stranding.
z_fence();
- // Check if the entry lists are empty.
+ // Check if the entry lists are empty (EntryList first - by convention).
load_and_test_long(temp, Address(currentHeader, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
z_brne(check_succ);
load_and_test_long(temp, Address(currentHeader, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
@@ -6014,21 +6016,6 @@ void MacroAssembler::zap_from_to(Register low, Register high, Register val, Regi
}
#endif // !PRODUCT
-SkipIfEqual::SkipIfEqual(MacroAssembler* masm, const bool* flag_addr, bool value, Register _rscratch) {
- _masm = masm;
- _masm->load_absolute_address(_rscratch, (address)flag_addr);
- _masm->load_and_test_int(_rscratch, Address(_rscratch));
- if (value) {
- _masm->z_brne(_label); // Skip if true, i.e. != 0.
- } else {
- _masm->z_bre(_label); // Skip if false, i.e. == 0.
- }
-}
-
-SkipIfEqual::~SkipIfEqual() {
- _masm->bind(_label);
-}
-
// Implements lightweight-locking.
// - obj: the object to be locked, contents preserved.
// - temp1, temp2: temporary registers, contents destroyed.
@@ -6508,7 +6495,7 @@ void MacroAssembler::compiler_fast_unlock_lightweight_object(Register obj, Regis
// We need a full fence after clearing owner to avoid stranding.
z_fence();
- // Check if the entry lists are empty.
+ // Check if the entry lists are empty (EntryList first - by convention).
load_and_test_long(tmp2, EntryList_address);
z_brne(check_succ);
load_and_test_long(tmp2, cxq_address);
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.hpp b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
index 5d3a4c2994091..061817a128913 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
@@ -1064,24 +1064,6 @@ class MacroAssembler: public Assembler {
};
-/**
- * class SkipIfEqual:
- *
- * Instantiating this class will result in assembly code being output that will
- * jump around any code emitted between the creation of the instance and it's
- * automatic destruction at the end of a scope block, depending on the value of
- * the flag passed to the constructor, which will be checked at run-time.
- */
-class SkipIfEqual {
- private:
- MacroAssembler* _masm;
- Label _label;
-
- public:
- SkipIfEqual(MacroAssembler*, const bool* flag_addr, bool value, Register _rscratch);
- ~SkipIfEqual();
-};
-
#ifdef ASSERT
// Return false (e.g. important for our impl. of virtual calls).
inline bool AbstractAssembler::pd_check_instruction_mark() { return false; }
diff --git a/src/hotspot/cpu/s390/matcher_s390.hpp b/src/hotspot/cpu/s390/matcher_s390.hpp
index 6c6cae3c58fc3..d8b1ae68f6f50 100644
--- a/src/hotspot/cpu/s390/matcher_s390.hpp
+++ b/src/hotspot/cpu/s390/matcher_s390.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2017, 2022 SAP SE. All rights reserved.
+ * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -63,11 +63,16 @@
return true;
}
- // Suppress CMOVL. Conditional move available on z/Architecture only from z196 onwards. Not exploited yet.
- static int long_cmove_cost() { return ConditionalMoveLimit; }
+ // Use conditional move (CMOVL)
+ static int long_cmove_cost() {
+ // z196/z11 or later hardware support conditional moves
+ return VM_Version::has_LoadStoreConditional() ? 0 : ConditionalMoveLimit;
+ }
- // Suppress CMOVF. Conditional move available on z/Architecture only from z196 onwards. Not exploited yet.
- static int float_cmove_cost() { return ConditionalMoveLimit; }
+ static int float_cmove_cost() {
+ // z196/z11 or later hardware support conditional moves
+ return VM_Version::has_LoadStoreConditional() ? 0 : ConditionalMoveLimit;
+ }
// Set this as clone_shift_expressions.
static bool narrow_oop_use_complex_address() {
diff --git a/src/hotspot/cpu/s390/nativeInst_s390.cpp b/src/hotspot/cpu/s390/nativeInst_s390.cpp
index 9f083fa890454..4db73395d3a70 100644
--- a/src/hotspot/cpu/s390/nativeInst_s390.cpp
+++ b/src/hotspot/cpu/s390/nativeInst_s390.cpp
@@ -658,8 +658,8 @@ void NativeGeneralJump::insert_unconditional(address code_pos, address entry) {
void NativeGeneralJump::replace_mt_safe(address instr_addr, address code_buffer) {
assert(((intptr_t)instr_addr & (BytesPerWord-1)) == 0, "requirement for mt safe patching");
- // Bytes_after_jump cannot change, because we own the Patching_lock.
- assert(Patching_lock->owned_by_self(), "must hold lock to patch instruction");
+ // Bytes_after_jump cannot change, because we own the CodeCache_lock.
+ assert(CodeCache_lock->owned_by_self(), "must hold lock to patch instruction");
intptr_t bytes_after_jump = (*(intptr_t*)instr_addr) & 0x000000000000ffffL; // 2 bytes after jump.
intptr_t load_const_bytes = (*(intptr_t*)code_buffer) & 0xffffffffffff0000L;
*(intptr_t*)instr_addr = load_const_bytes | bytes_after_jump;
diff --git a/src/hotspot/cpu/s390/register_s390.hpp b/src/hotspot/cpu/s390/register_s390.hpp
index 931e899257e92..18af232e56970 100644
--- a/src/hotspot/cpu/s390/register_s390.hpp
+++ b/src/hotspot/cpu/s390/register_s390.hpp
@@ -448,4 +448,12 @@ constexpr Register Z_R0_scratch = Z_R0;
constexpr Register Z_R1_scratch = Z_R1;
constexpr FloatRegister Z_fscratch_1 = Z_F1;
+typedef AbstractRegSet RegSet;
+
+template <>
+inline Register AbstractRegSet::first() {
+ if (_bitset == 0) { return noreg; }
+ return as_Register(count_trailing_zeros(_bitset));
+}
+
#endif // CPU_S390_REGISTER_S390_HPP
diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad
index 1bc9484215005..8b897033aa55d 100644
--- a/src/hotspot/cpu/s390/s390.ad
+++ b/src/hotspot/cpu/s390/s390.ad
@@ -1477,10 +1477,6 @@ const RegMask* Matcher::predicate_reg_mask(void) {
return nullptr;
}
-const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
- return nullptr;
-}
-
// Vector calling convention not yet implemented.
bool Matcher::supports_vector_calling_convention(void) {
return false;
@@ -1644,6 +1640,10 @@ const RegMask Matcher::method_handle_invoke_SP_save_mask() {
// Should the matcher clone input 'm' of node 'n'?
bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
+ if (is_encode_and_store_pattern(n, m)) {
+ mstack.push(m, Visit);
+ return true;
+ }
return false;
}
@@ -3913,6 +3913,7 @@ instruct loadL_unaligned(iRegL dst, memory mem) %{
// Load Pointer
instruct loadP(iRegP dst, memory mem) %{
match(Set dst (LoadP mem));
+ predicate(n->as_Load()->barrier_data() == 0);
ins_cost(MEMORY_REF_COST);
size(Z_DISP3_SIZE);
format %{ "LG $dst,$mem\t # ptr" %}
@@ -3924,6 +3925,7 @@ instruct loadP(iRegP dst, memory mem) %{
// LoadP + CastP2L
instruct castP2X_loadP(iRegL dst, memory mem) %{
match(Set dst (CastP2X (LoadP mem)));
+ predicate(n->as_Load()->barrier_data() == 0);
ins_cost(MEMORY_REF_COST);
size(Z_DISP3_SIZE);
format %{ "LG $dst,$mem\t # ptr + p2x" %}
@@ -4220,28 +4222,6 @@ instruct storeB(memory mem, iRegI src) %{
ins_pipe(pipe_class_dummy);
%}
-instruct storeCM(memory mem, immI_0 src) %{
- match(Set mem (StoreCM mem src));
- ins_cost(MEMORY_REF_COST);
- // TODO: s390 port size(VARIABLE_SIZE);
- format %{ "STC(Y) $src,$mem\t # CMS card-mark byte (must be 0!)" %}
- ins_encode %{
- guarantee($mem$$index$$Register != Z_R0, "content will not be used.");
- if ($mem$$index$$Register != noreg) {
- // Can't use clear_mem --> load const zero and store character.
- __ load_const_optimized(Z_R0_scratch, (long)0);
- if (Immediate::is_uimm12($mem$$disp)) {
- __ z_stc(Z_R0_scratch, $mem$$Address);
- } else {
- __ z_stcy(Z_R0_scratch, $mem$$Address);
- }
- } else {
- __ clear_mem(Address($mem$$Address), 1);
- }
- %}
- ins_pipe(pipe_class_dummy);
-%}
-
// CHAR/SHORT
// Store Char/Short
@@ -4286,6 +4266,7 @@ instruct storeL(memory mem, iRegL src) %{
// Store Pointer
instruct storeP(memory dst, memoryRegP src) %{
match(Set dst (StoreP dst src));
+ predicate(n->as_Store()->barrier_data() == 0);
ins_cost(MEMORY_REF_COST);
size(Z_DISP3_SIZE);
format %{ "STG $src,$dst\t # ptr" %}
@@ -4388,6 +4369,7 @@ instruct memInitL(memoryRS mem, immL16 src) %{
// Move Immediate to 8-byte memory.
instruct memInitP(memoryRS mem, immP16 src) %{
match(Set mem (StoreP mem src));
+ predicate(n->as_Store()->barrier_data() == 0);
ins_cost(MEMORY_REF_COST);
size(6);
format %{ "MVGHI $mem,$src\t # direct mem init 8" %}
@@ -4417,6 +4399,7 @@ instruct negL_reg_reg(iRegL dst, immL_0 zero, iRegL src, flagsReg cr) %{
// Load narrow oop
instruct loadN(iRegN dst, memory mem) %{
match(Set dst (LoadN mem));
+ predicate(n->as_Load()->barrier_data() == 0);
ins_cost(MEMORY_REF_COST);
size(Z_DISP3_SIZE);
format %{ "LoadN $dst,$mem\t # (cOop)" %}
@@ -4480,7 +4463,7 @@ instruct loadConNKlass(iRegN dst, immNKlass src) %{
instruct decodeLoadN(iRegP dst, memory mem) %{
match(Set dst (DecodeN (LoadN mem)));
- predicate(false && (CompressedOops::base()==nullptr)&&(CompressedOops::shift()==0));
+ predicate(false && (CompressedOops::base()==nullptr) && (CompressedOops::shift()==0));
ins_cost(MEMORY_REF_COST);
size(Z_DISP3_SIZE);
format %{ "DecodeLoadN $dst,$mem\t # (cOop Load+Decode)" %}
@@ -4735,6 +4718,7 @@ instruct encodeP_NN_Ex(iRegN dst, iRegP src, flagsReg cr) %{
// Store Compressed Pointer
instruct storeN(memory mem, iRegN_P2N src) %{
match(Set mem (StoreN mem src));
+ predicate(n->as_Store()->barrier_data() == 0);
ins_cost(MEMORY_REF_COST);
size(Z_DISP_SIZE);
format %{ "ST $src,$mem\t # (cOop)" %}
@@ -5146,6 +5130,7 @@ instruct compareAndSwapL_bool(iRegP mem_ptr, rarg5RegL oldval, iRegL newval, iRe
instruct compareAndSwapP_bool(iRegP mem_ptr, rarg5RegP oldval, iRegP_N2P newval, iRegI res, flagsReg cr) %{
match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
+ predicate(n->as_LoadStore()->barrier_data() == 0);
effect(USE mem_ptr, USE_KILL oldval, KILL cr);
size(18);
format %{ "$res = CompareAndSwapP $oldval,$newval,$mem_ptr" %}
@@ -5156,6 +5141,7 @@ instruct compareAndSwapP_bool(iRegP mem_ptr, rarg5RegP oldval, iRegP_N2P newval,
instruct compareAndSwapN_bool(iRegP mem_ptr, rarg5RegN oldval, iRegN_P2N newval, iRegI res, flagsReg cr) %{
match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
+ predicate(n->as_LoadStore()->barrier_data() == 0);
effect(USE mem_ptr, USE_KILL oldval, KILL cr);
size(16);
format %{ "$res = CompareAndSwapN $oldval,$newval,$mem_ptr" %}
@@ -5443,6 +5429,7 @@ instruct xchgL_reg_mem(memoryRSY mem, iRegL dst, iRegL tmp, flagsReg cr) %{
%}
instruct xchgN_reg_mem(memoryRSY mem, iRegN dst, iRegI tmp, flagsReg cr) %{
+ predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set dst (GetAndSetN mem dst));
effect(KILL cr, TEMP tmp); // USE_DEF dst by match rule.
format %{ "XCHGN $dst,[$mem]\t # EXCHANGE (coop, atomic), temp $tmp" %}
@@ -5452,6 +5439,7 @@ instruct xchgN_reg_mem(memoryRSY mem, iRegN dst, iRegI tmp, flagsReg cr) %{
instruct xchgP_reg_mem(memoryRSY mem, iRegP dst, iRegL tmp, flagsReg cr) %{
match(Set dst (GetAndSetP mem dst));
+ predicate(n->as_LoadStore()->barrier_data() == 0);
effect(KILL cr, TEMP tmp); // USE_DEF dst by match rule.
format %{ "XCHGP $dst,[$mem]\t # EXCHANGE (oop, atomic), temp $tmp" %}
ins_encode(z_enc_SwapL(mem, dst, tmp));
@@ -5926,7 +5914,7 @@ instruct addP_regN_reg_imm20(iRegP dst, iRegP_N2P src1, iRegL src2, immL20 con)
instruct addP_mem_imm(memoryRSY mem, immL8 src, flagsReg cr) %{
match(Set mem (StoreP mem (AddP (LoadP mem) src)));
effect(KILL cr);
- predicate(VM_Version::has_MemWithImmALUOps());
+ predicate(VM_Version::has_MemWithImmALUOps() && n->as_LoadStore()->barrier_data() == 0);
ins_cost(MEMORY_REF_COST);
size(6);
format %{ "AGSI $mem,$src\t # direct mem add 8 (ptr)" %}
diff --git a/src/hotspot/cpu/s390/stubGenerator_s390.cpp b/src/hotspot/cpu/s390/stubGenerator_s390.cpp
index d878731cca51f..dd9ed4c95462b 100644
--- a/src/hotspot/cpu/s390/stubGenerator_s390.cpp
+++ b/src/hotspot/cpu/s390/stubGenerator_s390.cpp
@@ -3053,6 +3053,29 @@ class StubGenerator: public StubCodeGenerator {
return start;
}
+ // load Method* target of MethodHandle
+ // Z_ARG1 = jobject receiver
+ // Z_method = Method* result
+ address generate_upcall_stub_load_target() {
+ StubCodeMark mark(this, "StubRoutines", "upcall_stub_load_target");
+ address start = __ pc();
+
+ __ resolve_global_jobject(Z_ARG1, Z_tmp_1, Z_tmp_2);
+ // Load target method from receiver
+ __ load_heap_oop(Z_method, Address(Z_ARG1, java_lang_invoke_MethodHandle::form_offset()),
+ noreg, noreg, IS_NOT_NULL);
+ __ load_heap_oop(Z_method, Address(Z_method, java_lang_invoke_LambdaForm::vmentry_offset()),
+ noreg, noreg, IS_NOT_NULL);
+ __ load_heap_oop(Z_method, Address(Z_method, java_lang_invoke_MemberName::method_offset()),
+ noreg, noreg, IS_NOT_NULL);
+ __ z_lg(Z_method, Address(Z_method, java_lang_invoke_ResolvedMethodName::vmtarget_offset()));
+ __ z_stg(Z_method, Address(Z_thread, JavaThread::callee_target_offset())); // just in case callee is deoptimized
+
+ __ z_br(Z_R14);
+
+ return start;
+ }
+
void generate_initial_stubs() {
// Generates all stubs and initializes the entry points.
@@ -3110,6 +3133,7 @@ class StubGenerator: public StubCodeGenerator {
}
StubRoutines::_upcall_stub_exception_handler = generate_upcall_stub_exception_handler();
+ StubRoutines::_upcall_stub_load_target = generate_upcall_stub_load_target();
}
void generate_compiler_stubs() {
diff --git a/src/hotspot/cpu/s390/upcallLinker_s390.cpp b/src/hotspot/cpu/s390/upcallLinker_s390.cpp
index 734b4e89c7cb2..8baad40a519a4 100644
--- a/src/hotspot/cpu/s390/upcallLinker_s390.cpp
+++ b/src/hotspot/cpu/s390/upcallLinker_s390.cpp
@@ -23,6 +23,7 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "classfile/javaClasses.hpp"
#include "logging/logStream.hpp"
#include "memory/resourceArea.hpp"
#include "prims/upcallLinker.hpp"
@@ -116,7 +117,7 @@ static void restore_callee_saved_registers(MacroAssembler* _masm, const ABIDescr
static const int upcall_stub_code_base_size = 1024;
static const int upcall_stub_size_per_arg = 16; // arg save & restore + move
-address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
+address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature,
BasicType* out_sig_bt, int total_out_args,
BasicType ret_type,
jobject jabi, jobject jconv,
@@ -206,7 +207,6 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
__ block_comment("on_entry {");
__ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, UpcallLinker::on_entry));
__ z_aghik(Z_ARG1, Z_SP, frame_data_offset);
- __ load_const_optimized(Z_ARG2, (intptr_t)receiver);
__ call(call_target_address);
__ z_lgr(Z_thread, Z_RET);
__ block_comment("} on_entry");
@@ -216,12 +216,11 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
arg_shuffle.generate(_masm, shuffle_reg, abi._shadow_space_bytes, frame::z_jit_out_preserve_size);
__ block_comment("} argument_shuffle");
- __ block_comment("receiver {");
- __ get_vm_result(Z_ARG1);
- __ block_comment("} receiver");
-
- __ load_const_optimized(Z_method, (intptr_t)entry);
- __ z_stg(Z_method, Address(Z_thread, in_bytes(JavaThread::callee_target_offset())));
+ __ block_comment("load_target {");
+ __ load_const_optimized(Z_ARG1, (intptr_t)receiver);
+ __ load_const_optimized(call_target_address, StubRoutines::upcall_stub_load_target());
+ __ call(call_target_address); // load taget Method* into Z_method
+ __ block_comment("} load_target");
__ z_lg(call_target_address, Address(Z_method, in_bytes(Method::from_compiled_offset())));
__ call(call_target_address);
@@ -274,7 +273,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
#ifndef PRODUCT
stringStream ss;
- ss.print("upcall_stub_%s", entry->signature()->as_C_string());
+ ss.print("upcall_stub_%s", signature->as_C_string());
const char* name = _masm->code_string(ss.as_string());
#else // PRODUCT
const char* name = "upcall_stub";
diff --git a/src/hotspot/cpu/x86/assembler_x86.cpp b/src/hotspot/cpu/x86/assembler_x86.cpp
index c1679cd111f5a..e4ab99bf1c36f 100644
--- a/src/hotspot/cpu/x86/assembler_x86.cpp
+++ b/src/hotspot/cpu/x86/assembler_x86.cpp
@@ -557,6 +557,14 @@ bool Assembler::needs_rex2(Register reg1, Register reg2, Register reg3) {
return rex2;
}
+#ifndef PRODUCT
+bool Assembler::needs_evex(XMMRegister reg1, XMMRegister reg2, XMMRegister reg3) {
+ return (reg1->is_valid() && reg1->encoding() >= 16) ||
+ (reg2->is_valid() && reg2->encoding() >= 16) ||
+ (reg3->is_valid() && reg3->encoding() >= 16);
+}
+#endif
+
bool Assembler::needs_eevex(Register reg1, Register reg2, Register reg3) {
return needs_rex2(reg1, reg2, reg3);
}
@@ -3525,7 +3533,7 @@ void Assembler::vmaskmovpd(Address dst, XMMRegister src, XMMRegister mask, int v
// Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
void Assembler::evmovdqub(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
- assert(VM_Version::supports_avx512vlbw(), "");
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
attributes.set_embedded_opmask_register_specifier(mask);
attributes.set_is_evex_instruction();
@@ -3542,7 +3550,7 @@ void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
}
void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
- assert(VM_Version::supports_avx512vlbw(), "");
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionMark im(this);
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
@@ -3562,7 +3570,7 @@ void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
}
void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
- assert(VM_Version::supports_avx512vlbw(), "");
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
assert(src != xnoreg, "sanity");
InstructionMark im(this);
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
@@ -3577,13 +3585,18 @@ void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool mer
emit_operand(src, dst, 0);
}
+void Assembler::evmovdquw(XMMRegister dst, XMMRegister src, int vector_len) {
+ // Unmasked instruction
+ evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
+}
+
void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
// Unmasked instruction
evmovdquw(dst, k0, src, /*merge*/ false, vector_len);
}
void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
- assert(VM_Version::supports_avx512vlbw(), "");
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionMark im(this);
InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
@@ -3603,7 +3616,7 @@ void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
}
void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
- assert(VM_Version::supports_avx512vlbw(), "");
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
assert(src != xnoreg, "sanity");
InstructionMark im(this);
InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
@@ -3618,6 +3631,19 @@ void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, bool mer
emit_operand(src, dst, 0);
}
+void Assembler::evmovdquw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
+ attributes.set_embedded_opmask_register_specifier(mask);
+ attributes.set_is_evex_instruction();
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
+ emit_int16(0x6F, (0xC0 | encode));
+}
+
+
void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
// Unmasked instruction
evmovdqul(dst, k0, src, /*merge*/ false, vector_len);
@@ -4439,11 +4465,6 @@ void Assembler::enotl(Register dst, Register src) {
emit_int16((unsigned char)0xF7, (0xD0 | encode));
}
-void Assembler::orw(Register dst, Register src) {
- (void)prefix_and_encode(dst->encoding(), src->encoding());
- emit_arith(0x0B, 0xC0, dst, src);
-}
-
void Assembler::eorw(Register dst, Register src1, Register src2, bool no_flags) {
InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
(void) evex_prefix_and_encode_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C, &attributes, no_flags);
@@ -4738,22 +4759,6 @@ void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_l
emit_int24(0x01, (0xC0 | encode), imm8);
}
-void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
- assert(VM_Version::supports_evex(), "");
- InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
- attributes.set_is_evex_instruction();
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
- emit_int16(0x76, (0xC0 | encode));
-}
-
-void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
- assert(VM_Version::supports_avx512_vbmi(), "");
- InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
- attributes.set_is_evex_instruction();
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
- emit_int16(0x7D, (0xC0 | encode));
-}
-
void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) {
assert(VM_Version::supports_avx512_vbmi(), "");
InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
@@ -4826,6 +4831,7 @@ void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister src1, Address src2, int ve
// In this context, kdst is written the mask used to process the equal components
void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
assert(VM_Version::supports_avx512bw(), "");
+ assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_is_evex_instruction();
int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
@@ -4833,7 +4839,8 @@ void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int
}
void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
- assert(VM_Version::supports_avx512vlbw(), "");
+ assert(VM_Version::supports_avx512bw(), "");
+ assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
InstructionMark im(this);
InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
@@ -4845,7 +4852,8 @@ void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vect
}
void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
- assert(VM_Version::supports_avx512vlbw(), "");
+ assert(VM_Version::supports_avx512bw(), "");
+ assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
InstructionMark im(this);
InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
@@ -4858,16 +4866,34 @@ void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Addre
emit_operand(as_Register(dst_enc), src, 0);
}
+void Assembler::evpcmpub(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
+ assert(VM_Version::supports_avx512bw(), "");
+ assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
+ InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
+ emit_int24(0x3E, (0xC0 | encode), vcc);
+}
+
void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
- assert(VM_Version::supports_avx512vlbw(), "");
+ assert(VM_Version::supports_avx512bw(), "");
+ assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_is_evex_instruction();
int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
emit_int24(0x3E, (0xC0 | encode), vcc);
}
+void Assembler::evpcmpud(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
+ assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
+ InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
+ emit_int24(0x1E, (0xC0 | encode), vcc);
+}
+
void Assembler::evpcmpuq(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
- assert(VM_Version::supports_avx512vl(), "");
+ assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_is_evex_instruction();
int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
@@ -4875,7 +4901,8 @@ void Assembler::evpcmpuq(KRegister kdst, XMMRegister nds, XMMRegister src, Compa
}
void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
- assert(VM_Version::supports_avx512vlbw(), "");
+ assert(VM_Version::supports_avx512bw(), "");
+ assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
InstructionMark im(this);
InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
@@ -4889,6 +4916,7 @@ void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, Compariso
void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
assert(VM_Version::supports_avx512bw(), "");
+ assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
InstructionMark im(this);
InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_is_evex_instruction();
@@ -4900,7 +4928,8 @@ void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vect
}
void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
- assert(VM_Version::supports_avx512vlbw(), "");
+ assert(VM_Version::supports_avx512bw(), "");
+ assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "");
InstructionMark im(this);
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
@@ -6767,6 +6796,27 @@ void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
emit_int16((unsigned char)0xCD, (0xC0 | encode));
}
+void Assembler::sha512msg1(XMMRegister dst, XMMRegister src) {
+ assert(VM_Version::supports_sha512() && VM_Version::supports_avx(), "");
+ InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
+ emit_int16((unsigned char)0xCC, (0xC0 | encode));
+}
+
+void Assembler::sha512msg2(XMMRegister dst, XMMRegister src) {
+ assert(VM_Version::supports_sha512() && VM_Version::supports_avx(), "");
+ InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
+ emit_int16((unsigned char)0xCD, (0xC0 | encode));
+}
+
+void Assembler::sha512rnds2(XMMRegister dst, XMMRegister nds, XMMRegister src) {
+ assert(VM_Version::supports_sha512() && VM_Version::supports_avx(), "");
+ InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
+ emit_int16((unsigned char)0xCB, (0xC0 | encode));
+}
+
void Assembler::shll(Register dst, int imm8) {
assert(isShiftCount(imm8), "illegal shift count");
int encode = prefix_and_encode(dst->encoding());
@@ -8353,6 +8403,161 @@ void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector
emit_operand(dst, src, 0);
}
+void Assembler::vpaddsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xEC, (0xC0 | encode));
+}
+
+void Assembler::vpaddsb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xEC);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::vpaddsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xED, (0xC0 | encode));
+}
+
+void Assembler::vpaddsw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xED);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::vpaddusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xDC, (0xC0 | encode));
+}
+
+void Assembler::vpaddusb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xDC);
+ emit_operand(dst, src, 0);
+}
+
+
+void Assembler::vpaddusw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xDD, (0xC0 | encode));
+}
+
+void Assembler::vpaddusw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xDD);
+ emit_operand(dst, src, 0);
+}
+
+
+void Assembler::vpsubsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xE8, (0xC0 | encode));
+}
+
+void Assembler::vpsubsb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xE8);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::vpsubsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xE9, (0xC0 | encode));
+}
+
+void Assembler::vpsubsw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xE9);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xD8, (0xC0 | encode));
+}
+
+void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xD8);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::vpsubusw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xD9, (0xC0 | encode));
+}
+
+void Assembler::vpsubusw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xD9);
+ emit_operand(dst, src, 0);
+}
+
+
void Assembler::psubb(XMMRegister dst, XMMRegister src) {
NOT_LP64(assert(VM_Version::supports_sse2(), ""));
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
@@ -8382,13 +8587,6 @@ void Assembler::psubq(XMMRegister dst, XMMRegister src) {
emit_int8((0xC0 | encode));
}
-void Assembler::vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
- assert(UseAVX > 0, "requires some form of AVX");
- InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int16((unsigned char)0xD8, (0xC0 | encode));
-}
-
void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
assert(UseAVX > 0, "requires some form of AVX");
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
@@ -8518,6 +8716,15 @@ void Assembler::vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int
emit_int16((unsigned char)0xF4, (0xC0 | encode));
}
+void Assembler::vpmuldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
+ (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_evex()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_rex_vex_w_reverted();
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x28, (0xC0 | encode));
+}
+
void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
assert(UseAVX > 0, "requires some form of AVX");
InstructionMark im(this);
@@ -8565,14 +8772,6 @@ void Assembler::vpminsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int v
emit_int16(0x38, (0xC0 | encode));
}
-void Assembler::vpminub(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
- assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
- (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
- InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int16(0xDA, (0xC0 | encode));
-}
-
void Assembler::pminsw(XMMRegister dst, XMMRegister src) {
assert(VM_Version::supports_sse2(), "");
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
@@ -8718,43 +8917,383 @@ void Assembler::vmaxpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int ve
emit_int16(0x5F, (0xC0 | encode));
}
-// Shift packed integers left by specified number of bits.
-void Assembler::psllw(XMMRegister dst, int shift) {
- NOT_LP64(assert(VM_Version::supports_sse2(), ""));
- InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
- // XMM6 is for /6 encoding: 66 0F 71 /6 ib
- int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
-}
-
-void Assembler::pslld(XMMRegister dst, int shift) {
- NOT_LP64(assert(VM_Version::supports_sse2(), ""));
- InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
- // XMM6 is for /6 encoding: 66 0F 72 /6 ib
- int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
-}
-
-void Assembler::psllq(XMMRegister dst, int shift) {
- NOT_LP64(assert(VM_Version::supports_sse2(), ""));
- InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
- // XMM6 is for /6 encoding: 66 0F 73 /6 ib
- int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
+void Assembler::vpminub(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds, src) || VM_Version::supports_avx512bw(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xDA, (0xC0 | encode));
}
-void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
- NOT_LP64(assert(VM_Version::supports_sse2(), ""));
- InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
- int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int16((unsigned char)0xF1, (0xC0 | encode));
+void Assembler::vpminub(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xDA);
+ emit_operand(dst, src, 0);
}
-void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
- NOT_LP64(assert(VM_Version::supports_sse2(), ""));
- InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
- int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int16((unsigned char)0xF2, (0xC0 | encode));
+void Assembler::evpminub(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xDA, (0xC0 | encode));
+}
+
+void Assembler::evpminub(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xDA);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::vpminuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x3A, (0xC0 | encode));
+}
+
+void Assembler::vpminuw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
+ assert(!needs_evex(dst, nds) || VM_Version::supports_avx512bw(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int8((unsigned char)0x3A);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evpminuw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x3A, (0xC0 | encode));
+}
+
+void Assembler::evpminuw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int8(0x3A);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::vpminud(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x3B, (0xC0 | encode));
+}
+
+void Assembler::vpminud(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int8((unsigned char)0x3B);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evpminud(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_evex(), "");
+ assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x3B, (0xC0 | encode));
+}
+
+void Assembler::evpminud(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+ assert(VM_Version::supports_evex(), "");
+ assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int8(0x3B);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evpminuq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_evex(), "");
+ assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x3B, (0xC0 | encode));
+}
+
+void Assembler::evpminuq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+ assert(VM_Version::supports_evex(), "");
+ assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int8(0x3B);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::vpmaxub(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
+ (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xDE, (0xC0 | encode));
+}
+
+void Assembler::vpmaxub(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
+ assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
+ (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xDE);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evpmaxub(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xDE, (0xC0 | encode));
+}
+
+void Assembler::evpmaxub(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xDE);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::vpmaxuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
+ (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x3E, (0xC0 | encode));
+}
+
+void Assembler::vpmaxuw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
+ assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
+ (vector_len == AVX_256bit ? VM_Version::supports_avx2() : VM_Version::supports_avx512bw()), "");
+ assert(UseAVX > 0 && (vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int8((unsigned char)0x3E);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evpmaxuw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x3E, (0xC0 | encode));
+}
+
+void Assembler::evpmaxuw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int8(0x3E);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::vpmaxud(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(UseAVX > 0, "");
+ assert((vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds, src) || VM_Version::supports_avx512vl())), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x3F, (0xC0 | encode));
+}
+
+void Assembler::vpmaxud(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
+ assert(UseAVX > 0, "");
+ assert((vector_len == Assembler::AVX_512bit || (!needs_evex(dst, nds) || VM_Version::supports_avx512vl())), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int8((unsigned char)0x3F);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evpmaxud(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_evex(), "");
+ assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x3F, (0xC0 | encode));
+}
+
+void Assembler::evpmaxud(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+ assert(VM_Version::supports_evex(), "");
+ assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int8(0x3F);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evpmaxuq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_evex(), "");
+ assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x3F, (0xC0 | encode));
+}
+
+void Assembler::evpmaxuq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+ assert(VM_Version::supports_evex(), "");
+ assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_NObit);
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int8(0x3F);
+ emit_operand(dst, src, 0);
+}
+
+// Shift packed integers left by specified number of bits.
+void Assembler::psllw(XMMRegister dst, int shift) {
+ NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ // XMM6 is for /6 encoding: 66 0F 71 /6 ib
+ int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
+}
+
+void Assembler::pslld(XMMRegister dst, int shift) {
+ NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ // XMM6 is for /6 encoding: 66 0F 72 /6 ib
+ int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
+}
+
+void Assembler::psllq(XMMRegister dst, int shift) {
+ NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ // XMM6 is for /6 encoding: 66 0F 73 /6 ib
+ int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
+}
+
+void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
+ NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
+ int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xF1, (0xC0 | encode));
+}
+
+void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
+ NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xF2, (0xC0 | encode));
}
void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
@@ -10080,8 +10619,178 @@ void Assembler::vpunpcklqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, i
emit_int16(0x6C, (0xC0 | encode));
}
-// xmm/mem sourced byte/word/dword/qword replicate
-void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+// xmm/mem sourced byte/word/dword/qword replicate
+void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xFC, (0xC0 | encode));
+}
+
+void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+ InstructionMark im(this);
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xFC);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xFD, (0xC0 | encode));
+}
+
+void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+ InstructionMark im(this);
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xFD);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_evex(), "");
+ assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xFE, (0xC0 | encode));
+}
+
+void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+ InstructionMark im(this);
+ assert(VM_Version::supports_evex(), "");
+ assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xFE);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_evex(), "");
+ assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xD4, (0xC0 | encode));
+}
+
+void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+ InstructionMark im(this);
+ assert(VM_Version::supports_evex(), "");
+ assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xD4);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_evex(), "");
+ assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
+ emit_int16(0x58, (0xC0 | encode));
+}
+
+void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+ InstructionMark im(this);
+ assert(VM_Version::supports_evex(), "");
+ assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
+ emit_int8(0x58);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_evex(), "");
+ assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16(0x58, (0xC0 | encode));
+}
+
+void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+ InstructionMark im(this);
+ assert(VM_Version::supports_evex(), "");
+ assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8(0x58);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
attributes.set_is_evex_instruction();
@@ -10090,10 +10799,10 @@ void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMReg
attributes.reset_is_clear_context();
}
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int16((unsigned char)0xFC, (0xC0 | encode));
+ emit_int16((unsigned char)0xF8, (0xC0 | encode));
}
-void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
InstructionMark im(this);
assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
@@ -10104,11 +10813,11 @@ void Assembler::evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Addres
attributes.reset_is_clear_context();
}
vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int8((unsigned char)0xFC);
+ emit_int8((unsigned char)0xF8);
emit_operand(dst, src, 0);
}
-void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
attributes.set_is_evex_instruction();
@@ -10117,10 +10826,10 @@ void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMReg
attributes.reset_is_clear_context();
}
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int16((unsigned char)0xFD, (0xC0 | encode));
+ emit_int16((unsigned char)0xF9, (0xC0 | encode));
}
-void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
InstructionMark im(this);
assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
@@ -10131,11 +10840,11 @@ void Assembler::evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Addres
attributes.reset_is_clear_context();
}
vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int8((unsigned char)0xFD);
+ emit_int8((unsigned char)0xF9);
emit_operand(dst, src, 0);
}
-void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
assert(VM_Version::supports_evex(), "");
assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
@@ -10145,10 +10854,10 @@ void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMReg
attributes.reset_is_clear_context();
}
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int16((unsigned char)0xFE, (0xC0 | encode));
+ emit_int16((unsigned char)0xFA, (0xC0 | encode));
}
-void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
InstructionMark im(this);
assert(VM_Version::supports_evex(), "");
assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
@@ -10160,11 +10869,11 @@ void Assembler::evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Addres
attributes.reset_is_clear_context();
}
vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int8((unsigned char)0xFE);
+ emit_int8((unsigned char)0xFA);
emit_operand(dst, src, 0);
}
-void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
assert(VM_Version::supports_evex(), "");
assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
@@ -10174,10 +10883,10 @@ void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMReg
attributes.reset_is_clear_context();
}
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int16((unsigned char)0xD4, (0xC0 | encode));
+ emit_int16((unsigned char)0xFB, (0xC0 | encode));
}
-void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
InstructionMark im(this);
assert(VM_Version::supports_evex(), "");
assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
@@ -10189,11 +10898,11 @@ void Assembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Addres
attributes.reset_is_clear_context();
}
vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int8((unsigned char)0xD4);
+ emit_int8((unsigned char)0xFB);
emit_operand(dst, src, 0);
}
-void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
assert(VM_Version::supports_evex(), "");
assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
@@ -10203,10 +10912,10 @@ void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMReg
attributes.reset_is_clear_context();
}
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
- emit_int16(0x58, (0xC0 | encode));
+ emit_int16(0x5C, (0xC0 | encode));
}
-void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
InstructionMark im(this);
assert(VM_Version::supports_evex(), "");
assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
@@ -10218,11 +10927,11 @@ void Assembler::evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Addres
attributes.reset_is_clear_context();
}
vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
- emit_int8(0x58);
+ emit_int8(0x5C);
emit_operand(dst, src, 0);
}
-void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
assert(VM_Version::supports_evex(), "");
assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
@@ -10232,10 +10941,10 @@ void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMReg
attributes.reset_is_clear_context();
}
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int16(0x58, (0xC0 | encode));
+ emit_int16(0x5C, (0xC0 | encode));
}
-void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
InstructionMark im(this);
assert(VM_Version::supports_evex(), "");
assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
@@ -10247,11 +10956,11 @@ void Assembler::evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Addres
attributes.reset_is_clear_context();
}
vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int8(0x58);
+ emit_int8(0x5C);
emit_operand(dst, src, 0);
}
-void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+void Assembler::evpaddsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
attributes.set_is_evex_instruction();
@@ -10260,10 +10969,10 @@ void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMReg
attributes.reset_is_clear_context();
}
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int16((unsigned char)0xF8, (0xC0 | encode));
+ emit_int16((unsigned char)0xEC, (0xC0 | encode));
}
-void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+void Assembler::evpaddsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
InstructionMark im(this);
assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
@@ -10274,11 +10983,11 @@ void Assembler::evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Addres
attributes.reset_is_clear_context();
}
vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int8((unsigned char)0xF8);
+ emit_int8((unsigned char)0xEC);
emit_operand(dst, src, 0);
}
-void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+void Assembler::evpaddsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
attributes.set_is_evex_instruction();
@@ -10287,10 +10996,10 @@ void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMReg
attributes.reset_is_clear_context();
}
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int16((unsigned char)0xF9, (0xC0 | encode));
+ emit_int16((unsigned char)0xED, (0xC0 | encode));
}
-void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+void Assembler::evpaddsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
InstructionMark im(this);
assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
@@ -10301,13 +11010,12 @@ void Assembler::evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Addres
attributes.reset_is_clear_context();
}
vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int8((unsigned char)0xF9);
+ emit_int8((unsigned char)0xED);
emit_operand(dst, src, 0);
}
-void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
- assert(VM_Version::supports_evex(), "");
- assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+void Assembler::evpaddusb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
attributes.set_is_evex_instruction();
attributes.set_embedded_opmask_register_specifier(mask);
@@ -10315,13 +11023,12 @@ void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMReg
attributes.reset_is_clear_context();
}
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int16((unsigned char)0xFA, (0xC0 | encode));
+ emit_int16((unsigned char)0xDC, (0xC0 | encode));
}
-void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+void Assembler::evpaddusb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
InstructionMark im(this);
- assert(VM_Version::supports_evex(), "");
- assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
attributes.set_is_evex_instruction();
@@ -10330,28 +11037,26 @@ void Assembler::evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Addres
attributes.reset_is_clear_context();
}
vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int8((unsigned char)0xFA);
+ emit_int8((unsigned char)0xDC);
emit_operand(dst, src, 0);
}
-void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
- assert(VM_Version::supports_evex(), "");
- assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
- InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+void Assembler::evpaddusw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
attributes.set_is_evex_instruction();
attributes.set_embedded_opmask_register_specifier(mask);
if (merge) {
attributes.reset_is_clear_context();
}
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int16((unsigned char)0xFB, (0xC0 | encode));
+ emit_int16((unsigned char)0xDD, (0xC0 | encode));
}
-void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+void Assembler::evpaddusw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
InstructionMark im(this);
- assert(VM_Version::supports_evex(), "");
- assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
- InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
attributes.set_is_evex_instruction();
attributes.set_embedded_opmask_register_specifier(mask);
@@ -10359,27 +11064,25 @@ void Assembler::evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Addres
attributes.reset_is_clear_context();
}
vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int8((unsigned char)0xFB);
+ emit_int8((unsigned char)0xDD);
emit_operand(dst, src, 0);
}
-void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
- assert(VM_Version::supports_evex(), "");
- assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+void Assembler::evpsubsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
attributes.set_is_evex_instruction();
attributes.set_embedded_opmask_register_specifier(mask);
if (merge) {
attributes.reset_is_clear_context();
}
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
- emit_int16(0x5C, (0xC0 | encode));
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xE8, (0xC0 | encode));
}
-void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+void Assembler::evpsubsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
InstructionMark im(this);
- assert(VM_Version::supports_evex(), "");
- assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
attributes.set_is_evex_instruction();
@@ -10387,29 +11090,27 @@ void Assembler::evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Addres
if (merge) {
attributes.reset_is_clear_context();
}
- vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
- emit_int8(0x5C);
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xE8);
emit_operand(dst, src, 0);
}
-void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
- assert(VM_Version::supports_evex(), "");
- assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
- InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+void Assembler::evpsubsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
attributes.set_is_evex_instruction();
attributes.set_embedded_opmask_register_specifier(mask);
if (merge) {
attributes.reset_is_clear_context();
}
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int16(0x5C, (0xC0 | encode));
+ emit_int16((unsigned char)0xE9, (0xC0 | encode));
}
-void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+void Assembler::evpsubsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
InstructionMark im(this);
- assert(VM_Version::supports_evex(), "");
- assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
- InstructionAttr attributes(vector_len, /* vex_w */ true,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
attributes.set_is_evex_instruction();
attributes.set_embedded_opmask_register_specifier(mask);
@@ -10417,7 +11118,62 @@ void Assembler::evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Addres
attributes.reset_is_clear_context();
}
vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
- emit_int8(0x5C);
+ emit_int8((unsigned char)0xE9);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evpsubusb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xD8, (0xC0 | encode));
+}
+
+void Assembler::evpsubusb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+ InstructionMark im(this);
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xD8);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evpsubusw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xD9, (0xC0 | encode));
+}
+
+
+void Assembler::evpsubusw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+ InstructionMark im(this);
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV,/* input_size_in_bits */ EVEX_NObit);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int8((unsigned char)0xD9);
emit_operand(dst, src, 0);
}
@@ -10504,6 +11260,18 @@ void Assembler::evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, Addre
emit_operand(dst, src, 0);
}
+void Assembler::evpmulhw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false,/* legacy_mode */ false, /* no_mask_reg */ false,/* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
+ emit_int16((unsigned char)0xE5, (0xC0 | encode));
+}
+
void Assembler::evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
assert(VM_Version::supports_evex(), "");
assert(vector_len == AVX_512bit || VM_Version::supports_avx512vl(), "");
@@ -11686,6 +12454,19 @@ void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
emit_operand(dst, src, 0);
}
+void Assembler::vbroadcasti128(XMMRegister dst, Address src, int vector_len) {
+ assert(VM_Version::supports_avx2(), "");
+ assert(vector_len == AVX_256bit, "");
+ assert(dst != xnoreg, "sanity");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
+ // swap src<->dst for encoding
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int8(0x5A);
+ emit_operand(dst, src, 0);
+}
+
// scalar single/double precision replicate
// duplicate single precision data from src into programmed locations in dest : requires AVX512VL
@@ -16103,3 +16884,84 @@ void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_b
_input_size_in_bits = input_size_in_bits;
}
}
+
+void Assembler::evpermi2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(VM_Version::supports_avx512_vbmi() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x75, (0xC0 | encode));
+}
+
+void Assembler::evpermi2w(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(VM_Version::supports_avx512bw() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x75, (0xC0 | encode));
+}
+
+void Assembler::evpermi2d(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x76, (0xC0 | encode));
+}
+
+void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x76, (0xC0 | encode));
+}
+
+void Assembler::evpermi2ps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x77, (0xC0 | encode));
+}
+
+void Assembler::evpermi2pd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x77, (0xC0 | encode));
+}
+
+void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(VM_Version::supports_avx512_vbmi(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x7D, (0xC0 | encode));
+}
+
+void Assembler::evpermt2w(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(vector_len <= AVX_256bit ? VM_Version::supports_avx512vlbw() : VM_Version::supports_avx512bw(), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x7D, (0xC0 | encode));
+}
+
+void Assembler::evpermt2d(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x7E, (0xC0 | encode));
+}
+
+void Assembler::evpermt2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
+ assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
+ InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x7E, (0xC0 | encode));
+}
+
diff --git a/src/hotspot/cpu/x86/assembler_x86.hpp b/src/hotspot/cpu/x86/assembler_x86.hpp
index eace7bb9cc169..420c28254d53f 100644
--- a/src/hotspot/cpu/x86/assembler_x86.hpp
+++ b/src/hotspot/cpu/x86/assembler_x86.hpp
@@ -780,6 +780,7 @@ class Assembler : public AbstractAssembler {
bool needs_eevex(Register reg1, Register reg2 = noreg, Register reg3 = noreg);
bool needs_eevex(int enc1, int enc2 = -1, int enc3 = -1);
+ NOT_PRODUCT(bool needs_evex(XMMRegister reg1, XMMRegister reg2 = xnoreg, XMMRegister reg3 = xnoreg);)
void rex_prefix(Address adr, XMMRegister xreg,
VexSimdPrefix pre, VexOpcode opc, bool rex_w);
@@ -1756,6 +1757,7 @@ class Assembler : public AbstractAssembler {
void evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len);
void evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len);
+ void evmovdquw(XMMRegister dst, XMMRegister src, int vector_len);
void evmovdquw(XMMRegister dst, Address src, int vector_len);
void evmovdquw(Address dst, XMMRegister src, int vector_len);
void evmovdquw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len);
@@ -1901,7 +1903,6 @@ class Assembler : public AbstractAssembler {
#endif
void btq(Register dst, Register src);
- void orw(Register dst, Register src);
void eorw(Register dst, Register src1, Register src2, bool no_flags);
void orl(Address dst, int32_t imm32);
@@ -1962,9 +1963,17 @@ class Assembler : public AbstractAssembler {
void vpermilps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len);
void vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len);
+ void evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len);
+ void evpermi2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+ void evpermi2w(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+ void evpermi2d(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+ void evpermi2ps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+ void evpermi2pd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
- void evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len);
+ void evpermt2w(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+ void evpermt2d(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+ void evpermt2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void pause();
@@ -1988,9 +1997,12 @@ class Assembler : public AbstractAssembler {
void evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len);
void evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len);
+ void evpcmpub(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len);
+
void evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len);
void evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len);
+ void evpcmpud(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len);
void evpcmpuq(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len);
void pcmpeqw(XMMRegister dst, XMMRegister src);
@@ -2340,6 +2352,9 @@ class Assembler : public AbstractAssembler {
void sha256rnds2(XMMRegister dst, XMMRegister src);
void sha256msg1(XMMRegister dst, XMMRegister src);
void sha256msg2(XMMRegister dst, XMMRegister src);
+ void sha512rnds2(XMMRegister dst, XMMRegister nds, XMMRegister src);
+ void sha512msg1(XMMRegister dst, XMMRegister src);
+ void sha512msg2(XMMRegister dst, XMMRegister src);
void shldl(Register dst, Register src);
void eshldl(Register dst, Register src1, Register src2, bool no_flags);
@@ -2671,6 +2686,40 @@ class Assembler : public AbstractAssembler {
void vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
void vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
+ // Saturating packed insturctions.
+ void vpaddsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+ void vpaddsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+ void vpaddusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+ void vpaddusw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+ void evpaddsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
+ void evpaddsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
+ void evpaddusb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
+ void evpaddusw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
+ void vpsubsb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+ void vpsubsw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+ void vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+ void vpsubusw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+ void evpsubsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
+ void evpsubsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
+ void evpsubusb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
+ void evpsubusw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
+ void vpaddsb(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
+ void vpaddsw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
+ void vpaddusb(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
+ void vpaddusw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
+ void evpaddsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+ void evpaddsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+ void evpaddusb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+ void evpaddusw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+ void vpsubsb(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
+ void vpsubsw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
+ void vpsubusb(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
+ void vpsubusw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
+ void evpsubsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+ void evpsubsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+ void evpsubusb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+ void evpsubusw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+
// Leaf level assembler routines for masked operations.
void evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
void evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
@@ -2696,6 +2745,7 @@ class Assembler : public AbstractAssembler {
void evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
void evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
void evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+ void evpmulhw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
void evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
void evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
void evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
@@ -2814,7 +2864,6 @@ class Assembler : public AbstractAssembler {
void psubw(XMMRegister dst, XMMRegister src);
void psubd(XMMRegister dst, XMMRegister src);
void psubq(XMMRegister dst, XMMRegister src);
- void vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
@@ -2832,6 +2881,7 @@ class Assembler : public AbstractAssembler {
void vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void evpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
+ void vpmuldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
void vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
void evpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
@@ -2840,7 +2890,6 @@ class Assembler : public AbstractAssembler {
// Minimum of packed integers
void pminsb(XMMRegister dst, XMMRegister src);
void vpminsb(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
- void vpminub(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
void pminsw(XMMRegister dst, XMMRegister src);
void vpminsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
void pminsd(XMMRegister dst, XMMRegister src);
@@ -2864,6 +2913,38 @@ class Assembler : public AbstractAssembler {
void maxpd(XMMRegister dst, XMMRegister src);
void vmaxpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
+ // Unsigned maximum packed integers.
+ void vpmaxub(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
+ void vpmaxuw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
+ void vpmaxud(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
+ void vpmaxub(XMMRegister dst, XMMRegister src1, Address src2, int vector_len);
+ void vpmaxuw(XMMRegister dst, XMMRegister src1, Address src2, int vector_len);
+ void vpmaxud(XMMRegister dst, XMMRegister src1, Address src2, int vector_len);
+ void evpmaxub(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
+ void evpmaxuw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
+ void evpmaxud(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
+ void evpmaxuq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
+ void evpmaxub(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+ void evpmaxuw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+ void evpmaxud(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+ void evpmaxuq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+
+ // Unsigned minimum packed integers.
+ void vpminub(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
+ void vpminuw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
+ void vpminud(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
+ void vpminub(XMMRegister dst, XMMRegister src1, Address src2, int vector_len);
+ void vpminuw(XMMRegister dst, XMMRegister src1, Address src2, int vector_len);
+ void vpminud(XMMRegister dst, XMMRegister src1, Address src2, int vector_len);
+ void evpminub(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
+ void evpminuw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
+ void evpminud(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
+ void evpminuq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
+ void evpminub(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+ void evpminuw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+ void evpminud(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+ void evpminuq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+
// Shift left packed integers
void psllw(XMMRegister dst, int shift);
void pslld(XMMRegister dst, int shift);
@@ -3030,6 +3111,7 @@ class Assembler : public AbstractAssembler {
void evbroadcasti32x4(XMMRegister dst, Address src, int vector_len);
void evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len);
void evbroadcasti64x2(XMMRegister dst, Address src, int vector_len);
+ void vbroadcasti128(XMMRegister dst, Address src, int vector_len);
// scalar single/double/128bit precision replicate
void vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len);
diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
index c3444d5a5abce..64265a9690940 100644
--- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
@@ -1333,10 +1333,7 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch
}
#endif
- if (!(UseZGC && !ZGenerational)) {
- // Load barrier has not yet been applied, so ZGC can't verify the oop here
- __ verify_oop(dest->as_register());
- }
+ __ verify_oop(dest->as_register());
}
}
@@ -1578,6 +1575,7 @@ void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
if (op->init_check()) {
add_debug_info_for_null_check_here(op->stub()->info());
+ // init_state needs acquire, but x86 is TSO, and so we are already good.
__ cmpb(Address(op->klass()->as_register(),
InstanceKlass::init_state_offset()),
InstanceKlass::fully_initialized);
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
index 839745f76ec6a..61c8036b1cec9 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
@@ -477,9 +477,9 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
// StoreLoad achieves this.
membar(StoreLoad);
- // Check if the entry lists are empty.
- movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
- orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
+ // Check if the entry lists are empty (EntryList first - by convention).
+ movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
+ orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
jccb(Assembler::zero, LSuccess); // If so we are done.
// Check if there is a successor.
@@ -806,9 +806,9 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register reg_rax,
// StoreLoad achieves this.
membar(StoreLoad);
- // Check if the entry lists are empty.
- movptr(reg_rax, cxq_address);
- orptr(reg_rax, EntryList_address);
+ // Check if the entry lists are empty (EntryList first - by convention).
+ movptr(reg_rax, EntryList_address);
+ orptr(reg_rax, cxq_address);
jccb(Assembler::zero, unlocked); // If so we are done.
// Check if there is a successor.
@@ -822,7 +822,7 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register reg_rax,
}
movptr(Address(thread, JavaThread::unlocked_inflated_monitor_offset()), monitor);
- testl(monitor, monitor); // Fast Unlock ZF = 0
+ orl(t, 1); // Fast Unlock ZF = 0
jmpb(slow_path);
// Recursive unlock.
@@ -837,8 +837,9 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register reg_rax,
#ifdef ASSERT
// Check that unlocked label is reached with ZF set.
Label zf_correct;
+ Label zf_bad_zero;
jcc(Assembler::zero, zf_correct);
- stop("Fast Unlock ZF != 1");
+ jmp(zf_bad_zero);
#endif
bind(slow_path);
@@ -847,8 +848,10 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register reg_rax,
}
#ifdef ASSERT
// Check that stub->continuation() label is reached with ZF not set.
- jccb(Assembler::notZero, zf_correct);
+ jcc(Assembler::notZero, zf_correct);
stop("Fast Unlock ZF != 0");
+ bind(zf_bad_zero);
+ stop("Fast Unlock ZF != 1");
bind(zf_correct);
#endif
// C2 uses the value of ZF to determine the continuation.
@@ -936,6 +939,72 @@ void C2_MacroAssembler::pminmax(int opcode, BasicType elem_bt, XMMRegister dst,
}
}
+void C2_MacroAssembler::vpuminmax(int opcode, BasicType elem_bt, XMMRegister dst,
+ XMMRegister src1, Address src2, int vlen_enc) {
+ assert(opcode == Op_UMinV || opcode == Op_UMaxV, "sanity");
+ if (opcode == Op_UMinV) {
+ switch(elem_bt) {
+ case T_BYTE: vpminub(dst, src1, src2, vlen_enc); break;
+ case T_SHORT: vpminuw(dst, src1, src2, vlen_enc); break;
+ case T_INT: vpminud(dst, src1, src2, vlen_enc); break;
+ case T_LONG: evpminuq(dst, k0, src1, src2, false, vlen_enc); break;
+ default: fatal("Unsupported type %s", type2name(elem_bt)); break;
+ }
+ } else {
+ assert(opcode == Op_UMaxV, "required");
+ switch(elem_bt) {
+ case T_BYTE: vpmaxub(dst, src1, src2, vlen_enc); break;
+ case T_SHORT: vpmaxuw(dst, src1, src2, vlen_enc); break;
+ case T_INT: vpmaxud(dst, src1, src2, vlen_enc); break;
+ case T_LONG: evpmaxuq(dst, k0, src1, src2, false, vlen_enc); break;
+ default: fatal("Unsupported type %s", type2name(elem_bt)); break;
+ }
+ }
+}
+
+void C2_MacroAssembler::vpuminmaxq(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc) {
+ // T1 = -1
+ vpcmpeqq(xtmp1, xtmp1, xtmp1, vlen_enc);
+ // T1 = -1 << 63
+ vpsllq(xtmp1, xtmp1, 63, vlen_enc);
+ // Convert SRC2 to signed value i.e. T2 = T1 + SRC2
+ vpaddq(xtmp2, xtmp1, src2, vlen_enc);
+ // Convert SRC1 to signed value i.e. T1 = T1 + SRC1
+ vpaddq(xtmp1, xtmp1, src1, vlen_enc);
+ // Mask = T2 > T1
+ vpcmpgtq(xtmp1, xtmp2, xtmp1, vlen_enc);
+ if (opcode == Op_UMaxV) {
+ // Res = Mask ? Src2 : Src1
+ vpblendvb(dst, src1, src2, xtmp1, vlen_enc);
+ } else {
+ // Res = Mask ? Src1 : Src2
+ vpblendvb(dst, src2, src1, xtmp1, vlen_enc);
+ }
+}
+
+void C2_MacroAssembler::vpuminmax(int opcode, BasicType elem_bt, XMMRegister dst,
+ XMMRegister src1, XMMRegister src2, int vlen_enc) {
+ assert(opcode == Op_UMinV || opcode == Op_UMaxV, "sanity");
+ if (opcode == Op_UMinV) {
+ switch(elem_bt) {
+ case T_BYTE: vpminub(dst, src1, src2, vlen_enc); break;
+ case T_SHORT: vpminuw(dst, src1, src2, vlen_enc); break;
+ case T_INT: vpminud(dst, src1, src2, vlen_enc); break;
+ case T_LONG: evpminuq(dst, k0, src1, src2, false, vlen_enc); break;
+ default: fatal("Unsupported type %s", type2name(elem_bt)); break;
+ }
+ } else {
+ assert(opcode == Op_UMaxV, "required");
+ switch(elem_bt) {
+ case T_BYTE: vpmaxub(dst, src1, src2, vlen_enc); break;
+ case T_SHORT: vpmaxuw(dst, src1, src2, vlen_enc); break;
+ case T_INT: vpmaxud(dst, src1, src2, vlen_enc); break;
+ case T_LONG: evpmaxuq(dst, k0, src1, src2, false, vlen_enc); break;
+ default: fatal("Unsupported type %s", type2name(elem_bt)); break;
+ }
+ }
+}
+
void C2_MacroAssembler::vpminmax(int opcode, BasicType elem_bt,
XMMRegister dst, XMMRegister src1, XMMRegister src2,
int vlen_enc) {
@@ -2359,6 +2428,10 @@ void C2_MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, X
MacroAssembler::evmovdqu(type, kmask, dst, src, merge, vector_len);
}
+void C2_MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, XMMRegister src, bool merge, int vector_len) {
+ MacroAssembler::evmovdqu(type, kmask, dst, src, merge, vector_len);
+}
+
void C2_MacroAssembler::vmovmask(BasicType elem_bt, XMMRegister dst, Address src, XMMRegister mask,
int vec_enc) {
switch(elem_bt) {
@@ -2657,7 +2730,6 @@ void C2_MacroAssembler::vectortest(BasicType bt, XMMRegister src1, XMMRegister s
}
void C2_MacroAssembler::vpadd(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc) {
- assert(UseAVX >= 2, "required");
#ifdef ASSERT
bool is_bw = ((elem_bt == T_BYTE) || (elem_bt == T_SHORT));
bool is_bw_supported = VM_Version::supports_avx512bw();
@@ -4631,7 +4703,126 @@ void C2_MacroAssembler::evmasked_op(int ideal_opc, BasicType eType, KRegister ma
case Op_RotateLeftV:
evrold(eType, dst, mask, src1, imm8, merge, vlen_enc); break;
default:
- fatal("Unsupported masked operation"); break;
+ fatal("Unsupported operation %s", NodeClassNames[ideal_opc]);
+ break;
+ }
+}
+
+void C2_MacroAssembler::evmasked_saturating_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1,
+ XMMRegister src2, bool is_unsigned, bool merge, int vlen_enc) {
+ if (is_unsigned) {
+ evmasked_saturating_unsigned_op(ideal_opc, elem_bt, mask, dst, src1, src2, merge, vlen_enc);
+ } else {
+ evmasked_saturating_signed_op(ideal_opc, elem_bt, mask, dst, src1, src2, merge, vlen_enc);
+ }
+}
+
+void C2_MacroAssembler::evmasked_saturating_signed_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst,
+ XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
+ switch (elem_bt) {
+ case T_BYTE:
+ if (ideal_opc == Op_SaturatingAddV) {
+ evpaddsb(dst, mask, src1, src2, merge, vlen_enc);
+ } else {
+ assert(ideal_opc == Op_SaturatingSubV, "");
+ evpsubsb(dst, mask, src1, src2, merge, vlen_enc);
+ }
+ break;
+ case T_SHORT:
+ if (ideal_opc == Op_SaturatingAddV) {
+ evpaddsw(dst, mask, src1, src2, merge, vlen_enc);
+ } else {
+ assert(ideal_opc == Op_SaturatingSubV, "");
+ evpsubsw(dst, mask, src1, src2, merge, vlen_enc);
+ }
+ break;
+ default:
+ fatal("Unsupported type %s", type2name(elem_bt));
+ break;
+ }
+}
+
+void C2_MacroAssembler::evmasked_saturating_unsigned_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst,
+ XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
+ switch (elem_bt) {
+ case T_BYTE:
+ if (ideal_opc == Op_SaturatingAddV) {
+ evpaddusb(dst, mask, src1, src2, merge, vlen_enc);
+ } else {
+ assert(ideal_opc == Op_SaturatingSubV, "");
+ evpsubusb(dst, mask, src1, src2, merge, vlen_enc);
+ }
+ break;
+ case T_SHORT:
+ if (ideal_opc == Op_SaturatingAddV) {
+ evpaddusw(dst, mask, src1, src2, merge, vlen_enc);
+ } else {
+ assert(ideal_opc == Op_SaturatingSubV, "");
+ evpsubusw(dst, mask, src1, src2, merge, vlen_enc);
+ }
+ break;
+ default:
+ fatal("Unsupported type %s", type2name(elem_bt));
+ break;
+ }
+}
+
+void C2_MacroAssembler::evmasked_saturating_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1,
+ Address src2, bool is_unsigned, bool merge, int vlen_enc) {
+ if (is_unsigned) {
+ evmasked_saturating_unsigned_op(ideal_opc, elem_bt, mask, dst, src1, src2, merge, vlen_enc);
+ } else {
+ evmasked_saturating_signed_op(ideal_opc, elem_bt, mask, dst, src1, src2, merge, vlen_enc);
+ }
+}
+
+void C2_MacroAssembler::evmasked_saturating_signed_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst,
+ XMMRegister src1, Address src2, bool merge, int vlen_enc) {
+ switch (elem_bt) {
+ case T_BYTE:
+ if (ideal_opc == Op_SaturatingAddV) {
+ evpaddsb(dst, mask, src1, src2, merge, vlen_enc);
+ } else {
+ assert(ideal_opc == Op_SaturatingSubV, "");
+ evpsubsb(dst, mask, src1, src2, merge, vlen_enc);
+ }
+ break;
+ case T_SHORT:
+ if (ideal_opc == Op_SaturatingAddV) {
+ evpaddsw(dst, mask, src1, src2, merge, vlen_enc);
+ } else {
+ assert(ideal_opc == Op_SaturatingSubV, "");
+ evpsubsw(dst, mask, src1, src2, merge, vlen_enc);
+ }
+ break;
+ default:
+ fatal("Unsupported type %s", type2name(elem_bt));
+ break;
+ }
+}
+
+void C2_MacroAssembler::evmasked_saturating_unsigned_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst,
+ XMMRegister src1, Address src2, bool merge, int vlen_enc) {
+ switch (elem_bt) {
+ case T_BYTE:
+ if (ideal_opc == Op_SaturatingAddV) {
+ evpaddusb(dst, mask, src1, src2, merge, vlen_enc);
+ } else {
+ assert(ideal_opc == Op_SaturatingSubV, "");
+ evpsubusb(dst, mask, src1, src2, merge, vlen_enc);
+ }
+ break;
+ case T_SHORT:
+ if (ideal_opc == Op_SaturatingAddV) {
+ evpaddusw(dst, mask, src1, src2, merge, vlen_enc);
+ } else {
+ assert(ideal_opc == Op_SaturatingSubV, "");
+ evpsubusw(dst, mask, src1, src2, merge, vlen_enc);
+ }
+ break;
+ default:
+ fatal("Unsupported type %s", type2name(elem_bt));
+ break;
}
}
@@ -4721,6 +4912,10 @@ void C2_MacroAssembler::evmasked_op(int ideal_opc, BasicType eType, KRegister ma
evpmaxs(eType, dst, mask, src1, src2, merge, vlen_enc); break;
case Op_MinV:
evpmins(eType, dst, mask, src1, src2, merge, vlen_enc); break;
+ case Op_UMinV:
+ evpminu(eType, dst, mask, src1, src2, merge, vlen_enc); break;
+ case Op_UMaxV:
+ evpmaxu(eType, dst, mask, src1, src2, merge, vlen_enc); break;
case Op_XorV:
evxor(eType, dst, mask, src1, src2, merge, vlen_enc); break;
case Op_OrV:
@@ -4728,7 +4923,8 @@ void C2_MacroAssembler::evmasked_op(int ideal_opc, BasicType eType, KRegister ma
case Op_AndV:
evand(eType, dst, mask, src1, src2, merge, vlen_enc); break;
default:
- fatal("Unsupported masked operation"); break;
+ fatal("Unsupported operation %s", NodeClassNames[ideal_opc]);
+ break;
}
}
@@ -4781,6 +4977,10 @@ void C2_MacroAssembler::evmasked_op(int ideal_opc, BasicType eType, KRegister ma
evpmaxs(eType, dst, mask, src1, src2, merge, vlen_enc); break;
case Op_MinV:
evpmins(eType, dst, mask, src1, src2, merge, vlen_enc); break;
+ case Op_UMaxV:
+ evpmaxu(eType, dst, mask, src1, src2, merge, vlen_enc); break;
+ case Op_UMinV:
+ evpminu(eType, dst, mask, src1, src2, merge, vlen_enc); break;
case Op_XorV:
evxor(eType, dst, mask, src1, src2, merge, vlen_enc); break;
case Op_OrV:
@@ -4788,7 +4988,8 @@ void C2_MacroAssembler::evmasked_op(int ideal_opc, BasicType eType, KRegister ma
case Op_AndV:
evand(eType, dst, mask, src1, src2, merge, vlen_enc); break;
default:
- fatal("Unsupported masked operation"); break;
+ fatal("Unsupported operation %s", NodeClassNames[ideal_opc]);
+ break;
}
}
@@ -6475,3 +6676,409 @@ void C2_MacroAssembler::vector_rearrange_int_float(BasicType bt, XMMRegister dst
vpermps(dst, shuffle, src, vlen_enc);
}
}
+
+void C2_MacroAssembler::vector_saturating_op(int ideal_opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc) {
+ switch(elem_bt) {
+ case T_BYTE:
+ if (ideal_opc == Op_SaturatingAddV) {
+ vpaddsb(dst, src1, src2, vlen_enc);
+ } else {
+ assert(ideal_opc == Op_SaturatingSubV, "");
+ vpsubsb(dst, src1, src2, vlen_enc);
+ }
+ break;
+ case T_SHORT:
+ if (ideal_opc == Op_SaturatingAddV) {
+ vpaddsw(dst, src1, src2, vlen_enc);
+ } else {
+ assert(ideal_opc == Op_SaturatingSubV, "");
+ vpsubsw(dst, src1, src2, vlen_enc);
+ }
+ break;
+ default:
+ fatal("Unsupported type %s", type2name(elem_bt));
+ break;
+ }
+}
+
+void C2_MacroAssembler::vector_saturating_unsigned_op(int ideal_opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc) {
+ switch(elem_bt) {
+ case T_BYTE:
+ if (ideal_opc == Op_SaturatingAddV) {
+ vpaddusb(dst, src1, src2, vlen_enc);
+ } else {
+ assert(ideal_opc == Op_SaturatingSubV, "");
+ vpsubusb(dst, src1, src2, vlen_enc);
+ }
+ break;
+ case T_SHORT:
+ if (ideal_opc == Op_SaturatingAddV) {
+ vpaddusw(dst, src1, src2, vlen_enc);
+ } else {
+ assert(ideal_opc == Op_SaturatingSubV, "");
+ vpsubusw(dst, src1, src2, vlen_enc);
+ }
+ break;
+ default:
+ fatal("Unsupported type %s", type2name(elem_bt));
+ break;
+ }
+}
+
+void C2_MacroAssembler::vector_sub_dq_saturating_unsigned_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1,
+ XMMRegister src2, KRegister ktmp, int vlen_enc) {
+ // For unsigned subtraction, overflow happens when magnitude of second input is greater than first input.
+ // overflow_mask = Inp1 Inp1 + MIN_VALUE < Inp2 + MIN_VALUE
+ vpgenmin_value(elem_bt, xtmp1, xtmp1, vlen_enc, true);
+ vpadd(elem_bt, xtmp2, src1, xtmp1, vlen_enc);
+ vpadd(elem_bt, xtmp1, src2, xtmp1, vlen_enc);
+
+ vpcmpgt(elem_bt, xtmp2, xtmp1, xtmp2, vlen_enc);
+
+ // Res = INP1 - INP2 (non-commutative and non-associative)
+ vpsub(elem_bt, dst, src1, src2, vlen_enc);
+ // Res = Mask ? Zero : Res
+ vpxor(xtmp1, xtmp1, xtmp1, vlen_enc);
+ vpblendvb(dst, dst, xtmp1, xtmp2, vlen_enc);
+}
+
+void C2_MacroAssembler::vector_add_dq_saturating_unsigned_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp, int vlen_enc) {
+ // Unsigned values ranges comprise of only +ve numbers, thus there exist only an upper bound saturation.
+ // overflow_mask = (SRC1 + SRC2) >> 31 == 1
+//
+// We empirically determined its semantic equivalence to following reduced expression
+// overflow_mask = (a + b) = Res + MIN_VALUE
+ vpadd(elem_bt, xtmp2, xtmp2, dst, vlen_enc);
+ // Compute overflow detection mask = Res<1> >> 31(I)/63(L)) == 1
+ vpxor(xtmp1, dst, src1, vlen_enc);
+ vpxor(xtmp2, dst, src2, vlen_enc);
+ vpand(xtmp2, xtmp1, xtmp2, vlen_enc);
+ } else {
+ assert(ideal_opc == Op_SaturatingSubV, "");
+ // res = src1 - src2
+ vpsub(elem_bt, dst, src1, src2, vlen_enc);
+ // Overflow occurs when both inputs have opposite polarity and
+ // result polarity does not comply with first input polarity.
+ // overflow = ((src1 ^ src2) & (res ^ src1) >>> 31(I)/63(L)) == 1;
+ vpxor(xtmp1, src1, src2, vlen_enc);
+ vpxor(xtmp2, dst, src1, vlen_enc);
+ vpand(xtmp2, xtmp1, xtmp2, vlen_enc);
+ }
+
+ // Compute overflow detection mask.
+ evpmov_vec_to_mask(elem_bt, ktmp1, xtmp2, xtmp2, xtmp1, vlen_enc);
+ // Note: xtmp1 hold -1 in all its lanes after above call.
+
+ // Compute mask based on first input polarity.
+ evpmov_vec_to_mask(elem_bt, ktmp2, src1, xtmp2, xtmp1, vlen_enc, true);
+
+ vpgenmax_value(elem_bt, xtmp2, xtmp1, vlen_enc, true);
+ vpgenmin_value(elem_bt, xtmp1, xtmp1, vlen_enc);
+
+ // Compose a vector of saturating (MAX/MIN) values, where lanes corresponding to
+ // set bits in first input polarity mask holds a min value.
+ evpblend(elem_bt, xtmp2, ktmp2, xtmp2, xtmp1, true, vlen_enc);
+ // Blend destination lanes with saturated values using overflow detection mask.
+ evpblend(elem_bt, dst, ktmp1, dst, xtmp2, true, vlen_enc);
+}
+
+
+void C2_MacroAssembler::vector_addsub_dq_saturating_avx(int ideal_opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1,
+ XMMRegister src2, XMMRegister xtmp1, XMMRegister xtmp2,
+ XMMRegister xtmp3, XMMRegister xtmp4, int vlen_enc) {
+ assert(elem_bt == T_INT || elem_bt == T_LONG, "");
+ // Addition/Subtraction happens over two's compliment representation of numbers and is agnostic to signed'ness.
+ // Overflow detection based on Hacker's delight section 2-13.
+ if (ideal_opc == Op_SaturatingAddV) {
+ // res = src1 + src2
+ vpadd(elem_bt, dst, src1, src2, vlen_enc);
+ // Overflow occurs if result polarity does not comply with equivalent polarity inputs.
+ // overflow = (((res ^ src1) & (res ^ src2)) >>> 31(I)/63(L)) == 1
+ vpxor(xtmp1, dst, src1, vlen_enc);
+ vpxor(xtmp2, dst, src2, vlen_enc);
+ vpand(xtmp2, xtmp1, xtmp2, vlen_enc);
+ } else {
+ assert(ideal_opc == Op_SaturatingSubV, "");
+ // res = src1 - src2
+ vpsub(elem_bt, dst, src1, src2, vlen_enc);
+ // Overflow occurs when both inputs have opposite polarity and
+ // result polarity does not comply with first input polarity.
+ // overflow = ((src1 ^ src2) & (res ^ src1) >>> 31(I)/63(L)) == 1;
+ vpxor(xtmp1, src1, src2, vlen_enc);
+ vpxor(xtmp2, dst, src1, vlen_enc);
+ vpand(xtmp2, xtmp1, xtmp2, vlen_enc);
+ }
+
+ // Sign-extend to compute overflow detection mask.
+ vpsign_extend_dq(elem_bt, xtmp3, xtmp2, vlen_enc);
+
+ vpcmpeqd(xtmp1, xtmp1, xtmp1, vlen_enc);
+ vpgenmax_value(elem_bt, xtmp2, xtmp1, vlen_enc);
+ vpgenmin_value(elem_bt, xtmp1, xtmp1, vlen_enc);
+
+ // Compose saturating min/max vector using first input polarity mask.
+ vpsign_extend_dq(elem_bt, xtmp4, src1, vlen_enc);
+ vpblendvb(xtmp1, xtmp2, xtmp1, xtmp4, vlen_enc);
+
+ // Blend result with saturating vector using overflow detection mask.
+ vpblendvb(dst, dst, xtmp1, xtmp3, vlen_enc);
+}
+
+void C2_MacroAssembler::vector_saturating_op(int ideal_opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, Address src2, int vlen_enc) {
+ switch(elem_bt) {
+ case T_BYTE:
+ if (ideal_opc == Op_SaturatingAddV) {
+ vpaddsb(dst, src1, src2, vlen_enc);
+ } else {
+ assert(ideal_opc == Op_SaturatingSubV, "");
+ vpsubsb(dst, src1, src2, vlen_enc);
+ }
+ break;
+ case T_SHORT:
+ if (ideal_opc == Op_SaturatingAddV) {
+ vpaddsw(dst, src1, src2, vlen_enc);
+ } else {
+ assert(ideal_opc == Op_SaturatingSubV, "");
+ vpsubsw(dst, src1, src2, vlen_enc);
+ }
+ break;
+ default:
+ fatal("Unsupported type %s", type2name(elem_bt));
+ break;
+ }
+}
+
+void C2_MacroAssembler::vector_saturating_unsigned_op(int ideal_opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, Address src2, int vlen_enc) {
+ switch(elem_bt) {
+ case T_BYTE:
+ if (ideal_opc == Op_SaturatingAddV) {
+ vpaddusb(dst, src1, src2, vlen_enc);
+ } else {
+ assert(ideal_opc == Op_SaturatingSubV, "");
+ vpsubusb(dst, src1, src2, vlen_enc);
+ }
+ break;
+ case T_SHORT:
+ if (ideal_opc == Op_SaturatingAddV) {
+ vpaddusw(dst, src1, src2, vlen_enc);
+ } else {
+ assert(ideal_opc == Op_SaturatingSubV, "");
+ vpsubusw(dst, src1, src2, vlen_enc);
+ }
+ break;
+ default:
+ fatal("Unsupported type %s", type2name(elem_bt));
+ break;
+ }
+}
+
+void C2_MacroAssembler::select_from_two_vectors_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1,
+ XMMRegister src2, int vlen_enc) {
+ switch(elem_bt) {
+ case T_BYTE:
+ evpermi2b(dst, src1, src2, vlen_enc);
+ break;
+ case T_SHORT:
+ evpermi2w(dst, src1, src2, vlen_enc);
+ break;
+ case T_INT:
+ evpermi2d(dst, src1, src2, vlen_enc);
+ break;
+ case T_LONG:
+ evpermi2q(dst, src1, src2, vlen_enc);
+ break;
+ case T_FLOAT:
+ evpermi2ps(dst, src1, src2, vlen_enc);
+ break;
+ case T_DOUBLE:
+ evpermi2pd(dst, src1, src2, vlen_enc);
+ break;
+ default:
+ fatal("Unsupported type %s", type2name(elem_bt));
+ break;
+ }
+}
+
+void C2_MacroAssembler::vector_saturating_op(int ideal_opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, bool is_unsigned, int vlen_enc) {
+ if (is_unsigned) {
+ vector_saturating_unsigned_op(ideal_opc, elem_bt, dst, src1, src2, vlen_enc);
+ } else {
+ vector_saturating_op(ideal_opc, elem_bt, dst, src1, src2, vlen_enc);
+ }
+}
+
+void C2_MacroAssembler::vector_saturating_op(int ideal_opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, Address src2, bool is_unsigned, int vlen_enc) {
+ if (is_unsigned) {
+ vector_saturating_unsigned_op(ideal_opc, elem_bt, dst, src1, src2, vlen_enc);
+ } else {
+ vector_saturating_op(ideal_opc, elem_bt, dst, src1, src2, vlen_enc);
+ }
+}
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
index af57546b3d143..3a36fd75e3f86 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
@@ -56,10 +56,21 @@
XMMRegister dst, XMMRegister src1, XMMRegister src2,
int vlen_enc);
+ void vpuminmax(int opcode, BasicType elem_bt,
+ XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ int vlen_enc);
+
+ void vpuminmax(int opcode, BasicType elem_bt,
+ XMMRegister dst, XMMRegister src1, Address src2,
+ int vlen_enc);
+
void vminmax_fp(int opcode, BasicType elem_bt,
XMMRegister dst, XMMRegister a, XMMRegister b,
XMMRegister tmp, XMMRegister atmp, XMMRegister btmp,
int vlen_enc);
+
+ void vpuminmaxq(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc);
+
void evminmax_fp(int opcode, BasicType elem_bt,
XMMRegister dst, XMMRegister a, XMMRegister b,
KRegister ktmp, XMMRegister atmp, XMMRegister btmp,
@@ -105,6 +116,7 @@
void evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len);
void evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len);
+ void evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, XMMRegister src, bool merge, int vector_len);
// extract
void extract(BasicType typ, Register dst, XMMRegister src, int idx);
@@ -505,4 +517,70 @@
void vgather8b_offset(BasicType elem_bt, XMMRegister dst, Register base, Register idx_base,
Register offset, Register rtmp, int vlen_enc);
+ void vector_saturating_op(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, bool is_unsigned, int vlen_enc);
+
+ void vector_saturating_op(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, Address src2, bool is_unsigned, int vlen_enc);
+
+ void vector_saturating_op(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc);
+
+ void vector_saturating_op(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, Address src2, int vlen_enc);
+
+ void vector_saturating_unsigned_op(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc);
+
+ void vector_saturating_unsigned_op(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, Address src2, int vlen_enc);
+
+ void vector_sub_dq_saturating_unsigned_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, KRegister ktmp, int vlen_enc);
+
+ void vector_sub_dq_saturating_unsigned_avx(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc);
+
+ void vector_add_dq_saturating_unsigned_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp, int vlen_enc);
+
+ void vector_add_dq_saturating_unsigned_avx(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, int vlen_enc);
+
+ void vector_addsub_dq_saturating_avx(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4, int vlen_enc);
+
+ void vector_addsub_dq_saturating_evex(int opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, int vlen_enc);
+
+ void evpmovd2m_emu(KRegister ktmp, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc, bool xtmp2_hold_M1 = false);
+
+ void evpmovq2m_emu(KRegister ktmp, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc, bool xtmp2_hold_M1 = false);
+
+ void vpsign_extend_dq(BasicType etype, XMMRegister dst, XMMRegister src, int vlen_enc);
+
+ void vpgenmin_value(BasicType etype, XMMRegister dst, XMMRegister allones, int vlen_enc, bool compute_allones = false);
+
+ void vpgenmax_value(BasicType etype, XMMRegister dst, XMMRegister allones, int vlen_enc, bool compute_allones = false);
+
+ void evpcmpu(BasicType etype, KRegister kmask, XMMRegister src1, XMMRegister src2, Assembler::ComparisonPredicate cond, int vlen_enc);
+
+ void vpcmpgt(BasicType etype, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc);
+
+ void evpmov_vec_to_mask(BasicType etype, KRegister ktmp, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
+ int vlen_enc, bool xtmp2_hold_M1 = false);
+
+ void evmasked_saturating_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ bool is_unsigned, bool merge, int vlen_enc);
+
+ void evmasked_saturating_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1, Address src2,
+ bool is_unsigned, bool merge, int vlen_enc);
+
+ void evmasked_saturating_signed_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ bool merge, int vlen_enc);
+
+ void evmasked_saturating_signed_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1, Address src2,
+ bool merge, int vlen_enc);
+
+ void evmasked_saturating_unsigned_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1,
+ XMMRegister src2, bool merge, int vlen_enc);
+
+ void evmasked_saturating_unsigned_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1,
+ Address src2, bool merge, int vlen_enc);
+
+ void select_from_two_vectors_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc);
+
#endif // CPU_X86_C2_MACROASSEMBLER_X86_HPP
diff --git a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp
index b52be627776b8..b6be4012519a0 100644
--- a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp
@@ -38,7 +38,10 @@
#include "c1/c1_LIRAssembler.hpp"
#include "c1/c1_MacroAssembler.hpp"
#include "gc/g1/c1/g1BarrierSetC1.hpp"
-#endif
+#endif // COMPILER1
+#ifdef COMPILER2
+#include "gc/g1/c2/g1BarrierSetC2.hpp"
+#endif // COMPILER2
#define __ masm->
@@ -160,6 +163,56 @@ void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorator
}
}
+static void generate_queue_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
+ const Register thread, const Register value, const Register temp) {
+ // This code assumes that buffer index is pointer sized.
+ STATIC_ASSERT(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t));
+ // Can we store a value in the given thread's buffer?
+ // (The index field is typed as size_t.)
+ __ movptr(temp, Address(thread, in_bytes(index_offset))); // temp := *(index address)
+ __ testptr(temp, temp); // index == 0?
+ __ jcc(Assembler::zero, runtime); // jump to runtime if index == 0 (full buffer)
+ // The buffer is not full, store value into it.
+ __ subptr(temp, wordSize); // temp := next index
+ __ movptr(Address(thread, in_bytes(index_offset)), temp); // *(index address) := next index
+ __ addptr(temp, Address(thread, in_bytes(buffer_offset))); // temp := buffer address + next index
+ __ movptr(Address(temp, 0), value); // *(buffer address + next index) := value
+}
+
+static void generate_pre_barrier_fast_path(MacroAssembler* masm,
+ const Register thread) {
+ Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
+ // Is marking active?
+ if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
+ __ cmpl(in_progress, 0);
+ } else {
+ assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
+ __ cmpb(in_progress, 0);
+ }
+}
+
+static void generate_pre_barrier_slow_path(MacroAssembler* masm,
+ const Register obj,
+ const Register pre_val,
+ const Register thread,
+ const Register tmp,
+ Label& done,
+ Label& runtime) {
+ // Do we need to load the previous value?
+ if (obj != noreg) {
+ __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
+ }
+ // Is the previous value null?
+ __ cmpptr(pre_val, NULL_WORD);
+ __ jcc(Assembler::equal, done);
+ generate_queue_insertion(masm,
+ G1ThreadLocalData::satb_mark_queue_index_offset(),
+ G1ThreadLocalData::satb_mark_queue_buffer_offset(),
+ runtime,
+ thread, pre_val, tmp);
+ __ jmp(done);
+}
+
void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
Register obj,
Register pre_val,
@@ -185,43 +238,10 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
assert(pre_val != rax, "check this code");
}
- Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
- Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
- Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
-
- // Is marking active?
- if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
- __ cmpl(in_progress, 0);
- } else {
- assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
- __ cmpb(in_progress, 0);
- }
- __ jcc(Assembler::equal, done);
-
- // Do we need to load the previous value?
- if (obj != noreg) {
- __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
- }
-
- // Is the previous value null?
- __ cmpptr(pre_val, NULL_WORD);
+ generate_pre_barrier_fast_path(masm, thread);
+ // If marking is not active (*(mark queue active address) == 0), jump to done
__ jcc(Assembler::equal, done);
-
- // Can we store original value in the thread's buffer?
- // Is index == 0?
- // (The index field is typed as size_t.)
-
- __ movptr(tmp, index); // tmp := *index_adr
- __ cmpptr(tmp, 0); // tmp == 0?
- __ jcc(Assembler::equal, runtime); // If yes, goto runtime
-
- __ subptr(tmp, wordSize); // tmp := tmp - wordSize
- __ movptr(index, tmp); // *index_adr := tmp
- __ addptr(tmp, buffer); // tmp := tmp + *buffer_adr
-
- // Record the previous value
- __ movptr(Address(tmp, 0), pre_val);
- __ jmp(done);
+ generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, done, runtime);
__ bind(runtime);
@@ -263,6 +283,54 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
__ bind(done);
}
+static void generate_post_barrier_fast_path(MacroAssembler* masm,
+ const Register store_addr,
+ const Register new_val,
+ const Register tmp,
+ const Register tmp2,
+ Label& done,
+ bool new_val_may_be_null) {
+ CardTableBarrierSet* ct = barrier_set_cast(BarrierSet::barrier_set());
+ // Does store cross heap regions?
+ __ movptr(tmp, store_addr); // tmp := store address
+ __ xorptr(tmp, new_val); // tmp := store address ^ new value
+ __ shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
+ __ jcc(Assembler::equal, done);
+ // Crosses regions, storing null?
+ if (new_val_may_be_null) {
+ __ cmpptr(new_val, NULL_WORD); // new value == null?
+ __ jcc(Assembler::equal, done);
+ }
+ // Storing region crossing non-null, is card young?
+ __ movptr(tmp, store_addr); // tmp := store address
+ __ shrptr(tmp, CardTable::card_shift()); // tmp := card address relative to card table base
+ // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
+ // a valid address and therefore is not properly handled by the relocation code.
+ __ movptr(tmp2, (intptr_t)ct->card_table()->byte_map_base()); // tmp2 := card table base address
+ __ addptr(tmp, tmp2); // tmp := card address
+ __ cmpb(Address(tmp, 0), G1CardTable::g1_young_card_val()); // *(card address) == young_card_val?
+}
+
+static void generate_post_barrier_slow_path(MacroAssembler* masm,
+ const Register thread,
+ const Register tmp,
+ const Register tmp2,
+ Label& done,
+ Label& runtime) {
+ __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad)); // StoreLoad membar
+ __ cmpb(Address(tmp, 0), G1CardTable::dirty_card_val()); // *(card address) == dirty_card_val?
+ __ jcc(Assembler::equal, done);
+ // Storing a region crossing, non-null oop, card is clean.
+ // Dirty card and log.
+ __ movb(Address(tmp, 0), G1CardTable::dirty_card_val()); // *(card address) := dirty_card_val
+ generate_queue_insertion(masm,
+ G1ThreadLocalData::dirty_card_queue_index_offset(),
+ G1ThreadLocalData::dirty_card_queue_buffer_offset(),
+ runtime,
+ thread, tmp, tmp2);
+ __ jmp(done);
+}
+
void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
Register store_addr,
Register new_val,
@@ -273,74 +341,125 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
assert(thread == r15_thread, "must be");
#endif // _LP64
- Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
- Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
-
- CardTableBarrierSet* ct =
- barrier_set_cast(BarrierSet::barrier_set());
-
Label done;
Label runtime;
- // Does store cross heap regions?
-
- __ movptr(tmp, store_addr);
- __ xorptr(tmp, new_val);
- __ shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes);
+ generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, done, true /* new_val_may_be_null */);
+ // If card is young, jump to done
__ jcc(Assembler::equal, done);
+ generate_post_barrier_slow_path(masm, thread, tmp, tmp2, done, runtime);
- // crosses regions, storing null?
+ __ bind(runtime);
+ // save the live input values
+ RegSet saved = RegSet::of(store_addr NOT_LP64(COMMA thread));
+ __ push_set(saved);
+ __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp, thread);
+ __ pop_set(saved);
- __ cmpptr(new_val, NULL_WORD);
- __ jcc(Assembler::equal, done);
+ __ bind(done);
+}
- // storing region crossing non-null, is card already dirty?
+#if defined(COMPILER2)
- const Register card_addr = tmp;
- const Register cardtable = tmp2;
+static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
+#ifdef _LP64
+ SaveLiveRegisters save_registers(masm, stub);
+ if (c_rarg0 != arg) {
+ __ mov(c_rarg0, arg);
+ }
+ __ mov(c_rarg1, r15_thread);
+ // rax is a caller-saved, non-argument-passing register, so it does not
+ // interfere with c_rarg0 or c_rarg1. If it contained any live value before
+ // entering this stub, it is saved at this point, and restored after the
+ // call. If it did not contain any live value, it is free to be used. In
+ // either case, it is safe to use it here as a call scratch register.
+ __ call(RuntimeAddress(runtime_path), rax);
+#else
+ Unimplemented();
+#endif // _LP64
+}
- __ movptr(card_addr, store_addr);
- __ shrptr(card_addr, CardTable::card_shift());
- // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
- // a valid address and therefore is not properly handled by the relocation code.
- __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
- __ addptr(card_addr, cardtable);
+void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register thread,
+ Register tmp,
+ G1PreBarrierStubC2* stub) {
+#ifdef _LP64
+ assert(thread == r15_thread, "must be");
+#endif // _LP64
+ assert(pre_val != noreg, "check this code");
+ if (obj != noreg) {
+ assert_different_registers(obj, pre_val, tmp);
+ }
- __ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val());
- __ jcc(Assembler::equal, done);
+ stub->initialize_registers(obj, pre_val, thread, tmp);
- __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
- __ cmpb(Address(card_addr, 0), G1CardTable::dirty_card_val());
- __ jcc(Assembler::equal, done);
+ generate_pre_barrier_fast_path(masm, thread);
+ // If marking is active (*(mark queue active address) != 0), jump to stub (slow path)
+ __ jcc(Assembler::notEqual, *stub->entry());
+ __ bind(*stub->continuation());
+}
- // storing a region crossing, non-null oop, card is clean.
- // dirty card and log.
+void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm,
+ G1PreBarrierStubC2* stub) const {
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ Label runtime;
+ Register obj = stub->obj();
+ Register pre_val = stub->pre_val();
+ Register thread = stub->thread();
+ Register tmp = stub->tmp1();
+ assert(stub->tmp2() == noreg, "not needed in this platform");
- __ movb(Address(card_addr, 0), G1CardTable::dirty_card_val());
+ __ bind(*stub->entry());
+ generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp, *stub->continuation(), runtime);
- // The code below assumes that buffer index is pointer sized.
- STATIC_ASSERT(in_bytes(G1DirtyCardQueue::byte_width_of_index()) == sizeof(intptr_t));
+ __ bind(runtime);
+ generate_c2_barrier_runtime_call(masm, stub, pre_val, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry));
+ __ jmp(*stub->continuation());
+}
- __ movptr(tmp2, queue_index);
- __ testptr(tmp2, tmp2);
- __ jcc(Assembler::zero, runtime);
- __ subptr(tmp2, wordSize);
- __ movptr(queue_index, tmp2);
- __ addptr(tmp2, buffer);
- __ movptr(Address(tmp2, 0), card_addr);
- __ jmp(done);
+void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp,
+ Register tmp2,
+ G1PostBarrierStubC2* stub) {
+#ifdef _LP64
+ assert(thread == r15_thread, "must be");
+#endif // _LP64
- __ bind(runtime);
- // save the live input values
- RegSet saved = RegSet::of(store_addr NOT_LP64(COMMA thread));
- __ push_set(saved);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
- __ pop_set(saved);
+ stub->initialize_registers(thread, tmp, tmp2);
- __ bind(done);
+ bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
+ generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, *stub->continuation(), new_val_may_be_null);
+ // If card is not young, jump to stub (slow path)
+ __ jcc(Assembler::notEqual, *stub->entry());
+
+ __ bind(*stub->continuation());
+}
+
+void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
+ G1PostBarrierStubC2* stub) const {
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ Label runtime;
+ Register thread = stub->thread();
+ Register tmp = stub->tmp1(); // tmp holds the card address.
+ Register tmp2 = stub->tmp2();
+ assert(stub->tmp3() == noreg, "not needed in this platform");
+
+ __ bind(*stub->entry());
+ generate_post_barrier_slow_path(masm, thread, tmp, tmp2, *stub->continuation(), runtime);
+
+ __ bind(runtime);
+ generate_c2_barrier_runtime_call(masm, stub, tmp, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
+ __ jmp(*stub->continuation());
}
+#endif // COMPILER2
+
void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
bool in_heap = (decorators & IN_HEAP) != 0;
diff --git a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.hpp
index a5695f5657a4a..4dbb1efd885ea 100644
--- a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.hpp
@@ -32,6 +32,9 @@ class LIR_Assembler;
class StubAssembler;
class G1PreBarrierStub;
class G1PostBarrierStub;
+class G1BarrierStubC2;
+class G1PreBarrierStubC2;
+class G1PostBarrierStubC2;
class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
protected:
@@ -65,6 +68,26 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp_thread);
+
+#ifdef COMPILER2
+ void g1_write_barrier_pre_c2(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register thread,
+ Register tmp,
+ G1PreBarrierStubC2* c2_stub);
+ void generate_c2_pre_barrier_stub(MacroAssembler* masm,
+ G1PreBarrierStubC2* stub) const;
+ void g1_write_barrier_post_c2(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp,
+ Register tmp2,
+ G1PostBarrierStubC2* c2_stub);
+ void generate_c2_post_barrier_stub(MacroAssembler* masm,
+ G1PostBarrierStubC2* stub) const;
+#endif // COMPILER2
};
#endif // CPU_X86_GC_G1_G1BARRIERSETASSEMBLER_X86_HPP
diff --git a/src/hotspot/cpu/x86/gc/g1/g1_x86_64.ad b/src/hotspot/cpu/x86/gc/g1/g1_x86_64.ad
new file mode 100644
index 0000000000000..8c1559f90f46d
--- /dev/null
+++ b/src/hotspot/cpu/x86/gc/g1/g1_x86_64.ad
@@ -0,0 +1,371 @@
+//
+// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+
+source_hpp %{
+
+#include "gc/g1/c2/g1BarrierSetC2.hpp"
+#include "gc/shared/gc_globals.hpp"
+
+%}
+
+source %{
+
+#include "gc/g1/g1BarrierSetAssembler_x86.hpp"
+#include "gc/g1/g1BarrierSetRuntime.hpp"
+
+static void write_barrier_pre(MacroAssembler* masm,
+ const MachNode* node,
+ Register obj,
+ Register pre_val,
+ Register tmp,
+ RegSet preserve = RegSet(),
+ RegSet no_preserve = RegSet()) {
+ if (!G1PreBarrierStubC2::needs_barrier(node)) {
+ return;
+ }
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
+ G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
+ for (RegSetIterator reg = preserve.begin(); *reg != noreg; ++reg) {
+ stub->preserve(*reg);
+ }
+ for (RegSetIterator reg = no_preserve.begin(); *reg != noreg; ++reg) {
+ stub->dont_preserve(*reg);
+ }
+ g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, r15_thread, tmp, stub);
+}
+
+static void write_barrier_post(MacroAssembler* masm,
+ const MachNode* node,
+ Register store_addr,
+ Register new_val,
+ Register tmp1,
+ Register tmp2) {
+ if (!G1PostBarrierStubC2::needs_barrier(node)) {
+ return;
+ }
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+ G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
+ G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
+ g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, r15_thread, tmp1, tmp2, stub);
+}
+
+%}
+
+instruct g1StoreP(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreP mem src));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(125); // XXX
+ format %{ "movq $mem, $src\t# ptr" %}
+ ins_encode %{
+ // Materialize the store address internally (as opposed to defining 'mem' as
+ // an indirect memory operand) to reduce the overhead of LCM when processing
+ // large basic blocks with many stores. Such basic blocks arise, for
+ // instance, from static initializations of large String arrays.
+ // The same holds for g1StoreN and g1EncodePAndStoreN.
+ __ lea($tmp1$$Register, $mem$$Address);
+ write_barrier_pre(masm, this,
+ $tmp1$$Register /* obj */,
+ $tmp2$$Register /* pre_val */,
+ $tmp3$$Register /* tmp */,
+ RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
+ __ movq(Address($tmp1$$Register, 0), $src$$Register);
+ write_barrier_post(masm, this,
+ $tmp1$$Register /* store_addr */,
+ $src$$Register /* new_val */,
+ $tmp3$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(ialu_mem_reg);
+%}
+
+instruct g1StoreN(memory mem, rRegN src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreN mem src));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(125); // XXX
+ format %{ "movl $mem, $src\t# ptr" %}
+ ins_encode %{
+ __ lea($tmp1$$Register, $mem$$Address);
+ write_barrier_pre(masm, this,
+ $tmp1$$Register /* obj */,
+ $tmp2$$Register /* pre_val */,
+ $tmp3$$Register /* tmp */,
+ RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
+ __ movl(Address($tmp1$$Register, 0), $src$$Register);
+ if ((barrier_data() & G1C2BarrierPost) != 0) {
+ __ movl($tmp2$$Register, $src$$Register);
+ if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
+ __ decode_heap_oop($tmp2$$Register);
+ } else {
+ __ decode_heap_oop_not_null($tmp2$$Register);
+ }
+ }
+ write_barrier_post(masm, this,
+ $tmp1$$Register /* store_addr */,
+ $tmp2$$Register /* new_val */,
+ $tmp3$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(ialu_mem_reg);
+%}
+
+instruct g1EncodePAndStoreN(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
+ match(Set mem (StoreN mem (EncodeP src)));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ ins_cost(125); // XXX
+ format %{ "encode_heap_oop $src\n\t"
+ "movl $mem, $src\t# ptr" %}
+ ins_encode %{
+ __ lea($tmp1$$Register, $mem$$Address);
+ write_barrier_pre(masm, this,
+ $tmp1$$Register /* obj */,
+ $tmp2$$Register /* pre_val */,
+ $tmp3$$Register /* tmp */,
+ RegSet::of($tmp1$$Register, $src$$Register) /* preserve */);
+ __ movq($tmp2$$Register, $src$$Register);
+ if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
+ __ encode_heap_oop($tmp2$$Register);
+ } else {
+ __ encode_heap_oop_not_null($tmp2$$Register);
+ }
+ __ movl(Address($tmp1$$Register, 0), $tmp2$$Register);
+ write_barrier_post(masm, this,
+ $tmp1$$Register /* store_addr */,
+ $src$$Register /* new_val */,
+ $tmp3$$Register /* tmp1 */,
+ $tmp2$$Register /* tmp2 */);
+ %}
+ ins_pipe(ialu_mem_reg);
+%}
+
+instruct g1CompareAndExchangeP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegP oldval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set oldval (CompareAndExchangeP mem (Binary oldval newval)));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ format %{ "lock\n\t"
+ "cmpxchgq $newval, $mem" %}
+ ins_encode %{
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ // Pass $oldval to the pre-barrier (instead of loading from $mem), because
+ // $oldval is the only value that can be overwritten.
+ // The same holds for g1CompareAndSwapP.
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $oldval$$Register /* pre_val */,
+ $tmp3$$Register /* tmp */,
+ RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */);
+ __ movq($tmp1$$Register, $newval$$Register);
+ __ lock();
+ __ cmpxchgq($tmp1$$Register, Address($mem$$Register, 0));
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_cmpxchg);
+%}
+
+instruct g1CompareAndExchangeN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegN oldval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set oldval (CompareAndExchangeN mem (Binary oldval newval)));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ format %{ "lock\n\t"
+ "cmpxchgq $newval, $mem" %}
+ ins_encode %{
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp2$$Register /* pre_val */,
+ $tmp3$$Register /* tmp */,
+ RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */);
+ __ movl($tmp1$$Register, $newval$$Register);
+ __ lock();
+ __ cmpxchgl($tmp1$$Register, Address($mem$$Register, 0));
+ __ decode_heap_oop($tmp1$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_cmpxchg);
+%}
+
+instruct g1CompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegP oldval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL oldval, KILL cr);
+ format %{ "lock\n\t"
+ "cmpxchgq $newval, $mem\n\t"
+ "sete $res\n\t"
+ "movzbl $res, $res" %}
+ ins_encode %{
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $oldval$$Register /* pre_val */,
+ $tmp3$$Register /* tmp */,
+ RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ movq($tmp1$$Register, $newval$$Register);
+ __ lock();
+ __ cmpxchgq($tmp1$$Register, Address($mem$$Register, 0));
+ __ setb(Assembler::equal, $res$$Register);
+ __ movzbl($res$$Register, $res$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_cmpxchg);
+%}
+
+instruct g1CompareAndSwapN(rRegI res, indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rax_RegN oldval, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set res (CompareAndSwapN mem (Binary oldval newval)));
+ match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
+ effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL oldval, KILL cr);
+ format %{ "lock\n\t"
+ "cmpxchgq $newval, $mem\n\t"
+ "sete $res\n\t"
+ "movzbl $res, $res" %}
+ ins_encode %{
+ assert_different_registers($oldval$$Register, $mem$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp2$$Register /* pre_val */,
+ $tmp3$$Register /* tmp */,
+ RegSet::of($mem$$Register, $newval$$Register, $oldval$$Register) /* preserve */,
+ RegSet::of($res$$Register) /* no_preserve */);
+ __ movl($tmp1$$Register, $newval$$Register);
+ __ lock();
+ __ cmpxchgl($tmp1$$Register, Address($mem$$Register, 0));
+ __ setb(Assembler::equal, $res$$Register);
+ __ movzbl($res$$Register, $res$$Register);
+ __ decode_heap_oop($tmp1$$Register);
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_cmpxchg);
+%}
+
+instruct g1GetAndSetP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set newval (GetAndSetP mem newval));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ format %{ "xchgq $newval, $mem" %}
+ ins_encode %{
+ assert_different_registers($mem$$Register, $newval$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp2$$Register /* pre_val */,
+ $tmp3$$Register /* tmp */,
+ RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
+ __ movq($tmp1$$Register, $newval$$Register);
+ __ xchgq($newval$$Register, Address($mem$$Register, 0));
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_cmpxchg);
+%}
+
+instruct g1GetAndSetN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
+ match(Set newval (GetAndSetN mem newval));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+ format %{ "xchgq $newval, $mem" %}
+ ins_encode %{
+ assert_different_registers($mem$$Register, $newval$$Register);
+ write_barrier_pre(masm, this,
+ $mem$$Register /* obj */,
+ $tmp2$$Register /* pre_val */,
+ $tmp3$$Register /* tmp */,
+ RegSet::of($mem$$Register, $newval$$Register) /* preserve */);
+ __ movl($tmp1$$Register, $newval$$Register);
+ __ decode_heap_oop($tmp1$$Register);
+ __ xchgl($newval$$Register, Address($mem$$Register, 0));
+ write_barrier_post(masm, this,
+ $mem$$Register /* store_addr */,
+ $tmp1$$Register /* new_val */,
+ $tmp2$$Register /* tmp1 */,
+ $tmp3$$Register /* tmp2 */);
+ %}
+ ins_pipe(pipe_cmpxchg);
+%}
+
+instruct g1LoadP(rRegP dst, memory mem, rRegP tmp, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
+ match(Set dst (LoadP mem));
+ effect(TEMP dst, TEMP tmp, KILL cr);
+ ins_cost(125); // XXX
+ format %{ "movq $dst, $mem\t# ptr" %}
+ ins_encode %{
+ __ movq($dst$$Register, $mem$$Address);
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $dst$$Register /* pre_val */,
+ $tmp$$Register /* tmp */);
+ %}
+ ins_pipe(ialu_reg_mem); // XXX
+%}
+
+instruct g1LoadN(rRegN dst, memory mem, rRegP tmp1, rRegP tmp2, rFlagsReg cr)
+%{
+ predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
+ match(Set dst (LoadN mem));
+ effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
+ ins_cost(125); // XXX
+ format %{ "movl $dst, $mem\t# compressed ptr" %}
+ ins_encode %{
+ __ movl($dst$$Register, $mem$$Address);
+ __ movl($tmp1$$Register, $dst$$Register);
+ __ decode_heap_oop($tmp1$$Register);
+ write_barrier_pre(masm, this,
+ noreg /* obj */,
+ $tmp1$$Register /* pre_val */,
+ $tmp2$$Register /* tmp */);
+ %}
+ ins_pipe(ialu_reg_mem); // XXX
+%}
diff --git a/src/hotspot/cpu/x86/gc/x/xBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/x/xBarrierSetAssembler_x86.cpp
deleted file mode 100644
index a7dc34b17b1f6..0000000000000
--- a/src/hotspot/cpu/x86/gc/x/xBarrierSetAssembler_x86.cpp
+++ /dev/null
@@ -1,734 +0,0 @@
-/*
- * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "asm/macroAssembler.inline.hpp"
-#include "code/codeBlob.hpp"
-#include "code/vmreg.inline.hpp"
-#include "gc/x/xBarrier.inline.hpp"
-#include "gc/x/xBarrierSet.hpp"
-#include "gc/x/xBarrierSetAssembler.hpp"
-#include "gc/x/xBarrierSetRuntime.hpp"
-#include "gc/x/xThreadLocalData.hpp"
-#include "memory/resourceArea.hpp"
-#include "runtime/sharedRuntime.hpp"
-#include "utilities/macros.hpp"
-#ifdef COMPILER1
-#include "c1/c1_LIRAssembler.hpp"
-#include "c1/c1_MacroAssembler.hpp"
-#include "gc/x/c1/xBarrierSetC1.hpp"
-#endif // COMPILER1
-#ifdef COMPILER2
-#include "gc/x/c2/xBarrierSetC2.hpp"
-#endif // COMPILER2
-
-#ifdef PRODUCT
-#define BLOCK_COMMENT(str) /* nothing */
-#else
-#define BLOCK_COMMENT(str) __ block_comment(str)
-#endif
-
-#undef __
-#define __ masm->
-
-static void call_vm(MacroAssembler* masm,
- address entry_point,
- Register arg0,
- Register arg1) {
- // Setup arguments
- if (arg1 == c_rarg0) {
- if (arg0 == c_rarg1) {
- __ xchgptr(c_rarg1, c_rarg0);
- } else {
- __ movptr(c_rarg1, arg1);
- __ movptr(c_rarg0, arg0);
- }
- } else {
- if (arg0 != c_rarg0) {
- __ movptr(c_rarg0, arg0);
- }
- if (arg1 != c_rarg1) {
- __ movptr(c_rarg1, arg1);
- }
- }
-
- // Call VM
- __ MacroAssembler::call_VM_leaf_base(entry_point, 2);
-}
-
-void XBarrierSetAssembler::load_at(MacroAssembler* masm,
- DecoratorSet decorators,
- BasicType type,
- Register dst,
- Address src,
- Register tmp1,
- Register tmp_thread) {
- if (!XBarrierSet::barrier_needed(decorators, type)) {
- // Barrier not needed
- BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
- return;
- }
-
- BLOCK_COMMENT("XBarrierSetAssembler::load_at {");
-
- // Allocate scratch register
- Register scratch = tmp1;
- if (tmp1 == noreg) {
- scratch = r12;
- __ push(scratch);
- }
-
- assert_different_registers(dst, scratch);
-
- Label done;
-
- //
- // Fast Path
- //
-
- // Load address
- __ lea(scratch, src);
-
- // Load oop at address
- __ movptr(dst, Address(scratch, 0));
-
- // Test address bad mask
- __ testptr(dst, address_bad_mask_from_thread(r15_thread));
- __ jcc(Assembler::zero, done);
-
- //
- // Slow path
- //
-
- // Save registers
- __ push(rax);
- __ push(rcx);
- __ push(rdx);
- __ push(rdi);
- __ push(rsi);
- __ push(r8);
- __ push(r9);
- __ push(r10);
- __ push(r11);
-
- // We may end up here from generate_native_wrapper, then the method may have
- // floats as arguments, and we must spill them before calling the VM runtime
- // leaf. From the interpreter all floats are passed on the stack.
- assert(Argument::n_float_register_parameters_j == 8, "Assumption");
- const int xmm_size = wordSize * 2;
- const int xmm_spill_size = xmm_size * Argument::n_float_register_parameters_j;
- __ subptr(rsp, xmm_spill_size);
- __ movdqu(Address(rsp, xmm_size * 7), xmm7);
- __ movdqu(Address(rsp, xmm_size * 6), xmm6);
- __ movdqu(Address(rsp, xmm_size * 5), xmm5);
- __ movdqu(Address(rsp, xmm_size * 4), xmm4);
- __ movdqu(Address(rsp, xmm_size * 3), xmm3);
- __ movdqu(Address(rsp, xmm_size * 2), xmm2);
- __ movdqu(Address(rsp, xmm_size * 1), xmm1);
- __ movdqu(Address(rsp, xmm_size * 0), xmm0);
-
- // Call VM
- call_vm(masm, XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), dst, scratch);
-
- __ movdqu(xmm0, Address(rsp, xmm_size * 0));
- __ movdqu(xmm1, Address(rsp, xmm_size * 1));
- __ movdqu(xmm2, Address(rsp, xmm_size * 2));
- __ movdqu(xmm3, Address(rsp, xmm_size * 3));
- __ movdqu(xmm4, Address(rsp, xmm_size * 4));
- __ movdqu(xmm5, Address(rsp, xmm_size * 5));
- __ movdqu(xmm6, Address(rsp, xmm_size * 6));
- __ movdqu(xmm7, Address(rsp, xmm_size * 7));
- __ addptr(rsp, xmm_spill_size);
-
- __ pop(r11);
- __ pop(r10);
- __ pop(r9);
- __ pop(r8);
- __ pop(rsi);
- __ pop(rdi);
- __ pop(rdx);
- __ pop(rcx);
-
- if (dst == rax) {
- __ addptr(rsp, wordSize);
- } else {
- __ movptr(dst, rax);
- __ pop(rax);
- }
-
- __ bind(done);
-
- // Restore scratch register
- if (tmp1 == noreg) {
- __ pop(scratch);
- }
-
- BLOCK_COMMENT("} XBarrierSetAssembler::load_at");
-}
-
-#ifdef ASSERT
-
-void XBarrierSetAssembler::store_at(MacroAssembler* masm,
- DecoratorSet decorators,
- BasicType type,
- Address dst,
- Register src,
- Register tmp1,
- Register tmp2,
- Register tmp3) {
- BLOCK_COMMENT("XBarrierSetAssembler::store_at {");
-
- // Verify oop store
- if (is_reference_type(type)) {
- // Note that src could be noreg, which means we
- // are storing null and can skip verification.
- if (src != noreg) {
- Label done;
- __ testptr(src, address_bad_mask_from_thread(r15_thread));
- __ jcc(Assembler::zero, done);
- __ stop("Verify oop store failed");
- __ should_not_reach_here();
- __ bind(done);
- }
- }
-
- // Store value
- BarrierSetAssembler::store_at(masm, decorators, type, dst, src, tmp1, tmp2, tmp3);
-
- BLOCK_COMMENT("} XBarrierSetAssembler::store_at");
-}
-
-#endif // ASSERT
-
-void XBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
- DecoratorSet decorators,
- BasicType type,
- Register src,
- Register dst,
- Register count) {
- if (!XBarrierSet::barrier_needed(decorators, type)) {
- // Barrier not needed
- return;
- }
-
- BLOCK_COMMENT("XBarrierSetAssembler::arraycopy_prologue {");
-
- // Save registers
- __ pusha();
-
- // Call VM
- call_vm(masm, XBarrierSetRuntime::load_barrier_on_oop_array_addr(), src, count);
-
- // Restore registers
- __ popa();
-
- BLOCK_COMMENT("} XBarrierSetAssembler::arraycopy_prologue");
-}
-
-void XBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
- Register jni_env,
- Register obj,
- Register tmp,
- Label& slowpath) {
- BLOCK_COMMENT("XBarrierSetAssembler::try_resolve_jobject_in_native {");
-
- // Resolve jobject
- BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
-
- // Test address bad mask
- __ testptr(obj, address_bad_mask_from_jni_env(jni_env));
- __ jcc(Assembler::notZero, slowpath);
-
- BLOCK_COMMENT("} XBarrierSetAssembler::try_resolve_jobject_in_native");
-}
-
-#ifdef COMPILER1
-
-#undef __
-#define __ ce->masm()->
-
-void XBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
- LIR_Opr ref) const {
- __ testptr(ref->as_register(), address_bad_mask_from_thread(r15_thread));
-}
-
-void XBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
- XLoadBarrierStubC1* stub) const {
- // Stub entry
- __ bind(*stub->entry());
-
- Register ref = stub->ref()->as_register();
- Register ref_addr = noreg;
- Register tmp = noreg;
-
- if (stub->tmp()->is_valid()) {
- // Load address into tmp register
- ce->leal(stub->ref_addr(), stub->tmp());
- ref_addr = tmp = stub->tmp()->as_pointer_register();
- } else {
- // Address already in register
- ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
- }
-
- assert_different_registers(ref, ref_addr, noreg);
-
- // Save rax unless it is the result or tmp register
- if (ref != rax && tmp != rax) {
- __ push(rax);
- }
-
- // Setup arguments and call runtime stub
- __ subptr(rsp, 2 * BytesPerWord);
- ce->store_parameter(ref_addr, 1);
- ce->store_parameter(ref, 0);
- __ call(RuntimeAddress(stub->runtime_stub()));
- __ addptr(rsp, 2 * BytesPerWord);
-
- // Verify result
- __ verify_oop(rax);
-
- // Move result into place
- if (ref != rax) {
- __ movptr(ref, rax);
- }
-
- // Restore rax unless it is the result or tmp register
- if (ref != rax && tmp != rax) {
- __ pop(rax);
- }
-
- // Stub exit
- __ jmp(*stub->continuation());
-}
-
-#undef __
-#define __ sasm->
-
-void XBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
- DecoratorSet decorators) const {
- // Enter and save registers
- __ enter();
- __ save_live_registers_no_oop_map(true /* save_fpu_registers */);
-
- // Setup arguments
- __ load_parameter(1, c_rarg1);
- __ load_parameter(0, c_rarg0);
-
- // Call VM
- __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1);
-
- // Restore registers and return
- __ restore_live_registers_except_rax(true /* restore_fpu_registers */);
- __ leave();
- __ ret(0);
-}
-
-#endif // COMPILER1
-
-#ifdef COMPILER2
-
-OptoReg::Name XBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
- if (!OptoReg::is_reg(opto_reg)) {
- return OptoReg::Bad;
- }
-
- const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
- if (vm_reg->is_XMMRegister()) {
- opto_reg &= ~15;
- switch (node->ideal_reg()) {
- case Op_VecX:
- opto_reg |= 2;
- break;
- case Op_VecY:
- opto_reg |= 4;
- break;
- case Op_VecZ:
- opto_reg |= 8;
- break;
- default:
- opto_reg |= 1;
- break;
- }
- }
-
- return opto_reg;
-}
-
-// We use the vec_spill_helper from the x86.ad file to avoid reinventing this wheel
-extern void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
- int stack_offset, int reg, uint ireg, outputStream* st);
-
-#undef __
-#define __ _masm->
-
-class XSaveLiveRegisters {
-private:
- struct XMMRegisterData {
- XMMRegister _reg;
- int _size;
-
- // Used by GrowableArray::find()
- bool operator == (const XMMRegisterData& other) {
- return _reg == other._reg;
- }
- };
-
- MacroAssembler* const _masm;
- GrowableArray _gp_registers;
- GrowableArray _opmask_registers;
- GrowableArray _xmm_registers;
- int _spill_size;
- int _spill_offset;
-
- static int xmm_compare_register_size(XMMRegisterData* left, XMMRegisterData* right) {
- if (left->_size == right->_size) {
- return 0;
- }
-
- return (left->_size < right->_size) ? -1 : 1;
- }
-
- static int xmm_slot_size(OptoReg::Name opto_reg) {
- // The low order 4 bytes denote what size of the XMM register is live
- return (opto_reg & 15) << 3;
- }
-
- static uint xmm_ideal_reg_for_size(int reg_size) {
- switch (reg_size) {
- case 8:
- return Op_VecD;
- case 16:
- return Op_VecX;
- case 32:
- return Op_VecY;
- case 64:
- return Op_VecZ;
- default:
- fatal("Invalid register size %d", reg_size);
- return 0;
- }
- }
-
- bool xmm_needs_vzeroupper() const {
- return _xmm_registers.is_nonempty() && _xmm_registers.at(0)._size > 16;
- }
-
- void xmm_register_save(const XMMRegisterData& reg_data) {
- const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg());
- const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size);
- _spill_offset -= reg_data._size;
- C2_MacroAssembler c2_masm(__ code());
- vec_spill_helper(&c2_masm, false /* is_load */, _spill_offset, opto_reg, ideal_reg, tty);
- }
-
- void xmm_register_restore(const XMMRegisterData& reg_data) {
- const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg());
- const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size);
- C2_MacroAssembler c2_masm(__ code());
- vec_spill_helper(&c2_masm, true /* is_load */, _spill_offset, opto_reg, ideal_reg, tty);
- _spill_offset += reg_data._size;
- }
-
- void gp_register_save(Register reg) {
- _spill_offset -= 8;
- __ movq(Address(rsp, _spill_offset), reg);
- }
-
- void opmask_register_save(KRegister reg) {
- _spill_offset -= 8;
- __ kmov(Address(rsp, _spill_offset), reg);
- }
-
- void gp_register_restore(Register reg) {
- __ movq(reg, Address(rsp, _spill_offset));
- _spill_offset += 8;
- }
-
- void opmask_register_restore(KRegister reg) {
- __ kmov(reg, Address(rsp, _spill_offset));
- _spill_offset += 8;
- }
-
- void initialize(XLoadBarrierStubC2* stub) {
- // Create mask of caller saved registers that need to
- // be saved/restored if live
- RegMask caller_saved;
- caller_saved.Insert(OptoReg::as_OptoReg(rax->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(rcx->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(rdx->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(rsi->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(rdi->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r8->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r9->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r10->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r11->as_VMReg()));
- caller_saved.Remove(OptoReg::as_OptoReg(stub->ref()->as_VMReg()));
-
- if (UseAPX) {
- caller_saved.Insert(OptoReg::as_OptoReg(r16->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r17->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r18->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r19->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r20->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r21->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r22->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r23->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r24->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r25->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r26->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r27->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r28->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r29->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r30->as_VMReg()));
- caller_saved.Insert(OptoReg::as_OptoReg(r31->as_VMReg()));
- }
-
- // Create mask of live registers
- RegMask live = stub->live();
- if (stub->tmp() != noreg) {
- live.Insert(OptoReg::as_OptoReg(stub->tmp()->as_VMReg()));
- }
-
- int gp_spill_size = 0;
- int opmask_spill_size = 0;
- int xmm_spill_size = 0;
-
- // Record registers that needs to be saved/restored
- RegMaskIterator rmi(live);
- while (rmi.has_next()) {
- const OptoReg::Name opto_reg = rmi.next();
- const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
-
- if (vm_reg->is_Register()) {
- if (caller_saved.Member(opto_reg)) {
- _gp_registers.append(vm_reg->as_Register());
- gp_spill_size += 8;
- }
- } else if (vm_reg->is_KRegister()) {
- // All opmask registers are caller saved, thus spill the ones
- // which are live.
- if (_opmask_registers.find(vm_reg->as_KRegister()) == -1) {
- _opmask_registers.append(vm_reg->as_KRegister());
- opmask_spill_size += 8;
- }
- } else if (vm_reg->is_XMMRegister()) {
- // We encode in the low order 4 bits of the opto_reg, how large part of the register is live
- const VMReg vm_reg_base = OptoReg::as_VMReg(opto_reg & ~15);
- const int reg_size = xmm_slot_size(opto_reg);
- const XMMRegisterData reg_data = { vm_reg_base->as_XMMRegister(), reg_size };
- const int reg_index = _xmm_registers.find(reg_data);
- if (reg_index == -1) {
- // Not previously appended
- _xmm_registers.append(reg_data);
- xmm_spill_size += reg_size;
- } else {
- // Previously appended, update size
- const int reg_size_prev = _xmm_registers.at(reg_index)._size;
- if (reg_size > reg_size_prev) {
- _xmm_registers.at_put(reg_index, reg_data);
- xmm_spill_size += reg_size - reg_size_prev;
- }
- }
- } else {
- fatal("Unexpected register type");
- }
- }
-
- // Sort by size, largest first
- _xmm_registers.sort(xmm_compare_register_size);
-
- // On Windows, the caller reserves stack space for spilling register arguments
- const int arg_spill_size = frame::arg_reg_save_area_bytes;
-
- // Stack pointer must be 16 bytes aligned for the call
- _spill_offset = _spill_size = align_up(xmm_spill_size + gp_spill_size + opmask_spill_size + arg_spill_size, 16);
- }
-
-public:
- XSaveLiveRegisters(MacroAssembler* masm, XLoadBarrierStubC2* stub) :
- _masm(masm),
- _gp_registers(),
- _opmask_registers(),
- _xmm_registers(),
- _spill_size(0),
- _spill_offset(0) {
-
- //
- // Stack layout after registers have been spilled:
- //
- // | ... | original rsp, 16 bytes aligned
- // ------------------
- // | zmm0 high |
- // | ... |
- // | zmm0 low | 16 bytes aligned
- // | ... |
- // | ymm1 high |
- // | ... |
- // | ymm1 low | 16 bytes aligned
- // | ... |
- // | xmmN high |
- // | ... |
- // | xmmN low | 8 bytes aligned
- // | reg0 | 8 bytes aligned
- // | reg1 |
- // | ... |
- // | regN | new rsp, if 16 bytes aligned
- // | | else new rsp, 16 bytes aligned
- // ------------------
- //
-
- // Figure out what registers to save/restore
- initialize(stub);
-
- // Allocate stack space
- if (_spill_size > 0) {
- __ subptr(rsp, _spill_size);
- }
-
- // Save XMM/YMM/ZMM registers
- for (int i = 0; i < _xmm_registers.length(); i++) {
- xmm_register_save(_xmm_registers.at(i));
- }
-
- if (xmm_needs_vzeroupper()) {
- __ vzeroupper();
- }
-
- // Save general purpose registers
- for (int i = 0; i < _gp_registers.length(); i++) {
- gp_register_save(_gp_registers.at(i));
- }
-
- // Save opmask registers
- for (int i = 0; i < _opmask_registers.length(); i++) {
- opmask_register_save(_opmask_registers.at(i));
- }
- }
-
- ~XSaveLiveRegisters() {
- // Restore opmask registers
- for (int i = _opmask_registers.length() - 1; i >= 0; i--) {
- opmask_register_restore(_opmask_registers.at(i));
- }
-
- // Restore general purpose registers
- for (int i = _gp_registers.length() - 1; i >= 0; i--) {
- gp_register_restore(_gp_registers.at(i));
- }
-
- __ vzeroupper();
-
- // Restore XMM/YMM/ZMM registers
- for (int i = _xmm_registers.length() - 1; i >= 0; i--) {
- xmm_register_restore(_xmm_registers.at(i));
- }
-
- // Free stack space
- if (_spill_size > 0) {
- __ addptr(rsp, _spill_size);
- }
- }
-};
-
-class XSetupArguments {
-private:
- MacroAssembler* const _masm;
- const Register _ref;
- const Address _ref_addr;
-
-public:
- XSetupArguments(MacroAssembler* masm, XLoadBarrierStubC2* stub) :
- _masm(masm),
- _ref(stub->ref()),
- _ref_addr(stub->ref_addr()) {
-
- // Setup arguments
- if (_ref_addr.base() == noreg) {
- // No self healing
- if (_ref != c_rarg0) {
- __ movq(c_rarg0, _ref);
- }
- __ xorq(c_rarg1, c_rarg1);
- } else {
- // Self healing
- if (_ref == c_rarg0) {
- __ lea(c_rarg1, _ref_addr);
- } else if (_ref != c_rarg1) {
- __ lea(c_rarg1, _ref_addr);
- __ movq(c_rarg0, _ref);
- } else if (_ref_addr.base() != c_rarg0 && _ref_addr.index() != c_rarg0) {
- __ movq(c_rarg0, _ref);
- __ lea(c_rarg1, _ref_addr);
- } else {
- __ xchgq(c_rarg0, c_rarg1);
- if (_ref_addr.base() == c_rarg0) {
- __ lea(c_rarg1, Address(c_rarg1, _ref_addr.index(), _ref_addr.scale(), _ref_addr.disp()));
- } else if (_ref_addr.index() == c_rarg0) {
- __ lea(c_rarg1, Address(_ref_addr.base(), c_rarg1, _ref_addr.scale(), _ref_addr.disp()));
- } else {
- ShouldNotReachHere();
- }
- }
- }
- }
-
- ~XSetupArguments() {
- // Transfer result
- if (_ref != rax) {
- __ movq(_ref, rax);
- }
- }
-};
-
-#undef __
-#define __ masm->
-
-void XBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, XLoadBarrierStubC2* stub) const {
- BLOCK_COMMENT("XLoadBarrierStubC2");
-
- // Stub entry
- __ bind(*stub->entry());
-
- {
- XSaveLiveRegisters save_live_registers(masm, stub);
- XSetupArguments setup_arguments(masm, stub);
- __ call(RuntimeAddress(stub->slow_path()));
- }
-
- // Stub exit
- __ jmp(*stub->continuation());
-}
-
-#endif // COMPILER2
-
-#undef __
-#define __ masm->
-
-void XBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
- // Check if metadata bits indicate a bad oop
- __ testptr(obj, Address(r15_thread, XThreadLocalData::address_bad_mask_offset()));
- __ jcc(Assembler::notZero, error);
- BarrierSetAssembler::check_oop(masm, obj, tmp1, tmp2, error);
-}
-
-#undef __
diff --git a/src/hotspot/cpu/x86/gc/x/xBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/x/xBarrierSetAssembler_x86.hpp
deleted file mode 100644
index 52034ab786ec2..0000000000000
--- a/src/hotspot/cpu/x86/gc/x/xBarrierSetAssembler_x86.hpp
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#ifndef CPU_X86_GC_X_XBARRIERSETASSEMBLER_X86_HPP
-#define CPU_X86_GC_X_XBARRIERSETASSEMBLER_X86_HPP
-
-#include "code/vmreg.hpp"
-#include "oops/accessDecorators.hpp"
-#ifdef COMPILER2
-#include "opto/optoreg.hpp"
-#endif // COMPILER2
-
-class MacroAssembler;
-
-#ifdef COMPILER1
-class LIR_Assembler;
-class LIR_Opr;
-class StubAssembler;
-#endif // COMPILER1
-
-#ifdef COMPILER2
-class Node;
-#endif // COMPILER2
-
-#ifdef COMPILER1
-class XLoadBarrierStubC1;
-#endif // COMPILER1
-
-#ifdef COMPILER2
-class XLoadBarrierStubC2;
-#endif // COMPILER2
-
-class XBarrierSetAssembler : public XBarrierSetAssemblerBase {
-public:
- virtual void load_at(MacroAssembler* masm,
- DecoratorSet decorators,
- BasicType type,
- Register dst,
- Address src,
- Register tmp1,
- Register tmp_thread);
-
-#ifdef ASSERT
- virtual void store_at(MacroAssembler* masm,
- DecoratorSet decorators,
- BasicType type,
- Address dst,
- Register src,
- Register tmp1,
- Register tmp2,
- Register tmp3);
-#endif // ASSERT
-
- virtual void arraycopy_prologue(MacroAssembler* masm,
- DecoratorSet decorators,
- BasicType type,
- Register src,
- Register dst,
- Register count);
-
- virtual void try_resolve_jobject_in_native(MacroAssembler* masm,
- Register jni_env,
- Register obj,
- Register tmp,
- Label& slowpath);
-
-#ifdef COMPILER1
- void generate_c1_load_barrier_test(LIR_Assembler* ce,
- LIR_Opr ref) const;
-
- void generate_c1_load_barrier_stub(LIR_Assembler* ce,
- XLoadBarrierStubC1* stub) const;
-
- void generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
- DecoratorSet decorators) const;
-#endif // COMPILER1
-
-#ifdef COMPILER2
- OptoReg::Name refine_register(const Node* node,
- OptoReg::Name opto_reg);
-
- void generate_c2_load_barrier_stub(MacroAssembler* masm,
- XLoadBarrierStubC2* stub) const;
-#endif // COMPILER2
-
- void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);
-};
-
-#endif // CPU_X86_GC_X_XBARRIERSETASSEMBLER_X86_HPP
diff --git a/src/hotspot/cpu/x86/gc/x/xGlobals_x86.cpp b/src/hotspot/cpu/x86/gc/x/xGlobals_x86.cpp
deleted file mode 100644
index baa99ddd60db7..0000000000000
--- a/src/hotspot/cpu/x86/gc/x/xGlobals_x86.cpp
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/shared/gc_globals.hpp"
-#include "gc/x/xGlobals.hpp"
-#include "runtime/globals.hpp"
-#include "utilities/globalDefinitions.hpp"
-#include "utilities/powerOfTwo.hpp"
-
-//
-// The heap can have three different layouts, depending on the max heap size.
-//
-// Address Space & Pointer Layout 1
-// --------------------------------
-//
-// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB)
-// . .
-// . .
-// . .
-// +--------------------------------+ 0x0000014000000000 (20TB)
-// | Remapped View |
-// +--------------------------------+ 0x0000010000000000 (16TB)
-// . .
-// +--------------------------------+ 0x00000c0000000000 (12TB)
-// | Marked1 View |
-// +--------------------------------+ 0x0000080000000000 (8TB)
-// | Marked0 View |
-// +--------------------------------+ 0x0000040000000000 (4TB)
-// . .
-// +--------------------------------+ 0x0000000000000000
-//
-// 6 4 4 4 4
-// 3 6 5 2 1 0
-// +--------------------+----+-----------------------------------------------+
-// |00000000 00000000 00|1111|11 11111111 11111111 11111111 11111111 11111111|
-// +--------------------+----+-----------------------------------------------+
-// | | |
-// | | * 41-0 Object Offset (42-bits, 4TB address space)
-// | |
-// | * 45-42 Metadata Bits (4-bits) 0001 = Marked0 (Address view 4-8TB)
-// | 0010 = Marked1 (Address view 8-12TB)
-// | 0100 = Remapped (Address view 16-20TB)
-// | 1000 = Finalizable (Address view N/A)
-// |
-// * 63-46 Fixed (18-bits, always zero)
-//
-//
-// Address Space & Pointer Layout 2
-// --------------------------------
-//
-// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB)
-// . .
-// . .
-// . .
-// +--------------------------------+ 0x0000280000000000 (40TB)
-// | Remapped View |
-// +--------------------------------+ 0x0000200000000000 (32TB)
-// . .
-// +--------------------------------+ 0x0000180000000000 (24TB)
-// | Marked1 View |
-// +--------------------------------+ 0x0000100000000000 (16TB)
-// | Marked0 View |
-// +--------------------------------+ 0x0000080000000000 (8TB)
-// . .
-// +--------------------------------+ 0x0000000000000000
-//
-// 6 4 4 4 4
-// 3 7 6 3 2 0
-// +------------------+-----+------------------------------------------------+
-// |00000000 00000000 0|1111|111 11111111 11111111 11111111 11111111 11111111|
-// +-------------------+----+------------------------------------------------+
-// | | |
-// | | * 42-0 Object Offset (43-bits, 8TB address space)
-// | |
-// | * 46-43 Metadata Bits (4-bits) 0001 = Marked0 (Address view 8-16TB)
-// | 0010 = Marked1 (Address view 16-24TB)
-// | 0100 = Remapped (Address view 32-40TB)
-// | 1000 = Finalizable (Address view N/A)
-// |
-// * 63-47 Fixed (17-bits, always zero)
-//
-//
-// Address Space & Pointer Layout 3
-// --------------------------------
-//
-// +--------------------------------+ 0x00007FFFFFFFFFFF (127TB)
-// . .
-// . .
-// . .
-// +--------------------------------+ 0x0000500000000000 (80TB)
-// | Remapped View |
-// +--------------------------------+ 0x0000400000000000 (64TB)
-// . .
-// +--------------------------------+ 0x0000300000000000 (48TB)
-// | Marked1 View |
-// +--------------------------------+ 0x0000200000000000 (32TB)
-// | Marked0 View |
-// +--------------------------------+ 0x0000100000000000 (16TB)
-// . .
-// +--------------------------------+ 0x0000000000000000
-//
-// 6 4 4 4 4
-// 3 8 7 4 3 0
-// +------------------+----+-------------------------------------------------+
-// |00000000 00000000 |1111|1111 11111111 11111111 11111111 11111111 11111111|
-// +------------------+----+-------------------------------------------------+
-// | | |
-// | | * 43-0 Object Offset (44-bits, 16TB address space)
-// | |
-// | * 47-44 Metadata Bits (4-bits) 0001 = Marked0 (Address view 16-32TB)
-// | 0010 = Marked1 (Address view 32-48TB)
-// | 0100 = Remapped (Address view 64-80TB)
-// | 1000 = Finalizable (Address view N/A)
-// |
-// * 63-48 Fixed (16-bits, always zero)
-//
-
-size_t XPlatformAddressOffsetBits() {
- const size_t min_address_offset_bits = 42; // 4TB
- const size_t max_address_offset_bits = 44; // 16TB
- const size_t address_offset = round_up_power_of_2(MaxHeapSize * XVirtualToPhysicalRatio);
- const size_t address_offset_bits = log2i_exact(address_offset);
- return clamp(address_offset_bits, min_address_offset_bits, max_address_offset_bits);
-}
-
-size_t XPlatformAddressMetadataShift() {
- return XPlatformAddressOffsetBits();
-}
diff --git a/src/hotspot/cpu/x86/gc/x/xGlobals_x86.hpp b/src/hotspot/cpu/x86/gc/x/xGlobals_x86.hpp
deleted file mode 100644
index dd00d4ddadcf1..0000000000000
--- a/src/hotspot/cpu/x86/gc/x/xGlobals_x86.hpp
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#ifndef CPU_X86_GC_X_XGLOBALS_X86_HPP
-#define CPU_X86_GC_X_XGLOBALS_X86_HPP
-
-const size_t XPlatformHeapViews = 3;
-const size_t XPlatformCacheLineSize = 64;
-
-size_t XPlatformAddressOffsetBits();
-size_t XPlatformAddressMetadataShift();
-
-#endif // CPU_X86_GC_X_XGLOBALS_X86_HPP
diff --git a/src/hotspot/cpu/x86/gc/x/x_x86_64.ad b/src/hotspot/cpu/x86/gc/x/x_x86_64.ad
deleted file mode 100644
index ba4b3cb6df05f..0000000000000
--- a/src/hotspot/cpu/x86/gc/x/x_x86_64.ad
+++ /dev/null
@@ -1,156 +0,0 @@
-//
-// Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
-// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-//
-// This code is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License version 2 only, as
-// published by the Free Software Foundation.
-//
-// This code is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-// version 2 for more details (a copy is included in the LICENSE file that
-// accompanied this code).
-//
-// You should have received a copy of the GNU General Public License version
-// 2 along with this work; if not, write to the Free Software Foundation,
-// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-//
-// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-// or visit www.oracle.com if you need additional information or have any
-// questions.
-//
-
-source_hpp %{
-
-#include "gc/shared/gc_globals.hpp"
-#include "gc/x/c2/xBarrierSetC2.hpp"
-#include "gc/x/xThreadLocalData.hpp"
-
-%}
-
-source %{
-
-#include "c2_intelJccErratum_x86.hpp"
-
-static void x_load_barrier(MacroAssembler* masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data) {
- if (barrier_data == XLoadBarrierElided) {
- return;
- }
- XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data);
- {
- IntelJccErratumAlignment intel_alignment(masm, 10 /* jcc_size */);
- __ testptr(ref, Address(r15_thread, XThreadLocalData::address_bad_mask_offset()));
- __ jcc(Assembler::notZero, *stub->entry());
- }
- __ bind(*stub->continuation());
-}
-
-static void x_load_barrier_cmpxchg(MacroAssembler* masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, Label& good) {
- XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, XLoadBarrierStrong);
- {
- IntelJccErratumAlignment intel_alignment(masm, 10 /* jcc_size */);
- __ testptr(ref, Address(r15_thread, XThreadLocalData::address_bad_mask_offset()));
- __ jcc(Assembler::zero, good);
- }
- {
- IntelJccErratumAlignment intel_alignment(masm, 5 /* jcc_size */);
- __ jmp(*stub->entry());
- }
- __ bind(*stub->continuation());
-}
-
-static void x_cmpxchg_common(MacroAssembler* masm, const MachNode* node, Register mem_reg, Register newval, Register tmp) {
- // Compare value (oldval) is in rax
- const Address mem = Address(mem_reg, 0);
-
- if (node->barrier_data() != XLoadBarrierElided) {
- __ movptr(tmp, rax);
- }
-
- __ lock();
- __ cmpxchgptr(newval, mem);
-
- if (node->barrier_data() != XLoadBarrierElided) {
- Label good;
- x_load_barrier_cmpxchg(masm, node, mem, rax, tmp, good);
- __ movptr(rax, tmp);
- __ lock();
- __ cmpxchgptr(newval, mem);
- __ bind(good);
- }
-}
-
-%}
-
-// Load Pointer
-instruct xLoadP(rRegP dst, memory mem, rFlagsReg cr)
-%{
- predicate(UseZGC && !ZGenerational && n->as_Load()->barrier_data() != 0);
- match(Set dst (LoadP mem));
- effect(KILL cr, TEMP dst);
-
- ins_cost(125);
-
- format %{ "movq $dst, $mem" %}
-
- ins_encode %{
- __ movptr($dst$$Register, $mem$$Address);
- x_load_barrier(masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, barrier_data());
- %}
-
- ins_pipe(ialu_reg_mem);
-%}
-
-instruct xCompareAndExchangeP(indirect mem, rax_RegP oldval, rRegP newval, rRegP tmp, rFlagsReg cr) %{
- match(Set oldval (CompareAndExchangeP mem (Binary oldval newval)));
- predicate(UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
- effect(KILL cr, TEMP tmp);
-
- format %{ "lock\n\t"
- "cmpxchgq $newval, $mem" %}
-
- ins_encode %{
- precond($oldval$$Register == rax);
- x_cmpxchg_common(masm, this, $mem$$Register, $newval$$Register, $tmp$$Register);
- %}
-
- ins_pipe(pipe_cmpxchg);
-%}
-
-instruct xCompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp, rFlagsReg cr, rax_RegP oldval) %{
- match(Set res (CompareAndSwapP mem (Binary oldval newval)));
- match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
- predicate(UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
- effect(KILL cr, KILL oldval, TEMP tmp);
-
- format %{ "lock\n\t"
- "cmpxchgq $newval, $mem\n\t"
- "setcc $res \t# emits sete + movzbl or setzue for APX" %}
-
- ins_encode %{
- precond($oldval$$Register == rax);
- x_cmpxchg_common(masm, this, $mem$$Register, $newval$$Register, $tmp$$Register);
- if (barrier_data() != XLoadBarrierElided) {
- __ cmpptr($tmp$$Register, rax);
- }
- __ setcc(Assembler::equal, $res$$Register);
- %}
-
- ins_pipe(pipe_cmpxchg);
-%}
-
-instruct xXChgP(indirect mem, rRegP newval, rFlagsReg cr) %{
- match(Set newval (GetAndSetP mem newval));
- predicate(UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() != 0);
- effect(KILL cr);
-
- format %{ "xchgq $newval, $mem" %}
-
- ins_encode %{
- __ xchgptr($newval$$Register, Address($mem$$Register, 0));
- x_load_barrier(masm, this, Address(noreg, 0), $newval$$Register, noreg /* tmp */, barrier_data());
- %}
-
- ins_pipe(pipe_cmpxchg);
-%}
diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
index 65d7c1e3303ba..3795b1fc176ac 100644
--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
@@ -363,8 +363,12 @@ static void emit_store_fast_path_check_c2(MacroAssembler* masm, Address ref_addr
}
static bool is_c2_compilation() {
+#ifdef COMPILER2
CompileTask* task = ciEnv::current()->task();
return task != nullptr && is_c2_compile(task->comp_level());
+#else
+ return false;
+#endif
}
void ZBarrierSetAssembler::store_barrier_fast(MacroAssembler* masm,
@@ -1260,6 +1264,8 @@ void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm,
__ call(RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_native_oop_field_without_healing_addr()));
} else if (stub->is_atomic()) {
__ call(RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr()));
+ } else if (stub->is_nokeepalive()) {
+ __ call(RuntimeAddress(ZBarrierSetRuntime::no_keepalive_store_barrier_on_oop_field_without_healing_addr()));
} else {
__ call(RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr()));
}
diff --git a/src/hotspot/cpu/x86/gc/z/z_x86_64.ad b/src/hotspot/cpu/x86/gc/z/z_x86_64.ad
index 455d622acdf17..9555cadd0227d 100644
--- a/src/hotspot/cpu/x86/gc/z/z_x86_64.ad
+++ b/src/hotspot/cpu/x86/gc/z/z_x86_64.ad
@@ -91,7 +91,8 @@ static void z_store_barrier(MacroAssembler* masm, const MachNode* node, Address
}
} else {
bool is_native = (node->barrier_data() & ZBarrierNative) != 0;
- ZStoreBarrierStubC2* const stub = ZStoreBarrierStubC2::create(node, ref_addr, rnew_zaddress, rnew_zpointer, is_native, is_atomic);
+ bool is_nokeepalive = (node->barrier_data() & ZBarrierNoKeepalive) != 0;
+ ZStoreBarrierStubC2* const stub = ZStoreBarrierStubC2::create(node, ref_addr, rnew_zaddress, rnew_zpointer, is_native, is_atomic, is_nokeepalive);
ZBarrierSetAssembler* bs_asm = ZBarrierSet::assembler();
bs_asm->store_barrier_fast(masm, ref_addr, rnew_zaddress, rnew_zpointer, true /* in_nmethod */, is_atomic, *stub->entry(), *stub->continuation());
}
@@ -114,7 +115,7 @@ operand no_rax_RegP()
// Load Pointer
instruct zLoadP(rRegP dst, memory mem, rFlagsReg cr)
%{
- predicate(UseZGC && ZGenerational && n->as_Load()->barrier_data() != 0);
+ predicate(UseZGC && n->as_Load()->barrier_data() != 0);
match(Set dst (LoadP mem));
effect(TEMP dst, KILL cr);
@@ -133,7 +134,7 @@ instruct zLoadP(rRegP dst, memory mem, rFlagsReg cr)
// Load Pointer and Null Check
instruct zLoadPNullCheck(rFlagsReg cr, memory op, immP0 zero)
%{
- predicate(UseZGC && ZGenerational && n->in(1)->as_Load()->barrier_data() != 0);
+ predicate(UseZGC && n->in(1)->as_Load()->barrier_data() != 0);
match(Set cr (CmpP (LoadP op) zero));
ins_cost(500); // XXX
@@ -149,7 +150,7 @@ instruct zLoadPNullCheck(rFlagsReg cr, memory op, immP0 zero)
// Store Pointer
instruct zStoreP(memory mem, any_RegP src, rRegP tmp, rFlagsReg cr)
%{
- predicate(UseZGC && ZGenerational && n->as_Store()->barrier_data() != 0);
+ predicate(UseZGC && n->as_Store()->barrier_data() != 0);
match(Set mem (StoreP mem src));
effect(TEMP tmp, KILL cr);
@@ -165,7 +166,7 @@ instruct zStoreP(memory mem, any_RegP src, rRegP tmp, rFlagsReg cr)
// Store Null Pointer
instruct zStorePNull(memory mem, immP0 zero, rRegP tmp, rFlagsReg cr)
%{
- predicate(UseZGC && ZGenerational && n->as_Store()->barrier_data() != 0);
+ predicate(UseZGC && n->as_Store()->barrier_data() != 0);
match(Set mem (StoreP mem zero));
effect(TEMP tmp, KILL cr);
@@ -184,7 +185,7 @@ instruct zStorePNull(memory mem, immP0 zero, rRegP tmp, rFlagsReg cr)
instruct zCompareAndExchangeP(indirect mem, no_rax_RegP newval, rRegP tmp, rax_RegP oldval, rFlagsReg cr) %{
match(Set oldval (CompareAndExchangeP mem (Binary oldval newval)));
- predicate(UseZGC && ZGenerational && n->as_LoadStore()->barrier_data() != 0);
+ predicate(UseZGC && n->as_LoadStore()->barrier_data() != 0);
effect(TEMP tmp, KILL cr);
format %{ "lock\n\t"
@@ -207,7 +208,7 @@ instruct zCompareAndExchangeP(indirect mem, no_rax_RegP newval, rRegP tmp, rax_R
instruct zCompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp, rax_RegP oldval, rFlagsReg cr) %{
match(Set res (CompareAndSwapP mem (Binary oldval newval)));
match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
- predicate(UseZGC && ZGenerational && n->as_LoadStore()->barrier_data() != 0);
+ predicate(UseZGC && n->as_LoadStore()->barrier_data() != 0);
effect(TEMP tmp, KILL oldval, KILL cr);
format %{ "lock\n\t"
@@ -229,7 +230,7 @@ instruct zCompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp, rax_
instruct zXChgP(indirect mem, rRegP newval, rRegP tmp, rFlagsReg cr) %{
match(Set newval (GetAndSetP mem newval));
- predicate(UseZGC && ZGenerational && n->as_LoadStore()->barrier_data() != 0);
+ predicate(UseZGC && n->as_LoadStore()->barrier_data() != 0);
effect(TEMP tmp, KILL cr);
format %{ "xchgq $newval, $mem" %}
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
index 893ae4e844ba4..55c4e29b8a33b 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
@@ -3482,6 +3482,17 @@ void MacroAssembler::vpbroadcastd(XMMRegister dst, AddressLiteral src, int vecto
}
}
+void MacroAssembler::vbroadcasti128(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
+ assert(rscratch != noreg || always_reachable(src), "missing");
+
+ if (reachable(src)) {
+ Assembler::vbroadcasti128(dst, as_Address(src), vector_len);
+ } else {
+ lea(rscratch, src);
+ Assembler::vbroadcasti128(dst, Address(rscratch, 0), vector_len);
+ }
+}
+
void MacroAssembler::vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
assert(rscratch != noreg || always_reachable(src), "missing");
@@ -5084,7 +5095,8 @@ void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fa
L_slow_path = &L_fallthrough;
}
- // Fast path check: class is fully initialized
+ // Fast path check: class is fully initialized.
+ // init_state needs acquire, but x86 is TSO, and so we are already good.
cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
jcc(Assembler::equal, *L_fast_path);
@@ -9302,6 +9314,30 @@ void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len
bind(done);
}
+void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, XMMRegister src, bool merge, int vector_len) {
+ switch(type) {
+ case T_BYTE:
+ case T_BOOLEAN:
+ evmovdqub(dst, kmask, src, merge, vector_len);
+ break;
+ case T_CHAR:
+ case T_SHORT:
+ evmovdquw(dst, kmask, src, merge, vector_len);
+ break;
+ case T_INT:
+ case T_FLOAT:
+ evmovdqul(dst, kmask, src, merge, vector_len);
+ break;
+ case T_LONG:
+ case T_DOUBLE:
+ evmovdquq(dst, kmask, src, merge, vector_len);
+ break;
+ default:
+ fatal("Unexpected type argument %s", type2name(type));
+ break;
+ }
+}
+
void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len) {
switch(type) {
@@ -9493,6 +9529,66 @@ void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMM
}
}
+void MacroAssembler::evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+ switch(type) {
+ case T_BYTE:
+ evpminub(dst, mask, nds, src, merge, vector_len); break;
+ case T_SHORT:
+ evpminuw(dst, mask, nds, src, merge, vector_len); break;
+ case T_INT:
+ evpminud(dst, mask, nds, src, merge, vector_len); break;
+ case T_LONG:
+ evpminuq(dst, mask, nds, src, merge, vector_len); break;
+ default:
+ fatal("Unexpected type argument %s", type2name(type)); break;
+ }
+}
+
+void MacroAssembler::evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
+ switch(type) {
+ case T_BYTE:
+ evpmaxub(dst, mask, nds, src, merge, vector_len); break;
+ case T_SHORT:
+ evpmaxuw(dst, mask, nds, src, merge, vector_len); break;
+ case T_INT:
+ evpmaxud(dst, mask, nds, src, merge, vector_len); break;
+ case T_LONG:
+ evpmaxuq(dst, mask, nds, src, merge, vector_len); break;
+ default:
+ fatal("Unexpected type argument %s", type2name(type)); break;
+ }
+}
+
+void MacroAssembler::evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ switch(type) {
+ case T_BYTE:
+ evpminub(dst, mask, nds, src, merge, vector_len); break;
+ case T_SHORT:
+ evpminuw(dst, mask, nds, src, merge, vector_len); break;
+ case T_INT:
+ evpminud(dst, mask, nds, src, merge, vector_len); break;
+ case T_LONG:
+ evpminuq(dst, mask, nds, src, merge, vector_len); break;
+ default:
+ fatal("Unexpected type argument %s", type2name(type)); break;
+ }
+}
+
+void MacroAssembler::evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
+ switch(type) {
+ case T_BYTE:
+ evpmaxub(dst, mask, nds, src, merge, vector_len); break;
+ case T_SHORT:
+ evpmaxuw(dst, mask, nds, src, merge, vector_len); break;
+ case T_INT:
+ evpmaxud(dst, mask, nds, src, merge, vector_len); break;
+ case T_LONG:
+ evpmaxuq(dst, mask, nds, src, merge, vector_len); break;
+ default:
+ fatal("Unexpected type argument %s", type2name(type)); break;
+ }
+}
+
void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
switch(type) {
case T_BYTE:
@@ -10201,17 +10297,6 @@ Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond)
ShouldNotReachHere(); return Assembler::overflow;
}
-SkipIfEqual::SkipIfEqual(
- MacroAssembler* masm, const bool* flag_addr, bool value, Register rscratch) {
- _masm = masm;
- _masm->cmp8(ExternalAddress((address)flag_addr), value, rscratch);
- _masm->jcc(Assembler::equal, _label);
-}
-
-SkipIfEqual::~SkipIfEqual() {
- _masm->bind(_label);
-}
-
// 32-bit Windows has its own fast-path implementation
// of get_thread
#if !defined(WIN32) || defined(_LP64)
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
index 2ce4fc40e90e0..d508feed93c28 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
@@ -1118,6 +1118,7 @@ class MacroAssembler: public Assembler {
XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
Register buf, Register state, Register ofs, Register limit, Register rsp, bool multi_block,
XMMRegister shuf_mask);
+ void sha512_update_ni_x1(Register arg_hash, Register arg_msg, Register ofs, Register limit, bool multi_block);
#endif // _LP64
void fast_md5(Register buf, Address state, Address ofs, Address limit,
@@ -1216,6 +1217,9 @@ class MacroAssembler: public Assembler {
void addpd(XMMRegister dst, Address src) { Assembler::addpd(dst, src); }
void addpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+ using Assembler::vbroadcasti128;
+ void vbroadcasti128(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
+
using Assembler::vbroadcastsd;
void vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
@@ -1278,6 +1282,7 @@ class MacroAssembler: public Assembler {
// AVX512 Unaligned
void evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len);
void evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len);
+ void evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, XMMRegister src, bool merge, int vector_len);
void evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) { Assembler::evmovdqub(dst, src, vector_len); }
void evmovdqub(XMMRegister dst, Address src, int vector_len) { Assembler::evmovdqub(dst, src, vector_len); }
@@ -1291,6 +1296,7 @@ class MacroAssembler: public Assembler {
void evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { Assembler::evmovdqub(dst, mask, src, merge, vector_len); }
void evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
+ void evmovdquw(XMMRegister dst, XMMRegister src, int vector_len) { Assembler::evmovdquw(dst, src, vector_len); }
void evmovdquw(Address dst, XMMRegister src, int vector_len) { Assembler::evmovdquw(dst, src, vector_len); }
void evmovdquw(XMMRegister dst, Address src, int vector_len) { Assembler::evmovdquw(dst, src, vector_len); }
@@ -1501,6 +1507,8 @@ class MacroAssembler: public Assembler {
void vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vpmulld(dst, nds, src, vector_len); }
void vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
+ void vpmuldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vpmuldq(dst, nds, src, vector_len); }
+
void vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
@@ -1510,9 +1518,13 @@ class MacroAssembler: public Assembler {
void vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
void vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
+ void evpsrad(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
+ void evpsrad(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
+
void evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
void evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
+ using Assembler::evpsllw;
void evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
if (!is_varshift) {
Assembler::evpsllw(dst, mask, nds, src, merge, vector_len);
@@ -1557,6 +1569,7 @@ class MacroAssembler: public Assembler {
Assembler::evpsrlvq(dst, mask, nds, src, merge, vector_len);
}
}
+ using Assembler::evpsraw;
void evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
if (!is_varshift) {
Assembler::evpsraw(dst, mask, nds, src, merge, vector_len);
@@ -1564,6 +1577,7 @@ class MacroAssembler: public Assembler {
Assembler::evpsravw(dst, mask, nds, src, merge, vector_len);
}
}
+ using Assembler::evpsrad;
void evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
if (!is_varshift) {
Assembler::evpsrad(dst, mask, nds, src, merge, vector_len);
@@ -1585,6 +1599,11 @@ class MacroAssembler: public Assembler {
void evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
void evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+ void evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
+ void evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
+ void evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+ void evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
+
void vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
void vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
@@ -2158,22 +2177,4 @@ class MacroAssembler: public Assembler {
#endif
};
-/**
- * class SkipIfEqual:
- *
- * Instantiating this class will result in assembly code being output that will
- * jump around any code emitted between the creation of the instance and it's
- * automatic destruction at the end of a scope block, depending on the value of
- * the flag passed to the constructor, which will be checked at run-time.
- */
-class SkipIfEqual {
- private:
- MacroAssembler* _masm;
- Label _label;
-
- public:
- SkipIfEqual(MacroAssembler*, const bool* flag_addr, bool value, Register rscratch);
- ~SkipIfEqual();
-};
-
#endif // CPU_X86_MACROASSEMBLER_X86_HPP
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp b/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp
index 090de71425f17..e7d728c2e9672 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp
@@ -1519,5 +1519,184 @@ void MacroAssembler::sha512_AVX2(XMMRegister msg, XMMRegister state0, XMMRegiste
}
}
+//Implemented using Intel IpSec implementation (intel-ipsec-mb on github)
+void MacroAssembler::sha512_update_ni_x1(Register arg_hash, Register arg_msg, Register ofs, Register limit, bool multi_block) {
+ Label done_hash, block_loop;
+ address K512_W = StubRoutines::x86::k512_W_addr();
+
+ vbroadcasti128(xmm15, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_addr_sha512()), Assembler::AVX_256bit, r10);
+
+ //load current hash value and transform
+ vmovdqu(xmm0, Address(arg_hash));
+ vmovdqu(xmm1, Address(arg_hash, 32));
+ //ymm0 = D C B A, ymm1 = H G F E
+ vperm2i128(xmm2, xmm0, xmm1, 0x20);
+ vperm2i128(xmm3, xmm0, xmm1, 0x31);
+ //ymm2 = F E B A, ymm3 = H G D C
+ vpermq(xmm13, xmm2, 0x1b, Assembler::AVX_256bit);
+ vpermq(xmm14, xmm3, 0x1b, Assembler::AVX_256bit);
+ //ymm13 = A B E F, ymm14 = C D G H
+
+ lea(rax, ExternalAddress(K512_W));
+ align(32);
+ bind(block_loop);
+ vmovdqu(xmm11, xmm13);//ABEF
+ vmovdqu(xmm12, xmm14);//CDGH
+
+ //R0 - R3
+ vmovdqu(xmm0, Address(arg_msg, 0 * 32));
+ vpshufb(xmm3, xmm0, xmm15, Assembler::AVX_256bit);//ymm0 / ymm3 = W[0..3]
+ vpaddq(xmm0, xmm3, Address(rax, 0 * 32), Assembler::AVX_256bit);
+ sha512rnds2(xmm12, xmm11, xmm0);
+ vperm2i128(xmm0, xmm0, xmm0, 0x01);
+ sha512rnds2(xmm11, xmm12, xmm0);
+
+ //R4 - R7
+ vmovdqu(xmm0, Address(arg_msg, 1 * 32));
+ vpshufb(xmm4, xmm0, xmm15, Assembler::AVX_256bit);//ymm0 / ymm4 = W[4..7]
+ vpaddq(xmm0, xmm4, Address(rax, 1 * 32), Assembler::AVX_256bit);
+ sha512rnds2(xmm12, xmm11, xmm0);
+ vperm2i128(xmm0, xmm0, xmm0, 0x01);
+ sha512rnds2(xmm11, xmm12, xmm0);
+ sha512msg1(xmm3, xmm4); //ymm3 = W[0..3] + S0(W[1..4])
+
+ //R8 - R11
+ vmovdqu(xmm0, Address(arg_msg, 2 * 32));
+ vpshufb(xmm5, xmm0, xmm15, Assembler::AVX_256bit);//ymm0 / ymm5 = W[8..11]
+ vpaddq(xmm0, xmm5, Address(rax, 2 * 32), Assembler::AVX_256bit);
+ sha512rnds2(xmm12, xmm11, xmm0);
+ vperm2i128(xmm0, xmm0, xmm0, 0x01);
+ sha512rnds2(xmm11, xmm12, xmm0);
+ sha512msg1(xmm4, xmm5);//ymm4 = W[4..7] + S0(W[5..8])
+
+ //R12 - R15
+ vmovdqu(xmm0, Address(arg_msg, 3 * 32));
+ vpshufb(xmm6, xmm0, xmm15, Assembler::AVX_256bit); //ymm0 / ymm6 = W[12..15]
+ vpaddq(xmm0, xmm6, Address(rax, 3 * 32), Assembler::AVX_256bit);
+ vpermq(xmm8, xmm6, 0x1b, Assembler::AVX_256bit); //ymm8 = W[12] W[13] W[14] W[15]
+ vpermq(xmm9, xmm5, 0x39, Assembler::AVX_256bit); //ymm9 = W[8] W[11] W[10] W[9]
+ vpblendd(xmm8, xmm8, xmm9, 0x3f, Assembler::AVX_256bit); //ymm8 = W[12] W[11] W[10] W[9]
+ vpaddq(xmm3, xmm3, xmm8, Assembler::AVX_256bit);
+ sha512msg2(xmm3, xmm6);//W[16..19] = xmm3 + W[9..12] + S1(W[14..17])
+ sha512rnds2(xmm12, xmm11, xmm0);
+ vperm2i128(xmm0, xmm0, xmm0, 0x01);
+ sha512rnds2(xmm11, xmm12, xmm0);
+ sha512msg1(xmm5, xmm6); //ymm5 = W[8..11] + S0(W[9..12])
+
+ //R16 - R19, R32 - R35, R48 - R51
+ for (int i = 4, j = 3; j > 0; j--) {
+ vpaddq(xmm0, xmm3, Address(rax, i * 32), Assembler::AVX_256bit);
+ vpermq(xmm8, xmm3, 0x1b, Assembler::AVX_256bit);//ymm8 = W[16] W[17] W[18] W[19]
+ vpermq(xmm9, xmm6, 0x39, Assembler::AVX_256bit);//ymm9 = W[12] W[15] W[14] W[13]
+ vpblendd(xmm7, xmm8, xmm9, 0x3f, Assembler::AVX_256bit);//xmm7 = W[16] W[15] W[14] W[13]
+ vpaddq(xmm4, xmm4, xmm7, Assembler::AVX_256bit);//ymm4 = W[4..7] + S0(W[5..8]) + W[13..16]
+ sha512msg2(xmm4, xmm3);//ymm4 += S1(W[14..17])
+ sha512rnds2(xmm12, xmm11, xmm0);
+ vperm2i128(xmm0, xmm0, xmm0, 0x01);
+ sha512rnds2(xmm11, xmm12, xmm0);
+ sha512msg1(xmm6, xmm3); //ymm6 = W[12..15] + S0(W[13..16])
+ i += 1;
+ //R20 - R23, R36 - R39, R52 - R55
+ vpaddq(xmm0, xmm4, Address(rax, i * 32), Assembler::AVX_256bit);
+ vpermq(xmm8, xmm4, 0x1b, Assembler::AVX_256bit);//ymm8 = W[20] W[21] W[22] W[23]
+ vpermq(xmm9, xmm3, 0x39, Assembler::AVX_256bit);//ymm9 = W[16] W[19] W[18] W[17]
+ vpblendd(xmm7, xmm8, xmm9, 0x3f, Assembler::AVX_256bit);//ymm7 = W[20] W[19] W[18] W[17]
+ vpaddq(xmm5, xmm5, xmm7, Assembler::AVX_256bit);//ymm5 = W[8..11] + S0(W[9..12]) + W[17..20]
+ sha512msg2(xmm5, xmm4);//ymm5 += S1(W[18..21])
+ sha512rnds2(xmm12, xmm11, xmm0);
+ vperm2i128(xmm0, xmm0, xmm0, 0x01);
+ sha512rnds2(xmm11, xmm12, xmm0);
+ sha512msg1(xmm3, xmm4); //ymm3 = W[16..19] + S0(W[17..20])
+ i += 1;
+ //R24 - R27, R40 - R43, R56 - R59
+ vpaddq(xmm0, xmm5, Address(rax, i * 32), Assembler::AVX_256bit);
+ vpermq(xmm8, xmm5, 0x1b, Assembler::AVX_256bit);//ymm8 = W[24] W[25] W[26] W[27]
+ vpermq(xmm9, xmm4, 0x39, Assembler::AVX_256bit);//ymm9 = W[20] W[23] W[22] W[21]
+ vpblendd(xmm7, xmm8, xmm9, 0x3f, Assembler::AVX_256bit);//ymm7 = W[24] W[23] W[22] W[21]
+ vpaddq(xmm6, xmm6, xmm7, Assembler::AVX_256bit);//ymm6 = W[12..15] + S0(W[13..16]) + W[21..24]
+ sha512msg2(xmm6, xmm5);//ymm6 += S1(W[22..25])
+ sha512rnds2(xmm12, xmm11, xmm0);
+ vperm2i128(xmm0, xmm0, xmm0, 0x01);
+ sha512rnds2(xmm11, xmm12, xmm0);
+ sha512msg1(xmm4, xmm5);//ymm4 = W[20..23] + S0(W[21..24])
+ i += 1;
+ //R28 - R31, R44 - R47, R60 - R63
+ vpaddq(xmm0, xmm6, Address(rax, i * 32), Assembler::AVX_256bit);
+ vpermq(xmm8, xmm6, 0x1b, Assembler::AVX_256bit);//ymm8 = W[28] W[29] W[30] W[31]
+ vpermq(xmm9, xmm5, 0x39, Assembler::AVX_256bit);//ymm9 = W[24] W[27] W[26] W[25]
+ vpblendd(xmm7, xmm8, xmm9, 0x3f, Assembler::AVX_256bit);//ymm7 = W[28] W[27] W[26] W[25]
+ vpaddq(xmm3, xmm3, xmm7, Assembler::AVX_256bit);//ymm3 = W[16..19] + S0(W[17..20]) + W[25..28]
+ sha512msg2(xmm3, xmm6); //ymm3 += S1(W[26..29])
+ sha512rnds2(xmm12, xmm11, xmm0);
+ vperm2i128(xmm0, xmm0, xmm0, 0x01);
+ sha512rnds2(xmm11, xmm12, xmm0);
+ sha512msg1(xmm5, xmm6);//ymm5 = W[24..27] + S0(W[25..28])
+ i += 1;
+ }
+ //R64 - R67
+ vpaddq(xmm0, xmm3, Address(rax, 16 * 32), Assembler::AVX_256bit);
+ vpermq(xmm8, xmm3, 0x1b, Assembler::AVX_256bit);//ymm8 = W[64] W[65] W[66] W[67]
+ vpermq(xmm9, xmm6, 0x39, Assembler::AVX_256bit);//ymm9 = W[60] W[63] W[62] W[61]
+ vpblendd(xmm7, xmm8, xmm9, 0x3f, Assembler::AVX_256bit);//ymm7 = W[64] W[63] W[62] W[61]
+ vpaddq(xmm4, xmm4, xmm7, Assembler::AVX_256bit);//ymm4 = W[52..55] + S0(W[53..56]) + W[61..64]
+ sha512msg2(xmm4, xmm3);//ymm4 += S1(W[62..65])
+ sha512rnds2(xmm12, xmm11, xmm0);
+ vperm2i128(xmm0, xmm0, xmm0, 0x01);
+ sha512rnds2(xmm11, xmm12, xmm0);
+ sha512msg1(xmm6, xmm3);//ymm6 = W[60..63] + S0(W[61..64])
+
+ //R68 - R71
+ vpaddq(xmm0, xmm4, Address(rax, 17 * 32), Assembler::AVX_256bit);
+ vpermq(xmm8, xmm4, 0x1b, Assembler::AVX_256bit);//ymm8 = W[68] W[69] W[70] W[71]
+ vpermq(xmm9, xmm3, 0x39, Assembler::AVX_256bit);//ymm9 = W[64] W[67] W[66] W[65]
+ vpblendd(xmm7, xmm8, xmm9, 0x3f, Assembler::AVX_256bit);//ymm7 = W[68] W[67] W[66] W[65]
+ vpaddq(xmm5, xmm5, xmm7, Assembler::AVX_256bit);//ymm5 = W[56..59] + S0(W[57..60]) + W[65..68]
+ sha512msg2(xmm5, xmm4);//ymm5 += S1(W[66..69])
+ sha512rnds2(xmm12, xmm11, xmm0);
+ vperm2i128(xmm0, xmm0, xmm0, 0x01);
+ sha512rnds2(xmm11, xmm12, xmm0);
+
+ //R72 - R75
+ vpaddq(xmm0, xmm5, Address(rax, 18 * 32), Assembler::AVX_256bit);
+ vpermq(xmm8, xmm5, 0x1b, Assembler::AVX_256bit);//ymm8 = W[72] W[73] W[74] W[75]
+ vpermq(xmm9, xmm4, 0x39, Assembler::AVX_256bit);//ymm9 = W[68] W[71] W[70] W[69]
+ vpblendd(xmm7, xmm8, xmm9, 0x3f, Assembler::AVX_256bit);//ymm7 = W[72] W[71] W[70] W[69]
+ vpaddq(xmm6, xmm6, xmm7, Assembler::AVX_256bit);//ymm6 = W[60..63] + S0(W[61..64]) + W[69..72]
+ sha512msg2(xmm6, xmm5);//ymm6 += S1(W[70..73])
+ sha512rnds2(xmm12, xmm11, xmm0);
+ vperm2i128(xmm0, xmm0, xmm0, 0x01);
+ sha512rnds2(xmm11, xmm12, xmm0);
+
+ //R76 - R79
+ vpaddq(xmm0, xmm6, Address(rax, 19 * 32), Assembler::AVX_256bit);
+ sha512rnds2(xmm12, xmm11, xmm0);
+ vperm2i128(xmm0, xmm0, xmm0, 0x01);
+ sha512rnds2(xmm11, xmm12, xmm0);
+
+ //update hash value
+ vpaddq(xmm14, xmm14, xmm12, Assembler::AVX_256bit);
+ vpaddq(xmm13, xmm13, xmm11, Assembler::AVX_256bit);
+
+ if (multi_block) {
+ addptr(arg_msg, 4 * 32);
+ addptr(ofs, 128);
+ cmpptr(ofs, limit);
+ jcc(Assembler::belowEqual, block_loop);
+ movptr(rax, ofs); //return ofs
+ }
+
+ //store the hash value back in memory
+ //xmm13 = ABEF
+ //xmm14 = CDGH
+ vperm2i128(xmm1, xmm13, xmm14, 0x31);
+ vperm2i128(xmm2, xmm13, xmm14, 0x20);
+ vpermq(xmm1, xmm1, 0xb1, Assembler::AVX_256bit);//ymm1 = D C B A
+ vpermq(xmm2, xmm2, 0xb1, Assembler::AVX_256bit);//ymm2 = H G F E
+ vmovdqu(Address(arg_hash, 0 * 32), xmm1);
+ vmovdqu(Address(arg_hash, 1 * 32), xmm2);
+
+ bind(done_hash);
+}
+
#endif //#ifdef _LP64
diff --git a/src/hotspot/cpu/x86/nativeInst_x86.cpp b/src/hotspot/cpu/x86/nativeInst_x86.cpp
index 395c3219809de..d5021c29ed6b0 100644
--- a/src/hotspot/cpu/x86/nativeInst_x86.cpp
+++ b/src/hotspot/cpu/x86/nativeInst_x86.cpp
@@ -81,7 +81,7 @@ void NativeCall::insert(address code_pos, address entry) {
// (spinlock). Then patches the last byte, and then atomically replaces
// the jmp's with the first 4 byte of the new instruction.
void NativeCall::replace_mt_safe(address instr_addr, address code_buffer) {
- assert(Patching_lock->is_locked() ||
+ assert(CodeCache_lock->is_locked() ||
SafepointSynchronize::is_at_safepoint(), "concurrent code patching");
assert (instr_addr != nullptr, "illegal address for code patching");
@@ -144,7 +144,7 @@ void NativeCall::set_destination_mt_safe(address dest) {
debug_only(verify());
// Make sure patching code is locked. No two threads can patch at the same
// time but one may be executing this code.
- assert(Patching_lock->is_locked() || SafepointSynchronize::is_at_safepoint() ||
+ assert(CodeCache_lock->is_locked() || SafepointSynchronize::is_at_safepoint() ||
CompiledICLocker::is_safe(instruction_address()), "concurrent code patching");
// Both C1 and C2 should now be generating code which aligns the patched address
// to be within a single cache line.
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
index 4f37dc31d0305..93b1618024e2d 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
@@ -24,6 +24,7 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
+#include "classfile/javaClasses.hpp"
#include "classfile/vmIntrinsics.hpp"
#include "compiler/oopMap.hpp"
#include "gc/shared/barrierSet.hpp"
@@ -1557,7 +1558,7 @@ address StubGenerator::generate_sha256_implCompress(bool multi_block, const char
address StubGenerator::generate_sha512_implCompress(bool multi_block, const char *name) {
assert(VM_Version::supports_avx2(), "");
- assert(VM_Version::supports_bmi2(), "");
+ assert(VM_Version::supports_bmi2() || VM_Version::supports_sha512(), "");
__ align(CodeEntryAlignment);
StubCodeMark mark(this, "StubRoutines", name);
address start = __ pc();
@@ -1567,22 +1568,24 @@ address StubGenerator::generate_sha512_implCompress(bool multi_block, const char
Register ofs = c_rarg2;
Register limit = c_rarg3;
- const XMMRegister msg = xmm0;
- const XMMRegister state0 = xmm1;
- const XMMRegister state1 = xmm2;
- const XMMRegister msgtmp0 = xmm3;
- const XMMRegister msgtmp1 = xmm4;
- const XMMRegister msgtmp2 = xmm5;
- const XMMRegister msgtmp3 = xmm6;
- const XMMRegister msgtmp4 = xmm7;
-
- const XMMRegister shuf_mask = xmm8;
-
__ enter();
- __ sha512_AVX2(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
- buf, state, ofs, limit, rsp, multi_block, shuf_mask);
-
+ if (VM_Version::supports_sha512()) {
+ __ sha512_update_ni_x1(state, buf, ofs, limit, multi_block);
+ } else {
+ const XMMRegister msg = xmm0;
+ const XMMRegister state0 = xmm1;
+ const XMMRegister state1 = xmm2;
+ const XMMRegister msgtmp0 = xmm3;
+ const XMMRegister msgtmp1 = xmm4;
+ const XMMRegister msgtmp2 = xmm5;
+ const XMMRegister msgtmp3 = xmm6;
+ const XMMRegister msgtmp4 = xmm7;
+
+ const XMMRegister shuf_mask = xmm8;
+ __ sha512_AVX2(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
+ buf, state, ofs, limit, rsp, multi_block, shuf_mask);
+ }
__ vzeroupper();
__ leave();
__ ret(0);
@@ -3796,6 +3799,28 @@ address StubGenerator::generate_upcall_stub_exception_handler() {
return start;
}
+// load Method* target of MethodHandle
+// j_rarg0 = jobject receiver
+// rbx = result
+address StubGenerator::generate_upcall_stub_load_target() {
+ StubCodeMark mark(this, "StubRoutines", "upcall_stub_load_target");
+ address start = __ pc();
+
+ __ resolve_global_jobject(j_rarg0, r15_thread, rscratch1);
+ // Load target method from receiver
+ __ load_heap_oop(rbx, Address(j_rarg0, java_lang_invoke_MethodHandle::form_offset()), rscratch1);
+ __ load_heap_oop(rbx, Address(rbx, java_lang_invoke_LambdaForm::vmentry_offset()), rscratch1);
+ __ load_heap_oop(rbx, Address(rbx, java_lang_invoke_MemberName::method_offset()), rscratch1);
+ __ access_load_at(T_ADDRESS, IN_HEAP, rbx,
+ Address(rbx, java_lang_invoke_ResolvedMethodName::vmtarget_offset()),
+ noreg, noreg);
+ __ movptr(Address(r15_thread, JavaThread::callee_target_offset()), rbx); // just in case callee is deoptimized
+
+ __ ret(0);
+
+ return start;
+}
+
address StubGenerator::generate_lookup_secondary_supers_table_stub(u1 super_klass_index) {
StubCodeMark mark(this, "StubRoutines", "lookup_secondary_supers_table");
@@ -3955,6 +3980,7 @@ void StubGenerator::generate_final_stubs() {
}
StubRoutines::_upcall_stub_exception_handler = generate_upcall_stub_exception_handler();
+ StubRoutines::_upcall_stub_load_target = generate_upcall_stub_load_target();
}
void StubGenerator::generate_compiler_stubs() {
@@ -4006,6 +4032,8 @@ void StubGenerator::generate_compiler_stubs() {
generate_chacha_stubs();
+ generate_sha3_stubs();
+
#ifdef COMPILER2
if ((UseAVX == 2) && EnableX86ECoreOpts) {
generate_string_indexof(StubRoutines::_string_indexof_array);
@@ -4160,41 +4188,41 @@ void StubGenerator::generate_compiler_stubs() {
log_info(library)("Loaded library %s, handle " INTPTR_FORMAT, JNI_LIB_PREFIX "jsvml" JNI_LIB_SUFFIX, p2i(libjsvml));
if (UseAVX > 2) {
- for (int op = 0; op < VectorSupport::NUM_SVML_OP; op++) {
- int vop = VectorSupport::VECTOR_OP_SVML_START + op;
+ for (int op = 0; op < VectorSupport::NUM_VECTOR_OP_MATH; op++) {
+ int vop = VectorSupport::VECTOR_OP_MATH_START + op;
if ((!VM_Version::supports_avx512dq()) &&
(vop == VectorSupport::VECTOR_OP_LOG || vop == VectorSupport::VECTOR_OP_LOG10 || vop == VectorSupport::VECTOR_OP_POW)) {
continue;
}
- snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf16_ha_z0", VectorSupport::svmlname[op]);
+ snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf16_ha_z0", VectorSupport::mathname[op]);
StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_512][op] = (address)os::dll_lookup(libjsvml, ebuf);
- snprintf(ebuf, sizeof(ebuf), "__jsvml_%s8_ha_z0", VectorSupport::svmlname[op]);
+ snprintf(ebuf, sizeof(ebuf), "__jsvml_%s8_ha_z0", VectorSupport::mathname[op]);
StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_512][op] = (address)os::dll_lookup(libjsvml, ebuf);
}
}
const char* avx_sse_str = (UseAVX >= 2) ? "l9" : ((UseAVX == 1) ? "e9" : "ex");
- for (int op = 0; op < VectorSupport::NUM_SVML_OP; op++) {
- int vop = VectorSupport::VECTOR_OP_SVML_START + op;
+ for (int op = 0; op < VectorSupport::NUM_VECTOR_OP_MATH; op++) {
+ int vop = VectorSupport::VECTOR_OP_MATH_START + op;
if (vop == VectorSupport::VECTOR_OP_POW) {
continue;
}
- snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf4_ha_%s", VectorSupport::svmlname[op], avx_sse_str);
+ snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf4_ha_%s", VectorSupport::mathname[op], avx_sse_str);
StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_64][op] = (address)os::dll_lookup(libjsvml, ebuf);
- snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf4_ha_%s", VectorSupport::svmlname[op], avx_sse_str);
+ snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf4_ha_%s", VectorSupport::mathname[op], avx_sse_str);
StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_128][op] = (address)os::dll_lookup(libjsvml, ebuf);
- snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf8_ha_%s", VectorSupport::svmlname[op], avx_sse_str);
+ snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf8_ha_%s", VectorSupport::mathname[op], avx_sse_str);
StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_256][op] = (address)os::dll_lookup(libjsvml, ebuf);
- snprintf(ebuf, sizeof(ebuf), "__jsvml_%s1_ha_%s", VectorSupport::svmlname[op], avx_sse_str);
+ snprintf(ebuf, sizeof(ebuf), "__jsvml_%s1_ha_%s", VectorSupport::mathname[op], avx_sse_str);
StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_64][op] = (address)os::dll_lookup(libjsvml, ebuf);
- snprintf(ebuf, sizeof(ebuf), "__jsvml_%s2_ha_%s", VectorSupport::svmlname[op], avx_sse_str);
+ snprintf(ebuf, sizeof(ebuf), "__jsvml_%s2_ha_%s", VectorSupport::mathname[op], avx_sse_str);
StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_128][op] = (address)os::dll_lookup(libjsvml, ebuf);
- snprintf(ebuf, sizeof(ebuf), "__jsvml_%s4_ha_%s", VectorSupport::svmlname[op], avx_sse_str);
+ snprintf(ebuf, sizeof(ebuf), "__jsvml_%s4_ha_%s", VectorSupport::mathname[op], avx_sse_str);
StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_256][op] = (address)os::dll_lookup(libjsvml, ebuf);
}
}
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp
index 71777fbfffea2..c6fa31c5213b0 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp
@@ -497,6 +497,10 @@ class StubGenerator: public StubCodeGenerator {
address generate_intpoly_montgomeryMult_P256();
address generate_intpoly_assign();
+ // SHA3 stubs
+ void generate_sha3_stubs();
+ address generate_sha3_implCompress(bool multiBlock, const char *name);
+
// BASE64 stubs
address base64_shuffle_addr();
@@ -620,6 +624,7 @@ class StubGenerator: public StubCodeGenerator {
// shared exception handler for FFM upcall stubs
address generate_upcall_stub_exception_handler();
+ address generate_upcall_stub_load_target();
// Specialized stub implementations for UseSecondarySupersTable.
address generate_lookup_secondary_supers_table_stub(u1 super_klass_index);
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp
new file mode 100644
index 0000000000000..49c39226708e3
--- /dev/null
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp
@@ -0,0 +1,326 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "asm/assembler.hpp"
+#include "asm/assembler.inline.hpp"
+#include "runtime/stubRoutines.hpp"
+#include "macroAssembler_x86.hpp"
+#include "stubGenerator_x86_64.hpp"
+
+#define __ _masm->
+
+#ifdef PRODUCT
+#define BLOCK_COMMENT(str) /* nothing */
+#else
+#define BLOCK_COMMENT(str) __ block_comment(str)
+#endif // PRODUCT
+
+#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
+
+// Constants
+ATTRIBUTE_ALIGNED(64) static const uint64_t round_consts_arr[24] = {
+ 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL,
+ 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L,
+ 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL,
+ 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL,
+ 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L,
+ 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L,
+ 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L,
+ 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L
+ };
+
+ATTRIBUTE_ALIGNED(64) static const uint64_t permsAndRots[] = {
+ // permutation in combined rho and pi
+ 9, 2, 11, 0, 1, 2, 3, 4, // step 1 and 3
+ 8, 1, 9, 2, 11, 4, 12, 0, // step 2
+ 9, 2, 10, 3, 11, 4, 12, 0, // step 4
+ 8, 9, 2, 3, 4, 5, 6, 7, // step 5
+ 0, 8, 9, 10, 15, 0, 0, 0, // step 6
+ 4, 5, 8, 9, 6, 7, 10, 11, // step 7 and 8
+ 0, 1, 2, 3, 13, 0, 0, 0, // step 9
+ 2, 3, 0, 1, 11, 0, 0, 0, // step 10
+ 4, 5, 6, 7, 14, 0, 0, 0, // step 11
+ 14, 15, 12, 13, 4, 0, 0, 0, // step 12
+ // size of rotations (after step 5)
+ 1, 6, 62, 55, 28, 20, 27, 36,
+ 3, 45, 10, 15, 25, 8, 39, 41,
+ 44, 43, 21, 18, 2, 61, 56, 14,
+ // rotation of row elements
+ 12, 8, 9, 10, 11, 5, 6, 7,
+ 9, 10, 11, 12, 8, 5, 6, 7
+};
+
+static address round_constsAddr() {
+ return (address) round_consts_arr;
+}
+
+static address permsAndRotsAddr() {
+ return (address) permsAndRots;
+}
+
+void StubGenerator::generate_sha3_stubs() {
+ if (UseSHA3Intrinsics) {
+ StubRoutines::_sha3_implCompress = generate_sha3_implCompress(false,"sha3_implCompress");
+ StubRoutines::_sha3_implCompressMB = generate_sha3_implCompress(true, "sha3_implCompressMB");
+ }
+}
+
+// Arguments:
+//
+// Inputs:
+// c_rarg0 - byte[] source+offset
+// c_rarg1 - long[] SHA3.state
+// c_rarg2 - int block_size
+// c_rarg3 - int offset
+// c_rarg4 - int limit
+//
+address StubGenerator::generate_sha3_implCompress(bool multiBlock, const char *name) {
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, "StubRoutines", name);
+ address start = __ pc();
+
+ const Register buf = c_rarg0;
+ const Register state = c_rarg1;
+ const Register block_size = c_rarg2;
+ const Register ofs = c_rarg3;
+#ifndef _WIN64
+ const Register limit = c_rarg4;
+#else
+ const Address limit_mem(rbp, 6 * wordSize);
+ const Register limit = r12;
+#endif
+
+ const Register permsAndRots = r10;
+ const Register round_consts = r11;
+ const Register constant2use = r13;
+ const Register roundsLeft = r14;
+
+ Label sha3_loop;
+ Label rounds24_loop, block104, block136, block144, block168;
+
+ __ enter();
+
+ __ push(r12);
+ __ push(r13);
+ __ push(r14);
+
+#ifdef _WIN64
+ // on win64, fill limit from stack position
+ __ movptr(limit, limit_mem);
+#endif
+
+ __ lea(permsAndRots, ExternalAddress(permsAndRotsAddr()));
+ __ lea(round_consts, ExternalAddress(round_constsAddr()));
+
+ // set up the masks
+ __ movl(rax, 0x1F);
+ __ kmovwl(k5, rax);
+ __ kshiftrwl(k4, k5, 1);
+ __ kshiftrwl(k3, k5, 2);
+ __ kshiftrwl(k2, k5, 3);
+ __ kshiftrwl(k1, k5, 4);
+
+ // load the state
+ __ evmovdquq(xmm0, k5, Address(state, 0), false, Assembler::AVX_512bit);
+ __ evmovdquq(xmm1, k5, Address(state, 40), false, Assembler::AVX_512bit);
+ __ evmovdquq(xmm2, k5, Address(state, 80), false, Assembler::AVX_512bit);
+ __ evmovdquq(xmm3, k5, Address(state, 120), false, Assembler::AVX_512bit);
+ __ evmovdquq(xmm4, k5, Address(state, 160), false, Assembler::AVX_512bit);
+
+ // load the permutation and rotation constants
+ __ evmovdquq(xmm17, Address(permsAndRots, 0), Assembler::AVX_512bit);
+ __ evmovdquq(xmm18, Address(permsAndRots, 64), Assembler::AVX_512bit);
+ __ evmovdquq(xmm19, Address(permsAndRots, 128), Assembler::AVX_512bit);
+ __ evmovdquq(xmm20, Address(permsAndRots, 192), Assembler::AVX_512bit);
+ __ evmovdquq(xmm21, Address(permsAndRots, 256), Assembler::AVX_512bit);
+ __ evmovdquq(xmm22, Address(permsAndRots, 320), Assembler::AVX_512bit);
+ __ evmovdquq(xmm23, Address(permsAndRots, 384), Assembler::AVX_512bit);
+ __ evmovdquq(xmm24, Address(permsAndRots, 448), Assembler::AVX_512bit);
+ __ evmovdquq(xmm25, Address(permsAndRots, 512), Assembler::AVX_512bit);
+ __ evmovdquq(xmm26, Address(permsAndRots, 576), Assembler::AVX_512bit);
+ __ evmovdquq(xmm27, Address(permsAndRots, 640), Assembler::AVX_512bit);
+ __ evmovdquq(xmm28, Address(permsAndRots, 704), Assembler::AVX_512bit);
+ __ evmovdquq(xmm29, Address(permsAndRots, 768), Assembler::AVX_512bit);
+ __ evmovdquq(xmm30, Address(permsAndRots, 832), Assembler::AVX_512bit);
+ __ evmovdquq(xmm31, Address(permsAndRots, 896), Assembler::AVX_512bit);
+
+ __ BIND(sha3_loop);
+
+ // there will be 24 keccak rounds
+ __ movl(roundsLeft, 24);
+ // load round_constants base
+ __ movptr(constant2use, round_consts);
+
+ // load input: 72, 104, 136, 144 or 168 bytes
+ // i.e. 5+4, 2*5+3, 3*5+2, 3*5+3 or 4*5+1 longs
+ __ evpxorq(xmm0, k5, xmm0, Address(buf, 0), true, Assembler::AVX_512bit);
+
+ // if(blockSize == 72) SHA3-512
+ __ cmpl(block_size, 72);
+ __ jcc(Assembler::notEqual, block104);
+ __ evpxorq(xmm1, k4, xmm1, Address(buf, 40), true, Assembler::AVX_512bit);
+ __ jmp(rounds24_loop);
+
+ // if(blockSize == 104) SHA3-384
+ __ BIND(block104);
+ __ cmpl(block_size, 104);
+ __ jcc(Assembler::notEqual, block136);
+ __ evpxorq(xmm1, k5, xmm1, Address(buf, 40), true, Assembler::AVX_512bit);
+ __ evpxorq(xmm2, k3, xmm2, Address(buf, 80), true, Assembler::AVX_512bit);
+ __ jmp(rounds24_loop);
+
+ // if(blockSize == 136) SHA3-256 and SHAKE256
+ __ BIND(block136);
+ __ cmpl(block_size, 136);
+ __ jcc(Assembler::notEqual, block144);
+ __ evpxorq(xmm1, k5, xmm1, Address(buf, 40), true, Assembler::AVX_512bit);
+ __ evpxorq(xmm2, k5, xmm2, Address(buf, 80), true, Assembler::AVX_512bit);
+ __ evpxorq(xmm3, k2, xmm3, Address(buf, 120), true, Assembler::AVX_512bit);
+ __ jmp(rounds24_loop);
+
+ // if(blockSize == 144) SHA3-224
+ __ BIND(block144);
+ __ cmpl(block_size, 144);
+ __ jcc(Assembler::notEqual, block168);
+ __ evpxorq(xmm1, k5, xmm1, Address(buf, 40), true, Assembler::AVX_512bit);
+ __ evpxorq(xmm2, k5, xmm2, Address(buf, 80), true, Assembler::AVX_512bit);
+ __ evpxorq(xmm3, k3, xmm3, Address(buf, 120), true, Assembler::AVX_512bit);
+ __ jmp(rounds24_loop);
+
+ // if(blockSize == 168) SHAKE128
+ __ BIND(block168);
+ __ evpxorq(xmm1, k5, xmm1, Address(buf, 40), true, Assembler::AVX_512bit);
+ __ evpxorq(xmm2, k5, xmm2, Address(buf, 80), true, Assembler::AVX_512bit);
+ __ evpxorq(xmm3, k5, xmm3, Address(buf, 120), true, Assembler::AVX_512bit);
+ __ evpxorq(xmm4, k1, xmm4, Address(buf, 160), true, Assembler::AVX_512bit);
+
+ // The 24 rounds of the keccak transformation.
+ // The implementation closely follows the Java version, with the state
+ // array "rows" in the lowest 5 64-bit slots of zmm0 - zmm4, i.e.
+ // each row of the SHA3 specification is located in one zmm register.
+ __ BIND(rounds24_loop);
+ __ subl(roundsLeft, 1);
+
+ __ evmovdquw(xmm5, xmm0, Assembler::AVX_512bit);
+ // vpternlogq(x, 150, y, z) does x = x ^ y ^ z
+ __ vpternlogq(xmm5, 150, xmm1, xmm2, Assembler::AVX_512bit);
+ __ vpternlogq(xmm5, 150, xmm3, xmm4, Assembler::AVX_512bit);
+ // Now the "c row", i.e. c0-c4 are in zmm5.
+ // Rotate each element of the c row by one bit to zmm6, call the
+ // rotated version c'.
+ __ evprolq(xmm6, xmm5, 1, Assembler::AVX_512bit);
+ // Rotate elementwise the c row so that c4 becomes c0,
+ // c0 becomes c1, etc.
+ __ evpermt2q(xmm5, xmm30, xmm5, Assembler::AVX_512bit);
+ // rotate elementwise the c' row so that c'0 becomes c'4,
+ // c'1 becomes c'0, etc.
+ __ evpermt2q(xmm6, xmm31, xmm6, Assembler::AVX_512bit);
+ __ vpternlogq(xmm0, 150, xmm5, xmm6, Assembler::AVX_512bit);
+ __ vpternlogq(xmm1, 150, xmm5, xmm6, Assembler::AVX_512bit);
+ __ vpternlogq(xmm2, 150, xmm5, xmm6, Assembler::AVX_512bit);
+ __ vpternlogq(xmm3, 150, xmm5, xmm6, Assembler::AVX_512bit);
+ __ vpternlogq(xmm4, 150, xmm5, xmm6, Assembler::AVX_512bit);
+ // Now the theta mapping has been finished.
+
+ // Do the cyclical permutation of the 24 moving state elements
+ // and the required rotations within each element (the combined
+ // rho and sigma steps).
+ __ evpermt2q(xmm4, xmm17, xmm3, Assembler::AVX_512bit);
+ __ evpermt2q(xmm3, xmm18, xmm2, Assembler::AVX_512bit);
+ __ evpermt2q(xmm2, xmm17, xmm1, Assembler::AVX_512bit);
+ __ evpermt2q(xmm1, xmm19, xmm0, Assembler::AVX_512bit);
+ __ evpermt2q(xmm4, xmm20, xmm2, Assembler::AVX_512bit);
+ // The 24 moving elements are now in zmm1, zmm3 and zmm4,
+ // do the rotations now.
+ __ evprolvq(xmm1, xmm1, xmm27, Assembler::AVX_512bit);
+ __ evprolvq(xmm3, xmm3, xmm28, Assembler::AVX_512bit);
+ __ evprolvq(xmm4, xmm4, xmm29, Assembler::AVX_512bit);
+ __ evmovdquw(xmm2, xmm1, Assembler::AVX_512bit);
+ __ evmovdquw(xmm5, xmm3, Assembler::AVX_512bit);
+ __ evpermt2q(xmm0, xmm21, xmm4, Assembler::AVX_512bit);
+ __ evpermt2q(xmm1, xmm22, xmm3, Assembler::AVX_512bit);
+ __ evpermt2q(xmm5, xmm22, xmm2, Assembler::AVX_512bit);
+ __ evmovdquw(xmm3, xmm1, Assembler::AVX_512bit);
+ __ evmovdquw(xmm2, xmm5, Assembler::AVX_512bit);
+ __ evpermt2q(xmm1, xmm23, xmm4, Assembler::AVX_512bit);
+ __ evpermt2q(xmm2, xmm24, xmm4, Assembler::AVX_512bit);
+ __ evpermt2q(xmm3, xmm25, xmm4, Assembler::AVX_512bit);
+ __ evpermt2q(xmm4, xmm26, xmm5, Assembler::AVX_512bit);
+ // The combined rho and sigma steps are done.
+
+ // Do the chi step (the same operation on all 5 rows).
+ // vpternlogq(x, 180, y, z) does x = x ^ (y & ~z).
+ __ evpermt2q(xmm5, xmm31, xmm0, Assembler::AVX_512bit);
+ __ evpermt2q(xmm6, xmm31, xmm5, Assembler::AVX_512bit);
+ __ vpternlogq(xmm0, 180, xmm6, xmm5, Assembler::AVX_512bit);
+
+ __ evpermt2q(xmm5, xmm31, xmm1, Assembler::AVX_512bit);
+ __ evpermt2q(xmm6, xmm31, xmm5, Assembler::AVX_512bit);
+ __ vpternlogq(xmm1, 180, xmm6, xmm5, Assembler::AVX_512bit);
+
+ // xor the round constant into a0 (the lowest 64 bits of zmm0
+ __ evpxorq(xmm0, k1, xmm0, Address(constant2use, 0), true, Assembler::AVX_512bit);
+ __ addptr(constant2use, 8);
+
+ __ evpermt2q(xmm5, xmm31, xmm2, Assembler::AVX_512bit);
+ __ evpermt2q(xmm6, xmm31, xmm5, Assembler::AVX_512bit);
+ __ vpternlogq(xmm2, 180, xmm6, xmm5, Assembler::AVX_512bit);
+
+ __ evpermt2q(xmm5, xmm31, xmm3, Assembler::AVX_512bit);
+ __ evpermt2q(xmm6, xmm31, xmm5, Assembler::AVX_512bit);
+ __ vpternlogq(xmm3, 180, xmm6, xmm5, Assembler::AVX_512bit);
+
+ __ evpermt2q(xmm5, xmm31, xmm4, Assembler::AVX_512bit);
+ __ evpermt2q(xmm6, xmm31, xmm5, Assembler::AVX_512bit);
+ __ vpternlogq(xmm4, 180, xmm6, xmm5, Assembler::AVX_512bit);
+ __ cmpl(roundsLeft, 0);
+ __ jcc(Assembler::notEqual, rounds24_loop);
+
+ if (multiBlock) {
+ __ addptr(buf, block_size);
+ __ addl(ofs, block_size);
+ __ cmpl(ofs, limit);
+ __ jcc(Assembler::lessEqual, sha3_loop);
+ __ movq(rax, ofs); // return ofs
+ } else {
+ __ xorq(rax, rax); // return 0
+ }
+
+ // store the state
+ __ evmovdquq(Address(state, 0), k5, xmm0, true, Assembler::AVX_512bit);
+ __ evmovdquq(Address(state, 40), k5, xmm1, true, Assembler::AVX_512bit);
+ __ evmovdquq(Address(state, 80), k5, xmm2, true, Assembler::AVX_512bit);
+ __ evmovdquq(Address(state, 120), k5, xmm3, true, Assembler::AVX_512bit);
+ __ evmovdquq(Address(state, 160), k5, xmm4, true, Assembler::AVX_512bit);
+
+ __ pop(r14);
+ __ pop(r13);
+ __ pop(r12);
+
+ __ leave(); // required for proper stackwalking of RuntimeStub frame
+ __ ret(0);
+
+ return start;
+}
diff --git a/src/hotspot/cpu/x86/templateTable_x86.cpp b/src/hotspot/cpu/x86/templateTable_x86.cpp
index 5e783225fcbfc..527d961259ecc 100644
--- a/src/hotspot/cpu/x86/templateTable_x86.cpp
+++ b/src/hotspot/cpu/x86/templateTable_x86.cpp
@@ -4048,6 +4048,7 @@ void TemplateTable::_new() {
__ push(rcx); // save the contexts of klass for initializing the header
// make sure klass is initialized
+ // init_state needs acquire, but x86 is TSO, and so we are already good.
#ifdef _LP64
assert(VM_Version::supports_fast_class_init_checks(), "must support fast class initialization checks");
__ clinit_barrier(rcx, r15_thread, nullptr /*L_fast_path*/, &slow_case);
diff --git a/src/hotspot/cpu/x86/upcallLinker_x86_32.cpp b/src/hotspot/cpu/x86/upcallLinker_x86_32.cpp
index e5075e180d9d6..d795c751d02b5 100644
--- a/src/hotspot/cpu/x86/upcallLinker_x86_32.cpp
+++ b/src/hotspot/cpu/x86/upcallLinker_x86_32.cpp
@@ -24,7 +24,7 @@
#include "precompiled.hpp"
#include "prims/upcallLinker.hpp"
-address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
+address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature,
BasicType* out_sig_bt, int total_out_args,
BasicType ret_type,
jobject jabi, jobject jconv,
diff --git a/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp b/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp
index 82179f9022e92..bc261bfd93f44 100644
--- a/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp
+++ b/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp
@@ -23,7 +23,7 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
-#include "code/codeBlob.hpp"
+#include "classfile/javaClasses.hpp"
#include "code/codeBlob.hpp"
#include "code/vmreg.inline.hpp"
#include "compiler/disassembler.hpp"
@@ -169,10 +169,10 @@ static void restore_callee_saved_registers(MacroAssembler* _masm, const ABIDescr
__ block_comment("} restore_callee_saved_regs ");
}
-static const int upcall_stub_code_base_size = 1024;
+static const int upcall_stub_code_base_size = 1200;
static const int upcall_stub_size_per_arg = 16;
-address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
+address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature,
BasicType* out_sig_bt, int total_out_args,
BasicType ret_type,
jobject jabi, jobject jconv,
@@ -281,7 +281,6 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
__ block_comment("{ on_entry");
__ vzeroupper();
__ lea(c_rarg0, Address(rsp, frame_data_offset));
- __ movptr(c_rarg1, (intptr_t)receiver);
// stack already aligned
__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, UpcallLinker::on_entry)));
__ movptr(r15_thread, rax);
@@ -297,12 +296,10 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
arg_shuffle.generate(_masm, shuffle_reg, abi._shadow_space_bytes, 0);
__ block_comment("} argument shuffle");
- __ block_comment("{ receiver ");
- __ get_vm_result(j_rarg0, r15_thread);
- __ block_comment("} receiver ");
-
- __ mov_metadata(rbx, entry);
- __ movptr(Address(r15_thread, JavaThread::callee_target_offset()), rbx); // just in case callee is deoptimized
+ __ block_comment("{ load target ");
+ __ movptr(j_rarg0, (intptr_t)receiver);
+ __ call(RuntimeAddress(StubRoutines::upcall_stub_load_target())); // puts target Method* in rbx
+ __ block_comment("} load target ");
__ push_cont_fastpath();
@@ -377,7 +374,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
#ifndef PRODUCT
stringStream ss;
- ss.print("upcall_stub_%s", entry->signature()->as_C_string());
+ ss.print("upcall_stub_%s", signature->as_C_string());
const char* name = _masm->code_string(ss.freeze());
#else // PRODUCT
const char* name = "upcall_stub";
diff --git a/src/hotspot/cpu/x86/vm_version_x86.cpp b/src/hotspot/cpu/x86/vm_version_x86.cpp
index 2549feb8a4069..f8c5de551cddb 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.cpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.cpp
@@ -437,6 +437,7 @@ class VM_Version_StubGenerator: public StubCodeGenerator {
__ cmpl(rax, 0x80000);
__ jcc(Assembler::notEqual, vector_save_restore);
+#ifndef PRODUCT
bool save_apx = UseAPX;
VM_Version::set_apx_cpuFeatures();
UseAPX = true;
@@ -453,6 +454,7 @@ class VM_Version_StubGenerator: public StubCodeGenerator {
__ movq(Address(rsi, 8), r31);
UseAPX = save_apx;
+#endif
#endif
__ bind(vector_save_restore);
//
@@ -1058,6 +1060,7 @@ void VM_Version::get_processor_features() {
_features &= ~CPU_AVX;
_features &= ~CPU_VZEROUPPER;
_features &= ~CPU_F16C;
+ _features &= ~CPU_SHA512;
}
if (logical_processors_per_package() == 1) {
@@ -1302,7 +1305,7 @@ void VM_Version::get_processor_features() {
#ifdef _LP64
// These are only supported on 64-bit
- if (UseSHA && supports_avx2() && supports_bmi2()) {
+ if (UseSHA && supports_avx2() && (supports_bmi2() || supports_sha512())) {
if (FLAG_IS_DEFAULT(UseSHA512Intrinsics)) {
FLAG_SET_DEFAULT(UseSHA512Intrinsics, true);
}
@@ -1313,9 +1316,16 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UseSHA512Intrinsics, false);
}
- if (UseSHA3Intrinsics) {
- warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
- FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
+#ifdef _LP64
+ if (supports_evex() && supports_avx512bw()) {
+ if (FLAG_IS_DEFAULT(UseSHA3Intrinsics)) {
+ UseSHA3Intrinsics = true;
+ }
+ } else
+#endif
+ if (UseSHA3Intrinsics) {
+ warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
+ FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
}
if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics)) {
@@ -3005,6 +3015,8 @@ uint64_t VM_Version::CpuidInfo::feature_flags() const {
xem_xcr0_eax.bits.ymm != 0) {
result |= CPU_AVX;
result |= CPU_VZEROUPPER;
+ if (sefsl1_cpuid7_eax.bits.sha512 != 0)
+ result |= CPU_SHA512;
if (std_cpuid1_ecx.bits.f16c != 0)
result |= CPU_F16C;
if (sef_cpuid7_ebx.bits.avx2 != 0) {
diff --git a/src/hotspot/cpu/x86/vm_version_x86.hpp b/src/hotspot/cpu/x86/vm_version_x86.hpp
index d58b5a9c09967..791f4a1fec72e 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.hpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.hpp
@@ -283,7 +283,8 @@ class VM_Version : public Abstract_VM_Version {
union SefCpuid7SubLeaf1Eax {
uint32_t value;
struct {
- uint32_t : 23,
+ uint32_t sha512 : 1,
+ : 22,
avx_ifma : 1,
: 8;
} bits;
@@ -415,7 +416,8 @@ class VM_Version : public Abstract_VM_Version {
decl(CET_SS, "cet_ss", 57) /* Control Flow Enforcement - Shadow Stack */ \
decl(AVX512_IFMA, "avx512_ifma", 58) /* Integer Vector FMA instructions*/ \
decl(AVX_IFMA, "avx_ifma", 59) /* 256-bit VEX-coded variant of AVX512-IFMA*/ \
- decl(APX_F, "apx_f", 60) /* Intel Advanced Performance Extensions*/
+ decl(APX_F, "apx_f", 60) /* Intel Advanced Performance Extensions*/\
+ decl(SHA512, "sha512", 61) /* SHA512 instructions*/
#define DECLARE_CPU_FEATURE_FLAG(id, name, bit) CPU_##id = (1ULL << bit),
CPU_FEATURE_FLAGS(DECLARE_CPU_FEATURE_FLAG)
@@ -638,9 +640,10 @@ class VM_Version : public Abstract_VM_Version {
LP64_ONLY(static void clear_apx_test_state());
static void clean_cpuFeatures() { _features = 0; }
- static void set_avx_cpuFeatures() { _features = (CPU_SSE | CPU_SSE2 | CPU_AVX | CPU_VZEROUPPER ); }
- static void set_evex_cpuFeatures() { _features = (CPU_AVX512F | CPU_SSE | CPU_SSE2 | CPU_VZEROUPPER ); }
+ static void set_avx_cpuFeatures() { _features |= (CPU_SSE | CPU_SSE2 | CPU_AVX | CPU_VZEROUPPER ); }
+ static void set_evex_cpuFeatures() { _features |= (CPU_AVX512F | CPU_SSE | CPU_SSE2 | CPU_VZEROUPPER ); }
static void set_apx_cpuFeatures() { _features |= CPU_APX_F; }
+ static void set_bmi_cpuFeatures() { _features |= (CPU_BMI1 | CPU_BMI2 | CPU_LZCNT); }
// Initialization
static void initialize();
@@ -757,6 +760,7 @@ class VM_Version : public Abstract_VM_Version {
static bool supports_ospke() { return (_features & CPU_OSPKE) != 0; }
static bool supports_cet_ss() { return (_features & CPU_CET_SS) != 0; }
static bool supports_cet_ibt() { return (_features & CPU_CET_IBT) != 0; }
+ static bool supports_sha512() { return (_features & CPU_SHA512) != 0; }
//
// Feature identification not affected by VM flags
diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad
index 2b29dd14e4b27..e5222d3863202 100644
--- a/src/hotspot/cpu/x86/x86.ad
+++ b/src/hotspot/cpu/x86/x86.ad
@@ -1765,6 +1765,12 @@ bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
return false;
}
break;
+ case Op_UMinV:
+ case Op_UMaxV:
+ if (UseAVX == 0) {
+ return false;
+ }
+ break;
case Op_MaxV:
case Op_MinV:
if (UseSSE < 4 && is_integral_type(bt)) {
@@ -1935,6 +1941,29 @@ bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
return false;
}
break;
+ case Op_SaturatingAddV:
+ case Op_SaturatingSubV:
+ if (UseAVX < 1) {
+ return false; // Implementation limitation
+ }
+ if (is_subword_type(bt) && size_in_bits == 512 && !VM_Version::supports_avx512bw()) {
+ return false;
+ }
+ break;
+ case Op_SelectFromTwoVector:
+ if (size_in_bits < 128 || (size_in_bits < 512 && !VM_Version::supports_avx512vl())) {
+ return false;
+ }
+ if (bt == T_SHORT && !VM_Version::supports_avx512bw()) {
+ return false;
+ }
+ if (bt == T_BYTE && !VM_Version::supports_avx512_vbmi()) {
+ return false;
+ }
+ if ((bt == T_INT || bt == T_FLOAT || bt == T_DOUBLE) && !VM_Version::supports_evex()) {
+ return false;
+ }
+ break;
case Op_MaskAll:
if (!VM_Version::supports_evex()) {
return false;
@@ -2111,6 +2140,8 @@ bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType
case Op_MaxV:
case Op_MinV:
+ case Op_UMinV:
+ case Op_UMaxV:
if (is_subword_type(bt) && !VM_Version::supports_avx512bw()) {
return false; // Implementation limitation
}
@@ -2118,6 +2149,15 @@ bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType
return false; // Implementation limitation
}
return true;
+ case Op_SaturatingAddV:
+ case Op_SaturatingSubV:
+ if (!is_subword_type(bt)) {
+ return false;
+ }
+ if (size_in_bits < 128 || !VM_Version::supports_avx512bw()) {
+ return false; // Implementation limitation
+ }
+ return true;
case Op_VectorMaskCmp:
if (is_subword_type(bt) && !VM_Version::supports_avx512bw()) {
@@ -2231,10 +2271,6 @@ const RegMask* Matcher::predicate_reg_mask(void) {
return &_VECTMASK_REG_mask;
}
-const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
- return new TypeVectMask(elemTy, length);
-}
-
// Max vector size in bytes. 0 if not supported.
int Matcher::vector_width_in_bytes(BasicType bt) {
assert(is_java_primitive(bt), "only primitive type vectors");
@@ -2457,6 +2493,10 @@ bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
mstack.push(m, Visit); // m = ShiftCntV
return true;
}
+ if (is_encode_and_store_pattern(n, m)) {
+ mstack.push(m, Visit);
+ return true;
+ }
return false;
}
@@ -3672,6 +3712,7 @@ instruct vconvF2HF(vec dst, vec src) %{
%}
instruct vconvF2HF_mem_reg(memory mem, vec src) %{
+ predicate(n->as_StoreVector()->memory_size() >= 16);
match(Set mem (StoreVector mem (VectorCastF2HF src)));
format %{ "vcvtps2ph $mem,$src" %}
ins_encode %{
@@ -6477,6 +6518,80 @@ instruct evminmaxFP_reg_eavx(vec dst, vec a, vec b, vec atmp, vec btmp, kReg ktm
ins_pipe( pipe_slow );
%}
+// ------------------------------ Unsigned vector Min/Max ----------------------
+
+instruct vector_uminmax_reg(vec dst, vec a, vec b) %{
+ predicate(VM_Version::supports_avx512vl() || Matcher::vector_element_basic_type(n) != T_LONG);
+ match(Set dst (UMinV a b));
+ match(Set dst (UMaxV a b));
+ format %{ "vector_uminmax $dst,$a,$b\t!" %}
+ ins_encode %{
+ int opcode = this->ideal_Opcode();
+ int vlen_enc = vector_length_encoding(this);
+ BasicType elem_bt = Matcher::vector_element_basic_type(this);
+ assert(is_integral_type(elem_bt), "");
+ __ vpuminmax(opcode, elem_bt, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vlen_enc);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+instruct vector_uminmax_mem(vec dst, vec a, memory b) %{
+ predicate(VM_Version::supports_avx512vl() || Matcher::vector_element_basic_type(n) != T_LONG);
+ match(Set dst (UMinV a (LoadVector b)));
+ match(Set dst (UMaxV a (LoadVector b)));
+ format %{ "vector_uminmax $dst,$a,$b\t!" %}
+ ins_encode %{
+ int opcode = this->ideal_Opcode();
+ int vlen_enc = vector_length_encoding(this);
+ BasicType elem_bt = Matcher::vector_element_basic_type(this);
+ assert(is_integral_type(elem_bt), "");
+ __ vpuminmax(opcode, elem_bt, $dst$$XMMRegister, $a$$XMMRegister, $b$$Address, vlen_enc);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+instruct vector_uminmaxq_reg(vec dst, vec a, vec b, vec xtmp1, vec xtmp2) %{
+ predicate(!VM_Version::supports_avx512vl() && Matcher::vector_element_basic_type(n) == T_LONG);
+ match(Set dst (UMinV a b));
+ match(Set dst (UMaxV a b));
+ effect(TEMP xtmp1, TEMP xtmp2);
+ format %{ "vector_uminmaxq $dst,$a,$b\t! using xtmp1 and xtmp2 as TEMP" %}
+ ins_encode %{
+ int opcode = this->ideal_Opcode();
+ int vlen_enc = vector_length_encoding(this);
+ __ vpuminmaxq(opcode, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+instruct vector_uminmax_reg_masked(vec dst, vec src2, kReg mask) %{
+ match(Set dst (UMinV (Binary dst src2) mask));
+ match(Set dst (UMaxV (Binary dst src2) mask));
+ format %{ "vector_uminmax_masked $dst, $dst, $src2, $mask\t! umin/max masked operation" %}
+ ins_encode %{
+ int vlen_enc = vector_length_encoding(this);
+ BasicType bt = Matcher::vector_element_basic_type(this);
+ int opc = this->ideal_Opcode();
+ __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
+ $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+instruct vector_uminmax_mem_masked(vec dst, memory src2, kReg mask) %{
+ match(Set dst (UMinV (Binary dst (LoadVector src2)) mask));
+ match(Set dst (UMaxV (Binary dst (LoadVector src2)) mask));
+ format %{ "vector_uminmax_masked $dst, $dst, $src2, $mask\t! umin/max masked operation" %}
+ ins_encode %{
+ int vlen_enc = vector_length_encoding(this);
+ BasicType bt = Matcher::vector_element_basic_type(this);
+ int opc = this->ideal_Opcode();
+ __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
+ $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
// --------------------------------- Signum/CopySign ---------------------------
instruct signumF_reg(regF dst, regF zero, regF one, rFlagsReg cr) %{
@@ -10468,3 +10583,246 @@ instruct DoubleClassCheck_reg_reg_vfpclass(rRegI dst, regD src, kReg ktmp, rFlag
%}
ins_pipe(pipe_slow);
%}
+
+instruct vector_addsub_saturating_subword_reg(vec dst, vec src1, vec src2)
+%{
+ predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
+ n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
+ match(Set dst (SaturatingAddV src1 src2));
+ match(Set dst (SaturatingSubV src1 src2));
+ format %{ "vector_addsub_saturating_subword $dst, $src1, $src2" %}
+ ins_encode %{
+ int vlen_enc = vector_length_encoding(this);
+ BasicType elem_bt = Matcher::vector_element_basic_type(this);
+ __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
+ $src1$$XMMRegister, $src2$$XMMRegister, false, vlen_enc);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vector_addsub_saturating_unsigned_subword_reg(vec dst, vec src1, vec src2)
+%{
+ predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
+ n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
+ match(Set dst (SaturatingAddV src1 src2));
+ match(Set dst (SaturatingSubV src1 src2));
+ format %{ "vector_addsub_saturating_unsigned_subword $dst, $src1, $src2" %}
+ ins_encode %{
+ int vlen_enc = vector_length_encoding(this);
+ BasicType elem_bt = Matcher::vector_element_basic_type(this);
+ __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
+ $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vector_addsub_saturating_reg_evex(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2)
+%{
+ predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
+ n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned() &&
+ (Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()));
+ match(Set dst (SaturatingAddV src1 src2));
+ match(Set dst (SaturatingSubV src1 src2));
+ effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2);
+ format %{ "vector_addsub_saturating_evex $dst, $src1, $src2 \t! using $xtmp1, $xtmp2, $ktmp1 and $ktmp2 as TEMP" %}
+ ins_encode %{
+ int vlen_enc = vector_length_encoding(this);
+ BasicType elem_bt = Matcher::vector_element_basic_type(this);
+ __ vector_addsub_dq_saturating_evex(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
+ $src1$$XMMRegister, $src2$$XMMRegister,
+ $xtmp1$$XMMRegister, $xtmp2$$XMMRegister,
+ $ktmp1$$KRegister, $ktmp2$$KRegister, vlen_enc);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vector_addsub_saturating_reg_avx(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4)
+%{
+ predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
+ n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned() &&
+ Matcher::vector_length_in_bytes(n) <= 32 && !VM_Version::supports_avx512vl());
+ match(Set dst (SaturatingAddV src1 src2));
+ match(Set dst (SaturatingSubV src1 src2));
+ effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4);
+ format %{ "vector_addsub_saturating_avx $dst, $src1, $src2 \t! using $xtmp1, $xtmp2, $xtmp3 and $xtmp4 as TEMP" %}
+ ins_encode %{
+ int vlen_enc = vector_length_encoding(this);
+ BasicType elem_bt = Matcher::vector_element_basic_type(this);
+ __ vector_addsub_dq_saturating_avx(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister, $src1$$XMMRegister,
+ $src2$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister,
+ $xtmp3$$XMMRegister, $xtmp4$$XMMRegister, vlen_enc);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vector_add_saturating_unsigned_reg_evex(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, kReg ktmp)
+%{
+ predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
+ n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
+ (Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()));
+ match(Set dst (SaturatingAddV src1 src2));
+ effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp);
+ format %{ "vector_add_saturating_unsigned_evex $dst, $src1, $src2 \t! using $xtmp1, $xtmp2 and $ktmp as TEMP" %}
+ ins_encode %{
+ int vlen_enc = vector_length_encoding(this);
+ BasicType elem_bt = Matcher::vector_element_basic_type(this);
+ __ vector_add_dq_saturating_unsigned_evex(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
+ $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp$$KRegister, vlen_enc);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vector_add_saturating_unsigned_reg_avx(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, vec xtmp3)
+%{
+ predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
+ n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
+ Matcher::vector_length_in_bytes(n) <= 32 && !VM_Version::supports_avx512vl());
+ match(Set dst (SaturatingAddV src1 src2));
+ effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3);
+ format %{ "vector_add_saturating_unsigned_avx $dst, $src1, $src2 \t! using $xtmp1, $xtmp2 and $xtmp3 as TEMP" %}
+ ins_encode %{
+ int vlen_enc = vector_length_encoding(this);
+ BasicType elem_bt = Matcher::vector_element_basic_type(this);
+ __ vector_add_dq_saturating_unsigned_avx(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
+ $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, vlen_enc);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vector_sub_saturating_unsigned_reg_evex(vec dst, vec src1, vec src2, kReg ktmp)
+%{
+ predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
+ n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
+ (Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()));
+ match(Set dst (SaturatingSubV src1 src2));
+ effect(TEMP ktmp);
+ format %{ "vector_sub_saturating_unsigned_evex $dst, $src1, $src2 \t! using $ktmp as TEMP" %}
+ ins_encode %{
+ int vlen_enc = vector_length_encoding(this);
+ BasicType elem_bt = Matcher::vector_element_basic_type(this);
+ __ vector_sub_dq_saturating_unsigned_evex(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister,
+ $src2$$XMMRegister, $ktmp$$KRegister, vlen_enc);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vector_sub_saturating_unsigned_reg_avx(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2)
+%{
+ predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
+ n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
+ Matcher::vector_length_in_bytes(n) <= 32 && !VM_Version::supports_avx512vl());
+ match(Set dst (SaturatingSubV src1 src2));
+ effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
+ format %{ "vector_sub_saturating_unsigned_avx $dst, $src1, $src2 \t! using $xtmp1 and $xtmp2 as TEMP" %}
+ ins_encode %{
+ int vlen_enc = vector_length_encoding(this);
+ BasicType elem_bt = Matcher::vector_element_basic_type(this);
+ __ vector_sub_dq_saturating_unsigned_avx(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
+ $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vector_addsub_saturating_subword_mem(vec dst, vec src1, memory src2)
+%{
+ predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
+ n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
+ match(Set dst (SaturatingAddV src1 (LoadVector src2)));
+ match(Set dst (SaturatingSubV src1 (LoadVector src2)));
+ format %{ "vector_addsub_saturating_subword $dst, $src1, $src2" %}
+ ins_encode %{
+ int vlen_enc = vector_length_encoding(this);
+ BasicType elem_bt = Matcher::vector_element_basic_type(this);
+ __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
+ $src1$$XMMRegister, $src2$$Address, false, vlen_enc);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vector_addsub_saturating_unsigned_subword_mem(vec dst, vec src1, memory src2)
+%{
+ predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
+ n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
+ match(Set dst (SaturatingAddV src1 (LoadVector src2)));
+ match(Set dst (SaturatingSubV src1 (LoadVector src2)));
+ format %{ "vector_addsub_saturating_unsigned_subword $dst, $src1, $src2" %}
+ ins_encode %{
+ int vlen_enc = vector_length_encoding(this);
+ BasicType elem_bt = Matcher::vector_element_basic_type(this);
+ __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
+ $src1$$XMMRegister, $src2$$Address, true, vlen_enc);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vector_addsub_saturating_subword_masked_reg(vec dst, vec src, kReg mask) %{
+ predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
+ n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
+ match(Set dst (SaturatingAddV (Binary dst src) mask));
+ match(Set dst (SaturatingSubV (Binary dst src) mask));
+ format %{ "vector_addsub_saturating_subword_masked $dst, $mask, $src" %}
+ ins_encode %{
+ int vlen_enc = vector_length_encoding(this);
+ BasicType elem_bt = Matcher::vector_element_basic_type(this);
+ __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
+ $dst$$XMMRegister, $src$$XMMRegister, false, true, vlen_enc);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+instruct vector_addsub_saturating_unsigned_subword_masked_reg(vec dst, vec src, kReg mask) %{
+ predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
+ n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
+ match(Set dst (SaturatingAddV (Binary dst src) mask));
+ match(Set dst (SaturatingSubV (Binary dst src) mask));
+ format %{ "vector_addsub_saturating_unsigned_subword_masked $dst, $mask, $src" %}
+ ins_encode %{
+ int vlen_enc = vector_length_encoding(this);
+ BasicType elem_bt = Matcher::vector_element_basic_type(this);
+ __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
+ $dst$$XMMRegister, $src$$XMMRegister, true, true, vlen_enc);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+instruct vector_addsub_saturating_subword_masked_mem(vec dst, memory src, kReg mask) %{
+ predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
+ n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
+ match(Set dst (SaturatingAddV (Binary dst (LoadVector src)) mask));
+ match(Set dst (SaturatingSubV (Binary dst (LoadVector src)) mask));
+ format %{ "vector_addsub_saturating_subword_masked $dst, $mask, $src" %}
+ ins_encode %{
+ int vlen_enc = vector_length_encoding(this);
+ BasicType elem_bt = Matcher::vector_element_basic_type(this);
+ __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
+ $dst$$XMMRegister, $src$$Address, false, true, vlen_enc);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+instruct vector_addsub_saturating_unsigned_subword_masked_mem(vec dst, memory src, kReg mask) %{
+ predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
+ n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
+ match(Set dst (SaturatingAddV (Binary dst (LoadVector src)) mask));
+ match(Set dst (SaturatingSubV (Binary dst (LoadVector src)) mask));
+ format %{ "vector_addsub_saturating_unsigned_subword_masked $dst, $mask, $src" %}
+ ins_encode %{
+ int vlen_enc = vector_length_encoding(this);
+ BasicType elem_bt = Matcher::vector_element_basic_type(this);
+ __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
+ $dst$$XMMRegister, $src$$Address, true, true, vlen_enc);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+instruct vector_selectfrom_twovectors_reg_evex(vec index, vec src1, vec src2)
+%{
+ match(Set index (SelectFromTwoVector (Binary index src1) src2));
+ format %{ "select_from_two_vector $index, $src1, $src2 \t!" %}
+ ins_encode %{
+ int vlen_enc = vector_length_encoding(this);
+ BasicType bt = Matcher::vector_element_basic_type(this);
+ __ select_from_two_vectors_evex(bt, $index$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
+ %}
+ ins_pipe(pipe_slow);
+%}
diff --git a/src/hotspot/cpu/x86/x86_32.ad b/src/hotspot/cpu/x86/x86_32.ad
index 7aa4f6a29a116..7c9695571daec 100644
--- a/src/hotspot/cpu/x86/x86_32.ad
+++ b/src/hotspot/cpu/x86/x86_32.ad
@@ -6322,17 +6322,6 @@ instruct storeImmB(memory mem, immI8 src) %{
ins_pipe( ialu_mem_imm );
%}
-// Store CMS card-mark Immediate
-instruct storeImmCM(memory mem, immI8 src) %{
- match(Set mem (StoreCM mem src));
-
- ins_cost(150);
- format %{ "MOV8 $mem,$src\t! CMS card-mark imm0" %}
- opcode(0xC6); /* C6 /0 */
- ins_encode( SetInstMark, OpcP, RMopc_Mem(0x00,mem), Con8or32(src), ClearInstMark);
- ins_pipe( ialu_mem_imm );
-%}
-
// Store Double
instruct storeDPR( memory mem, regDPR1 src) %{
predicate(UseSSE<=1);
diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad
index 1b271683bd60d..c3fa4c16e553f 100644
--- a/src/hotspot/cpu/x86/x86_64.ad
+++ b/src/hotspot/cpu/x86/x86_64.ad
@@ -4341,6 +4341,7 @@ instruct loadP(rRegP dst, memory mem)
// Load Compressed Pointer
instruct loadN(rRegN dst, memory mem)
%{
+ predicate(n->as_Load()->barrier_data() == 0);
match(Set dst (LoadN mem));
ins_cost(125); // XXX
@@ -5126,6 +5127,7 @@ instruct storeImmP(memory mem, immP31 src)
// Store Compressed Pointer
instruct storeN(memory mem, rRegN src)
%{
+ predicate(n->as_Store()->barrier_data() == 0);
match(Set mem (StoreN mem src));
ins_cost(125); // XXX
@@ -5150,7 +5152,7 @@ instruct storeNKlass(memory mem, rRegN src)
instruct storeImmN0(memory mem, immN0 zero)
%{
- predicate(CompressedOops::base() == nullptr);
+ predicate(CompressedOops::base() == nullptr && n->as_Store()->barrier_data() == 0);
match(Set mem (StoreN mem zero));
ins_cost(125); // XXX
@@ -5163,6 +5165,7 @@ instruct storeImmN0(memory mem, immN0 zero)
instruct storeImmN(memory mem, immN src)
%{
+ predicate(n->as_Store()->barrier_data() == 0);
match(Set mem (StoreN mem src));
ins_cost(150); // XXX
@@ -5295,32 +5298,6 @@ instruct storeImmB(memory mem, immI8 src)
ins_pipe(ialu_mem_imm);
%}
-// Store CMS card-mark Immediate
-instruct storeImmCM0_reg(memory mem, immI_0 zero)
-%{
- predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
- match(Set mem (StoreCM mem zero));
-
- ins_cost(125); // XXX
- format %{ "movb $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
- ins_encode %{
- __ movb($mem$$Address, r12);
- %}
- ins_pipe(ialu_mem_reg);
-%}
-
-instruct storeImmCM0(memory mem, immI_0 src)
-%{
- match(Set mem (StoreCM mem src));
-
- ins_cost(150); // XXX
- format %{ "movb $mem, $src\t# CMS card-mark byte 0" %}
- ins_encode %{
- __ movb($mem$$Address, $src$$constant);
- %}
- ins_pipe(ialu_mem_imm);
-%}
-
// Store Float
instruct storeF(memory mem, regF src)
%{
@@ -7162,6 +7139,7 @@ instruct compareAndSwapN(rRegI res,
memory mem_ptr,
rax_RegN oldval, rRegN newval,
rFlagsReg cr) %{
+ predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
effect(KILL cr, KILL oldval);
@@ -7249,6 +7227,7 @@ instruct compareAndExchangeN(
memory mem_ptr,
rax_RegN oldval, rRegN newval,
rFlagsReg cr) %{
+ predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
effect(KILL cr);
@@ -7470,6 +7449,7 @@ instruct xchgP( memory mem, rRegP newval) %{
%}
instruct xchgN( memory mem, rRegN newval) %{
+ predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set newval (GetAndSetN mem newval));
format %{ "XCHGL $newval,$mem]" %}
ins_encode %{
@@ -11659,6 +11639,7 @@ instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
%{
+ predicate(n->in(2)->as_Load()->barrier_data() == 0);
match(Set cr (CmpN src (LoadN mem)));
format %{ "cmpl $src, $mem\t# compressed ptr" %}
@@ -11680,6 +11661,7 @@ instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
%{
+ predicate(n->in(2)->as_Load()->barrier_data() == 0);
match(Set cr (CmpN src (LoadN mem)));
format %{ "cmpl $mem, $src\t# compressed ptr" %}
@@ -11720,7 +11702,8 @@ instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
%{
- predicate(CompressedOops::base() != nullptr);
+ predicate(CompressedOops::base() != nullptr &&
+ n->in(1)->as_Load()->barrier_data() == 0);
match(Set cr (CmpN (LoadN mem) zero));
ins_cost(500); // XXX
@@ -11733,7 +11716,8 @@ instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
%{
- predicate(CompressedOops::base() == nullptr);
+ predicate(CompressedOops::base() == nullptr &&
+ n->in(1)->as_Load()->barrier_data() == 0);
match(Set cr (CmpN (LoadN mem) zero));
format %{ "cmpl R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
diff --git a/src/hotspot/cpu/zero/upcallLinker_zero.cpp b/src/hotspot/cpu/zero/upcallLinker_zero.cpp
index 6447dac86c915..408ebc328205d 100644
--- a/src/hotspot/cpu/zero/upcallLinker_zero.cpp
+++ b/src/hotspot/cpu/zero/upcallLinker_zero.cpp
@@ -24,7 +24,7 @@
#include "precompiled.hpp"
#include "prims/upcallLinker.hpp"
-address UpcallLinker::make_upcall_stub(jobject mh, Method* entry,
+address UpcallLinker::make_upcall_stub(jobject mh, Symbol* signature,
BasicType* out_sig_bt, int total_out_args,
BasicType ret_type,
jobject jabi, jobject jconv,
diff --git a/src/hotspot/cpu/zero/vm_version_zero.cpp b/src/hotspot/cpu/zero/vm_version_zero.cpp
index 1fcf4b1086253..7312dd116468c 100644
--- a/src/hotspot/cpu/zero/vm_version_zero.cpp
+++ b/src/hotspot/cpu/zero/vm_version_zero.cpp
@@ -116,11 +116,6 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseVectorizedMismatchIntrinsic, false);
}
- if ((LockingMode != LM_LEGACY) && (LockingMode != LM_MONITOR)) {
- warning("Unsupported locking mode for this CPU.");
- FLAG_SET_DEFAULT(LockingMode, LM_LEGACY);
- }
-
// Enable error context decoding on known platforms
#if defined(IA32) || defined(AMD64) || defined(ARM) || \
defined(AARCH64) || defined(PPC) || defined(RISCV) || \
diff --git a/src/hotspot/cpu/zero/zeroInterpreter_zero.cpp b/src/hotspot/cpu/zero/zeroInterpreter_zero.cpp
index 2b53042ef1017..aab43e733964e 100644
--- a/src/hotspot/cpu/zero/zeroInterpreter_zero.cpp
+++ b/src/hotspot/cpu/zero/zeroInterpreter_zero.cpp
@@ -485,26 +485,30 @@ int ZeroInterpreter::native_entry(Method* method, intptr_t UNUSED, TRAPS) {
// Unlock if necessary
if (monitor) {
- BasicLock *lock = monitor->lock();
- markWord header = lock->displaced_header();
- oop rcvr = monitor->obj();
- monitor->set_obj(nullptr);
-
- bool dec_monitor_count = true;
- if (header.to_pointer() != nullptr) {
- markWord old_header = markWord::encode(lock);
- if (rcvr->cas_set_mark(header, old_header) != old_header) {
- monitor->set_obj(rcvr);
- dec_monitor_count = false;
- InterpreterRuntime::monitorexit(monitor);
+ bool success = false;
+ if (LockingMode == LM_LEGACY) {
+ BasicLock* lock = monitor->lock();
+ oop rcvr = monitor->obj();
+ monitor->set_obj(nullptr);
+ success = true;
+ markWord header = lock->displaced_header();
+ if (header.to_pointer() != nullptr) { // Check for recursive lock
+ markWord old_header = markWord::encode(lock);
+ if (rcvr->cas_set_mark(header, old_header) != old_header) {
+ monitor->set_obj(rcvr);
+ success = false;
+ }
+ }
+ if (success) {
+ THREAD->dec_held_monitor_count();
}
}
- if (dec_monitor_count) {
- THREAD->dec_held_monitor_count();
+ if (!success) {
+ InterpreterRuntime::monitorexit(monitor);
}
}
- unwind_and_return:
+ unwind_and_return:
// Unwind the current activation
thread->pop_zero_frame();
diff --git a/src/hotspot/os/aix/osThread_aix.cpp b/src/hotspot/os/aix/osThread_aix.cpp
index 4049d6b58b777..c424b044c09b9 100644
--- a/src/hotspot/os/aix/osThread_aix.cpp
+++ b/src/hotspot/os/aix/osThread_aix.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2015 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -23,32 +23,26 @@
*
*/
-// no precompiled headers
-
-#include "memory/allocation.inline.hpp"
-#include "runtime/handles.inline.hpp"
-#include "runtime/mutexLocker.hpp"
-#include "runtime/os.hpp"
+#include "precompiled.hpp"
+#include "memory/allocation.hpp"
+#include "runtime/mutex.hpp"
#include "runtime/osThread.hpp"
-#include "runtime/safepoint.hpp"
-#include "runtime/vmThread.hpp"
-
-void OSThread::pd_initialize() {
- _thread_id = 0;
- _kernel_thread_id = 0;
- _siginfo = nullptr;
- _ucontext = nullptr;
- _expanding_stack = 0;
- _alt_sig_stack = nullptr;
- _last_cpu_times.sys = _last_cpu_times.user = 0L;
+#include
+OSThread::OSThread()
+ : _thread_id(0),
+ _kernel_thread_id(0),
+ _caller_sigmask(),
+ sr(),
+ _siginfo(nullptr),
+ _ucontext(nullptr),
+ _expanding_stack(0),
+ _alt_sig_stack(nullptr),
+ _startThread_lock(new Monitor(Mutex::event, "startThread_lock")) {
sigemptyset(&_caller_sigmask);
-
- _startThread_lock = new Monitor(Mutex::event, "startThread_lock");
- assert(_startThread_lock != nullptr, "check");
}
-void OSThread::pd_destroy() {
+OSThread::~OSThread() {
delete _startThread_lock;
}
diff --git a/src/hotspot/os/aix/osThread_aix.hpp b/src/hotspot/os/aix/osThread_aix.hpp
index 5feb3c5799aa0..514e554101e88 100644
--- a/src/hotspot/os/aix/osThread_aix.hpp
+++ b/src/hotspot/os/aix/osThread_aix.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2013 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -26,22 +26,16 @@
#ifndef OS_AIX_OSTHREAD_AIX_HPP
#define OS_AIX_OSTHREAD_AIX_HPP
- public:
- typedef pthread_t thread_id_t;
-
- private:
- int _thread_type;
+#include "runtime/osThreadBase.hpp"
+#include "suspendResume_posix.hpp"
+#include "utilities/globalDefinitions.hpp"
- public:
+class OSThread : public OSThreadBase {
+ friend class VMStructs;
- int thread_type() const {
- return _thread_type;
- }
- void set_thread_type(int type) {
- _thread_type = type;
- }
+ typedef pthread_t thread_id_t;
- private:
+ thread_id_t _thread_id;
// On AIX, we use the pthread id as OSThread::thread_id and keep the kernel thread id
// separately for diagnostic purposes.
@@ -54,15 +48,20 @@
sigset_t _caller_sigmask; // Caller's signal mask
public:
+ OSThread();
+ ~OSThread();
// Methods to save/restore caller's signal mask
sigset_t caller_sigmask() const { return _caller_sigmask; }
void set_caller_sigmask(sigset_t sigmask) { _caller_sigmask = sigmask; }
-#ifndef PRODUCT
- // Used for debugging, return a unique integer for each thread.
- int thread_identifier() const { return _thread_id; }
-#endif
+ thread_id_t thread_id() const {
+ return _thread_id;
+ }
+ void set_thread_id(thread_id_t id) {
+ _thread_id = id;
+ }
+
tid_t kernel_thread_id() const {
return _kernel_thread_id;
}
@@ -71,7 +70,7 @@
}
pthread_t pthread_id() const {
- // Here: same as OSThread::thread_id()
+ // Here: same as thread_id()
return _thread_id;
}
@@ -79,7 +78,6 @@
// suspension support.
// ***************************************************************
- public:
// flags that support signal based suspend/resume on Aix are in a
// separate class to avoid confusion with many flags in OSThread that
// are used by VM level suspend/resume.
@@ -125,22 +123,10 @@
return _startThread_lock;
}
- // ***************************************************************
- // Platform dependent initialization and cleanup
- // ***************************************************************
-
- private:
-
- void pd_initialize();
- void pd_destroy();
-
- public:
-
- // The last measured values of cpu timing to prevent the "stale
- // value return" bug in thread_cpu_time.
- volatile struct {
- jlong sys;
- jlong user;
- } _last_cpu_times;
+ // Printing
+ uintx thread_id_for_printing() const override {
+ return (uintx)_thread_id;
+ }
+};
#endif // OS_AIX_OSTHREAD_AIX_HPP
diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp
index fd16a7984a6f3..842ab0c6eebd6 100644
--- a/src/hotspot/os/aix/os_aix.cpp
+++ b/src/hotspot/os/aix/os_aix.cpp
@@ -698,9 +698,6 @@ bool os::create_thread(Thread* thread, ThreadType thr_type,
return false;
}
- // Set the correct thread state.
- osthread->set_thread_type(thr_type);
-
// Initial state is ALLOCATED but not INITIALIZED
osthread->set_state(ALLOCATED);
@@ -1293,7 +1290,7 @@ void os::jvm_path(char *buf, jint buflen) {
Dl_info dlinfo;
int ret = dladdr(CAST_FROM_FN_PTR(void *, os::jvm_path), &dlinfo);
assert(ret != 0, "cannot locate libjvm");
- char* rp = os::Posix::realpath((char *)dlinfo.dli_fname, buf, buflen);
+ char* rp = os::realpath((char *)dlinfo.dli_fname, buf, buflen);
assert(rp != nullptr, "error in realpath(): maybe the 'path' argument is too long?");
if (Arguments::sun_java_launcher_is_altjvm()) {
@@ -1324,7 +1321,7 @@ void os::jvm_path(char *buf, jint buflen) {
}
assert(strstr(p, "/libjvm") == p, "invalid library name");
- rp = os::Posix::realpath(java_home_var, buf, buflen);
+ rp = os::realpath(java_home_var, buf, buflen);
if (rp == nullptr) {
return;
}
@@ -1345,7 +1342,7 @@ void os::jvm_path(char *buf, jint buflen) {
snprintf(buf + len, buflen-len, "/hotspot/libjvm.so");
} else {
// Go back to path of .so
- rp = os::Posix::realpath((char *)dlinfo.dli_fname, buf, buflen);
+ rp = os::realpath((char *)dlinfo.dli_fname, buf, buflen);
if (rp == nullptr) {
return;
}
@@ -2483,16 +2480,6 @@ int os::open(const char *path, int oflag, int mode) {
return fd;
}
-// return current position of file pointer
-jlong os::current_file_offset(int fd) {
- return (jlong)::lseek(fd, (off_t)0, SEEK_CUR);
-}
-
-// move file pointer to the specified offset
-jlong os::seek_to_file_offset(int fd, jlong offset) {
- return (jlong)::lseek(fd, (off_t)offset, SEEK_SET);
-}
-
// current_thread_cpu_time(bool) and thread_cpu_time(Thread*, bool)
// are used by JVM M&M and JVMTI to get user+sys or user CPU time
// of a thread.
diff --git a/src/hotspot/os/aix/vmStructs_aix.hpp b/src/hotspot/os/aix/vmStructs_aix.hpp
index 1a2f4c4bf6e21..f3bbc80e62c72 100644
--- a/src/hotspot/os/aix/vmStructs_aix.hpp
+++ b/src/hotspot/os/aix/vmStructs_aix.hpp
@@ -29,9 +29,20 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
+#define VM_STRUCTS_OS(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
+ \
+ /******************************/ \
+ /* Threads (NOTE: incomplete) */ \
+ /******************************/ \
+ nonstatic_field(OSThread, _thread_id, pthread_t) \
+
+#define VM_TYPES_OS(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
+ \
+ /**********************/ \
+ /* Posix Thread IDs */ \
+ /**********************/ \
+ \
+ declare_unsigned_integer_type(pthread_t)
#define VM_INT_CONSTANTS_OS(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
diff --git a/src/hotspot/os/bsd/gc/x/xPhysicalMemoryBacking_bsd.cpp b/src/hotspot/os/bsd/gc/x/xPhysicalMemoryBacking_bsd.cpp
deleted file mode 100644
index 2c64c3788d34d..0000000000000
--- a/src/hotspot/os/bsd/gc/x/xPhysicalMemoryBacking_bsd.cpp
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/shared/gcLogPrecious.hpp"
-#include "gc/x/xErrno.hpp"
-#include "gc/x/xGlobals.hpp"
-#include "gc/x/xLargePages.inline.hpp"
-#include "gc/x/xPhysicalMemory.inline.hpp"
-#include "gc/x/xPhysicalMemoryBacking_bsd.hpp"
-#include "logging/log.hpp"
-#include "runtime/globals.hpp"
-#include "runtime/os.hpp"
-#include "utilities/align.hpp"
-#include "utilities/debug.hpp"
-
-#include
-#include
-#include
-#include
-
-// The backing is represented by a reserved virtual address space, in which
-// we commit and uncommit physical memory. Multi-mapping the different heap
-// views is done by simply remapping the backing memory using mach_vm_remap().
-
-static int vm_flags_superpage() {
- if (!XLargePages::is_explicit()) {
- return 0;
- }
-
- const int page_size_in_megabytes = XGranuleSize >> 20;
- return page_size_in_megabytes << VM_FLAGS_SUPERPAGE_SHIFT;
-}
-
-static XErrno mremap(uintptr_t from_addr, uintptr_t to_addr, size_t size) {
- mach_vm_address_t remap_addr = to_addr;
- vm_prot_t remap_cur_prot;
- vm_prot_t remap_max_prot;
-
- // Remap memory to an additional location
- const kern_return_t res = mach_vm_remap(mach_task_self(),
- &remap_addr,
- size,
- 0 /* mask */,
- VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE | vm_flags_superpage(),
- mach_task_self(),
- from_addr,
- FALSE /* copy */,
- &remap_cur_prot,
- &remap_max_prot,
- VM_INHERIT_COPY);
-
- return (res == KERN_SUCCESS) ? XErrno(0) : XErrno(EINVAL);
-}
-
-XPhysicalMemoryBacking::XPhysicalMemoryBacking(size_t max_capacity) :
- _base(0),
- _initialized(false) {
-
- // Reserve address space for backing memory
- _base = (uintptr_t)os::reserve_memory(max_capacity);
- if (_base == 0) {
- // Failed
- log_error_pd(gc)("Failed to reserve address space for backing memory");
- return;
- }
-
- // Successfully initialized
- _initialized = true;
-}
-
-bool XPhysicalMemoryBacking::is_initialized() const {
- return _initialized;
-}
-
-void XPhysicalMemoryBacking::warn_commit_limits(size_t max_capacity) const {
- // Does nothing
-}
-
-bool XPhysicalMemoryBacking::commit_inner(size_t offset, size_t length) const {
- assert(is_aligned(offset, os::vm_page_size()), "Invalid offset");
- assert(is_aligned(length, os::vm_page_size()), "Invalid length");
-
- log_trace(gc, heap)("Committing memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
- offset / M, (offset + length) / M, length / M);
-
- const uintptr_t addr = _base + offset;
- const void* const res = mmap((void*)addr, length, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
- if (res == MAP_FAILED) {
- XErrno err;
- log_error(gc)("Failed to commit memory (%s)", err.to_string());
- return false;
- }
-
- // Success
- return true;
-}
-
-size_t XPhysicalMemoryBacking::commit(size_t offset, size_t length) const {
- // Try to commit the whole region
- if (commit_inner(offset, length)) {
- // Success
- return length;
- }
-
- // Failed, try to commit as much as possible
- size_t start = offset;
- size_t end = offset + length;
-
- for (;;) {
- length = align_down((end - start) / 2, XGranuleSize);
- if (length == 0) {
- // Done, don't commit more
- return start - offset;
- }
-
- if (commit_inner(start, length)) {
- // Success, try commit more
- start += length;
- } else {
- // Failed, try commit less
- end -= length;
- }
- }
-}
-
-size_t XPhysicalMemoryBacking::uncommit(size_t offset, size_t length) const {
- assert(is_aligned(offset, os::vm_page_size()), "Invalid offset");
- assert(is_aligned(length, os::vm_page_size()), "Invalid length");
-
- log_trace(gc, heap)("Uncommitting memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
- offset / M, (offset + length) / M, length / M);
-
- const uintptr_t start = _base + offset;
- const void* const res = mmap((void*)start, length, PROT_NONE, MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0);
- if (res == MAP_FAILED) {
- XErrno err;
- log_error(gc)("Failed to uncommit memory (%s)", err.to_string());
- return 0;
- }
-
- return length;
-}
-
-void XPhysicalMemoryBacking::map(uintptr_t addr, size_t size, uintptr_t offset) const {
- const XErrno err = mremap(_base + offset, addr, size);
- if (err) {
- fatal("Failed to remap memory (%s)", err.to_string());
- }
-}
-
-void XPhysicalMemoryBacking::unmap(uintptr_t addr, size_t size) const {
- // Note that we must keep the address space reservation intact and just detach
- // the backing memory. For this reason we map a new anonymous, non-accessible
- // and non-reserved page over the mapping instead of actually unmapping.
- const void* const res = mmap((void*)addr, size, PROT_NONE, MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0);
- if (res == MAP_FAILED) {
- XErrno err;
- fatal("Failed to map memory (%s)", err.to_string());
- }
-}
diff --git a/src/hotspot/os/bsd/gc/x/xPhysicalMemoryBacking_bsd.hpp b/src/hotspot/os/bsd/gc/x/xPhysicalMemoryBacking_bsd.hpp
deleted file mode 100644
index 8b4747026ff23..0000000000000
--- a/src/hotspot/os/bsd/gc/x/xPhysicalMemoryBacking_bsd.hpp
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#ifndef OS_BSD_GC_X_XPHYSICALMEMORYBACKING_BSD_HPP
-#define OS_BSD_GC_X_XPHYSICALMEMORYBACKING_BSD_HPP
-
-class XPhysicalMemoryBacking {
-private:
- uintptr_t _base;
- bool _initialized;
-
- bool commit_inner(size_t offset, size_t length) const;
-
-public:
- XPhysicalMemoryBacking(size_t max_capacity);
-
- bool is_initialized() const;
-
- void warn_commit_limits(size_t max_capacity) const;
-
- size_t commit(size_t offset, size_t length) const;
- size_t uncommit(size_t offset, size_t length) const;
-
- void map(uintptr_t addr, size_t size, uintptr_t offset) const;
- void unmap(uintptr_t addr, size_t size) const;
-};
-
-#endif // OS_BSD_GC_X_XPHYSICALMEMORYBACKING_BSD_HPP
diff --git a/src/hotspot/os/bsd/osThread_bsd.cpp b/src/hotspot/os/bsd/osThread_bsd.cpp
index 7b9ad1f76a855..d9624040bc740 100644
--- a/src/hotspot/os/bsd/osThread_bsd.cpp
+++ b/src/hotspot/os/bsd/osThread_bsd.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -22,30 +22,31 @@
*
*/
-// no precompiled headers
-#include "memory/allocation.inline.hpp"
-#include "runtime/mutexLocker.hpp"
+#include "precompiled.hpp"
+#include "memory/allocation.hpp"
+#include "runtime/mutex.hpp"
#include "runtime/osThread.hpp"
#include
-void OSThread::pd_initialize() {
+OSThread::OSThread()
+ : _thread_id(
#ifdef __APPLE__
- _thread_id = 0;
+ 0
#else
- _thread_id = nullptr;
+ nullptr
#endif
- _unique_thread_id = 0;
- _pthread_id = nullptr;
- _siginfo = nullptr;
- _ucontext = nullptr;
- _expanding_stack = 0;
- _alt_sig_stack = nullptr;
-
+ ),
+ _pthread_id(nullptr),
+ _unique_thread_id(0),
+ _caller_sigmask(),
+ sr(),
+ _siginfo(nullptr),
+ _ucontext(nullptr),
+ _expanding_stack(0),
+ _alt_sig_stack(nullptr),
+ _startThread_lock(new Monitor(Mutex::event, "startThread_lock")) {
sigemptyset(&_caller_sigmask);
-
- _startThread_lock = new Monitor(Mutex::event, "startThread_lock");
- assert(_startThread_lock !=nullptr, "check");
}
// Additional thread_id used to correlate threads in SA
@@ -64,6 +65,6 @@ void OSThread::set_unique_thread_id() {
#endif
}
-void OSThread::pd_destroy() {
+OSThread::~OSThread() {
delete _startThread_lock;
}
diff --git a/src/hotspot/os/bsd/osThread_bsd.hpp b/src/hotspot/os/bsd/osThread_bsd.hpp
index 11376835063c4..21bec3f3836e2 100644
--- a/src/hotspot/os/bsd/osThread_bsd.hpp
+++ b/src/hotspot/os/bsd/osThread_bsd.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,19 +25,12 @@
#ifndef OS_BSD_OSTHREAD_BSD_HPP
#define OS_BSD_OSTHREAD_BSD_HPP
- private:
- int _thread_type;
+#include "runtime/osThreadBase.hpp"
+#include "suspendResume_posix.hpp"
+#include "utilities/globalDefinitions.hpp"
- public:
-
- int thread_type() const {
- return _thread_type;
- }
- void set_thread_type(int type) {
- _thread_type = type;
- }
-
- private:
+class OSThread : public OSThreadBase {
+ friend class VMStructs;
#ifdef __APPLE__
typedef thread_t thread_id_t;
@@ -45,6 +38,8 @@
typedef pid_t thread_id_t;
#endif
+ thread_id_t _thread_id;
+
// _pthread_id is the pthread id, which is used by library calls
// (e.g. pthread_kill).
pthread_t _pthread_id;
@@ -57,15 +52,19 @@
sigset_t _caller_sigmask; // Caller's signal mask
public:
+ OSThread();
+ ~OSThread();
// Methods to save/restore caller's signal mask
sigset_t caller_sigmask() const { return _caller_sigmask; }
void set_caller_sigmask(sigset_t sigmask) { _caller_sigmask = sigmask; }
-#ifndef PRODUCT
- // Used for debugging, return a unique integer for each thread.
- intptr_t thread_identifier() const { return (intptr_t)_pthread_id; }
-#endif
+ thread_id_t thread_id() const {
+ return _thread_id;
+ }
+ void set_thread_id(thread_id_t id) {
+ _thread_id = id;
+ }
pthread_t pthread_id() const {
return _pthread_id;
@@ -80,7 +79,6 @@
// suspension support.
// ***************************************************************
-public:
// flags that support signal based suspend/resume on Bsd are in a
// separate class to avoid confusion with many flags in OSThread that
// are used by VM level suspend/resume.
@@ -126,17 +124,9 @@
return _startThread_lock;
}
- // ***************************************************************
- // Platform dependent initialization and cleanup
- // ***************************************************************
-
-private:
-
- void pd_initialize();
- void pd_destroy();
-
-// Reconciliation History
-// osThread_solaris.hpp 1.24 99/08/27 13:11:54
-// End
+ uintx thread_id_for_printing() const override {
+ return (uintx)_thread_id;
+ }
+};
#endif // OS_BSD_OSTHREAD_BSD_HPP
diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp
index 9ad7c35e6bdef..b50ebf2220390 100644
--- a/src/hotspot/os/bsd/os_bsd.cpp
+++ b/src/hotspot/os/bsd/os_bsd.cpp
@@ -633,9 +633,6 @@ bool os::create_thread(Thread* thread, ThreadType thr_type,
return false;
}
- // set the correct thread state
- osthread->set_thread_type(thr_type);
-
// Initial state is ALLOCATED but not INITIALIZED
osthread->set_state(ALLOCATED);
@@ -1509,7 +1506,7 @@ void os::jvm_path(char *buf, jint buflen) {
assert(ret, "cannot locate libjvm");
char *rp = nullptr;
if (ret && dli_fname[0] != '\0') {
- rp = os::Posix::realpath(dli_fname, buf, buflen);
+ rp = os::realpath(dli_fname, buf, buflen);
}
if (rp == nullptr) {
return;
@@ -1541,7 +1538,7 @@ void os::jvm_path(char *buf, jint buflen) {
p = strrchr(buf, '/');
assert(strstr(p, "/libjvm") == p, "invalid library name");
- rp = os::Posix::realpath(java_home_var, buf, buflen);
+ rp = os::realpath(java_home_var, buf, buflen);
if (rp == nullptr) {
return;
}
@@ -1575,7 +1572,7 @@ void os::jvm_path(char *buf, jint buflen) {
snprintf(buf + len, buflen-len, "/libjvm%s", JNI_LIB_SUFFIX);
} else {
// Fall back to path of current library
- rp = os::Posix::realpath(dli_fname, buf, buflen);
+ rp = os::realpath(dli_fname, buf, buflen);
if (rp == nullptr) {
return;
}
@@ -2400,16 +2397,6 @@ int os::open(const char *path, int oflag, int mode) {
return fd;
}
-// return current position of file pointer
-jlong os::current_file_offset(int fd) {
- return (jlong)::lseek(fd, (off_t)0, SEEK_CUR);
-}
-
-// move file pointer to the specified offset
-jlong os::seek_to_file_offset(int fd, jlong offset) {
- return (jlong)::lseek(fd, (off_t)offset, SEEK_SET);
-}
-
// current_thread_cpu_time(bool) and thread_cpu_time(Thread*, bool)
// are used by JVM M&M and JVMTI to get user+sys or user CPU time
// of a thread.
diff --git a/src/hotspot/os/bsd/vmStructs_bsd.hpp b/src/hotspot/os/bsd/vmStructs_bsd.hpp
index 84c1be77374d0..8c9c132e1c25c 100644
--- a/src/hotspot/os/bsd/vmStructs_bsd.hpp
+++ b/src/hotspot/os/bsd/vmStructs_bsd.hpp
@@ -31,9 +31,21 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
+#define VM_STRUCTS_OS(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
+ \
+ /******************************/ \
+ /* Threads (NOTE: incomplete) */ \
+ /******************************/ \
+ nonstatic_field(OSThread, _thread_id, OSThread::thread_id_t) \
+ nonstatic_field(OSThread, _unique_thread_id, uint64_t)
+
+#define VM_TYPES_OS(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
+ \
+ /**********************/ \
+ /* Thread IDs */ \
+ /**********************/ \
+ \
+ declare_unsigned_integer_type(OSThread::thread_id_t)
#define VM_INT_CONSTANTS_OS(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
diff --git a/src/hotspot/os/linux/cgroupV2Subsystem_linux.hpp b/src/hotspot/os/linux/cgroupV2Subsystem_linux.hpp
index 527573644a816..56dcadd670f82 100644
--- a/src/hotspot/os/linux/cgroupV2Subsystem_linux.hpp
+++ b/src/hotspot/os/linux/cgroupV2Subsystem_linux.hpp
@@ -64,16 +64,16 @@ class CgroupV2CpuController: public CgroupCpuController {
bool is_read_only() override {
return reader()->is_read_only();
}
- const char* subsystem_path() {
+ const char* subsystem_path() override {
return reader()->subsystem_path();
}
bool needs_hierarchy_adjustment() override {
return reader()->needs_hierarchy_adjustment();
}
- void set_subsystem_path(const char* cgroup_path) {
+ void set_subsystem_path(const char* cgroup_path) override {
reader()->set_subsystem_path(cgroup_path);
}
- const char* mount_point() { return reader()->mount_point(); }
+ const char* mount_point() override { return reader()->mount_point(); }
const char* cgroup_path() override { return reader()->cgroup_path(); }
};
@@ -97,16 +97,16 @@ class CgroupV2MemoryController final: public CgroupMemoryController {
bool is_read_only() override {
return reader()->is_read_only();
}
- const char* subsystem_path() {
+ const char* subsystem_path() override {
return reader()->subsystem_path();
}
bool needs_hierarchy_adjustment() override {
return reader()->needs_hierarchy_adjustment();
}
- void set_subsystem_path(const char* cgroup_path) {
+ void set_subsystem_path(const char* cgroup_path) override {
reader()->set_subsystem_path(cgroup_path);
}
- const char* mount_point() { return reader()->mount_point(); }
+ const char* mount_point() override { return reader()->mount_point(); }
const char* cgroup_path() override { return reader()->cgroup_path(); }
};
diff --git a/src/hotspot/os/linux/gc/x/xMountPoint_linux.cpp b/src/hotspot/os/linux/gc/x/xMountPoint_linux.cpp
deleted file mode 100644
index 96c0f2f92dbd2..0000000000000
--- a/src/hotspot/os/linux/gc/x/xMountPoint_linux.cpp
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/shared/gcLogPrecious.hpp"
-#include "gc/x/xArray.inline.hpp"
-#include "gc/x/xErrno.hpp"
-#include "gc/x/xMountPoint_linux.hpp"
-#include "runtime/globals.hpp"
-#include "runtime/os.hpp"
-#include "utilities/globalDefinitions.hpp"
-
-#include
-#include
-
-// Mount information, see proc(5) for more details.
-#define PROC_SELF_MOUNTINFO "/proc/self/mountinfo"
-
-XMountPoint::XMountPoint(const char* filesystem, const char** preferred_mountpoints) {
- if (AllocateHeapAt != nullptr) {
- // Use specified path
- _path = os::strdup(AllocateHeapAt, mtGC);
- } else {
- // Find suitable path
- _path = find_mountpoint(filesystem, preferred_mountpoints);
- }
-}
-
-XMountPoint::~XMountPoint() {
- os::free(_path);
- _path = nullptr;
-}
-
-char* XMountPoint::get_mountpoint(const char* line, const char* filesystem) const {
- char* line_mountpoint = nullptr;
- char* line_filesystem = nullptr;
-
- // Parse line and return a newly allocated string containing the mount point if
- // the line contains a matching filesystem and the mount point is accessible by
- // the current user.
- // sscanf, using %m, will return malloced memory. Need raw ::free, not os::free.
- if (sscanf(line, "%*u %*u %*u:%*u %*s %ms %*[^-]- %ms", &line_mountpoint, &line_filesystem) != 2 ||
- strcmp(line_filesystem, filesystem) != 0 ||
- access(line_mountpoint, R_OK|W_OK|X_OK) != 0) {
- // Not a matching or accessible filesystem
- ALLOW_C_FUNCTION(::free, ::free(line_mountpoint);)
- line_mountpoint = nullptr;
- }
-
- ALLOW_C_FUNCTION(::free, ::free(line_filesystem);)
-
- return line_mountpoint;
-}
-
-void XMountPoint::get_mountpoints(const char* filesystem, XArray* mountpoints) const {
- FILE* fd = os::fopen(PROC_SELF_MOUNTINFO, "r");
- if (fd == nullptr) {
- XErrno err;
- log_error_p(gc)("Failed to open %s: %s", PROC_SELF_MOUNTINFO, err.to_string());
- return;
- }
-
- char* line = nullptr;
- size_t length = 0;
-
- while (getline(&line, &length, fd) != -1) {
- char* const mountpoint = get_mountpoint(line, filesystem);
- if (mountpoint != nullptr) {
- mountpoints->append(mountpoint);
- }
- }
-
- // readline will return malloced memory. Need raw ::free, not os::free.
- ALLOW_C_FUNCTION(::free, ::free(line);)
- fclose(fd);
-}
-
-void XMountPoint::free_mountpoints(XArray* mountpoints) const {
- XArrayIterator iter(mountpoints);
- for (char* mountpoint; iter.next(&mountpoint);) {
- ALLOW_C_FUNCTION(::free, ::free(mountpoint);) // *not* os::free
- }
- mountpoints->clear();
-}
-
-char* XMountPoint::find_preferred_mountpoint(const char* filesystem,
- XArray* mountpoints,
- const char** preferred_mountpoints) const {
- // Find preferred mount point
- XArrayIterator iter1(mountpoints);
- for (char* mountpoint; iter1.next(&mountpoint);) {
- for (const char** preferred = preferred_mountpoints; *preferred != nullptr; preferred++) {
- if (!strcmp(mountpoint, *preferred)) {
- // Preferred mount point found
- return os::strdup(mountpoint, mtGC);
- }
- }
- }
-
- // Preferred mount point not found
- log_error_p(gc)("More than one %s filesystem found:", filesystem);
- XArrayIterator iter2(mountpoints);
- for (char* mountpoint; iter2.next(&mountpoint);) {
- log_error_p(gc)(" %s", mountpoint);
- }
-
- return nullptr;
-}
-
-char* XMountPoint::find_mountpoint(const char* filesystem, const char** preferred_mountpoints) const {
- char* path = nullptr;
- XArray mountpoints;
-
- get_mountpoints(filesystem, &mountpoints);
-
- if (mountpoints.length() == 0) {
- // No mount point found
- log_error_p(gc)("Failed to find an accessible %s filesystem", filesystem);
- } else if (mountpoints.length() == 1) {
- // One mount point found
- path = os::strdup(mountpoints.at(0), mtGC);
- } else {
- // More than one mount point found
- path = find_preferred_mountpoint(filesystem, &mountpoints, preferred_mountpoints);
- }
-
- free_mountpoints(&mountpoints);
-
- return path;
-}
-
-const char* XMountPoint::get() const {
- return _path;
-}
diff --git a/src/hotspot/os/linux/gc/x/xMountPoint_linux.hpp b/src/hotspot/os/linux/gc/x/xMountPoint_linux.hpp
deleted file mode 100644
index e0ca126e0667a..0000000000000
--- a/src/hotspot/os/linux/gc/x/xMountPoint_linux.hpp
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#ifndef OS_LINUX_GC_X_XMOUNTPOINT_LINUX_HPP
-#define OS_LINUX_GC_X_XMOUNTPOINT_LINUX_HPP
-
-#include "gc/x/xArray.hpp"
-#include "memory/allocation.hpp"
-
-class XMountPoint : public StackObj {
-private:
- char* _path;
-
- char* get_mountpoint(const char* line,
- const char* filesystem) const;
- void get_mountpoints(const char* filesystem,
- XArray* mountpoints) const;
- void free_mountpoints(XArray* mountpoints) const;
- char* find_preferred_mountpoint(const char* filesystem,
- XArray* mountpoints,
- const char** preferred_mountpoints) const;
- char* find_mountpoint(const char* filesystem,
- const char** preferred_mountpoints) const;
-
-public:
- XMountPoint(const char* filesystem, const char** preferred_mountpoints);
- ~XMountPoint();
-
- const char* get() const;
-};
-
-#endif // OS_LINUX_GC_X_XMOUNTPOINT_LINUX_HPP
diff --git a/src/hotspot/os/linux/gc/x/xNUMA_linux.cpp b/src/hotspot/os/linux/gc/x/xNUMA_linux.cpp
deleted file mode 100644
index 0cc557dde6e86..0000000000000
--- a/src/hotspot/os/linux/gc/x/xNUMA_linux.cpp
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "gc/x/xCPU.inline.hpp"
-#include "gc/x/xErrno.hpp"
-#include "gc/x/xNUMA.hpp"
-#include "gc/x/xSyscall_linux.hpp"
-#include "os_linux.hpp"
-#include "runtime/globals.hpp"
-#include "runtime/os.hpp"
-#include "utilities/debug.hpp"
-
-void XNUMA::pd_initialize() {
- _enabled = UseNUMA;
-}
-
-uint32_t XNUMA::count() {
- if (!_enabled) {
- // NUMA support not enabled
- return 1;
- }
-
- return os::Linux::numa_max_node() + 1;
-}
-
-uint32_t XNUMA::id() {
- if (!_enabled) {
- // NUMA support not enabled
- return 0;
- }
-
- return os::Linux::get_node_by_cpu(XCPU::id());
-}
-
-uint32_t XNUMA::memory_id(uintptr_t addr) {
- if (!_enabled) {
- // NUMA support not enabled, assume everything belongs to node zero
- return 0;
- }
-
- uint32_t id = (uint32_t)-1;
-
- if (XSyscall::get_mempolicy((int*)&id, nullptr, 0, (void*)addr, MPOL_F_NODE | MPOL_F_ADDR) == -1) {
- XErrno err;
- fatal("Failed to get NUMA id for memory at " PTR_FORMAT " (%s)", addr, err.to_string());
- }
-
- assert(id < count(), "Invalid NUMA id");
-
- return id;
-}
diff --git a/src/hotspot/os/linux/gc/x/xPhysicalMemoryBacking_linux.cpp b/src/hotspot/os/linux/gc/x/xPhysicalMemoryBacking_linux.cpp
deleted file mode 100644
index 35625f613d349..0000000000000
--- a/src/hotspot/os/linux/gc/x/xPhysicalMemoryBacking_linux.cpp
+++ /dev/null
@@ -1,724 +0,0 @@
-/*
- * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/shared/gcLogPrecious.hpp"
-#include "gc/x/xArray.inline.hpp"
-#include "gc/x/xErrno.hpp"
-#include "gc/x/xGlobals.hpp"
-#include "gc/x/xLargePages.inline.hpp"
-#include "gc/x/xMountPoint_linux.hpp"
-#include "gc/x/xNUMA.inline.hpp"
-#include "gc/x/xPhysicalMemoryBacking_linux.hpp"
-#include "gc/x/xSyscall_linux.hpp"
-#include "logging/log.hpp"
-#include "os_linux.hpp"
-#include "runtime/init.hpp"
-#include "runtime/os.hpp"
-#include "runtime/safefetch.hpp"
-#include "utilities/align.hpp"
-#include "utilities/debug.hpp"
-#include "utilities/growableArray.hpp"
-
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-
-//
-// Support for building on older Linux systems
-//
-
-// memfd_create(2) flags
-#ifndef MFD_CLOEXEC
-#define MFD_CLOEXEC 0x0001U
-#endif
-#ifndef MFD_HUGETLB
-#define MFD_HUGETLB 0x0004U
-#endif
-#ifndef MFD_HUGE_2MB
-#define MFD_HUGE_2MB 0x54000000U
-#endif
-
-// open(2) flags
-#ifndef O_CLOEXEC
-#define O_CLOEXEC 02000000
-#endif
-#ifndef O_TMPFILE
-#define O_TMPFILE (020000000 | O_DIRECTORY)
-#endif
-
-// fallocate(2) flags
-#ifndef FALLOC_FL_KEEP_SIZE
-#define FALLOC_FL_KEEP_SIZE 0x01
-#endif
-#ifndef FALLOC_FL_PUNCH_HOLE
-#define FALLOC_FL_PUNCH_HOLE 0x02
-#endif
-
-// Filesystem types, see statfs(2)
-#ifndef TMPFS_MAGIC
-#define TMPFS_MAGIC 0x01021994
-#endif
-#ifndef HUGETLBFS_MAGIC
-#define HUGETLBFS_MAGIC 0x958458f6
-#endif
-
-// Filesystem names
-#define XFILESYSTEM_TMPFS "tmpfs"
-#define XFILESYSTEM_HUGETLBFS "hugetlbfs"
-
-// Proc file entry for max map mount
-#define XFILENAME_PROC_MAX_MAP_COUNT "/proc/sys/vm/max_map_count"
-
-// Sysfs file for transparent huge page on tmpfs
-#define XFILENAME_SHMEM_ENABLED "/sys/kernel/mm/transparent_hugepage/shmem_enabled"
-
-// Java heap filename
-#define XFILENAME_HEAP "java_heap"
-
-// Preferred tmpfs mount points, ordered by priority
-static const char* z_preferred_tmpfs_mountpoints[] = {
- "/dev/shm",
- "/run/shm",
- nullptr
-};
-
-// Preferred hugetlbfs mount points, ordered by priority
-static const char* z_preferred_hugetlbfs_mountpoints[] = {
- "/dev/hugepages",
- "/hugepages",
- nullptr
-};
-
-static int z_fallocate_hugetlbfs_attempts = 3;
-static bool z_fallocate_supported = true;
-
-XPhysicalMemoryBacking::XPhysicalMemoryBacking(size_t max_capacity) :
- _fd(-1),
- _filesystem(0),
- _block_size(0),
- _available(0),
- _initialized(false) {
-
- // Create backing file
- _fd = create_fd(XFILENAME_HEAP);
- if (_fd == -1) {
- return;
- }
-
- // Truncate backing file
- while (ftruncate(_fd, max_capacity) == -1) {
- if (errno != EINTR) {
- XErrno err;
- log_error_p(gc)("Failed to truncate backing file (%s)", err.to_string());
- return;
- }
- }
-
- // Get filesystem statistics
- struct statfs buf;
- if (fstatfs(_fd, &buf) == -1) {
- XErrno err;
- log_error_p(gc)("Failed to determine filesystem type for backing file (%s)", err.to_string());
- return;
- }
-
- _filesystem = buf.f_type;
- _block_size = buf.f_bsize;
- _available = buf.f_bavail * _block_size;
-
- log_info_p(gc, init)("Heap Backing Filesystem: %s (" UINT64_FORMAT_X ")",
- is_tmpfs() ? XFILESYSTEM_TMPFS : is_hugetlbfs() ? XFILESYSTEM_HUGETLBFS : "other", _filesystem);
-
- // Make sure the filesystem type matches requested large page type
- if (XLargePages::is_transparent() && !is_tmpfs()) {
- log_error_p(gc)("-XX:+UseTransparentHugePages can only be enabled when using a %s filesystem",
- XFILESYSTEM_TMPFS);
- return;
- }
-
- if (XLargePages::is_transparent() && !tmpfs_supports_transparent_huge_pages()) {
- log_error_p(gc)("-XX:+UseTransparentHugePages on a %s filesystem not supported by kernel",
- XFILESYSTEM_TMPFS);
- return;
- }
-
- if (XLargePages::is_explicit() && !is_hugetlbfs()) {
- log_error_p(gc)("-XX:+UseLargePages (without -XX:+UseTransparentHugePages) can only be enabled "
- "when using a %s filesystem", XFILESYSTEM_HUGETLBFS);
- return;
- }
-
- if (!XLargePages::is_explicit() && is_hugetlbfs()) {
- log_error_p(gc)("-XX:+UseLargePages must be enabled when using a %s filesystem",
- XFILESYSTEM_HUGETLBFS);
- return;
- }
-
- // Make sure the filesystem block size is compatible
- if (XGranuleSize % _block_size != 0) {
- log_error_p(gc)("Filesystem backing the heap has incompatible block size (" SIZE_FORMAT ")",
- _block_size);
- return;
- }
-
- if (is_hugetlbfs() && _block_size != XGranuleSize) {
- log_error_p(gc)("%s filesystem has unexpected block size " SIZE_FORMAT " (expected " SIZE_FORMAT ")",
- XFILESYSTEM_HUGETLBFS, _block_size, XGranuleSize);
- return;
- }
-
- // Successfully initialized
- _initialized = true;
-}
-
-int XPhysicalMemoryBacking::create_mem_fd(const char* name) const {
- assert(XGranuleSize == 2 * M, "Granule size must match MFD_HUGE_2MB");
-
- // Create file name
- char filename[PATH_MAX];
- snprintf(filename, sizeof(filename), "%s%s", name, XLargePages::is_explicit() ? ".hugetlb" : "");
-
- // Create file
- const int extra_flags = XLargePages::is_explicit() ? (MFD_HUGETLB | MFD_HUGE_2MB) : 0;
- const int fd = XSyscall::memfd_create(filename, MFD_CLOEXEC | extra_flags);
- if (fd == -1) {
- XErrno err;
- log_debug_p(gc, init)("Failed to create memfd file (%s)",
- (XLargePages::is_explicit() && (err == EINVAL || err == ENODEV)) ?
- "Hugepages (2M) not available" : err.to_string());
- return -1;
- }
-
- log_info_p(gc, init)("Heap Backing File: /memfd:%s", filename);
-
- return fd;
-}
-
-int XPhysicalMemoryBacking::create_file_fd(const char* name) const {
- const char* const filesystem = XLargePages::is_explicit()
- ? XFILESYSTEM_HUGETLBFS
- : XFILESYSTEM_TMPFS;
- const char** const preferred_mountpoints = XLargePages::is_explicit()
- ? z_preferred_hugetlbfs_mountpoints
- : z_preferred_tmpfs_mountpoints;
-
- // Find mountpoint
- XMountPoint mountpoint(filesystem, preferred_mountpoints);
- if (mountpoint.get() == nullptr) {
- log_error_p(gc)("Use -XX:AllocateHeapAt to specify the path to a %s filesystem", filesystem);
- return -1;
- }
-
- // Try to create an anonymous file using the O_TMPFILE flag. Note that this
- // flag requires kernel >= 3.11. If this fails we fall back to open/unlink.
- const int fd_anon = os::open(mountpoint.get(), O_TMPFILE|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR);
- if (fd_anon == -1) {
- XErrno err;
- log_debug_p(gc, init)("Failed to create anonymous file in %s (%s)", mountpoint.get(),
- (err == EINVAL ? "Not supported" : err.to_string()));
- } else {
- // Get inode number for anonymous file
- struct stat stat_buf;
- if (fstat(fd_anon, &stat_buf) == -1) {
- XErrno err;
- log_error_pd(gc)("Failed to determine inode number for anonymous file (%s)", err.to_string());
- return -1;
- }
-
- log_info_p(gc, init)("Heap Backing File: %s/#" UINT64_FORMAT, mountpoint.get(), (uint64_t)stat_buf.st_ino);
-
- return fd_anon;
- }
-
- log_debug_p(gc, init)("Falling back to open/unlink");
-
- // Create file name
- char filename[PATH_MAX];
- snprintf(filename, sizeof(filename), "%s/%s.%d", mountpoint.get(), name, os::current_process_id());
-
- // Create file
- const int fd = os::open(filename, O_CREAT|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR);
- if (fd == -1) {
- XErrno err;
- log_error_p(gc)("Failed to create file %s (%s)", filename, err.to_string());
- return -1;
- }
-
- // Unlink file
- if (unlink(filename) == -1) {
- XErrno err;
- log_error_p(gc)("Failed to unlink file %s (%s)", filename, err.to_string());
- return -1;
- }
-
- log_info_p(gc, init)("Heap Backing File: %s", filename);
-
- return fd;
-}
-
-int XPhysicalMemoryBacking::create_fd(const char* name) const {
- if (AllocateHeapAt == nullptr) {
- // If the path is not explicitly specified, then we first try to create a memfd file
- // instead of looking for a tmpfd/hugetlbfs mount point. Note that memfd_create() might
- // not be supported at all (requires kernel >= 3.17), or it might not support large
- // pages (requires kernel >= 4.14). If memfd_create() fails, then we try to create a
- // file on an accessible tmpfs or hugetlbfs mount point.
- const int fd = create_mem_fd(name);
- if (fd != -1) {
- return fd;
- }
-
- log_debug_p(gc)("Falling back to searching for an accessible mount point");
- }
-
- return create_file_fd(name);
-}
-
-bool XPhysicalMemoryBacking::is_initialized() const {
- return _initialized;
-}
-
-void XPhysicalMemoryBacking::warn_available_space(size_t max_capacity) const {
- // Note that the available space on a tmpfs or a hugetlbfs filesystem
- // will be zero if no size limit was specified when it was mounted.
- if (_available == 0) {
- // No size limit set, skip check
- log_info_p(gc, init)("Available space on backing filesystem: N/A");
- return;
- }
-
- log_info_p(gc, init)("Available space on backing filesystem: " SIZE_FORMAT "M", _available / M);
-
- // Warn if the filesystem doesn't currently have enough space available to hold
- // the max heap size. The max heap size will be capped if we later hit this limit
- // when trying to expand the heap.
- if (_available < max_capacity) {
- log_warning_p(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****");
- log_warning_p(gc)("Not enough space available on the backing filesystem to hold the current max Java heap");
- log_warning_p(gc)("size (" SIZE_FORMAT "M). Please adjust the size of the backing filesystem accordingly "
- "(available", max_capacity / M);
- log_warning_p(gc)("space is currently " SIZE_FORMAT "M). Continuing execution with the current filesystem "
- "size could", _available / M);
- log_warning_p(gc)("lead to a premature OutOfMemoryError being thrown, due to failure to commit memory.");
- }
-}
-
-void XPhysicalMemoryBacking::warn_max_map_count(size_t max_capacity) const {
- const char* const filename = XFILENAME_PROC_MAX_MAP_COUNT;
- FILE* const file = os::fopen(filename, "r");
- if (file == nullptr) {
- // Failed to open file, skip check
- log_debug_p(gc, init)("Failed to open %s", filename);
- return;
- }
-
- size_t actual_max_map_count = 0;
- const int result = fscanf(file, SIZE_FORMAT, &actual_max_map_count);
- fclose(file);
- if (result != 1) {
- // Failed to read file, skip check
- log_debug_p(gc, init)("Failed to read %s", filename);
- return;
- }
-
- // The required max map count is impossible to calculate exactly since subsystems
- // other than ZGC are also creating memory mappings, and we have no control over that.
- // However, ZGC tends to create the most mappings and dominate the total count.
- // In the worst cases, ZGC will map each granule three times, i.e. once per heap view.
- // We speculate that we need another 20% to allow for non-ZGC subsystems to map memory.
- const size_t required_max_map_count = (max_capacity / XGranuleSize) * 3 * 1.2;
- if (actual_max_map_count < required_max_map_count) {
- log_warning_p(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****");
- log_warning_p(gc)("The system limit on number of memory mappings per process might be too low for the given");
- log_warning_p(gc)("max Java heap size (" SIZE_FORMAT "M). Please adjust %s to allow for at",
- max_capacity / M, filename);
- log_warning_p(gc)("least " SIZE_FORMAT " mappings (current limit is " SIZE_FORMAT "). Continuing execution "
- "with the current", required_max_map_count, actual_max_map_count);
- log_warning_p(gc)("limit could lead to a premature OutOfMemoryError being thrown, due to failure to map memory.");
- }
-}
-
-void XPhysicalMemoryBacking::warn_commit_limits(size_t max_capacity) const {
- // Warn if available space is too low
- warn_available_space(max_capacity);
-
- // Warn if max map count is too low
- warn_max_map_count(max_capacity);
-}
-
-bool XPhysicalMemoryBacking::is_tmpfs() const {
- return _filesystem == TMPFS_MAGIC;
-}
-
-bool XPhysicalMemoryBacking::is_hugetlbfs() const {
- return _filesystem == HUGETLBFS_MAGIC;
-}
-
-bool XPhysicalMemoryBacking::tmpfs_supports_transparent_huge_pages() const {
- // If the shmem_enabled file exists and is readable then we
- // know the kernel supports transparent huge pages for tmpfs.
- return access(XFILENAME_SHMEM_ENABLED, R_OK) == 0;
-}
-
-XErrno XPhysicalMemoryBacking::fallocate_compat_mmap_hugetlbfs(size_t offset, size_t length, bool touch) const {
- // On hugetlbfs, mapping a file segment will fail immediately, without
- // the need to touch the mapped pages first, if there aren't enough huge
- // pages available to back the mapping.
- void* const addr = mmap(nullptr, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, offset);
- if (addr == MAP_FAILED) {
- // Failed
- return errno;
- }
-
- // Once mapped, the huge pages are only reserved. We need to touch them
- // to associate them with the file segment. Note that we can not punch
- // hole in file segments which only have reserved pages.
- if (touch) {
- char* const start = (char*)addr;
- char* const end = start + length;
- os::pretouch_memory(start, end, _block_size);
- }
-
- // Unmap again. From now on, the huge pages that were mapped are allocated
- // to this file. There's no risk of getting a SIGBUS when mapping and
- // touching these pages again.
- if (munmap(addr, length) == -1) {
- // Failed
- return errno;
- }
-
- // Success
- return 0;
-}
-
-static bool safe_touch_mapping(void* addr, size_t length, size_t page_size) {
- char* const start = (char*)addr;
- char* const end = start + length;
-
- // Touching a mapping that can't be backed by memory will generate a
- // SIGBUS. By using SafeFetch32 any SIGBUS will be safely caught and
- // handled. On tmpfs, doing a fetch (rather than a store) is enough
- // to cause backing pages to be allocated (there's no zero-page to
- // worry about).
- for (char *p = start; p < end; p += page_size) {
- if (SafeFetch32((int*)p, -1) == -1) {
- // Failed
- return false;
- }
- }
-
- // Success
- return true;
-}
-
-XErrno XPhysicalMemoryBacking::fallocate_compat_mmap_tmpfs(size_t offset, size_t length) const {
- // On tmpfs, we need to touch the mapped pages to figure out
- // if there are enough pages available to back the mapping.
- void* const addr = mmap(nullptr, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, offset);
- if (addr == MAP_FAILED) {
- // Failed
- return errno;
- }
-
- // Advise mapping to use transparent huge pages
- os::realign_memory((char*)addr, length, XGranuleSize);
-
- // Touch the mapping (safely) to make sure it's backed by memory
- const bool backed = safe_touch_mapping(addr, length, _block_size);
-
- // Unmap again. If successfully touched, the backing memory will
- // be allocated to this file. There's no risk of getting a SIGBUS
- // when mapping and touching these pages again.
- if (munmap(addr, length) == -1) {
- // Failed
- return errno;
- }
-
- // Success
- return backed ? 0 : ENOMEM;
-}
-
-XErrno XPhysicalMemoryBacking::fallocate_compat_pwrite(size_t offset, size_t length) const {
- uint8_t data = 0;
-
- // Allocate backing memory by writing to each block
- for (size_t pos = offset; pos < offset + length; pos += _block_size) {
- if (pwrite(_fd, &data, sizeof(data), pos) == -1) {
- // Failed
- return errno;
- }
- }
-
- // Success
- return 0;
-}
-
-XErrno XPhysicalMemoryBacking::fallocate_fill_hole_compat(size_t offset, size_t length) const {
- // fallocate(2) is only supported by tmpfs since Linux 3.5, and by hugetlbfs
- // since Linux 4.3. When fallocate(2) is not supported we emulate it using
- // mmap/munmap (for hugetlbfs and tmpfs with transparent huge pages) or pwrite
- // (for tmpfs without transparent huge pages and other filesystem types).
- if (XLargePages::is_explicit()) {
- return fallocate_compat_mmap_hugetlbfs(offset, length, false /* touch */);
- } else if (XLargePages::is_transparent()) {
- return fallocate_compat_mmap_tmpfs(offset, length);
- } else {
- return fallocate_compat_pwrite(offset, length);
- }
-}
-
-XErrno XPhysicalMemoryBacking::fallocate_fill_hole_syscall(size_t offset, size_t length) const {
- const int mode = 0; // Allocate
- const int res = XSyscall::fallocate(_fd, mode, offset, length);
- if (res == -1) {
- // Failed
- return errno;
- }
-
- // Success
- return 0;
-}
-
-XErrno XPhysicalMemoryBacking::fallocate_fill_hole(size_t offset, size_t length) const {
- // Using compat mode is more efficient when allocating space on hugetlbfs.
- // Note that allocating huge pages this way will only reserve them, and not
- // associate them with segments of the file. We must guarantee that we at
- // some point touch these segments, otherwise we can not punch hole in them.
- // Also note that we need to use compat mode when using transparent huge pages,
- // since we need to use madvise(2) on the mapping before the page is allocated.
- if (z_fallocate_supported && !XLargePages::is_enabled()) {
- const XErrno err = fallocate_fill_hole_syscall(offset, length);
- if (!err) {
- // Success
- return 0;
- }
-
- if (err != ENOSYS && err != EOPNOTSUPP) {
- // Failed
- return err;
- }
-
- // Not supported
- log_debug_p(gc)("Falling back to fallocate() compatibility mode");
- z_fallocate_supported = false;
- }
-
- return fallocate_fill_hole_compat(offset, length);
-}
-
-XErrno XPhysicalMemoryBacking::fallocate_punch_hole(size_t offset, size_t length) const {
- if (XLargePages::is_explicit()) {
- // We can only punch hole in pages that have been touched. Non-touched
- // pages are only reserved, and not associated with any specific file
- // segment. We don't know which pages have been previously touched, so
- // we always touch them here to guarantee that we can punch hole.
- const XErrno err = fallocate_compat_mmap_hugetlbfs(offset, length, true /* touch */);
- if (err) {
- // Failed
- return err;
- }
- }
-
- const int mode = FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE;
- if (XSyscall::fallocate(_fd, mode, offset, length) == -1) {
- // Failed
- return errno;
- }
-
- // Success
- return 0;
-}
-
-XErrno XPhysicalMemoryBacking::split_and_fallocate(bool punch_hole, size_t offset, size_t length) const {
- // Try first half
- const size_t offset0 = offset;
- const size_t length0 = align_up(length / 2, _block_size);
- const XErrno err0 = fallocate(punch_hole, offset0, length0);
- if (err0) {
- return err0;
- }
-
- // Try second half
- const size_t offset1 = offset0 + length0;
- const size_t length1 = length - length0;
- const XErrno err1 = fallocate(punch_hole, offset1, length1);
- if (err1) {
- return err1;
- }
-
- // Success
- return 0;
-}
-
-XErrno XPhysicalMemoryBacking::fallocate(bool punch_hole, size_t offset, size_t length) const {
- assert(is_aligned(offset, _block_size), "Invalid offset");
- assert(is_aligned(length, _block_size), "Invalid length");
-
- const XErrno err = punch_hole ? fallocate_punch_hole(offset, length) : fallocate_fill_hole(offset, length);
- if (err == EINTR && length > _block_size) {
- // Calling fallocate(2) with a large length can take a long time to
- // complete. When running profilers, such as VTune, this syscall will
- // be constantly interrupted by signals. Expanding the file in smaller
- // steps avoids this problem.
- return split_and_fallocate(punch_hole, offset, length);
- }
-
- return err;
-}
-
-bool XPhysicalMemoryBacking::commit_inner(size_t offset, size_t length) const {
- log_trace(gc, heap)("Committing memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
- offset / M, (offset + length) / M, length / M);
-
-retry:
- const XErrno err = fallocate(false /* punch_hole */, offset, length);
- if (err) {
- if (err == ENOSPC && !is_init_completed() && XLargePages::is_explicit() && z_fallocate_hugetlbfs_attempts-- > 0) {
- // If we fail to allocate during initialization, due to lack of space on
- // the hugetlbfs filesystem, then we wait and retry a few times before
- // giving up. Otherwise there is a risk that running JVMs back-to-back
- // will fail, since there is a delay between process termination and the
- // huge pages owned by that process being returned to the huge page pool
- // and made available for new allocations.
- log_debug_p(gc, init)("Failed to commit memory (%s), retrying", err.to_string());
-
- // Wait and retry in one second, in the hope that huge pages will be
- // available by then.
- sleep(1);
- goto retry;
- }
-
- // Failed
- log_error_p(gc)("Failed to commit memory (%s)", err.to_string());
- return false;
- }
-
- // Success
- return true;
-}
-
-static int offset_to_node(size_t offset) {
- const GrowableArray* mapping = os::Linux::numa_nindex_to_node();
- const size_t nindex = (offset >> XGranuleSizeShift) % mapping->length();
- return mapping->at((int)nindex);
-}
-
-size_t XPhysicalMemoryBacking::commit_numa_interleaved(size_t offset, size_t length) const {
- size_t committed = 0;
-
- // Commit one granule at a time, so that each granule
- // can be allocated from a different preferred node.
- while (committed < length) {
- const size_t granule_offset = offset + committed;
-
- // Setup NUMA policy to allocate memory from a preferred node
- os::Linux::numa_set_preferred(offset_to_node(granule_offset));
-
- if (!commit_inner(granule_offset, XGranuleSize)) {
- // Failed
- break;
- }
-
- committed += XGranuleSize;
- }
-
- // Restore NUMA policy
- os::Linux::numa_set_preferred(-1);
-
- return committed;
-}
-
-size_t XPhysicalMemoryBacking::commit_default(size_t offset, size_t length) const {
- // Try to commit the whole region
- if (commit_inner(offset, length)) {
- // Success
- return length;
- }
-
- // Failed, try to commit as much as possible
- size_t start = offset;
- size_t end = offset + length;
-
- for (;;) {
- length = align_down((end - start) / 2, XGranuleSize);
- if (length < XGranuleSize) {
- // Done, don't commit more
- return start - offset;
- }
-
- if (commit_inner(start, length)) {
- // Success, try commit more
- start += length;
- } else {
- // Failed, try commit less
- end -= length;
- }
- }
-}
-
-size_t XPhysicalMemoryBacking::commit(size_t offset, size_t length) const {
- if (XNUMA::is_enabled() && !XLargePages::is_explicit()) {
- // To get granule-level NUMA interleaving when using non-large pages,
- // we must explicitly interleave the memory at commit/fallocate time.
- return commit_numa_interleaved(offset, length);
- }
-
- return commit_default(offset, length);
-}
-
-size_t XPhysicalMemoryBacking::uncommit(size_t offset, size_t length) const {
- log_trace(gc, heap)("Uncommitting memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
- offset / M, (offset + length) / M, length / M);
-
- const XErrno err = fallocate(true /* punch_hole */, offset, length);
- if (err) {
- log_error(gc)("Failed to uncommit memory (%s)", err.to_string());
- return 0;
- }
-
- return length;
-}
-
-void XPhysicalMemoryBacking::map(uintptr_t addr, size_t size, uintptr_t offset) const {
- const void* const res = mmap((void*)addr, size, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_SHARED, _fd, offset);
- if (res == MAP_FAILED) {
- XErrno err;
- fatal("Failed to map memory (%s)", err.to_string());
- }
-}
-
-void XPhysicalMemoryBacking::unmap(uintptr_t addr, size_t size) const {
- // Note that we must keep the address space reservation intact and just detach
- // the backing memory. For this reason we map a new anonymous, non-accessible
- // and non-reserved page over the mapping instead of actually unmapping.
- const void* const res = mmap((void*)addr, size, PROT_NONE, MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0);
- if (res == MAP_FAILED) {
- XErrno err;
- fatal("Failed to map memory (%s)", err.to_string());
- }
-}
diff --git a/src/hotspot/os/linux/gc/x/xPhysicalMemoryBacking_linux.hpp b/src/hotspot/os/linux/gc/x/xPhysicalMemoryBacking_linux.hpp
deleted file mode 100644
index 253a3f87ef427..0000000000000
--- a/src/hotspot/os/linux/gc/x/xPhysicalMemoryBacking_linux.hpp
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#ifndef OS_LINUX_GC_X_XPHYSICALMEMORYBACKING_LINUX_HPP
-#define OS_LINUX_GC_X_XPHYSICALMEMORYBACKING_LINUX_HPP
-
-class XErrno;
-
-class XPhysicalMemoryBacking {
-private:
- int _fd;
- size_t _size;
- uint64_t _filesystem;
- size_t _block_size;
- size_t _available;
- bool _initialized;
-
- void warn_available_space(size_t max_capacity) const;
- void warn_max_map_count(size_t max_capacity) const;
-
- int create_mem_fd(const char* name) const;
- int create_file_fd(const char* name) const;
- int create_fd(const char* name) const;
-
- bool is_tmpfs() const;
- bool is_hugetlbfs() const;
- bool tmpfs_supports_transparent_huge_pages() const;
-
- XErrno fallocate_compat_mmap_hugetlbfs(size_t offset, size_t length, bool touch) const;
- XErrno fallocate_compat_mmap_tmpfs(size_t offset, size_t length) const;
- XErrno fallocate_compat_pwrite(size_t offset, size_t length) const;
- XErrno fallocate_fill_hole_compat(size_t offset, size_t length) const;
- XErrno fallocate_fill_hole_syscall(size_t offset, size_t length) const;
- XErrno fallocate_fill_hole(size_t offset, size_t length) const;
- XErrno fallocate_punch_hole(size_t offset, size_t length) const;
- XErrno split_and_fallocate(bool punch_hole, size_t offset, size_t length) const;
- XErrno fallocate(bool punch_hole, size_t offset, size_t length) const;
-
- bool commit_inner(size_t offset, size_t length) const;
- size_t commit_numa_interleaved(size_t offset, size_t length) const;
- size_t commit_default(size_t offset, size_t length) const;
-
-public:
- XPhysicalMemoryBacking(size_t max_capacity);
-
- bool is_initialized() const;
-
- void warn_commit_limits(size_t max_capacity) const;
-
- size_t commit(size_t offset, size_t length) const;
- size_t uncommit(size_t offset, size_t length) const;
-
- void map(uintptr_t addr, size_t size, uintptr_t offset) const;
- void unmap(uintptr_t addr, size_t size) const;
-};
-
-#endif // OS_LINUX_GC_X_XPHYSICALMEMORYBACKING_LINUX_HPP
diff --git a/src/hotspot/os/linux/gc/x/xSyscall_linux.cpp b/src/hotspot/os/linux/gc/x/xSyscall_linux.cpp
deleted file mode 100644
index 6035eaae61bd1..0000000000000
--- a/src/hotspot/os/linux/gc/x/xSyscall_linux.cpp
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/x/xSyscall_linux.hpp"
-#include OS_CPU_HEADER(gc/x/xSyscall)
-
-#include
-
-int XSyscall::memfd_create(const char *name, unsigned int flags) {
- return syscall(SYS_memfd_create, name, flags);
-}
-
-int XSyscall::fallocate(int fd, int mode, size_t offset, size_t length) {
- return syscall(SYS_fallocate, fd, mode, offset, length);
-}
-
-long XSyscall::get_mempolicy(int* mode, unsigned long* nodemask, unsigned long maxnode, void* addr, unsigned long flags) {
- return syscall(SYS_get_mempolicy, mode, nodemask, maxnode, addr, flags);
-}
diff --git a/src/hotspot/os/linux/gc/x/xSyscall_linux.hpp b/src/hotspot/os/linux/gc/x/xSyscall_linux.hpp
deleted file mode 100644
index f16d2b2ffdcc6..0000000000000
--- a/src/hotspot/os/linux/gc/x/xSyscall_linux.hpp
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#ifndef OS_LINUX_GC_X_XSYSCALL_LINUX_HPP
-#define OS_LINUX_GC_X_XSYSCALL_LINUX_HPP
-
-#include "memory/allStatic.hpp"
-#include "utilities/globalDefinitions.hpp"
-
-// Flags for get_mempolicy()
-#ifndef MPOL_F_NODE
-#define MPOL_F_NODE (1<<0)
-#endif
-#ifndef MPOL_F_ADDR
-#define MPOL_F_ADDR (1<<1)
-#endif
-
-class XSyscall : public AllStatic {
-public:
- static int memfd_create(const char* name, unsigned int flags);
- static int fallocate(int fd, int mode, size_t offset, size_t length);
- static long get_mempolicy(int* mode, unsigned long* nodemask, unsigned long maxnode, void* addr, unsigned long flags);
-};
-
-#endif // OS_LINUX_GC_X_XSYSCALL_LINUX_HPP
diff --git a/src/hotspot/os/linux/osThread_linux.cpp b/src/hotspot/os/linux/osThread_linux.cpp
index 9c77cb32f6d1c..c9a44eb413f43 100644
--- a/src/hotspot/os/linux/osThread_linux.cpp
+++ b/src/hotspot/os/linux/osThread_linux.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -22,27 +22,26 @@
*
*/
-// no precompiled headers
-#include "memory/allocation.inline.hpp"
+#include "precompiled.hpp"
+#include "memory/allocation.hpp"
#include "runtime/mutex.hpp"
#include "runtime/osThread.hpp"
#include
-void OSThread::pd_initialize() {
- _thread_id = 0;
- _pthread_id = 0;
- _siginfo = nullptr;
- _ucontext = nullptr;
- _expanding_stack = 0;
- _alt_sig_stack = nullptr;
-
+OSThread::OSThread()
+ : _thread_id(0),
+ _pthread_id(0),
+ _caller_sigmask(),
+ sr(),
+ _siginfo(nullptr),
+ _ucontext(nullptr),
+ _expanding_stack(0),
+ _alt_sig_stack(nullptr),
+ _startThread_lock(new Monitor(Mutex::event, "startThread_lock")) {
sigemptyset(&_caller_sigmask);
-
- _startThread_lock = new Monitor(Mutex::event, "startThread_lock");
- assert(_startThread_lock !=nullptr, "check");
}
-void OSThread::pd_destroy() {
+OSThread::~OSThread() {
delete _startThread_lock;
}
diff --git a/src/hotspot/os/linux/osThread_linux.hpp b/src/hotspot/os/linux/osThread_linux.hpp
index a849673af62db..5ed2d07850a97 100644
--- a/src/hotspot/os/linux/osThread_linux.hpp
+++ b/src/hotspot/os/linux/osThread_linux.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -24,20 +24,17 @@
#ifndef OS_LINUX_OSTHREAD_LINUX_HPP
#define OS_LINUX_OSTHREAD_LINUX_HPP
- public:
- typedef pid_t thread_id_t;
- private:
- int _thread_type;
+#include "runtime/osThreadBase.hpp"
+#include "suspendResume_posix.hpp"
+#include "utilities/globalDefinitions.hpp"
- public:
+class OSThread : public OSThreadBase {
+ friend class VMStructs;
- int thread_type() const {
- return _thread_type;
- }
- void set_thread_type(int type) {
- _thread_type = type;
- }
+ typedef pid_t thread_id_t;
+
+ thread_id_t _thread_id;
// _pthread_id is the pthread id, which is used by library calls
// (e.g. pthread_kill).
@@ -46,15 +43,19 @@
sigset_t _caller_sigmask; // Caller's signal mask
public:
+ OSThread();
+ ~OSThread();
// Methods to save/restore caller's signal mask
sigset_t caller_sigmask() const { return _caller_sigmask; }
void set_caller_sigmask(sigset_t sigmask) { _caller_sigmask = sigmask; }
-#ifndef PRODUCT
- // Used for debugging, return a unique integer for each thread.
- int thread_identifier() const { return _thread_id; }
-#endif
+ thread_id_t thread_id() const {
+ return _thread_id;
+ }
+ void set_thread_id(thread_id_t id) {
+ _thread_id = id;
+ }
pthread_t pthread_id() const {
return _pthread_id;
@@ -67,7 +68,6 @@
// suspension support.
// ***************************************************************
-public:
// flags that support signal based suspend/resume on Linux are in a
// separate class to avoid confusion with many flags in OSThread that
// are used by VM level suspend/resume.
@@ -113,17 +113,10 @@
return _startThread_lock;
}
- // ***************************************************************
- // Platform dependent initialization and cleanup
- // ***************************************************************
-
-private:
-
- void pd_initialize();
- void pd_destroy();
-
-// Reconciliation History
-// osThread_solaris.hpp 1.24 99/08/27 13:11:54
-// End
+ // Printing
+ uintx thread_id_for_printing() const override {
+ return (uintx)_thread_id;
+ }
+};
#endif // OS_LINUX_OSTHREAD_LINUX_HPP
diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp
index 609317df45fc2..b2b9a798119d2 100644
--- a/src/hotspot/os/linux/os_linux.cpp
+++ b/src/hotspot/os/linux/os_linux.cpp
@@ -817,7 +817,7 @@ static void *thread_native_entry(Thread *thread) {
OSThread* osthread = thread->osthread();
Monitor* sync = osthread->startThread_lock();
- osthread->set_thread_id(checked_cast(os::current_thread_id()));
+ osthread->set_thread_id(checked_cast(os::current_thread_id()));
if (UseNUMA) {
int lgrp_id = os::numa_get_group_id();
@@ -974,9 +974,6 @@ bool os::create_thread(Thread* thread, ThreadType thr_type,
return false;
}
- // set the correct thread state
- osthread->set_thread_type(thr_type);
-
// Initial state is ALLOCATED but not INITIALIZED
osthread->set_state(ALLOCATED);
@@ -2763,7 +2760,7 @@ void os::jvm_path(char *buf, jint buflen) {
assert(ret, "cannot locate libjvm");
char *rp = nullptr;
if (ret && dli_fname[0] != '\0') {
- rp = os::Posix::realpath(dli_fname, buf, buflen);
+ rp = os::realpath(dli_fname, buf, buflen);
}
if (rp == nullptr) {
return;
@@ -2797,7 +2794,7 @@ void os::jvm_path(char *buf, jint buflen) {
}
assert(strstr(p, "/libjvm") == p, "invalid library name");
- rp = os::Posix::realpath(java_home_var, buf, buflen);
+ rp = os::realpath(java_home_var, buf, buflen);
if (rp == nullptr) {
return;
}
@@ -2818,7 +2815,7 @@ void os::jvm_path(char *buf, jint buflen) {
snprintf(buf + len, buflen-len, "/hotspot/libjvm.so");
} else {
// Go back to path of .so
- rp = os::Posix::realpath(dli_fname, buf, buflen);
+ rp = os::realpath(dli_fname, buf, buflen);
if (rp == nullptr) {
return;
}
@@ -5053,16 +5050,6 @@ int os::open(const char *path, int oflag, int mode) {
return fd;
}
-// return current position of file pointer
-jlong os::current_file_offset(int fd) {
- return (jlong)::lseek(fd, (off_t)0, SEEK_CUR);
-}
-
-// move file pointer to the specified offset
-jlong os::seek_to_file_offset(int fd, jlong offset) {
- return (jlong)::lseek(fd, (off_t)offset, SEEK_SET);
-}
-
static jlong slow_thread_cpu_time(Thread *thread, bool user_sys_cpu_time);
static jlong fast_cpu_time(Thread *thread) {
diff --git a/src/hotspot/os/linux/os_perf_linux.cpp b/src/hotspot/os/linux/os_perf_linux.cpp
index a31a58e55af93..996f83611b048 100644
--- a/src/hotspot/os/linux/os_perf_linux.cpp
+++ b/src/hotspot/os/linux/os_perf_linux.cpp
@@ -789,7 +789,7 @@ char* SystemProcessInterface::SystemProcesses::ProcessIterator::get_exe_path() {
jio_snprintf(buffer, PATH_MAX, "/proc/%s/exe", _entry->d_name);
buffer[PATH_MAX - 1] = '\0';
- return os::Posix::realpath(buffer, _exePath, PATH_MAX);
+ return os::realpath(buffer, _exePath, PATH_MAX);
}
char* SystemProcessInterface::SystemProcesses::ProcessIterator::allocate_string(const char* str) const {
diff --git a/src/hotspot/os/linux/vmStructs_linux.hpp b/src/hotspot/os/linux/vmStructs_linux.hpp
index 818f6bb188fe8..3b82ac58ac697 100644
--- a/src/hotspot/os/linux/vmStructs_linux.hpp
+++ b/src/hotspot/os/linux/vmStructs_linux.hpp
@@ -31,9 +31,22 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
+#define VM_STRUCTS_OS(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
+ \
+ /******************************/ \
+ /* Threads (NOTE: incomplete) */ \
+ /******************************/ \
+ nonstatic_field(OSThread, _thread_id, pid_t) \
+ nonstatic_field(OSThread, _pthread_id, pthread_t)
+
+#define VM_TYPES_OS(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
+ \
+ /**********************/ \
+ /* Posix Thread IDs */ \
+ /**********************/ \
+ \
+ declare_integer_type(pid_t) \
+ declare_unsigned_integer_type(pthread_t)
#define VM_INT_CONSTANTS_OS(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
diff --git a/src/hotspot/os/posix/gc/x/xArguments_posix.cpp b/src/hotspot/os/posix/gc/x/xArguments_posix.cpp
deleted file mode 100644
index 6df0a9bd07460..0000000000000
--- a/src/hotspot/os/posix/gc/x/xArguments_posix.cpp
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/x/xArguments.hpp"
-
-bool XArguments::is_os_supported() {
- return true;
-}
diff --git a/src/hotspot/os/posix/gc/x/xInitialize_posix.cpp b/src/hotspot/os/posix/gc/x/xInitialize_posix.cpp
deleted file mode 100644
index acf71e9890178..0000000000000
--- a/src/hotspot/os/posix/gc/x/xInitialize_posix.cpp
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/x/xInitialize.hpp"
-
-void XInitialize::pd_initialize() {
- // Does nothing
-}
diff --git a/src/hotspot/os/posix/gc/x/xUtils_posix.cpp b/src/hotspot/os/posix/gc/x/xUtils_posix.cpp
deleted file mode 100644
index eee3e5cfbe60d..0000000000000
--- a/src/hotspot/os/posix/gc/x/xUtils_posix.cpp
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/x/xUtils.hpp"
-#include "utilities/debug.hpp"
-#include "utilities/globalDefinitions.hpp"
-
-#include
-
-uintptr_t XUtils::alloc_aligned(size_t alignment, size_t size) {
- void* res = nullptr;
-
- // Use raw posix_memalign as long as we have no wrapper for it
- ALLOW_C_FUNCTION(::posix_memalign, int rc = posix_memalign(&res, alignment, size);)
- if (rc != 0) {
- fatal("posix_memalign() failed");
- }
-
- memset(res, 0, size);
-
- return (uintptr_t)res;
-}
diff --git a/src/hotspot/os/posix/gc/x/xVirtualMemory_posix.cpp b/src/hotspot/os/posix/gc/x/xVirtualMemory_posix.cpp
deleted file mode 100644
index e2422eb0978fc..0000000000000
--- a/src/hotspot/os/posix/gc/x/xVirtualMemory_posix.cpp
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/x/xAddress.inline.hpp"
-#include "gc/x/xVirtualMemory.hpp"
-#include "logging/log.hpp"
-
-#include
-#include
-
-void XVirtualMemoryManager::pd_initialize_before_reserve() {
- // Does nothing
-}
-
-void XVirtualMemoryManager::pd_initialize_after_reserve() {
- // Does nothing
-}
-
-bool XVirtualMemoryManager::pd_reserve(uintptr_t addr, size_t size) {
- const uintptr_t res = (uintptr_t)mmap((void*)addr, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0);
- if (res == (uintptr_t)MAP_FAILED) {
- // Failed to reserve memory
- return false;
- }
-
- if (res != addr) {
- // Failed to reserve memory at the requested address
- munmap((void*)res, size);
- return false;
- }
-
- // Success
- return true;
-}
-
-void XVirtualMemoryManager::pd_unreserve(uintptr_t addr, size_t size) {
- const int res = munmap((void*)addr, size);
- assert(res == 0, "Failed to unmap memory");
-}
diff --git a/src/hotspot/os/posix/os_posix.cpp b/src/hotspot/os/posix/os_posix.cpp
index 60efdeb2ef59a..75253843593c6 100644
--- a/src/hotspot/os/posix/os_posix.cpp
+++ b/src/hotspot/os/posix/os_posix.cpp
@@ -104,49 +104,44 @@ static int clock_tics_per_sec = 100;
size_t os::_os_min_stack_allowed = PTHREAD_STACK_MIN;
// Check core dump limit and report possible place where core can be found
-void os::check_dump_limit(char* buffer, size_t bufferSize) {
+void os::check_core_dump_prerequisites(char* buffer, size_t bufferSize, bool check_only) {
if (!FLAG_IS_DEFAULT(CreateCoredumpOnCrash) && !CreateCoredumpOnCrash) {
jio_snprintf(buffer, bufferSize, "CreateCoredumpOnCrash is disabled from command line");
VMError::record_coredump_status(buffer, false);
- return;
- }
-
- int n;
- struct rlimit rlim;
- bool success;
-
- char core_path[PATH_MAX];
- n = get_core_path(core_path, PATH_MAX);
-
- if (n <= 0) {
- jio_snprintf(buffer, bufferSize, "core.%d (may not exist)", current_process_id());
- success = true;
+ } else {
+ struct rlimit rlim;
+ bool success = true;
+ bool warn = true;
+ char core_path[PATH_MAX];
+ if (get_core_path(core_path, PATH_MAX) <= 0) {
+ jio_snprintf(buffer, bufferSize, "core.%d (may not exist)", current_process_id());
#ifdef LINUX
- } else if (core_path[0] == '"') { // redirect to user process
- jio_snprintf(buffer, bufferSize, "Core dumps may be processed with %s", core_path);
- success = true;
+ } else if (core_path[0] == '"') { // redirect to user process
+ jio_snprintf(buffer, bufferSize, "Core dumps may be processed with %s", core_path);
#endif
- } else if (getrlimit(RLIMIT_CORE, &rlim) != 0) {
- jio_snprintf(buffer, bufferSize, "%s (may not exist)", core_path);
- success = true;
- } else {
- switch(rlim.rlim_cur) {
- case RLIM_INFINITY:
- jio_snprintf(buffer, bufferSize, "%s", core_path);
- success = true;
- break;
- case 0:
- jio_snprintf(buffer, bufferSize, "Core dumps have been disabled. To enable core dumping, try \"ulimit -c unlimited\" before starting Java again");
- success = false;
- break;
- default:
- jio_snprintf(buffer, bufferSize, "%s (max size " UINT64_FORMAT " k). To ensure a full core dump, try \"ulimit -c unlimited\" before starting Java again", core_path, uint64_t(rlim.rlim_cur) / K);
- success = true;
- break;
+ } else if (getrlimit(RLIMIT_CORE, &rlim) != 0) {
+ jio_snprintf(buffer, bufferSize, "%s (may not exist)", core_path);
+ } else {
+ switch(rlim.rlim_cur) {
+ case RLIM_INFINITY:
+ jio_snprintf(buffer, bufferSize, "%s", core_path);
+ warn = false;
+ break;
+ case 0:
+ jio_snprintf(buffer, bufferSize, "Core dumps have been disabled. To enable core dumping, try \"ulimit -c unlimited\" before starting Java again");
+ success = false;
+ break;
+ default:
+ jio_snprintf(buffer, bufferSize, "%s (max size " UINT64_FORMAT " k). To ensure a full core dump, try \"ulimit -c unlimited\" before starting Java again", core_path, uint64_t(rlim.rlim_cur) / K);
+ break;
+ }
+ }
+ if (!check_only) {
+ VMError::record_coredump_status(buffer, success);
+ } else if (warn) {
+ warning("CreateCoredumpOnCrash specified, but %s", buffer);
}
}
-
- VMError::record_coredump_status(buffer, success);
}
bool os::committed_in_range(address start, size_t size, address& committed_start, size_t& committed_size) {
@@ -348,6 +343,16 @@ int os::create_file_for_heap(const char* dir) {
return fd;
}
+// return current position of file pointer
+jlong os::current_file_offset(int fd) {
+ return (jlong)::lseek(fd, (off_t)0, SEEK_CUR);
+}
+
+// move file pointer to the specified offset
+jlong os::seek_to_file_offset(int fd, jlong offset) {
+ return (jlong)::lseek(fd, (off_t)offset, SEEK_SET);
+}
+
// Is a (classpath) directory empty?
bool os::dir_is_empty(const char* path) {
DIR *dir = nullptr;
@@ -1024,10 +1029,10 @@ char* os::Posix::describe_pthread_attr(char* buf, size_t buflen, const pthread_a
return buf;
}
-char* os::Posix::realpath(const char* filename, char* outbuf, size_t outbuflen) {
+char* os::realpath(const char* filename, char* outbuf, size_t outbuflen) {
if (filename == nullptr || outbuf == nullptr || outbuflen < 1) {
- assert(false, "os::Posix::realpath: invalid arguments.");
+ assert(false, "os::realpath: invalid arguments.");
errno = EINVAL;
return nullptr;
}
@@ -1062,7 +1067,6 @@ char* os::Posix::realpath(const char* filename, char* outbuf, size_t outbuflen)
}
}
return result;
-
}
int os::stat(const char *path, struct stat *sbuf) {
diff --git a/src/hotspot/os/posix/os_posix.hpp b/src/hotspot/os/posix/os_posix.hpp
index d872a6dae899e..cac5b250cdfa0 100644
--- a/src/hotspot/os/posix/os_posix.hpp
+++ b/src/hotspot/os/posix/os_posix.hpp
@@ -73,13 +73,6 @@ class os::Posix {
// to buf with len buflen; buf is returned.
static char* describe_pthread_attr(char* buf, size_t buflen, const pthread_attr_t* attr);
- // A safe implementation of realpath which will not cause a buffer overflow if the resolved path
- // is longer than PATH_MAX.
- // On success, returns 'outbuf', which now contains the path.
- // On error, it will return null and set errno. The content of 'outbuf' is undefined.
- // On truncation error ('outbuf' too small), it will return null and set errno to ENAMETOOLONG.
- static char* realpath(const char* filename, char* outbuf, size_t outbuflen);
-
// Returns true if given uid is root.
static bool is_root(uid_t uid);
diff --git a/src/hotspot/os/posix/perfMemory_posix.cpp b/src/hotspot/os/posix/perfMemory_posix.cpp
index 4eb46169878cd..17bf63092c208 100644
--- a/src/hotspot/os/posix/perfMemory_posix.cpp
+++ b/src/hotspot/os/posix/perfMemory_posix.cpp
@@ -1086,7 +1086,7 @@ static char* mmap_create_shared(size_t size) {
static void unmap_shared(char* addr, size_t bytes) {
int res;
if (MemTracker::enabled()) {
- ThreadCritical tc;
+ NmtVirtualMemoryLocker ml;
res = ::munmap(addr, bytes);
if (res == 0) {
MemTracker::record_virtual_memory_release((address)addr, bytes);
diff --git a/src/hotspot/os/windows/attachListener_windows.cpp b/src/hotspot/os/windows/attachListener_windows.cpp
index 3f6ca941c209b..bfa377d52cf2f 100644
--- a/src/hotspot/os/windows/attachListener_windows.cpp
+++ b/src/hotspot/os/windows/attachListener_windows.cpp
@@ -32,17 +32,24 @@
#include // SIGBREAK
#include
-// The AttachListener thread services a queue of operations. It blocks in the dequeue
-// function until an operation is enqueued. A client enqueues an operation by creating
+// The AttachListener thread services a queue of operation requests. It blocks in the dequeue
+// function until a request is enqueued. A client enqueues a request by creating
// a thread in this process using the Win32 CreateRemoteThread function. That thread
// executes a small stub generated by the client. The stub invokes the
-// JVM_EnqueueOperation function which checks the operation parameters and enqueues
-// the operation to the queue serviced by the attach listener. The thread created by
+// JVM_EnqueueOperation or JVM_EnqueueOperation_v2 function which checks the operation parameters
+// and enqueues the operation request to the queue. The thread created by
// the client is a native thread and is restricted to a single page of stack. To keep
-// it simple operations are pre-allocated at initialization time. An enqueue thus
-// takes a preallocated operation, populates the operation parameters, adds it to
+// it simple operation requests are pre-allocated at initialization time. An enqueue thus
+// takes a preallocated request, populates the operation parameters, adds it to
// queue and wakes up the attach listener.
//
+// Differences between Attach API v1 and v2:
+// In v1 (jdk6+) client calls JVM_EnqueueOperation function and passes all operation parameters
+// as arguments of the function.
+// In v2 (jdk24+) client calls JVM_EnqueueOperation_v2 function and passes only pipe name.
+// Attach listeners connects to the pipe (in read/write mode) and reads all operation parameters
+// (the same way as other platform implementations read them using sockets).
+//
// When an operation has completed the attach listener is required to send the
// operation result and any result data to the client. In this implementation the
// client is a pipe server. In the enqueue operation it provides the name of pipe
@@ -55,8 +62,154 @@
// this wasn't worth worrying about.
-// forward reference
-class Win32AttachOperation;
+class PipeChannel : public AttachOperation::RequestReader, public AttachOperation::ReplyWriter {
+private:
+ HANDLE _hPipe;
+public:
+ PipeChannel() : _hPipe(INVALID_HANDLE_VALUE) {}
+ ~PipeChannel() {
+ close();
+ }
+
+ bool opened() const {
+ return _hPipe != INVALID_HANDLE_VALUE;
+ }
+
+ bool open(const char* pipe, bool write_only) {
+ _hPipe = ::CreateFile(pipe,
+ GENERIC_WRITE | (write_only ? 0 : GENERIC_READ),
+ 0, // no sharing
+ nullptr, // default security attributes
+ OPEN_EXISTING, // opens existing pipe
+ 0, // default attributes
+ nullptr); // no template file
+ if (_hPipe == INVALID_HANDLE_VALUE) {
+ log_error(attach)("could not open (%d) pipe %s", GetLastError(), pipe);
+ return false;
+ }
+ return true;
+ }
+
+ void close() {
+ if (opened()) {
+ CloseHandle(_hPipe);
+ _hPipe = INVALID_HANDLE_VALUE;
+ }
+ }
+
+ // RequestReader
+ int read(void* buffer, int size) override {
+ assert(opened(), "must be");
+ DWORD nread;
+ BOOL fSuccess = ReadFile(_hPipe,
+ buffer,
+ (DWORD)size,
+ &nread,
+ nullptr); // not overlapped
+ return fSuccess ? (int)nread : -1;
+ }
+
+ // ReplyWriter
+ int write(const void* buffer, int size) override {
+ assert(opened(), "must be");
+ DWORD written;
+ BOOL fSuccess = WriteFile(_hPipe,
+ buffer,
+ (DWORD)size,
+ &written,
+ nullptr); // not overlapped
+ return fSuccess ? (int)written : -1;
+ }
+
+ void flush() override {
+ assert(opened(), "must be");
+ FlushFileBuffers(_hPipe);
+ }
+};
+
+class Win32AttachOperation: public AttachOperation {
+public:
+ enum {
+ pipe_name_max = 256 // maximum pipe name
+ };
+
+private:
+ PipeChannel _pipe;
+
+public:
+ // for v1 pipe must be write-only
+ void open_pipe(const char* pipe_name, bool write_only) {
+ _pipe.open(pipe_name, write_only);
+ }
+
+ bool read_request() {
+ return AttachOperation::read_request(&_pipe);
+ }
+
+public:
+ void complete(jint result, bufferedStream* result_stream) override;
+};
+
+
+// Win32AttachOperationRequest is an element of AttachOperation request list.
+class Win32AttachOperationRequest {
+private:
+ AttachAPIVersion _ver;
+ char _name[AttachOperation::name_length_max + 1];
+ char _arg[AttachOperation::arg_count_max][AttachOperation::arg_length_max + 1];
+ char _pipe[Win32AttachOperation::pipe_name_max + 1];
+
+ Win32AttachOperationRequest* _next;
+
+ void set_value(char* dst, const char* str, size_t dst_size) {
+ if (str != nullptr) {
+ assert(strlen(str) < dst_size, "exceeds maximum length");
+ strncpy(dst, str, dst_size - 1);
+ dst[dst_size - 1] = '\0';
+ } else {
+ strcpy(dst, "");
+ }
+ }
+
+public:
+ void set(AttachAPIVersion ver, const char* pipename,
+ const char* cmd = nullptr,
+ const char* arg0 = nullptr,
+ const char* arg1 = nullptr,
+ const char* arg2 = nullptr) {
+ _ver = ver;
+ set_value(_name, cmd, sizeof(_name));
+ set_value(_arg[0], arg0, sizeof(_arg[0]));
+ set_value(_arg[1], arg1, sizeof(_arg[1]));
+ set_value(_arg[2], arg2, sizeof(_arg[2]));
+ set_value(_pipe, pipename, sizeof(_pipe));
+ }
+ AttachAPIVersion ver() const {
+ return _ver;
+ }
+ const char* cmd() const {
+ return _name;
+ }
+ const char* arg(int i) const {
+ return (i >= 0 && i < AttachOperation::arg_count_max) ? _arg[i] : nullptr;
+ }
+ const char* pipe() const {
+ return _pipe;
+ }
+
+ Win32AttachOperationRequest* next() const {
+ return _next;
+ }
+ void set_next(Win32AttachOperationRequest* next) {
+ _next = next;
+ }
+
+ // noarg constructor as operation is preallocated
+ Win32AttachOperationRequest() {
+ set(ATTACH_API_V1, "");
+ set_next(nullptr);
+ }
+};
class Win32AttachListener: AllStatic {
@@ -69,18 +222,18 @@ class Win32AttachListener: AllStatic {
static HANDLE _mutex;
// head of preallocated operations list
- static Win32AttachOperation* _avail;
+ static Win32AttachOperationRequest* _avail;
// head and tail of enqueue operations list
- static Win32AttachOperation* _head;
- static Win32AttachOperation* _tail;
+ static Win32AttachOperationRequest* _head;
+ static Win32AttachOperationRequest* _tail;
- static Win32AttachOperation* head() { return _head; }
- static void set_head(Win32AttachOperation* head) { _head = head; }
+ static Win32AttachOperationRequest* head() { return _head; }
+ static void set_head(Win32AttachOperationRequest* head) { _head = head; }
- static Win32AttachOperation* tail() { return _tail; }
- static void set_tail(Win32AttachOperation* tail) { _tail = tail; }
+ static Win32AttachOperationRequest* tail() { return _tail; }
+ static void set_tail(Win32AttachOperationRequest* tail) { _tail = tail; }
// A semaphore is used for communication about enqueued operations.
@@ -101,11 +254,12 @@ class Win32AttachListener: AllStatic {
static int init();
static HANDLE mutex() { return _mutex; }
- static Win32AttachOperation* available() { return _avail; }
- static void set_available(Win32AttachOperation* avail) { _avail = avail; }
+ static Win32AttachOperationRequest* available() { return _avail; }
+ static void set_available(Win32AttachOperationRequest* avail) { _avail = avail; }
// enqueue an operation to the end of the list
- static int enqueue(char* cmd, char* arg1, char* arg2, char* arg3, char* pipename);
+ static int enqueue(AttachAPIVersion ver, const char* cmd,
+ const char* arg1, const char* arg2, const char* arg3, const char* pipename);
// dequeue an operation from from head of the list
static Win32AttachOperation* dequeue();
@@ -114,48 +268,9 @@ class Win32AttachListener: AllStatic {
// statics
HANDLE Win32AttachListener::_mutex;
HANDLE Win32AttachListener::_enqueued_ops_semaphore;
-Win32AttachOperation* Win32AttachListener::_avail;
-Win32AttachOperation* Win32AttachListener::_head;
-Win32AttachOperation* Win32AttachListener::_tail;
-
-
-// Win32AttachOperation is an AttachOperation that additionally encapsulates the name
-// of a pipe which is used to send the operation reply/output to the client.
-// Win32AttachOperation can also be linked in a list.
-
-class Win32AttachOperation: public AttachOperation {
- private:
- friend class Win32AttachListener;
-
- enum {
- pipe_name_max = 256 // maximum pipe name
- };
-
- char _pipe[pipe_name_max + 1];
-
- const char* pipe() const { return _pipe; }
- void set_pipe(const char* pipe) {
- assert(strlen(pipe) <= pipe_name_max, "exceeds maximum length of pipe name");
- os::snprintf(_pipe, sizeof(_pipe), "%s", pipe);
- }
-
- HANDLE open_pipe();
- static BOOL write_pipe(HANDLE hPipe, char* buf, int len);
-
- Win32AttachOperation* _next;
-
- Win32AttachOperation* next() const { return _next; }
- void set_next(Win32AttachOperation* next) { _next = next; }
-
- // noarg constructor as operation is preallocated
- Win32AttachOperation() : AttachOperation("") {
- set_pipe("");
- set_next(nullptr);
- }
-
- public:
- void complete(jint result, bufferedStream* result_stream);
-};
+Win32AttachOperationRequest* Win32AttachListener::_avail;
+Win32AttachOperationRequest* Win32AttachListener::_head;
+Win32AttachOperationRequest* Win32AttachListener::_tail;
// Preallocate the maximum number of operations that can be enqueued.
@@ -171,18 +286,24 @@ int Win32AttachListener::init() {
set_available(nullptr);
for (int i=0; iset_next(available());
set_available(op);
}
+ AttachListener::set_supported_version(ATTACH_API_V2);
+
return 0;
}
// Enqueue an operation. This is called from a native thread that is not attached to VM.
// Also we need to be careful not to execute anything that results in more than a 4k stack.
//
-int Win32AttachListener::enqueue(char* cmd, char* arg0, char* arg1, char* arg2, char* pipename) {
+int Win32AttachListener::enqueue(AttachAPIVersion ver, const char* cmd,
+ const char* arg0, const char* arg1, const char* arg2, const char* pipename) {
+
+ log_debug(attach)("AttachListener::enqueue, ver = %d, cmd = %s", (int)ver, cmd);
+
// wait up to 10 seconds for listener to be up and running
int sleep_count = 0;
while (!AttachListener::is_initialized()) {
@@ -210,7 +331,7 @@ int Win32AttachListener::enqueue(char* cmd, char* arg0, char* arg1, char* arg2,
}
// try to get an operation from the available list
- Win32AttachOperation* op = available();
+ Win32AttachOperationRequest* op = available();
if (op != nullptr) {
set_available(op->next());
@@ -223,11 +344,7 @@ int Win32AttachListener::enqueue(char* cmd, char* arg0, char* arg1, char* arg2,
}
set_tail(op);
- op->set_name(cmd);
- op->set_arg(0, arg0);
- op->set_arg(1, arg1);
- op->set_arg(2, arg2);
- op->set_pipe(pipename);
+ op->set(ver, pipename, cmd, arg0, arg1, arg2);
// Increment number of enqueued operations.
// Side effect: Semaphore will be signaled and will release
@@ -236,6 +353,7 @@ int Win32AttachListener::enqueue(char* cmd, char* arg0, char* arg1, char* arg2,
::ReleaseSemaphore(enqueued_ops_semaphore(), 1, nullptr);
guarantee(not_exceeding_semaphore_maximum_count, "invariant");
}
+
::ReleaseMutex(mutex());
return (op != nullptr) ? 0 : ATTACH_ERROR_RESOURCE;
@@ -255,107 +373,63 @@ Win32AttachOperation* Win32AttachListener::dequeue() {
guarantee(res != WAIT_FAILED, "WaitForSingleObject failed with error code: %lu", GetLastError());
guarantee(res == WAIT_OBJECT_0, "WaitForSingleObject failed with return value: %lu", res);
+ Win32AttachOperation* op = nullptr;
+ Win32AttachOperationRequest* request = head();
+ if (request != nullptr) {
+ log_debug(attach)("AttachListener::dequeue, got request, ver = %d, cmd = %s", request->ver(), request->cmd());
- Win32AttachOperation* op = head();
- if (op != nullptr) {
- set_head(op->next());
+ set_head(request->next());
if (head() == nullptr) { // list is empty
set_tail(nullptr);
}
+
+ switch (request->ver()) {
+ case ATTACH_API_V1:
+ op = new Win32AttachOperation();
+ op->set_name(request->cmd());
+ for (int i = 0; i < AttachOperation::arg_count_max; i++) {
+ op->append_arg(request->arg(i));
+ }
+ op->open_pipe(request->pipe(), true/*write-only*/);
+ break;
+ case ATTACH_API_V2:
+ op = new Win32AttachOperation();
+ op->open_pipe(request->pipe(), false/*write-only*/);
+ if (!op->read_request()) {
+ log_error(attach)("AttachListener::dequeue, reading request ERROR");
+ delete op;
+ op = nullptr;
+ }
+ break;
+ default:
+ log_error(attach)("AttachListener::dequeue, unsupported version: %d", request->ver(), request->cmd());
+ break;
+ }
}
+ // put the operation back on the available list
+ request->set_next(Win32AttachListener::available());
+ Win32AttachListener::set_available(request);
+
::ReleaseMutex(mutex());
if (op != nullptr) {
+ log_debug(attach)("AttachListener::dequeue, return op: %s", op->name());
return op;
}
}
}
-
-// open the pipe to the client
-HANDLE Win32AttachOperation::open_pipe() {
- HANDLE hPipe = ::CreateFile( pipe(), // pipe name
- GENERIC_WRITE, // write only
- 0, // no sharing
- nullptr, // default security attributes
- OPEN_EXISTING, // opens existing pipe
- 0, // default attributes
- nullptr); // no template file
- return hPipe;
-}
-
-// write to the pipe
-BOOL Win32AttachOperation::write_pipe(HANDLE hPipe, char* buf, int len) {
- do {
- DWORD nwrote;
-
- BOOL fSuccess = WriteFile( hPipe, // pipe handle
- (LPCVOID)buf, // message
- (DWORD)len, // message length
- &nwrote, // bytes written
- nullptr); // not overlapped
- if (!fSuccess) {
- return fSuccess;
- }
- buf += nwrote;
- len -= nwrote;
- } while (len > 0);
- return TRUE;
-}
-
-// Complete the operation:
-// - open the pipe to the client
-// - write the operation result (a jint)
-// - write the operation output (the result stream)
-//
void Win32AttachOperation::complete(jint result, bufferedStream* result_stream) {
JavaThread* thread = JavaThread::current();
ThreadBlockInVM tbivm(thread);
- HANDLE hPipe = open_pipe();
- int lastError = (int)::GetLastError();
- if (hPipe != INVALID_HANDLE_VALUE) {
- BOOL fSuccess;
-
- char msg[32];
- os::snprintf(msg, sizeof(msg), "%d\n", result);
- msg[sizeof(msg) - 1] = '\0';
-
- fSuccess = write_pipe(hPipe, msg, (int)strlen(msg));
- if (fSuccess) {
- fSuccess = write_pipe(hPipe, (char*)result_stream->base(), (int)(result_stream->size()));
- }
- lastError = (int)::GetLastError();
-
- // Need to flush buffers
- FlushFileBuffers(hPipe);
- CloseHandle(hPipe);
+ write_reply(&_pipe, result, result_stream);
- if (fSuccess) {
- log_debug(attach)("wrote result of attach operation %s to pipe %s", name(), pipe());
- } else {
- log_error(attach)("failure (%d) writing result of operation %s to pipe %s", lastError, name(), pipe());
- }
- } else {
- log_error(attach)("could not open (%d) pipe %s to send result of operation %s", lastError, pipe(), name());
- }
-
- DWORD res = ::WaitForSingleObject(Win32AttachListener::mutex(), INFINITE);
- assert(res != WAIT_FAILED, "WaitForSingleObject failed with error code: %lu", GetLastError());
- assert(res == WAIT_OBJECT_0, "WaitForSingleObject failed with return value: %lu", res);
-
- if (res == WAIT_OBJECT_0) {
-
- // put the operation back on the available list
- set_next(Win32AttachListener::available());
- Win32AttachListener::set_available(this);
-
- ::ReleaseMutex(Win32AttachListener::mutex());
- }
+ delete this;
}
-// AttachOperation functions
+// AttachListener functions
AttachOperation* AttachListener::dequeue() {
JavaThread* thread = JavaThread::current();
@@ -404,8 +478,12 @@ void AttachListener::pd_detachall() {
// Native thread started by remote client executes this.
extern "C" {
JNIEXPORT jint JNICALL
- JVM_EnqueueOperation(char* cmd, char* arg0, char* arg1, char* arg2, char* pipename) {
- return (jint)Win32AttachListener::enqueue(cmd, arg0, arg1, arg2, pipename);
- }
+ JVM_EnqueueOperation(char* cmd, char* arg0, char* arg1, char* arg2, char* pipename) {
+ return (jint)Win32AttachListener::enqueue(ATTACH_API_V1, cmd, arg0, arg1, arg2, pipename);
+ }
+ JNIEXPORT jint JNICALL
+ JVM_EnqueueOperation_v2(char* pipename) {
+ return (jint)Win32AttachListener::enqueue(ATTACH_API_V2, "", "", "", "", pipename);
+ }
} // extern
diff --git a/src/hotspot/os/windows/gc/x/xArguments_windows.cpp b/src/hotspot/os/windows/gc/x/xArguments_windows.cpp
deleted file mode 100644
index fc5f7eccb911f..0000000000000
--- a/src/hotspot/os/windows/gc/x/xArguments_windows.cpp
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/x/xArguments.hpp"
-#include "gc/x/xSyscall_windows.hpp"
-
-bool XArguments::is_os_supported() {
- return XSyscall::is_supported();
-}
diff --git a/src/hotspot/os/windows/gc/x/xInitialize_windows.cpp b/src/hotspot/os/windows/gc/x/xInitialize_windows.cpp
deleted file mode 100644
index 99f6432803326..0000000000000
--- a/src/hotspot/os/windows/gc/x/xInitialize_windows.cpp
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/x/xInitialize.hpp"
-#include "gc/x/xSyscall_windows.hpp"
-
-void XInitialize::pd_initialize() {
- XSyscall::initialize();
-}
diff --git a/src/hotspot/os/windows/gc/x/xMapper_windows.cpp b/src/hotspot/os/windows/gc/x/xMapper_windows.cpp
deleted file mode 100644
index e69b6ec56e293..0000000000000
--- a/src/hotspot/os/windows/gc/x/xMapper_windows.cpp
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/x/xMapper_windows.hpp"
-#include "gc/x/xSyscall_windows.hpp"
-#include "logging/log.hpp"
-#include "utilities/debug.hpp"
-
-#include
-
-// Memory reservation, commit, views, and placeholders.
-//
-// To be able to up-front reserve address space for the heap views, and later
-// multi-map the heap views to the same physical memory, without ever losing the
-// reservation of the reserved address space, we use "placeholders".
-//
-// These placeholders block out the address space from being used by other parts
-// of the process. To commit memory in this address space, the placeholder must
-// be replaced by anonymous memory, or replaced by mapping a view against a
-// paging file mapping. We use the later to support multi-mapping.
-//
-// We want to be able to dynamically commit and uncommit the physical memory of
-// the heap (and also unmap ZPages), in granules of ZGranuleSize bytes. There is
-// no way to grow and shrink the committed memory of a paging file mapping.
-// Therefore, we create multiple granule-sized page file mappings. The memory is
-// committed by creating a page file mapping, map a view against it, commit the
-// memory, unmap the view. The memory will stay committed until all views are
-// unmapped, and the paging file mapping handle is closed.
-//
-// When replacing a placeholder address space reservation with a mapped view
-// against a paging file mapping, the virtual address space must exactly match
-// an existing placeholder's address and size. Therefore we only deal with
-// granule-sized placeholders at this layer. Higher layers that keep track of
-// reserved available address space can (and will) coalesce placeholders, but
-// they will be split before being used.
-
-#define fatal_error(msg, addr, size) \
- fatal(msg ": " PTR_FORMAT " " SIZE_FORMAT "M (%d)", \
- (addr), (size) / M, GetLastError())
-
-uintptr_t XMapper::reserve(uintptr_t addr, size_t size) {
- void* const res = XSyscall::VirtualAlloc2(
- GetCurrentProcess(), // Process
- (void*)addr, // BaseAddress
- size, // Size
- MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, // AllocationType
- PAGE_NOACCESS, // PageProtection
- nullptr, // ExtendedParameters
- 0 // ParameterCount
- );
-
- // Caller responsible for error handling
- return (uintptr_t)res;
-}
-
-void XMapper::unreserve(uintptr_t addr, size_t size) {
- const bool res = XSyscall::VirtualFreeEx(
- GetCurrentProcess(), // hProcess
- (void*)addr, // lpAddress
- size, // dwSize
- MEM_RELEASE // dwFreeType
- );
-
- if (!res) {
- fatal_error("Failed to unreserve memory", addr, size);
- }
-}
-
-HANDLE XMapper::create_paging_file_mapping(size_t size) {
- // Create mapping with SEC_RESERVE instead of SEC_COMMIT.
- //
- // We use MapViewOfFile3 for two different reasons:
- // 1) When committing memory for the created paging file
- // 2) When mapping a view of the memory created in (2)
- //
- // The non-platform code is only setup to deal with out-of-memory
- // errors in (1). By using SEC_RESERVE, we prevent MapViewOfFile3
- // from failing because of "commit limit" checks. To actually commit
- // memory in (1), a call to VirtualAlloc2 is done.
-
- HANDLE const res = XSyscall::CreateFileMappingW(
- INVALID_HANDLE_VALUE, // hFile
- nullptr, // lpFileMappingAttribute
- PAGE_READWRITE | SEC_RESERVE, // flProtect
- size >> 32, // dwMaximumSizeHigh
- size & 0xFFFFFFFF, // dwMaximumSizeLow
- nullptr // lpName
- );
-
- // Caller responsible for error handling
- return res;
-}
-
-bool XMapper::commit_paging_file_mapping(HANDLE file_handle, uintptr_t file_offset, size_t size) {
- const uintptr_t addr = map_view_no_placeholder(file_handle, file_offset, size);
- if (addr == 0) {
- log_error(gc)("Failed to map view of paging file mapping (%d)", GetLastError());
- return false;
- }
-
- const uintptr_t res = commit(addr, size);
- if (res != addr) {
- log_error(gc)("Failed to commit memory (%d)", GetLastError());
- }
-
- unmap_view_no_placeholder(addr, size);
-
- return res == addr;
-}
-
-uintptr_t XMapper::map_view_no_placeholder(HANDLE file_handle, uintptr_t file_offset, size_t size) {
- void* const res = XSyscall::MapViewOfFile3(
- file_handle, // FileMapping
- GetCurrentProcess(), // ProcessHandle
- nullptr, // BaseAddress
- file_offset, // Offset
- size, // ViewSize
- 0, // AllocationType
- PAGE_NOACCESS, // PageProtection
- nullptr, // ExtendedParameters
- 0 // ParameterCount
- );
-
- // Caller responsible for error handling
- return (uintptr_t)res;
-}
-
-void XMapper::unmap_view_no_placeholder(uintptr_t addr, size_t size) {
- const bool res = XSyscall::UnmapViewOfFile2(
- GetCurrentProcess(), // ProcessHandle
- (void*)addr, // BaseAddress
- 0 // UnmapFlags
- );
-
- if (!res) {
- fatal_error("Failed to unmap memory", addr, size);
- }
-}
-
-uintptr_t XMapper::commit(uintptr_t addr, size_t size) {
- void* const res = XSyscall::VirtualAlloc2(
- GetCurrentProcess(), // Process
- (void*)addr, // BaseAddress
- size, // Size
- MEM_COMMIT, // AllocationType
- PAGE_NOACCESS, // PageProtection
- nullptr, // ExtendedParameters
- 0 // ParameterCount
- );
-
- // Caller responsible for error handling
- return (uintptr_t)res;
-}
-
-HANDLE XMapper::create_and_commit_paging_file_mapping(size_t size) {
- HANDLE const file_handle = create_paging_file_mapping(size);
- if (file_handle == 0) {
- log_error(gc)("Failed to create paging file mapping (%d)", GetLastError());
- return 0;
- }
-
- const bool res = commit_paging_file_mapping(file_handle, 0 /* file_offset */, size);
- if (!res) {
- close_paging_file_mapping(file_handle);
- return 0;
- }
-
- return file_handle;
-}
-
-void XMapper::close_paging_file_mapping(HANDLE file_handle) {
- const bool res = CloseHandle(
- file_handle // hObject
- );
-
- if (!res) {
- fatal("Failed to close paging file handle (%d)", GetLastError());
- }
-}
-
-HANDLE XMapper::create_shared_awe_section() {
- MEM_EXTENDED_PARAMETER parameter = { 0 };
- parameter.Type = MemSectionExtendedParameterUserPhysicalFlags;
- parameter.ULong64 = 0;
-
- HANDLE section = XSyscall::CreateFileMapping2(
- INVALID_HANDLE_VALUE, // File
- nullptr, // SecurityAttributes
- SECTION_MAP_READ | SECTION_MAP_WRITE, // DesiredAccess
- PAGE_READWRITE, // PageProtection
- SEC_RESERVE | SEC_LARGE_PAGES, // AllocationAttributes
- 0, // MaximumSize
- nullptr, // Name
- ¶meter, // ExtendedParameters
- 1 // ParameterCount
- );
-
- if (section == nullptr) {
- fatal("Could not create shared AWE section (%d)", GetLastError());
- }
-
- return section;
-}
-
-uintptr_t XMapper::reserve_for_shared_awe(HANDLE awe_section, uintptr_t addr, size_t size) {
- MEM_EXTENDED_PARAMETER parameter = { 0 };
- parameter.Type = MemExtendedParameterUserPhysicalHandle;
- parameter.Handle = awe_section;
-
- void* const res = XSyscall::VirtualAlloc2(
- GetCurrentProcess(), // Process
- (void*)addr, // BaseAddress
- size, // Size
- MEM_RESERVE | MEM_PHYSICAL, // AllocationType
- PAGE_READWRITE, // PageProtection
- ¶meter, // ExtendedParameters
- 1 // ParameterCount
- );
-
- // Caller responsible for error handling
- return (uintptr_t)res;
-}
-
-void XMapper::unreserve_for_shared_awe(uintptr_t addr, size_t size) {
- bool res = VirtualFree(
- (void*)addr, // lpAddress
- 0, // dwSize
- MEM_RELEASE // dwFreeType
- );
-
- if (!res) {
- fatal("Failed to unreserve memory: " PTR_FORMAT " " SIZE_FORMAT "M (%d)",
- addr, size / M, GetLastError());
- }
-}
-
-void XMapper::split_placeholder(uintptr_t addr, size_t size) {
- const bool res = VirtualFree(
- (void*)addr, // lpAddress
- size, // dwSize
- MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER // dwFreeType
- );
-
- if (!res) {
- fatal_error("Failed to split placeholder", addr, size);
- }
-}
-
-void XMapper::coalesce_placeholders(uintptr_t addr, size_t size) {
- const bool res = VirtualFree(
- (void*)addr, // lpAddress
- size, // dwSize
- MEM_RELEASE | MEM_COALESCE_PLACEHOLDERS // dwFreeType
- );
-
- if (!res) {
- fatal_error("Failed to coalesce placeholders", addr, size);
- }
-}
-
-void XMapper::map_view_replace_placeholder(HANDLE file_handle, uintptr_t file_offset, uintptr_t addr, size_t size) {
- void* const res = XSyscall::MapViewOfFile3(
- file_handle, // FileMapping
- GetCurrentProcess(), // ProcessHandle
- (void*)addr, // BaseAddress
- file_offset, // Offset
- size, // ViewSize
- MEM_REPLACE_PLACEHOLDER, // AllocationType
- PAGE_READWRITE, // PageProtection
- nullptr, // ExtendedParameters
- 0 // ParameterCount
- );
-
- if (res == nullptr) {
- fatal_error("Failed to map memory", addr, size);
- }
-}
-
-void XMapper::unmap_view_preserve_placeholder(uintptr_t addr, size_t size) {
- const bool res = XSyscall::UnmapViewOfFile2(
- GetCurrentProcess(), // ProcessHandle
- (void*)addr, // BaseAddress
- MEM_PRESERVE_PLACEHOLDER // UnmapFlags
- );
-
- if (!res) {
- fatal_error("Failed to unmap memory", addr, size);
- }
-}
diff --git a/src/hotspot/os/windows/gc/x/xMapper_windows.hpp b/src/hotspot/os/windows/gc/x/xMapper_windows.hpp
deleted file mode 100644
index 0f266d3fab7c6..0000000000000
--- a/src/hotspot/os/windows/gc/x/xMapper_windows.hpp
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#ifndef OS_WINDOWS_GC_X_XMAPPER_WINDOWS_HPP
-#define OS_WINDOWS_GC_X_XMAPPER_WINDOWS_HPP
-
-#include "memory/allStatic.hpp"
-#include "utilities/globalDefinitions.hpp"
-
-#include
-
-class XMapper : public AllStatic {
-private:
- // Create paging file mapping
- static HANDLE create_paging_file_mapping(size_t size);
-
- // Commit paging file mapping
- static bool commit_paging_file_mapping(HANDLE file_handle, uintptr_t file_offset, size_t size);
-
- // Map a view anywhere without a placeholder
- static uintptr_t map_view_no_placeholder(HANDLE file_handle, uintptr_t file_offset, size_t size);
-
- // Unmap a view without preserving a placeholder
- static void unmap_view_no_placeholder(uintptr_t addr, size_t size);
-
- // Commit memory covering the given virtual address range
- static uintptr_t commit(uintptr_t addr, size_t size);
-
-public:
- // Reserve memory with a placeholder
- static uintptr_t reserve(uintptr_t addr, size_t size);
-
- // Unreserve memory
- static void unreserve(uintptr_t addr, size_t size);
-
- // Create and commit paging file mapping
- static HANDLE create_and_commit_paging_file_mapping(size_t size);
-
- // Close paging file mapping
- static void close_paging_file_mapping(HANDLE file_handle);
-
- // Create a shared AWE section
- static HANDLE create_shared_awe_section();
-
- // Reserve memory attached to the shared AWE section
- static uintptr_t reserve_for_shared_awe(HANDLE awe_section, uintptr_t addr, size_t size);
-
- // Unreserve memory attached to a shared AWE section
- static void unreserve_for_shared_awe(uintptr_t addr, size_t size);
-
- // Split a placeholder
- //
- // A view can only replace an entire placeholder, so placeholders need to be
- // split and coalesced to be the exact size of the new views.
- // [addr, addr + size) needs to be a proper sub-placeholder of an existing
- // placeholder.
- static void split_placeholder(uintptr_t addr, size_t size);
-
- // Coalesce a placeholder
- //
- // [addr, addr + size) is the new placeholder. A sub-placeholder needs to
- // exist within that range.
- static void coalesce_placeholders(uintptr_t addr, size_t size);
-
- // Map a view of the file handle and replace the placeholder covering the
- // given virtual address range
- static void map_view_replace_placeholder(HANDLE file_handle, uintptr_t file_offset, uintptr_t addr, size_t size);
-
- // Unmap the view and reinstate a placeholder covering the given virtual
- // address range
- static void unmap_view_preserve_placeholder(uintptr_t addr, size_t size);
-};
-
-#endif // OS_WINDOWS_GC_X_XMAPPER_WINDOWS_HPP
diff --git a/src/hotspot/os/windows/gc/x/xNUMA_windows.cpp b/src/hotspot/os/windows/gc/x/xNUMA_windows.cpp
deleted file mode 100644
index 47a84df962e92..0000000000000
--- a/src/hotspot/os/windows/gc/x/xNUMA_windows.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/x/xNUMA.hpp"
-
-void XNUMA::pd_initialize() {
- _enabled = false;
-}
-
-uint32_t XNUMA::count() {
- return 1;
-}
-
-uint32_t XNUMA::id() {
- return 0;
-}
-
-uint32_t XNUMA::memory_id(uintptr_t addr) {
- // NUMA support not enabled, assume everything belongs to node zero
- return 0;
-}
diff --git a/src/hotspot/os/windows/gc/x/xPhysicalMemoryBacking_windows.cpp b/src/hotspot/os/windows/gc/x/xPhysicalMemoryBacking_windows.cpp
deleted file mode 100644
index 92d47dfb7c84c..0000000000000
--- a/src/hotspot/os/windows/gc/x/xPhysicalMemoryBacking_windows.cpp
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/x/xGlobals.hpp"
-#include "gc/x/xGranuleMap.inline.hpp"
-#include "gc/x/xLargePages.inline.hpp"
-#include "gc/x/xMapper_windows.hpp"
-#include "gc/x/xPhysicalMemoryBacking_windows.hpp"
-#include "logging/log.hpp"
-#include "runtime/globals.hpp"
-#include "utilities/debug.hpp"
-
-class XPhysicalMemoryBackingImpl : public CHeapObj {
-public:
- virtual size_t commit(size_t offset, size_t size) = 0;
- virtual size_t uncommit(size_t offset, size_t size) = 0;
- virtual void map(uintptr_t addr, size_t size, size_t offset) const = 0;
- virtual void unmap(uintptr_t addr, size_t size) const = 0;
-};
-
-// Implements small pages (paged) support using placeholder reservation.
-//
-// The backing commits and uncommits physical memory, that can be
-// multi-mapped into the virtual address space. To support fine-graned
-// committing and uncommitting, each XGranuleSize'd chunk is mapped to
-// a separate paging file mapping.
-
-class XPhysicalMemoryBackingSmallPages : public XPhysicalMemoryBackingImpl {
-private:
- XGranuleMap _handles;
-
- HANDLE get_handle(uintptr_t offset) const {
- HANDLE const handle = _handles.get(offset);
- assert(handle != 0, "Should be set");
- return handle;
- }
-
- void put_handle(uintptr_t offset, HANDLE handle) {
- assert(handle != INVALID_HANDLE_VALUE, "Invalid handle");
- assert(_handles.get(offset) == 0, "Should be cleared");
- _handles.put(offset, handle);
- }
-
- void clear_handle(uintptr_t offset) {
- assert(_handles.get(offset) != 0, "Should be set");
- _handles.put(offset, 0);
- }
-
-public:
- XPhysicalMemoryBackingSmallPages(size_t max_capacity) :
- XPhysicalMemoryBackingImpl(),
- _handles(max_capacity) {}
-
- size_t commit(size_t offset, size_t size) {
- for (size_t i = 0; i < size; i += XGranuleSize) {
- HANDLE const handle = XMapper::create_and_commit_paging_file_mapping(XGranuleSize);
- if (handle == 0) {
- return i;
- }
-
- put_handle(offset + i, handle);
- }
-
- return size;
- }
-
- size_t uncommit(size_t offset, size_t size) {
- for (size_t i = 0; i < size; i += XGranuleSize) {
- HANDLE const handle = get_handle(offset + i);
- clear_handle(offset + i);
- XMapper::close_paging_file_mapping(handle);
- }
-
- return size;
- }
-
- void map(uintptr_t addr, size_t size, size_t offset) const {
- assert(is_aligned(offset, XGranuleSize), "Misaligned");
- assert(is_aligned(addr, XGranuleSize), "Misaligned");
- assert(is_aligned(size, XGranuleSize), "Misaligned");
-
- for (size_t i = 0; i < size; i += XGranuleSize) {
- HANDLE const handle = get_handle(offset + i);
- XMapper::map_view_replace_placeholder(handle, 0 /* offset */, addr + i, XGranuleSize);
- }
- }
-
- void unmap(uintptr_t addr, size_t size) const {
- assert(is_aligned(addr, XGranuleSize), "Misaligned");
- assert(is_aligned(size, XGranuleSize), "Misaligned");
-
- for (size_t i = 0; i < size; i += XGranuleSize) {
- XMapper::unmap_view_preserve_placeholder(addr + i, XGranuleSize);
- }
- }
-};
-
-// Implements Large Pages (locked) support using shared AWE physical memory.
-//
-// Shared AWE physical memory also works with small pages, but it has
-// a few drawbacks that makes it a no-go to use it at this point:
-//
-// 1) It seems to use 8 bytes of committed memory per *reserved* memory.
-// Given our scheme to use a large address space range this turns out to
-// use too much memory.
-//
-// 2) It requires memory locking privileges, even for small pages. This
-// has always been a requirement for large pages, and would be an extra
-// restriction for usage with small pages.
-//
-// Note: The large pages size is tied to our XGranuleSize.
-
-extern HANDLE XAWESection;
-
-class XPhysicalMemoryBackingLargePages : public XPhysicalMemoryBackingImpl {
-private:
- ULONG_PTR* const _page_array;
-
- static ULONG_PTR* alloc_page_array(size_t max_capacity) {
- const size_t npages = max_capacity / XGranuleSize;
- const size_t array_size = npages * sizeof(ULONG_PTR);
-
- return (ULONG_PTR*)os::malloc(array_size, mtGC);
- }
-
-public:
- XPhysicalMemoryBackingLargePages(size_t max_capacity) :
- XPhysicalMemoryBackingImpl(),
- _page_array(alloc_page_array(max_capacity)) {}
-
- size_t commit(size_t offset, size_t size) {
- const size_t index = offset >> XGranuleSizeShift;
- const size_t npages = size >> XGranuleSizeShift;
-
- size_t npages_res = npages;
- const bool res = AllocateUserPhysicalPages(XAWESection, &npages_res, &_page_array[index]);
- if (!res) {
- fatal("Failed to allocate physical memory " SIZE_FORMAT "M @ " PTR_FORMAT " (%d)",
- size / M, offset, GetLastError());
- } else {
- log_debug(gc)("Allocated physical memory: " SIZE_FORMAT "M @ " PTR_FORMAT, size / M, offset);
- }
-
- // AllocateUserPhysicalPages might not be able to allocate the requested amount of memory.
- // The allocated number of pages are written in npages_res.
- return npages_res << XGranuleSizeShift;
- }
-
- size_t uncommit(size_t offset, size_t size) {
- const size_t index = offset >> XGranuleSizeShift;
- const size_t npages = size >> XGranuleSizeShift;
-
- size_t npages_res = npages;
- const bool res = FreeUserPhysicalPages(XAWESection, &npages_res, &_page_array[index]);
- if (!res) {
- fatal("Failed to uncommit physical memory " SIZE_FORMAT "M @ " PTR_FORMAT " (%d)",
- size, offset, GetLastError());
- }
-
- return npages_res << XGranuleSizeShift;
- }
-
- void map(uintptr_t addr, size_t size, size_t offset) const {
- const size_t npages = size >> XGranuleSizeShift;
- const size_t index = offset >> XGranuleSizeShift;
-
- const bool res = MapUserPhysicalPages((char*)addr, npages, &_page_array[index]);
- if (!res) {
- fatal("Failed to map view " PTR_FORMAT " " SIZE_FORMAT "M @ " PTR_FORMAT " (%d)",
- addr, size / M, offset, GetLastError());
- }
- }
-
- void unmap(uintptr_t addr, size_t size) const {
- const size_t npages = size >> XGranuleSizeShift;
-
- const bool res = MapUserPhysicalPages((char*)addr, npages, nullptr);
- if (!res) {
- fatal("Failed to unmap view " PTR_FORMAT " " SIZE_FORMAT "M (%d)",
- addr, size / M, GetLastError());
- }
- }
-};
-
-static XPhysicalMemoryBackingImpl* select_impl(size_t max_capacity) {
- if (XLargePages::is_enabled()) {
- return new XPhysicalMemoryBackingLargePages(max_capacity);
- }
-
- return new XPhysicalMemoryBackingSmallPages(max_capacity);
-}
-
-XPhysicalMemoryBacking::XPhysicalMemoryBacking(size_t max_capacity) :
- _impl(select_impl(max_capacity)) {}
-
-bool XPhysicalMemoryBacking::is_initialized() const {
- return true;
-}
-
-void XPhysicalMemoryBacking::warn_commit_limits(size_t max_capacity) const {
- // Does nothing
-}
-
-size_t XPhysicalMemoryBacking::commit(size_t offset, size_t length) {
- log_trace(gc, heap)("Committing memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
- offset / M, (offset + length) / M, length / M);
-
- return _impl->commit(offset, length);
-}
-
-size_t XPhysicalMemoryBacking::uncommit(size_t offset, size_t length) {
- log_trace(gc, heap)("Uncommitting memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
- offset / M, (offset + length) / M, length / M);
-
- return _impl->uncommit(offset, length);
-}
-
-void XPhysicalMemoryBacking::map(uintptr_t addr, size_t size, size_t offset) const {
- assert(is_aligned(offset, XGranuleSize), "Misaligned: " PTR_FORMAT, offset);
- assert(is_aligned(addr, XGranuleSize), "Misaligned: " PTR_FORMAT, addr);
- assert(is_aligned(size, XGranuleSize), "Misaligned: " PTR_FORMAT, size);
-
- _impl->map(addr, size, offset);
-}
-
-void XPhysicalMemoryBacking::unmap(uintptr_t addr, size_t size) const {
- assert(is_aligned(addr, XGranuleSize), "Misaligned");
- assert(is_aligned(size, XGranuleSize), "Misaligned");
-
- _impl->unmap(addr, size);
-}
diff --git a/src/hotspot/os/windows/gc/x/xPhysicalMemoryBacking_windows.hpp b/src/hotspot/os/windows/gc/x/xPhysicalMemoryBacking_windows.hpp
deleted file mode 100644
index d6e123f21e51a..0000000000000
--- a/src/hotspot/os/windows/gc/x/xPhysicalMemoryBacking_windows.hpp
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#ifndef OS_WINDOWS_GC_X_XPHYSICALMEMORYBACKING_WINDOWS_HPP
-#define OS_WINDOWS_GC_X_XPHYSICALMEMORYBACKING_WINDOWS_HPP
-
-#include "utilities/globalDefinitions.hpp"
-
-#include
-
-class XPhysicalMemoryBackingImpl;
-
-class XPhysicalMemoryBacking {
-private:
- XPhysicalMemoryBackingImpl* _impl;
-
-public:
- XPhysicalMemoryBacking(size_t max_capacity);
-
- bool is_initialized() const;
-
- void warn_commit_limits(size_t max_capacity) const;
-
- size_t commit(size_t offset, size_t length);
- size_t uncommit(size_t offset, size_t length);
-
- void map(uintptr_t addr, size_t size, size_t offset) const;
- void unmap(uintptr_t addr, size_t size) const;
-};
-
-#endif // OS_WINDOWS_GC_X_XPHYSICALMEMORYBACKING_WINDOWS_HPP
diff --git a/src/hotspot/os/windows/gc/x/xSyscall_windows.cpp b/src/hotspot/os/windows/gc/x/xSyscall_windows.cpp
deleted file mode 100644
index f22966a54898d..0000000000000
--- a/src/hotspot/os/windows/gc/x/xSyscall_windows.cpp
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/shared/gcLogPrecious.hpp"
-#include "gc/x/xSyscall_windows.hpp"
-#include "runtime/java.hpp"
-#include "runtime/os.hpp"
-
-XSyscall::CreateFileMappingWFn XSyscall::CreateFileMappingW;
-XSyscall::CreateFileMapping2Fn XSyscall::CreateFileMapping2;
-XSyscall::VirtualAlloc2Fn XSyscall::VirtualAlloc2;
-XSyscall::VirtualFreeExFn XSyscall::VirtualFreeEx;
-XSyscall::MapViewOfFile3Fn XSyscall::MapViewOfFile3;
-XSyscall::UnmapViewOfFile2Fn XSyscall::UnmapViewOfFile2;
-
-static void* lookup_kernelbase_library() {
- const char* const name = "KernelBase";
- char ebuf[1024];
- void* const handle = os::dll_load(name, ebuf, sizeof(ebuf));
- if (handle == nullptr) {
- log_error_p(gc)("Failed to load library: %s", name);
- }
- return handle;
-}
-
-static void* lookup_kernelbase_symbol(const char* name) {
- static void* const handle = lookup_kernelbase_library();
- if (handle == nullptr) {
- return nullptr;
- }
- return os::dll_lookup(handle, name);
-}
-
-static bool has_kernelbase_symbol(const char* name) {
- return lookup_kernelbase_symbol(name) != nullptr;
-}
-
-template
-static void install_kernelbase_symbol(Fn*& fn, const char* name) {
- fn = reinterpret_cast(lookup_kernelbase_symbol(name));
-}
-
-template
-static void install_kernelbase_1803_symbol_or_exit(Fn*& fn, const char* name) {
- install_kernelbase_symbol(fn, name);
- if (fn == nullptr) {
- log_error_p(gc)("Failed to lookup symbol: %s", name);
- vm_exit_during_initialization("ZGC requires Windows version 1803 or later");
- }
-}
-
-void XSyscall::initialize() {
- // Required
- install_kernelbase_1803_symbol_or_exit(CreateFileMappingW, "CreateFileMappingW");
- install_kernelbase_1803_symbol_or_exit(VirtualAlloc2, "VirtualAlloc2");
- install_kernelbase_1803_symbol_or_exit(VirtualFreeEx, "VirtualFreeEx");
- install_kernelbase_1803_symbol_or_exit(MapViewOfFile3, "MapViewOfFile3");
- install_kernelbase_1803_symbol_or_exit(UnmapViewOfFile2, "UnmapViewOfFile2");
-
- // Optional - for large pages support
- install_kernelbase_symbol(CreateFileMapping2, "CreateFileMapping2");
-}
-
-bool XSyscall::is_supported() {
- // Available in Windows version 1803 and later
- return has_kernelbase_symbol("VirtualAlloc2");
-}
-
-bool XSyscall::is_large_pages_supported() {
- // Available in Windows version 1809 and later
- return has_kernelbase_symbol("CreateFileMapping2");
-}
diff --git a/src/hotspot/os/windows/gc/x/xSyscall_windows.hpp b/src/hotspot/os/windows/gc/x/xSyscall_windows.hpp
deleted file mode 100644
index 89ba2573b10cc..0000000000000
--- a/src/hotspot/os/windows/gc/x/xSyscall_windows.hpp
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#ifndef OS_WINDOWS_GC_X_XSYSCALL_WINDOWS_HPP
-#define OS_WINDOWS_GC_X_XSYSCALL_WINDOWS_HPP
-
-#include "utilities/globalDefinitions.hpp"
-
-#include
-#include
-
-class XSyscall {
-private:
- typedef HANDLE (*CreateFileMappingWFn)(HANDLE, LPSECURITY_ATTRIBUTES, DWORD, DWORD, DWORD, LPCWSTR);
- typedef HANDLE (*CreateFileMapping2Fn)(HANDLE, LPSECURITY_ATTRIBUTES, ULONG, ULONG, ULONG, ULONG64, PCWSTR, PMEM_EXTENDED_PARAMETER, ULONG);
- typedef PVOID (*VirtualAlloc2Fn)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER*, ULONG);
- typedef BOOL (*VirtualFreeExFn)(HANDLE, LPVOID, SIZE_T, DWORD);
- typedef PVOID (*MapViewOfFile3Fn)(HANDLE, HANDLE, PVOID, ULONG64, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER*, ULONG);
- typedef BOOL (*UnmapViewOfFile2Fn)(HANDLE, PVOID, ULONG);
-
-public:
- static CreateFileMappingWFn CreateFileMappingW;
- static CreateFileMapping2Fn CreateFileMapping2;
- static VirtualAlloc2Fn VirtualAlloc2;
- static VirtualFreeExFn VirtualFreeEx;
- static MapViewOfFile3Fn MapViewOfFile3;
- static UnmapViewOfFile2Fn UnmapViewOfFile2;
-
- static void initialize();
-
- static bool is_supported();
- static bool is_large_pages_supported();
-};
-
-#endif // OS_WINDOWS_GC_X_XSYSCALL_WINDOWS_HPP
diff --git a/src/hotspot/os/windows/gc/x/xUtils_windows.cpp b/src/hotspot/os/windows/gc/x/xUtils_windows.cpp
deleted file mode 100644
index 788da80834ab3..0000000000000
--- a/src/hotspot/os/windows/gc/x/xUtils_windows.cpp
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/x/xUtils.hpp"
-#include "utilities/debug.hpp"
-
-#include
-
-uintptr_t XUtils::alloc_aligned(size_t alignment, size_t size) {
- void* const res = _aligned_malloc(size, alignment);
-
- if (res == nullptr) {
- fatal("_aligned_malloc failed");
- }
-
- memset(res, 0, size);
-
- return (uintptr_t)res;
-}
diff --git a/src/hotspot/os/windows/gc/x/xVirtualMemory_windows.cpp b/src/hotspot/os/windows/gc/x/xVirtualMemory_windows.cpp
deleted file mode 100644
index a54f1e3cbaefc..0000000000000
--- a/src/hotspot/os/windows/gc/x/xVirtualMemory_windows.cpp
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#include "precompiled.hpp"
-#include "gc/x/xAddress.inline.hpp"
-#include "gc/x/xGlobals.hpp"
-#include "gc/x/xLargePages.inline.hpp"
-#include "gc/x/xMapper_windows.hpp"
-#include "gc/x/xSyscall_windows.hpp"
-#include "gc/x/xVirtualMemory.inline.hpp"
-#include "utilities/align.hpp"
-#include "utilities/debug.hpp"
-
-class XVirtualMemoryManagerImpl : public CHeapObj {
-public:
- virtual void initialize_before_reserve() {}
- virtual void initialize_after_reserve(XMemoryManager* manager) {}
- virtual bool reserve(uintptr_t addr, size_t size) = 0;
- virtual void unreserve(uintptr_t addr, size_t size) = 0;
-};
-
-// Implements small pages (paged) support using placeholder reservation.
-class XVirtualMemoryManagerSmallPages : public XVirtualMemoryManagerImpl {
-private:
- class PlaceholderCallbacks : public AllStatic {
- public:
- static void split_placeholder(uintptr_t start, size_t size) {
- XMapper::split_placeholder(XAddress::marked0(start), size);
- XMapper::split_placeholder(XAddress::marked1(start), size);
- XMapper::split_placeholder(XAddress::remapped(start), size);
- }
-
- static void coalesce_placeholders(uintptr_t start, size_t size) {
- XMapper::coalesce_placeholders(XAddress::marked0(start), size);
- XMapper::coalesce_placeholders(XAddress::marked1(start), size);
- XMapper::coalesce_placeholders(XAddress::remapped(start), size);
- }
-
- static void split_into_placeholder_granules(uintptr_t start, size_t size) {
- for (uintptr_t addr = start; addr < start + size; addr += XGranuleSize) {
- split_placeholder(addr, XGranuleSize);
- }
- }
-
- static void coalesce_into_one_placeholder(uintptr_t start, size_t size) {
- assert(is_aligned(size, XGranuleSize), "Must be granule aligned");
-
- if (size > XGranuleSize) {
- coalesce_placeholders(start, size);
- }
- }
-
- static void create_callback(const XMemory* area) {
- assert(is_aligned(area->size(), XGranuleSize), "Must be granule aligned");
- coalesce_into_one_placeholder(area->start(), area->size());
- }
-
- static void destroy_callback(const XMemory* area) {
- assert(is_aligned(area->size(), XGranuleSize), "Must be granule aligned");
- // Don't try split the last granule - VirtualFree will fail
- split_into_placeholder_granules(area->start(), area->size() - XGranuleSize);
- }
-
- static void shrink_from_front_callback(const XMemory* area, size_t size) {
- assert(is_aligned(size, XGranuleSize), "Must be granule aligned");
- split_into_placeholder_granules(area->start(), size);
- }
-
- static void shrink_from_back_callback(const XMemory* area, size_t size) {
- assert(is_aligned(size, XGranuleSize), "Must be granule aligned");
- // Don't try split the last granule - VirtualFree will fail
- split_into_placeholder_granules(area->end() - size, size - XGranuleSize);
- }
-
- static void grow_from_front_callback(const XMemory* area, size_t size) {
- assert(is_aligned(area->size(), XGranuleSize), "Must be granule aligned");
- coalesce_into_one_placeholder(area->start() - size, area->size() + size);
- }
-
- static void grow_from_back_callback(const XMemory* area, size_t size) {
- assert(is_aligned(area->size(), XGranuleSize), "Must be granule aligned");
- coalesce_into_one_placeholder(area->start(), area->size() + size);
- }
-
- static void register_with(XMemoryManager* manager) {
- // Each reserved virtual memory address area registered in _manager is
- // exactly covered by a single placeholder. Callbacks are installed so
- // that whenever a memory area changes, the corresponding placeholder
- // is adjusted.
- //
- // The create and grow callbacks are called when virtual memory is
- // returned to the memory manager. The new memory area is then covered
- // by a new single placeholder.
- //
- // The destroy and shrink callbacks are called when virtual memory is
- // allocated from the memory manager. The memory area is then is split
- // into granule-sized placeholders.
- //
- // See comment in zMapper_windows.cpp explaining why placeholders are
- // split into XGranuleSize sized placeholders.
-
- XMemoryManager::Callbacks callbacks;
-
- callbacks._create = &create_callback;
- callbacks._destroy = &destroy_callback;
- callbacks._shrink_from_front = &shrink_from_front_callback;
- callbacks._shrink_from_back = &shrink_from_back_callback;
- callbacks._grow_from_front = &grow_from_front_callback;
- callbacks._grow_from_back = &grow_from_back_callback;
-
- manager->register_callbacks(callbacks);
- }
- };
-
- virtual void initialize_after_reserve(XMemoryManager* manager) {
- PlaceholderCallbacks::register_with(manager);
- }
-
- virtual bool reserve(uintptr_t addr, size_t size) {
- const uintptr_t res = XMapper::reserve(addr, size);
-
- assert(res == addr || res == 0, "Should not reserve other memory than requested");
- return res == addr;
- }
-
- virtual void unreserve(uintptr_t addr, size_t size) {
- XMapper::unreserve(addr, size);
- }
-};
-
-// Implements Large Pages (locked) support using shared AWE physical memory.
-
-// XPhysicalMemory layer needs access to the section
-HANDLE XAWESection;
-
-class XVirtualMemoryManagerLargePages : public XVirtualMemoryManagerImpl {
-private:
- virtual void initialize_before_reserve() {
- XAWESection = XMapper::create_shared_awe_section();
- }
-
- virtual bool reserve(uintptr_t addr, size_t size) {
- const uintptr_t res = XMapper::reserve_for_shared_awe(XAWESection, addr, size);
-
- assert(res == addr || res == 0, "Should not reserve other memory than requested");
- return res == addr;
- }
-
- virtual void unreserve(uintptr_t addr, size_t size) {
- XMapper::unreserve_for_shared_awe(addr, size);
- }
-};
-
-static XVirtualMemoryManagerImpl* _impl = nullptr;
-
-void XVirtualMemoryManager::pd_initialize_before_reserve() {
- if (XLargePages::is_enabled()) {
- _impl = new XVirtualMemoryManagerLargePages();
- } else {
- _impl = new XVirtualMemoryManagerSmallPages();
- }
- _impl->initialize_before_reserve();
-}
-
-void XVirtualMemoryManager::pd_initialize_after_reserve() {
- _impl->initialize_after_reserve(&_manager);
-}
-
-bool XVirtualMemoryManager::pd_reserve(uintptr_t addr, size_t size) {
- return _impl->reserve(addr, size);
-}
-
-void XVirtualMemoryManager::pd_unreserve(uintptr_t addr, size_t size) {
- _impl->unreserve(addr, size);
-}
diff --git a/src/hotspot/os/windows/globals_windows.hpp b/src/hotspot/os/windows/globals_windows.hpp
index 78cbac6e9ccc5..8f0a6261cc0db 100644
--- a/src/hotspot/os/windows/globals_windows.hpp
+++ b/src/hotspot/os/windows/globals_windows.hpp
@@ -38,6 +38,10 @@
product(bool, UseAllWindowsProcessorGroups, false, \
"Use all processor groups on supported Windows versions") \
\
+product(bool, EnableAllLargePageSizesForWindows, false, \
+ "Enable support for multiple large page sizes on " \
+ "Windows Server") \
+ \
product(bool, UseOSErrorReporting, false, \
"Let VM fatal error propagate to the OS (ie. WER on Windows)")
diff --git a/src/hotspot/os/windows/osThread_windows.cpp b/src/hotspot/os/windows/osThread_windows.cpp
index 5f369bb7aa05b..922b4b0104be4 100644
--- a/src/hotspot/os/windows/osThread_windows.cpp
+++ b/src/hotspot/os/windows/osThread_windows.cpp
@@ -22,17 +22,17 @@
*
*/
-// no precompiled headers
-#include "runtime/os.hpp"
+#include "precompiled.hpp"
#include "runtime/osThread.hpp"
-void OSThread::pd_initialize() {
- set_thread_handle(nullptr);
- set_thread_id(0);
- set_interrupt_event(nullptr);
-}
+#include
+
+OSThread::OSThread()
+ : _thread_id(0),
+ _thread_handle(nullptr),
+ _interrupt_event(nullptr) {}
-void OSThread::pd_destroy() {
+OSThread::~OSThread() {
if (_interrupt_event != nullptr) {
CloseHandle(_interrupt_event);
}
diff --git a/src/hotspot/os/windows/osThread_windows.hpp b/src/hotspot/os/windows/osThread_windows.hpp
index 5bd07646b1718..e54783aef1c15 100644
--- a/src/hotspot/os/windows/osThread_windows.hpp
+++ b/src/hotspot/os/windows/osThread_windows.hpp
@@ -25,17 +25,29 @@
#ifndef OS_WINDOWS_OSTHREAD_WINDOWS_HPP
#define OS_WINDOWS_OSTHREAD_WINDOWS_HPP
- typedef void* HANDLE;
- public:
+#include "runtime/osThreadBase.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class OSThread : public OSThreadBase {
+ friend class VMStructs;
+
typedef unsigned long thread_id_t;
+ typedef void* HANDLE;
+
+ thread_id_t _thread_id;
- private:
// Win32-specific thread information
HANDLE _thread_handle; // Win32 thread handle
HANDLE _interrupt_event; // Event signalled on thread interrupt for use by
// Process.waitFor().
public:
+ OSThread();
+ ~OSThread();
+
+ thread_id_t thread_id() const { return _thread_id; }
+ void set_thread_id(thread_id_t id) { _thread_id = id; }
+
// The following will only apply in the Win32 implementation, and should only
// be visible in the concrete class, not this which should be an abstract base class
HANDLE thread_handle() const { return _thread_handle; }
@@ -45,13 +57,9 @@
// This is specialized on Windows to interact with the _interrupt_event.
void set_interrupted(bool z);
-#ifndef PRODUCT
- // Used for debugging, return a unique integer for each thread.
- int thread_identifier() const { return _thread_id; }
-#endif
-
- private:
- void pd_initialize();
- void pd_destroy();
+ uintx thread_id_for_printing() const override {
+ return (uintx)_thread_id;
+ }
+};
#endif // OS_WINDOWS_OSTHREAD_WINDOWS_HPP
diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp
index 817757f1ac6a2..71efb57e0f2c5 100644
--- a/src/hotspot/os/windows/os_windows.cpp
+++ b/src/hotspot/os/windows/os_windows.cpp
@@ -63,6 +63,7 @@
#include "runtime/sharedRuntime.hpp"
#include "runtime/statSampler.hpp"
#include "runtime/stubRoutines.hpp"
+#include "runtime/suspendedThreadTask.hpp"
#include "runtime/threadCritical.hpp"
#include "runtime/threads.hpp"
#include "runtime/timer.hpp"
@@ -1286,38 +1287,50 @@ void os::shutdown() {
static HANDLE dumpFile = nullptr;
-// Check if dump file can be created.
-void os::check_dump_limit(char* buffer, size_t buffsz) {
- bool status = true;
+// Check if core dump is active and if a core dump file can be created
+void os::check_core_dump_prerequisites(char* buffer, size_t bufferSize, bool check_only) {
if (!FLAG_IS_DEFAULT(CreateCoredumpOnCrash) && !CreateCoredumpOnCrash) {
- jio_snprintf(buffer, buffsz, "CreateCoredumpOnCrash is disabled from command line");
- status = false;
- }
-
+ jio_snprintf(buffer, bufferSize, "CreateCoredumpOnCrash is disabled from command line");
+ VMError::record_coredump_status(buffer, false);
+ } else {
+ bool success = true;
+ bool warn = true;
#ifndef ASSERT
- if (!os::win32::is_windows_server() && FLAG_IS_DEFAULT(CreateCoredumpOnCrash)) {
- jio_snprintf(buffer, buffsz, "Minidumps are not enabled by default on client versions of Windows");
- status = false;
- }
+ if (!os::win32::is_windows_server() && FLAG_IS_DEFAULT(CreateCoredumpOnCrash)) {
+ jio_snprintf(buffer, bufferSize, "Minidumps are not enabled by default on client versions of Windows");
+ success = false;
+ warn = true;
+ }
#endif
- if (status) {
- const char* cwd = get_current_directory(nullptr, 0);
- int pid = current_process_id();
- if (cwd != nullptr) {
- jio_snprintf(buffer, buffsz, "%s\\hs_err_pid%u.mdmp", cwd, pid);
- } else {
- jio_snprintf(buffer, buffsz, ".\\hs_err_pid%u.mdmp", pid);
+ if (success) {
+ if (!check_only) {
+ const char* cwd = get_current_directory(nullptr, 0);
+ int pid = current_process_id();
+ if (cwd != nullptr) {
+ jio_snprintf(buffer, bufferSize, "%s\\hs_err_pid%u.mdmp", cwd, pid);
+ } else {
+ jio_snprintf(buffer, bufferSize, ".\\hs_err_pid%u.mdmp", pid);
+ }
+
+ if (dumpFile == nullptr &&
+ (dumpFile = CreateFile(buffer, GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr))
+ == INVALID_HANDLE_VALUE) {
+ jio_snprintf(buffer, bufferSize, "Failed to create minidump file (0x%x).", GetLastError());
+ success = false;
+ }
+ } else {
+ // For now on Windows, there are no more checks that we can do
+ warn = false;
+ }
}
- if (dumpFile == nullptr &&
- (dumpFile = CreateFile(buffer, GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr))
- == INVALID_HANDLE_VALUE) {
- jio_snprintf(buffer, buffsz, "Failed to create minidump file (0x%x).", GetLastError());
- status = false;
+ if (!check_only) {
+ VMError::record_coredump_status(buffer, success);
+ } else if (warn) {
+ warning("CreateCoredumpOnCrash specified, but %s", buffer);
}
}
- VMError::record_coredump_status(buffer, status);
}
void os::abort(bool dump_core, void* siginfo, const void* context) {
@@ -3126,7 +3139,7 @@ class NUMANodeListHolder {
static size_t _large_page_size = 0;
-static bool request_lock_memory_privilege() {
+bool os::win32::request_lock_memory_privilege() {
HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE,
os::current_process_id());
@@ -3310,14 +3323,14 @@ static char* allocate_pages_individually(size_t bytes, char* addr, DWORD flags,
return p_buf;
}
-static size_t large_page_init_decide_size() {
+size_t os::win32::large_page_init_decide_size() {
// print a warning if any large page related flag is specified on command line
bool warn_on_failure = !FLAG_IS_DEFAULT(UseLargePages) ||
!FLAG_IS_DEFAULT(LargePageSizeInBytes);
-#define WARN(msg) if (warn_on_failure) { warning(msg); }
+#define WARN(...) if (warn_on_failure) { warning(__VA_ARGS__); }
- if (!request_lock_memory_privilege()) {
+ if (!os::win32::request_lock_memory_privilege()) {
WARN("JVM cannot use large page memory because it does not have enough privilege to lock pages in memory.");
return 0;
}
@@ -3328,15 +3341,26 @@ static size_t large_page_init_decide_size() {
return 0;
}
-#if defined(IA32) || defined(AMD64)
- if (size > 4*M || LargePageSizeInBytes > 4*M) {
+#if defined(IA32)
+ if (size > 4 * M || LargePageSizeInBytes > 4 * M) {
WARN("JVM cannot use large pages bigger than 4mb.");
return 0;
}
+#elif defined(AMD64)
+ if (!EnableAllLargePageSizesForWindows) {
+ if (size > 4 * M || LargePageSizeInBytes > 4 * M) {
+ WARN("JVM cannot use large pages bigger than 4mb.");
+ return 0;
+ }
+ }
#endif
- if (LargePageSizeInBytes > 0 && LargePageSizeInBytes % size == 0) {
- size = LargePageSizeInBytes;
+ if (LargePageSizeInBytes > 0) {
+ if (LargePageSizeInBytes % size == 0) {
+ size = LargePageSizeInBytes;
+ } else {
+ WARN("The specified large page size (%d) is not a multiple of the minimum large page size (%d), defaulting to minimum page size.", LargePageSizeInBytes, size);
+ }
}
#undef WARN
@@ -3349,12 +3373,23 @@ void os::large_page_init() {
return;
}
- _large_page_size = large_page_init_decide_size();
+ _large_page_size = os::win32::large_page_init_decide_size();
const size_t default_page_size = os::vm_page_size();
if (_large_page_size > default_page_size) {
+#if !defined(IA32)
+ if (EnableAllLargePageSizesForWindows) {
+ size_t min_size = GetLargePageMinimum();
+
+ // Populate _page_sizes with large page sizes less than or equal to _large_page_size, ensuring each page size is double the size of the previous one.
+ for (size_t page_size = min_size; page_size < _large_page_size; page_size *= 2) {
+ _page_sizes.add(page_size);
+ }
+ }
+#endif
+
_page_sizes.add(_large_page_size);
}
-
+ // Set UseLargePages based on whether a large page size was successfully determined
UseLargePages = _large_page_size != 0;
}
@@ -3618,7 +3653,6 @@ static char* reserve_large_pages_aligned(size_t size, size_t alignment, bool exe
char* os::pd_reserve_memory_special(size_t bytes, size_t alignment, size_t page_size, char* addr,
bool exec) {
assert(UseLargePages, "only for large pages");
- assert(page_size == os::large_page_size(), "Currently only support one large page size on Windows");
assert(is_aligned(addr, alignment), "Must be");
assert(is_aligned(addr, page_size), "Must be");
@@ -3627,11 +3661,17 @@ char* os::pd_reserve_memory_special(size_t bytes, size_t alignment, size_t page_
return nullptr;
}
+ // Ensure GetLargePageMinimum() returns a valid positive value
+ size_t large_page_min = GetLargePageMinimum();
+ if (large_page_min <= 0) {
+ return nullptr;
+ }
+
// The requested alignment can be larger than the page size, for example with G1
// the alignment is bound to the heap region size. So this reservation needs to
// ensure that the requested alignment is met. When there is a requested address
// this solves it self, since it must be properly aligned already.
- if (addr == nullptr && alignment > page_size) {
+ if (addr == nullptr && alignment > large_page_min) {
return reserve_large_pages_aligned(bytes, alignment, exec);
}
@@ -3785,7 +3825,8 @@ bool os::pd_release_memory(char* addr, size_t bytes) {
if (err != nullptr) {
log_warning(os)("bad release: [" PTR_FORMAT "-" PTR_FORMAT "): %s", p2i(start), p2i(end), err);
#ifdef ASSERT
- os::print_memory_mappings((char*)start, bytes, tty);
+ fileStream fs(stdout);
+ os::print_memory_mappings((char*)start, bytes, &fs);
assert(false, "bad release: [" PTR_FORMAT "-" PTR_FORMAT "): %s", p2i(start), p2i(end), err);
#endif
return false;
@@ -5357,6 +5398,28 @@ void os::funlockfile(FILE* fp) {
_unlock_file(fp);
}
+char* os::realpath(const char* filename, char* outbuf, size_t outbuflen) {
+
+ if (filename == nullptr || outbuf == nullptr || outbuflen < 1) {
+ assert(false, "os::realpath: invalid arguments.");
+ errno = EINVAL;
+ return nullptr;
+ }
+
+ char* result = nullptr;
+ ALLOW_C_FUNCTION(::_fullpath, char* p = ::_fullpath(nullptr, filename, 0);)
+ if (p != nullptr) {
+ if (strlen(p) < outbuflen) {
+ strcpy(outbuf, p);
+ result = outbuf;
+ } else {
+ errno = ENAMETOOLONG;
+ }
+ ALLOW_C_FUNCTION(::free, ::free(p);) // *not* os::free
+ }
+ return result;
+}
+
// Map a block of memory.
char* os::pd_map_memory(int fd, const char* file_name, size_t file_offset,
char *addr, size_t bytes, bool read_only,
@@ -5691,7 +5754,15 @@ void PlatformEvent::park() {
// TODO: consider a brief spin here, gated on the success of recent
// spin attempts by this thread.
while (_Event < 0) {
+ // The following code is only here to maintain the
+ // characteristics/performance from when an ObjectMonitor
+ // "responsible" thread used to issue timed parks.
+ HighResolutionInterval *phri = nullptr;
+ if (!ForceTimeHighResolution) {
+ phri = new HighResolutionInterval((jlong)1);
+ }
DWORD rv = ::WaitForSingleObject(_ParkHandle, INFINITE);
+ delete phri; // if it is null, harmless
assert(rv != WAIT_FAILED, "WaitForSingleObject failed with error code: %lu", GetLastError());
assert(rv == WAIT_OBJECT_0, "WaitForSingleObject failed with return value: %lu", rv);
}
@@ -5965,7 +6036,7 @@ static void do_resume(HANDLE* h) {
// retrieve a suspend/resume context capable handle
// from the tid. Caller validates handle return value.
void get_thread_handle_for_extended_context(HANDLE* h,
- OSThread::thread_id_t tid) {
+ DWORD tid) {
if (h != nullptr) {
*h = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT | THREAD_QUERY_INFORMATION, FALSE, tid);
}
diff --git a/src/hotspot/os/windows/os_windows.hpp b/src/hotspot/os/windows/os_windows.hpp
index 3bc5ab9eef1f3..1d5237243000b 100644
--- a/src/hotspot/os/windows/os_windows.hpp
+++ b/src/hotspot/os/windows/os_windows.hpp
@@ -65,6 +65,8 @@ class os::win32 {
static void setmode_streams();
static bool is_windows_11_or_greater();
static bool is_windows_server_2022_or_greater();
+ static bool request_lock_memory_privilege();
+ static size_t large_page_init_decide_size();
static int windows_major_version() {
assert(_major_version > 0, "windows version not initialized.");
return _major_version;
diff --git a/src/hotspot/os/windows/perfMemory_windows.cpp b/src/hotspot/os/windows/perfMemory_windows.cpp
index 06b057315cbdd..959be982fabbe 100644
--- a/src/hotspot/os/windows/perfMemory_windows.cpp
+++ b/src/hotspot/os/windows/perfMemory_windows.cpp
@@ -1803,7 +1803,7 @@ void PerfMemory::detach(char* addr, size_t bytes) {
if (MemTracker::enabled()) {
// it does not go through os api, the operation has to record from here
- ThreadCritical tc;
+ NmtVirtualMemoryLocker ml;
remove_file_mapping(addr);
MemTracker::record_virtual_memory_release((address)addr, bytes);
} else {
diff --git a/src/hotspot/os/windows/vmStructs_windows.hpp b/src/hotspot/os/windows/vmStructs_windows.hpp
index 2550e685f16e2..93f4ea7c8111d 100644
--- a/src/hotspot/os/windows/vmStructs_windows.hpp
+++ b/src/hotspot/os/windows/vmStructs_windows.hpp
@@ -29,9 +29,18 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
+#define VM_STRUCTS_OS(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
+ \
+ /******************************/ \
+ /* Threads (NOTE: incomplete) */ \
+ /******************************/ \
+ \
+ nonstatic_field(OSThread, _thread_id, OSThread::thread_id_t) \
+ unchecked_nonstatic_field(OSThread, _thread_handle, sizeof(HANDLE)) /* NOTE: no type */
+
+#define VM_TYPES_OS(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
+ \
+ declare_unsigned_integer_type(OSThread::thread_id_t)
#define VM_INT_CONSTANTS_OS(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
diff --git a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp
index b45d38650c076..f83aa6030622a 100644
--- a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp
+++ b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp
@@ -462,12 +462,6 @@ void os::print_tos_pc(outputStream *st, const void *context) {
address pc = os::Posix::ucontext_get_pc(uc);
print_instructions(st, pc);
st->cr();
-
- // Try to decode the instructions.
- st->print_cr("Decoded instructions: (pc=" PTR_FORMAT ")", p2i(pc));
- st->print("");
- // TODO: PPC port Disassembler::decode(pc, 16, 16, st);
- st->cr();
}
void os::print_register_info(outputStream *st, const void *context, int& continuation) {
diff --git a/src/hotspot/os_cpu/aix_ppc/vmStructs_aix_ppc.hpp b/src/hotspot/os_cpu/aix_ppc/vmStructs_aix_ppc.hpp
index 157d57f8e0fa2..123cd67248f86 100644
--- a/src/hotspot/os_cpu/aix_ppc/vmStructs_aix_ppc.hpp
+++ b/src/hotspot/os_cpu/aix_ppc/vmStructs_aix_ppc.hpp
@@ -30,21 +30,9 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
- \
- /******************************/ \
- /* Threads (NOTE: incomplete) */ \
- /******************************/ \
- nonstatic_field(OSThread, _thread_id, pthread_t) \
+#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
- \
- /**********************/ \
- /* Posix Thread IDs */ \
- /**********************/ \
- \
- declare_unsigned_integer_type(pthread_t)
+#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
diff --git a/src/hotspot/os_cpu/bsd_aarch64/vmStructs_bsd_aarch64.hpp b/src/hotspot/os_cpu/bsd_aarch64/vmStructs_bsd_aarch64.hpp
index 07b878106cfcd..c384afac7ecff 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/vmStructs_bsd_aarch64.hpp
+++ b/src/hotspot/os_cpu/bsd_aarch64/vmStructs_bsd_aarch64.hpp
@@ -31,22 +31,9 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
- \
- /******************************/ \
- /* Threads (NOTE: incomplete) */ \
- /******************************/ \
- nonstatic_field(OSThread, _thread_id, OSThread::thread_id_t) \
- nonstatic_field(OSThread, _unique_thread_id, uint64_t)
+#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
- \
- /**********************/ \
- /* Thread IDs */ \
- /**********************/ \
- \
- declare_unsigned_integer_type(OSThread::thread_id_t)
+#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
diff --git a/src/hotspot/os_cpu/bsd_x86/vmStructs_bsd_x86.hpp b/src/hotspot/os_cpu/bsd_x86/vmStructs_bsd_x86.hpp
index fb43541fa775a..b48ea82712ecd 100644
--- a/src/hotspot/os_cpu/bsd_x86/vmStructs_bsd_x86.hpp
+++ b/src/hotspot/os_cpu/bsd_x86/vmStructs_bsd_x86.hpp
@@ -29,22 +29,9 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
- \
- /******************************/ \
- /* Threads (NOTE: incomplete) */ \
- /******************************/ \
- nonstatic_field(OSThread, _thread_id, OSThread::thread_id_t) \
- nonstatic_field(OSThread, _unique_thread_id, uint64_t)
+#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
- \
- /**********************/ \
- /* Thread IDs */ \
- /**********************/ \
- \
- declare_unsigned_integer_type(OSThread::thread_id_t)
+#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
diff --git a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S
index 2724e78862de3..b3e7f64397970 100644
--- a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S
+++ b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S
@@ -26,6 +26,7 @@
.align 5
DECLARE_FUNC(aarch64_atomic_fetch_add_8_default_impl):
+ hint #0x22 // bti c
#ifdef __ARM_FEATURE_ATOMICS
ldaddal x1, x2, [x0]
#else
@@ -41,6 +42,7 @@ DECLARE_FUNC(aarch64_atomic_fetch_add_8_default_impl):
.align 5
DECLARE_FUNC(aarch64_atomic_fetch_add_4_default_impl):
+ hint #0x22 // bti c
#ifdef __ARM_FEATURE_ATOMICS
ldaddal w1, w2, [x0]
#else
@@ -56,8 +58,9 @@ DECLARE_FUNC(aarch64_atomic_fetch_add_4_default_impl):
.align 5
DECLARE_FUNC(aarch64_atomic_fetch_add_8_relaxed_default_impl):
+ hint #0x22 // bti c
#ifdef __ARM_FEATURE_ATOMICS
- ldadd x1, x2, [x0]
+ ldadd x1, x2, [x0]
#else
prfm pstl1strm, [x0]
0: ldxr x2, [x0]
@@ -70,8 +73,9 @@ DECLARE_FUNC(aarch64_atomic_fetch_add_8_relaxed_default_impl):
.align 5
DECLARE_FUNC(aarch64_atomic_fetch_add_4_relaxed_default_impl):
+ hint #0x22 // bti c
#ifdef __ARM_FEATURE_ATOMICS
- ldadd w1, w2, [x0]
+ ldadd w1, w2, [x0]
#else
prfm pstl1strm, [x0]
0: ldxr w2, [x0]
@@ -84,6 +88,7 @@ DECLARE_FUNC(aarch64_atomic_fetch_add_4_relaxed_default_impl):
.align 5
DECLARE_FUNC(aarch64_atomic_xchg_4_default_impl):
+ hint #0x22 // bti c
#ifdef __ARM_FEATURE_ATOMICS
swpal w1, w2, [x0]
#else
@@ -98,6 +103,7 @@ DECLARE_FUNC(aarch64_atomic_xchg_4_default_impl):
.align 5
DECLARE_FUNC(aarch64_atomic_xchg_8_default_impl):
+ hint #0x22 // bti c
#ifdef __ARM_FEATURE_ATOMICS
swpal x1, x2, [x0]
#else
@@ -112,6 +118,7 @@ DECLARE_FUNC(aarch64_atomic_xchg_8_default_impl):
.align 5
DECLARE_FUNC(aarch64_atomic_cmpxchg_1_default_impl):
+ hint #0x22 // bti c
#ifdef __ARM_FEATURE_ATOMICS
mov x3, x1
casalb w3, w2, [x0]
@@ -131,6 +138,7 @@ DECLARE_FUNC(aarch64_atomic_cmpxchg_1_default_impl):
.align 5
DECLARE_FUNC(aarch64_atomic_cmpxchg_4_default_impl):
+ hint #0x22 // bti c
#ifdef __ARM_FEATURE_ATOMICS
mov x3, x1
casal w3, w2, [x0]
@@ -149,6 +157,7 @@ DECLARE_FUNC(aarch64_atomic_cmpxchg_4_default_impl):
.align 5
DECLARE_FUNC(aarch64_atomic_cmpxchg_8_default_impl):
+ hint #0x22 // bti c
#ifdef __ARM_FEATURE_ATOMICS
mov x3, x1
casal x3, x2, [x0]
@@ -167,6 +176,7 @@ DECLARE_FUNC(aarch64_atomic_cmpxchg_8_default_impl):
.align 5
DECLARE_FUNC(aarch64_atomic_cmpxchg_4_release_default_impl):
+ hint #0x22 // bti c
#ifdef __ARM_FEATURE_ATOMICS
mov x3, x1
casl w3, w2, [x0]
@@ -183,6 +193,7 @@ DECLARE_FUNC(aarch64_atomic_cmpxchg_4_release_default_impl):
.align 5
DECLARE_FUNC(aarch64_atomic_cmpxchg_8_release_default_impl):
+ hint #0x22 // bti c
#ifdef __ARM_FEATURE_ATOMICS
mov x3, x1
casl x3, x2, [x0]
@@ -199,6 +210,7 @@ DECLARE_FUNC(aarch64_atomic_cmpxchg_8_release_default_impl):
.align 5
DECLARE_FUNC(aarch64_atomic_cmpxchg_4_seq_cst_default_impl):
+ hint #0x22 // bti c
#ifdef __ARM_FEATURE_ATOMICS
mov x3, x1
casal w3, w2, [x0]
@@ -215,6 +227,7 @@ DECLARE_FUNC(aarch64_atomic_cmpxchg_4_seq_cst_default_impl):
.align 5
DECLARE_FUNC(aarch64_atomic_cmpxchg_8_seq_cst_default_impl):
+ hint #0x22 // bti c
#ifdef __ARM_FEATURE_ATOMICS
mov x3, x1
casal x3, x2, [x0]
@@ -231,6 +244,7 @@ DECLARE_FUNC(aarch64_atomic_cmpxchg_8_seq_cst_default_impl):
.align 5
DECLARE_FUNC(aarch64_atomic_cmpxchg_1_relaxed_default_impl):
+ hint #0x22 // bti c
#ifdef __ARM_FEATURE_ATOMICS
mov x3, x1
casb w3, w2, [x0]
@@ -248,6 +262,7 @@ DECLARE_FUNC(aarch64_atomic_cmpxchg_1_relaxed_default_impl):
.align 5
DECLARE_FUNC(aarch64_atomic_cmpxchg_4_relaxed_default_impl):
+ hint #0x22 // bti c
#ifdef __ARM_FEATURE_ATOMICS
mov x3, x1
cas w3, w2, [x0]
@@ -264,6 +279,7 @@ DECLARE_FUNC(aarch64_atomic_cmpxchg_4_relaxed_default_impl):
.align 5
DECLARE_FUNC(aarch64_atomic_cmpxchg_8_relaxed_default_impl):
+ hint #0x22 // bti c
#ifdef __ARM_FEATURE_ATOMICS
mov x3, x1
cas x3, x2, [x0]
@@ -277,3 +293,35 @@ DECLARE_FUNC(aarch64_atomic_cmpxchg_8_relaxed_default_impl):
#endif
1: mov x0, x3
ret
+
+/* Emit .note.gnu.property section in case of PAC or BTI being enabled.
+ * For more details see "ELF for the Arm® 64-bit Architecture (AArch64)".
+ * https://github.com/ARM-software/abi-aa/blob/main/aaelf64/aaelf64.rst
+ */
+#ifdef __ARM_FEATURE_BTI_DEFAULT
+ #ifdef __ARM_FEATURE_PAC_DEFAULT
+ #define GNU_PROPERTY_AARCH64_FEATURE 3
+ #else
+ #define GNU_PROPERTY_AARCH64_FEATURE 1
+ #endif
+#else
+ #ifdef __ARM_FEATURE_PAC_DEFAULT
+ #define GNU_PROPERTY_AARCH64_FEATURE 2
+ #else
+ #define GNU_PROPERTY_AARCH64_FEATURE 0
+ #endif
+#endif
+
+#if (GNU_PROPERTY_AARCH64_FEATURE != 0)
+ .pushsection .note.gnu.property, "a"
+ .align 3
+ .long 4 /* name length */
+ .long 0x10 /* data length */
+ .long 5 /* note type: NT_GNU_PROPERTY_TYPE_0 */
+ .string "GNU" /* vendor name */
+ .long 0xc0000000 /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */
+ .long 4 /* pr_datasze */
+ .long GNU_PROPERTY_AARCH64_FEATURE
+ .long 0
+ .popsection
+#endif
diff --git a/src/hotspot/os_cpu/linux_aarch64/copy_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/copy_linux_aarch64.S
index f9702d5e55419..33fe81d920653 100644
--- a/src/hotspot/os_cpu/linux_aarch64/copy_linux_aarch64.S
+++ b/src/hotspot/os_cpu/linux_aarch64/copy_linux_aarch64.S
@@ -83,29 +83,41 @@ fwd_copy_drain:
br t0
.align 5
- ret // -8 == 0 words
+ // -8 == 0 words
+ hint #0x24 // bti j
+ ret
.align 5
- ldr t0, [s, #16] // -7 == 1 word
+ // -7 == 1 word
+ hint #0x24 // bti j
+ ldr t0, [s, #16]
str t0, [d, #16]
ret
.align 5
- ldp t0, t1, [s, #16] // -6 = 2 words
+ // -6 == 2 words
+ hint #0x24 // bti j
+ ldp t0, t1, [s, #16]
stp t0, t1, [d, #16]
ret
.align 5
- ldp t0, t1, [s, #16] // -5 = 3 words
+ // -5 == 3 words
+ hint #0x24 // bti j
+ ldp t0, t1, [s, #16]
ldr t2, [s, #32]
stp t0, t1, [d, #16]
str t2, [d, #32]
ret
.align 5
- ldp t0, t1, [s, #16] // -4 = 4 words
+ // -4 == 4 words
+ hint #0x24 // bti j
+ ldp t0, t1, [s, #16]
ldp t2, t3, [s, #32]
stp t0, t1, [d, #16]
stp t2, t3, [d, #32]
ret
.align 5
- ldp t0, t1, [s, #16] // -3 = 5 words
+ // -3 == 5 words
+ hint #0x24 // bti j
+ ldp t0, t1, [s, #16]
ldp t2, t3, [s, #32]
ldr t4, [s, #48]
stp t0, t1, [d, #16]
@@ -113,7 +125,9 @@ fwd_copy_drain:
str t4, [d, #48]
ret
.align 5
- ldp t0, t1, [s, #16] // -2 = 6 words
+ // -2 == 6 words
+ hint #0x24 // bti j
+ ldp t0, t1, [s, #16]
ldp t2, t3, [s, #32]
ldp t4, t5, [s, #48]
stp t0, t1, [d, #16]
@@ -121,18 +135,20 @@ fwd_copy_drain:
stp t4, t5, [d, #48]
ret
.align 5
- ldp t0, t1, [s, #16] // -1 = 7 words
+ // -1 == 7 words
+ hint #0x24 // bti j
+ ldp t0, t1, [s, #16]
ldp t2, t3, [s, #32]
ldp t4, t5, [s, #48]
ldr t6, [s, #64]
stp t0, t1, [d, #16]
stp t2, t3, [d, #32]
stp t4, t5, [d, #48]
- str t6, [d, #64]
- // Is always aligned here, code for 7 words is one instruction
+ // Is always aligned here, code for 7 words is two instructions
// too large so it just falls through.
.align 5
0:
+ str t6, [d, #64]
ret
.align 6
@@ -184,29 +200,41 @@ bwd_copy_drain:
br t0
.align 5
- ret // -8 == 0 words
+ // -8 == 0 words
+ hint #0x24 // bti j
+ ret
.align 5
- ldr t0, [s, #-8] // -7 == 1 word
+ // -7 == 1 word
+ hint #0x24 // bti j
+ ldr t0, [s, #-8]
str t0, [d, #-8]
ret
.align 5
- ldp t0, t1, [s, #-16] // -6 = 2 words
+ // -6 == 2 words
+ hint #0x24 // bti j
+ ldp t0, t1, [s, #-16]
stp t0, t1, [d, #-16]
ret
.align 5
- ldp t0, t1, [s, #-16] // -5 = 3 words
+ // -5 == 3 words
+ hint #0x24 // bti j
+ ldp t0, t1, [s, #-16]
ldr t2, [s, #-24]
stp t0, t1, [d, #-16]
str t2, [d, #-24]
ret
.align 5
- ldp t0, t1, [s, #-16] // -4 = 4 words
+ // -4 == 4 words
+ hint #0x24 // bti j
+ ldp t0, t1, [s, #-16]
ldp t2, t3, [s, #-32]
stp t0, t1, [d, #-16]
stp t2, t3, [d, #-32]
ret
.align 5
- ldp t0, t1, [s, #-16] // -3 = 5 words
+ // -3 == 5 words
+ hint #0x24 // bti j
+ ldp t0, t1, [s, #-16]
ldp t2, t3, [s, #-32]
ldr t4, [s, #-40]
stp t0, t1, [d, #-16]
@@ -214,7 +242,9 @@ bwd_copy_drain:
str t4, [d, #-40]
ret
.align 5
- ldp t0, t1, [s, #-16] // -2 = 6 words
+ // -2 == 6 words
+ hint #0x24 // bti j
+ ldp t0, t1, [s, #-16]
ldp t2, t3, [s, #-32]
ldp t4, t5, [s, #-48]
stp t0, t1, [d, #-16]
@@ -222,16 +252,50 @@ bwd_copy_drain:
stp t4, t5, [d, #-48]
ret
.align 5
- ldp t0, t1, [s, #-16] // -1 = 7 words
+ // -1 == 7 words
+ hint #0x24 // bti j
+ ldp t0, t1, [s, #-16]
ldp t2, t3, [s, #-32]
ldp t4, t5, [s, #-48]
ldr t6, [s, #-56]
stp t0, t1, [d, #-16]
stp t2, t3, [d, #-32]
stp t4, t5, [d, #-48]
- str t6, [d, #-56]
- // Is always aligned here, code for 7 words is one instruction
+ // Is always aligned here, code for 7 words is two instructions
// too large so it just falls through.
.align 5
0:
+ str t6, [d, #-56]
ret
+
+/* Emit .note.gnu.property section in case of PAC or BTI being enabled.
+ * For more details see "ELF for the Arm® 64-bit Architecture (AArch64)".
+ * https://github.com/ARM-software/abi-aa/blob/main/aaelf64/aaelf64.rst
+ */
+#ifdef __ARM_FEATURE_BTI_DEFAULT
+ #ifdef __ARM_FEATURE_PAC_DEFAULT
+ #define GNU_PROPERTY_AARCH64_FEATURE 3
+ #else
+ #define GNU_PROPERTY_AARCH64_FEATURE 1
+ #endif
+#else
+ #ifdef __ARM_FEATURE_PAC_DEFAULT
+ #define GNU_PROPERTY_AARCH64_FEATURE 2
+ #else
+ #define GNU_PROPERTY_AARCH64_FEATURE 0
+ #endif
+#endif
+
+#if (GNU_PROPERTY_AARCH64_FEATURE != 0)
+ .pushsection .note.gnu.property, "a"
+ .align 3
+ .long 4 /* name length */
+ .long 0x10 /* data length */
+ .long 5 /* note type: NT_GNU_PROPERTY_TYPE_0 */
+ .string "GNU" /* vendor name */
+ .long 0xc0000000 /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */
+ .long 4 /* pr_datasze */
+ .long GNU_PROPERTY_AARCH64_FEATURE
+ .long 0
+ .popsection
+#endif
diff --git a/src/hotspot/os_cpu/linux_aarch64/gc/x/xSyscall_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/gc/x/xSyscall_linux_aarch64.hpp
deleted file mode 100644
index b4c49f477a677..0000000000000
--- a/src/hotspot/os_cpu/linux_aarch64/gc/x/xSyscall_linux_aarch64.hpp
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#ifndef OS_CPU_LINUX_AARCH64_GC_X_XSYSCALL_LINUX_AARCH64_HPP
-#define OS_CPU_LINUX_AARCH64_GC_X_XSYSCALL_LINUX_AARCH64_HPP
-
-#include
-
-//
-// Support for building on older Linux systems
-//
-
-#ifndef SYS_memfd_create
-#define SYS_memfd_create 279
-#endif
-#ifndef SYS_fallocate
-#define SYS_fallocate 47
-#endif
-
-#endif // OS_CPU_LINUX_AARCH64_GC_X_XSYSCALL_LINUX_AARCH64_HPP
diff --git a/src/hotspot/os_cpu/linux_aarch64/safefetch_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/safefetch_linux_aarch64.S
index 82d989a9b8830..a26aab44ac43d 100644
--- a/src/hotspot/os_cpu/linux_aarch64/safefetch_linux_aarch64.S
+++ b/src/hotspot/os_cpu/linux_aarch64/safefetch_linux_aarch64.S
@@ -48,3 +48,35 @@ DECLARE_FUNC(_SafeFetchN_fault):
DECLARE_FUNC(_SafeFetchN_continuation):
mov x0, x1
ret
+
+/* Emit .note.gnu.property section in case of PAC or BTI being enabled.
+ * For more details see "ELF for the Arm® 64-bit Architecture (AArch64)".
+ * https://github.com/ARM-software/abi-aa/blob/main/aaelf64/aaelf64.rst
+ */
+#ifdef __ARM_FEATURE_BTI_DEFAULT
+ #ifdef __ARM_FEATURE_PAC_DEFAULT
+ #define GNU_PROPERTY_AARCH64_FEATURE 3
+ #else
+ #define GNU_PROPERTY_AARCH64_FEATURE 1
+ #endif
+#else
+ #ifdef __ARM_FEATURE_PAC_DEFAULT
+ #define GNU_PROPERTY_AARCH64_FEATURE 2
+ #else
+ #define GNU_PROPERTY_AARCH64_FEATURE 0
+ #endif
+#endif
+
+#if (GNU_PROPERTY_AARCH64_FEATURE != 0)
+ .pushsection .note.gnu.property, "a"
+ .align 3
+ .long 4 /* name length */
+ .long 0x10 /* data length */
+ .long 5 /* note type: NT_GNU_PROPERTY_TYPE_0 */
+ .string "GNU" /* vendor name */
+ .long 0xc0000000 /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */
+ .long 4 /* pr_datasze */
+ .long GNU_PROPERTY_AARCH64_FEATURE
+ .long 0
+ .popsection
+#endif
diff --git a/src/hotspot/os_cpu/linux_aarch64/threadLS_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/threadLS_linux_aarch64.S
index ca31585871dd1..55a252301d793 100644
--- a/src/hotspot/os_cpu/linux_aarch64/threadLS_linux_aarch64.S
+++ b/src/hotspot/os_cpu/linux_aarch64/threadLS_linux_aarch64.S
@@ -44,3 +44,35 @@ DECLARE_FUNC(_ZN10JavaThread25aarch64_get_thread_helperEv):
ret
.size _ZN10JavaThread25aarch64_get_thread_helperEv, .-_ZN10JavaThread25aarch64_get_thread_helperEv
+
+/* Emit .note.gnu.property section in case of PAC or BTI being enabled.
+ * For more details see "ELF for the Arm® 64-bit Architecture (AArch64)".
+ * https://github.com/ARM-software/abi-aa/blob/main/aaelf64/aaelf64.rst
+ */
+#ifdef __ARM_FEATURE_BTI_DEFAULT
+ #ifdef __ARM_FEATURE_PAC_DEFAULT
+ #define GNU_PROPERTY_AARCH64_FEATURE 3
+ #else
+ #define GNU_PROPERTY_AARCH64_FEATURE 1
+ #endif
+#else
+ #ifdef __ARM_FEATURE_PAC_DEFAULT
+ #define GNU_PROPERTY_AARCH64_FEATURE 2
+ #else
+ #define GNU_PROPERTY_AARCH64_FEATURE 0
+ #endif
+#endif
+
+#if (GNU_PROPERTY_AARCH64_FEATURE != 0)
+ .pushsection .note.gnu.property, "a"
+ .align 3
+ .long 4 /* name length */
+ .long 0x10 /* data length */
+ .long 5 /* note type: NT_GNU_PROPERTY_TYPE_0 */
+ .string "GNU" /* vendor name */
+ .long 0xc0000000 /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */
+ .long 4 /* pr_datasze */
+ .long GNU_PROPERTY_AARCH64_FEATURE
+ .long 0
+ .popsection
+#endif
diff --git a/src/hotspot/os_cpu/linux_aarch64/vmStructs_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/vmStructs_linux_aarch64.hpp
index f2ad002996b5c..3c8e9c4441477 100644
--- a/src/hotspot/os_cpu/linux_aarch64/vmStructs_linux_aarch64.hpp
+++ b/src/hotspot/os_cpu/linux_aarch64/vmStructs_linux_aarch64.hpp
@@ -30,23 +30,9 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
- \
- /******************************/ \
- /* Threads (NOTE: incomplete) */ \
- /******************************/ \
- nonstatic_field(OSThread, _thread_id, OSThread::thread_id_t) \
- nonstatic_field(OSThread, _pthread_id, pthread_t)
+#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
- \
- /**********************/ \
- /* Posix Thread IDs */ \
- /**********************/ \
- \
- declare_integer_type(OSThread::thread_id_t) \
- declare_unsigned_integer_type(pthread_t)
+#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
diff --git a/src/hotspot/os_cpu/linux_arm/vmStructs_linux_arm.hpp b/src/hotspot/os_cpu/linux_arm/vmStructs_linux_arm.hpp
index 9b4bd0faf0ad9..120726bf55fcd 100644
--- a/src/hotspot/os_cpu/linux_arm/vmStructs_linux_arm.hpp
+++ b/src/hotspot/os_cpu/linux_arm/vmStructs_linux_arm.hpp
@@ -29,22 +29,9 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
- \
- /******************************/ \
- /* Threads (NOTE: incomplete) */ \
- /******************************/ \
- nonstatic_field(OSThread, _thread_id, OSThread::thread_id_t) \
- nonstatic_field(OSThread, _pthread_id, pthread_t)
+#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
- \
- /**********************/ \
- /* Posix Thread IDs */ \
- /**********************/ \
- \
- declare_integer_type(OSThread::thread_id_t) \
- declare_unsigned_integer_type(pthread_t)
+#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
diff --git a/src/hotspot/os_cpu/linux_ppc/vmStructs_linux_ppc.hpp b/src/hotspot/os_cpu/linux_ppc/vmStructs_linux_ppc.hpp
index 9464c35977078..ae948c7303101 100644
--- a/src/hotspot/os_cpu/linux_ppc/vmStructs_linux_ppc.hpp
+++ b/src/hotspot/os_cpu/linux_ppc/vmStructs_linux_ppc.hpp
@@ -30,23 +30,9 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
- \
- /******************************/ \
- /* Threads (NOTE: incomplete) */ \
- /******************************/ \
- nonstatic_field(OSThread, _thread_id, pid_t) \
- nonstatic_field(OSThread, _pthread_id, pthread_t)
+#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
- \
- /**********************/ \
- /* Posix Thread IDs */ \
- /**********************/ \
- \
- declare_integer_type(pid_t) \
- declare_unsigned_integer_type(pthread_t)
+#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
diff --git a/src/hotspot/os_cpu/linux_riscv/gc/x/xSyscall_linux_riscv.hpp b/src/hotspot/os_cpu/linux_riscv/gc/x/xSyscall_linux_riscv.hpp
deleted file mode 100644
index bfd49b0bf4e4d..0000000000000
--- a/src/hotspot/os_cpu/linux_riscv/gc/x/xSyscall_linux_riscv.hpp
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef OS_CPU_LINUX_RISCV_GC_X_XSYSCALL_LINUX_RISCV_HPP
-#define OS_CPU_LINUX_RISCV_GC_X_XSYSCALL_LINUX_RISCV_HPP
-
-#include
-
-//
-// Support for building on older Linux systems
-//
-
-#ifndef SYS_memfd_create
-#define SYS_memfd_create 279
-#endif
-#ifndef SYS_fallocate
-#define SYS_fallocate 47
-#endif
-
-#endif // OS_CPU_LINUX_RISCV_GC_X_XSYSCALL_LINUX_RISCV_HPP
diff --git a/src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp b/src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp
index df4a2e347ccac..2020e2fdb2429 100644
--- a/src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp
+++ b/src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp
@@ -74,7 +74,7 @@
#define RISCV_HWPROBE_EXT_ZFHMIN (1 << 28)
#define RISCV_HWPROBE_EXT_ZIHINTNTL (1 << 29)
#define RISCV_HWPROBE_EXT_ZVFH (1 << 30)
-#define RISCV_HWPROBE_EXT_ZVFHMIN (1 << 31)
+#define RISCV_HWPROBE_EXT_ZVFHMIN (1ULL << 31)
#define RISCV_HWPROBE_EXT_ZFA (1ULL << 32)
#define RISCV_HWPROBE_EXT_ZTSO (1ULL << 33)
#define RISCV_HWPROBE_EXT_ZACAS (1ULL << 34)
@@ -178,6 +178,9 @@ void RiscvHwprobe::add_features_from_query_result() {
if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZFH)) {
VM_Version::ext_Zfh.enable_feature();
}
+ if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZVFH)) {
+ VM_Version::ext_Zvfh.enable_feature();
+ }
if (is_valid(RISCV_HWPROBE_KEY_CPUPERF_0)) {
VM_Version::unaligned_access.enable_feature(
query[RISCV_HWPROBE_KEY_CPUPERF_0].value & RISCV_HWPROBE_MISALIGNED_MASK);
diff --git a/src/hotspot/os_cpu/linux_riscv/vmStructs_linux_riscv.hpp b/src/hotspot/os_cpu/linux_riscv/vmStructs_linux_riscv.hpp
index 6cf7683a58602..3946394c19b1f 100644
--- a/src/hotspot/os_cpu/linux_riscv/vmStructs_linux_riscv.hpp
+++ b/src/hotspot/os_cpu/linux_riscv/vmStructs_linux_riscv.hpp
@@ -30,23 +30,9 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
- \
- /******************************/ \
- /* Threads (NOTE: incomplete) */ \
- /******************************/ \
- nonstatic_field(OSThread, _thread_id, OSThread::thread_id_t) \
- nonstatic_field(OSThread, _pthread_id, pthread_t)
+#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
- \
- /**********************/ \
- /* Posix Thread IDs */ \
- /**********************/ \
- \
- declare_integer_type(OSThread::thread_id_t) \
- declare_unsigned_integer_type(pthread_t)
+#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
diff --git a/src/hotspot/os_cpu/linux_s390/vmStructs_linux_s390.hpp b/src/hotspot/os_cpu/linux_s390/vmStructs_linux_s390.hpp
index 0442510fa247a..a0fb5eb1a6ab9 100644
--- a/src/hotspot/os_cpu/linux_s390/vmStructs_linux_s390.hpp
+++ b/src/hotspot/os_cpu/linux_s390/vmStructs_linux_s390.hpp
@@ -30,23 +30,9 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
- \
- /******************************/ \
- /* Threads (NOTE: incomplete) */ \
- /******************************/ \
- nonstatic_field(OSThread, _thread_id, pid_t) \
- nonstatic_field(OSThread, _pthread_id, pthread_t)
+#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
- \
- /**********************/ \
- /* Posix Thread IDs */ \
- /**********************/ \
- \
- declare_integer_type(pid_t) \
- declare_unsigned_integer_type(pthread_t)
+#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
diff --git a/src/hotspot/os_cpu/linux_x86/gc/x/xSyscall_linux_x86.hpp b/src/hotspot/os_cpu/linux_x86/gc/x/xSyscall_linux_x86.hpp
deleted file mode 100644
index 2709b373b2812..0000000000000
--- a/src/hotspot/os_cpu/linux_x86/gc/x/xSyscall_linux_x86.hpp
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-#ifndef OS_CPU_LINUX_X86_GC_X_XSYSCALL_LINUX_X86_HPP
-#define OS_CPU_LINUX_X86_GC_X_XSYSCALL_LINUX_X86_HPP
-
-#include
-
-//
-// Support for building on older Linux systems
-//
-
-#ifndef SYS_memfd_create
-#define SYS_memfd_create 319
-#endif
-#ifndef SYS_fallocate
-#define SYS_fallocate 285
-#endif
-
-#endif // OS_CPU_LINUX_X86_GC_X_XSYSCALL_LINUX_X86_HPP
diff --git a/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp b/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp
index 0d5d07fc8a804..8fdcbe63c7e8b 100644
--- a/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp
+++ b/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp
@@ -544,6 +544,21 @@ void os::print_context(outputStream *st, const void *context) {
st->print(", ERR=" INTPTR_FORMAT, (intptr_t)uc->uc_mcontext.gregs[REG_ERR]);
st->cr();
st->print(" TRAPNO=" INTPTR_FORMAT, (intptr_t)uc->uc_mcontext.gregs[REG_TRAPNO]);
+ // Add XMM registers + MXCSR. Note that C2 uses XMM to spill GPR values including pointers.
+ st->cr();
+ st->cr();
+ // Sanity check: fpregs should point into the context.
+ if ((address)uc->uc_mcontext.fpregs < (address)uc ||
+ pointer_delta(uc->uc_mcontext.fpregs, uc, 1) >= sizeof(ucontext_t)) {
+ st->print_cr("bad uc->uc_mcontext.fpregs: " INTPTR_FORMAT " (uc: " INTPTR_FORMAT ")",
+ p2i(uc->uc_mcontext.fpregs), p2i(uc));
+ } else {
+ for (int i = 0; i < 16; ++i) {
+ const int64_t* xmm_val_addr = (int64_t*)&(uc->uc_mcontext.fpregs->_xmm[i]);
+ st->print_cr("XMM[%d]=" INTPTR_FORMAT " " INTPTR_FORMAT, i, xmm_val_addr[1], xmm_val_addr[0]);
+ }
+ st->print(" MXCSR=" UINT32_FORMAT_X_0, uc->uc_mcontext.fpregs->mxcsr);
+ }
#else
st->print( "EAX=" INTPTR_FORMAT, uc->uc_mcontext.gregs[REG_EAX]);
st->print(", EBX=" INTPTR_FORMAT, uc->uc_mcontext.gregs[REG_EBX]);
diff --git a/src/hotspot/os_cpu/linux_x86/vmStructs_linux_x86.hpp b/src/hotspot/os_cpu/linux_x86/vmStructs_linux_x86.hpp
index 277486549c035..8f6d365723700 100644
--- a/src/hotspot/os_cpu/linux_x86/vmStructs_linux_x86.hpp
+++ b/src/hotspot/os_cpu/linux_x86/vmStructs_linux_x86.hpp
@@ -29,23 +29,9 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
- \
- /******************************/ \
- /* Threads (NOTE: incomplete) */ \
- /******************************/ \
- nonstatic_field(OSThread, _thread_id, OSThread::thread_id_t) \
- nonstatic_field(OSThread, _pthread_id, pthread_t)
+#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
- \
- /**********************/ \
- /* Posix Thread IDs */ \
- /**********************/ \
- \
- declare_integer_type(OSThread::thread_id_t) \
- declare_unsigned_integer_type(pthread_t)
+#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
diff --git a/src/hotspot/os_cpu/windows_aarch64/vmStructs_windows_aarch64.hpp b/src/hotspot/os_cpu/windows_aarch64/vmStructs_windows_aarch64.hpp
index 220787823dc69..18a5588b743b9 100644
--- a/src/hotspot/os_cpu/windows_aarch64/vmStructs_windows_aarch64.hpp
+++ b/src/hotspot/os_cpu/windows_aarch64/vmStructs_windows_aarch64.hpp
@@ -29,18 +29,9 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
- \
- /******************************/ \
- /* Threads (NOTE: incomplete) */ \
- /******************************/ \
- \
- nonstatic_field(OSThread, _thread_id, OSThread::thread_id_t) \
- unchecked_nonstatic_field(OSThread, _thread_handle, sizeof(HANDLE)) /* NOTE: no type */
+#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
- \
- declare_unsigned_integer_type(OSThread::thread_id_t)
+#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
diff --git a/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp b/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp
index 62022d780a20b..de59a74cc24cc 100644
--- a/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp
+++ b/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp
@@ -437,6 +437,15 @@ void os::print_context(outputStream *st, const void *context) {
st->cr();
st->print( "RIP=" INTPTR_FORMAT, uc->Rip);
st->print(", EFLAGS=" INTPTR_FORMAT, uc->EFlags);
+ // Add XMM registers + MXCSR. Note that C2 uses XMM to spill GPR values including pointers.
+ st->cr();
+ st->cr();
+ for (int i = 0; i < 16; ++i) {
+ const uint64_t *xmm = ((const uint64_t*)&(uc->Xmm0)) + 2 * i;
+ st->print_cr("XMM[%d]=" INTPTR_FORMAT " " INTPTR_FORMAT,
+ i, xmm[1], xmm[0]);
+ }
+ st->print(" MXCSR=" UINT32_FORMAT_X_0, uc->MxCsr);
#else
st->print( "EAX=" INTPTR_FORMAT, uc->Eax);
st->print(", EBX=" INTPTR_FORMAT, uc->Ebx);
diff --git a/src/hotspot/os_cpu/windows_x86/vmStructs_windows_x86.hpp b/src/hotspot/os_cpu/windows_x86/vmStructs_windows_x86.hpp
index 9f50a7ed9ae29..985a6a331daba 100644
--- a/src/hotspot/os_cpu/windows_x86/vmStructs_windows_x86.hpp
+++ b/src/hotspot/os_cpu/windows_x86/vmStructs_windows_x86.hpp
@@ -29,18 +29,9 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
- \
- /******************************/ \
- /* Threads (NOTE: incomplete) */ \
- /******************************/ \
- \
- nonstatic_field(OSThread, _thread_id, OSThread::thread_id_t) \
- unchecked_nonstatic_field(OSThread, _thread_handle, sizeof(HANDLE)) /* NOTE: no type */
+#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
- \
- declare_unsigned_integer_type(OSThread::thread_id_t)
+#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
diff --git a/src/hotspot/share/adlc/forms.cpp b/src/hotspot/share/adlc/forms.cpp
index 068d745254e3d..c34a73ea1e13f 100644
--- a/src/hotspot/share/adlc/forms.cpp
+++ b/src/hotspot/share/adlc/forms.cpp
@@ -276,7 +276,6 @@ Form::DataType Form::is_load_from_memory(const char *opType) const {
Form::DataType Form::is_store_to_memory(const char *opType) const {
if( strcmp(opType,"StoreB")==0) return Form::idealB;
- if( strcmp(opType,"StoreCM")==0) return Form::idealB;
if( strcmp(opType,"StoreC")==0) return Form::idealC;
if( strcmp(opType,"StoreD")==0) return Form::idealD;
if( strcmp(opType,"StoreF")==0) return Form::idealF;
diff --git a/src/hotspot/share/adlc/formsopt.cpp b/src/hotspot/share/adlc/formsopt.cpp
index e1e4ed96c2ea0..5de8974e2c096 100644
--- a/src/hotspot/share/adlc/formsopt.cpp
+++ b/src/hotspot/share/adlc/formsopt.cpp
@@ -171,9 +171,13 @@ int RegisterForm::RegMask_Size() {
// on the stack (stack registers) up to some interesting limit. Methods
// that need more parameters will NOT be compiled. On Intel, the limit
// is something like 90+ parameters.
- // Add a few (3 words == 96 bits) for incoming & outgoing arguments to calls.
- // Round up to the next doubleword size.
- return (words_for_regs + 3 + 1) & ~1;
+ // - Add a few (3 words == 96 bits) for incoming & outgoing arguments to
+ // calls.
+ // - Round up to the next doubleword size.
+ // - Add one more word to accommodate a reasonable number of stack locations
+ // in the register mask regardless of how much slack is created by rounding.
+ // This was found necessary after adding 16 new registers for APX.
+ return (words_for_regs + 3 + 1 + 1) & ~1;
}
void RegisterForm::dump() { // Debug printer
diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp
index 15bc7ddc67d60..dfa414ef56484 100644
--- a/src/hotspot/share/adlc/formssel.cpp
+++ b/src/hotspot/share/adlc/formssel.cpp
@@ -3654,7 +3654,6 @@ int MatchNode::needs_ideal_memory_edge(FormDict &globals) const {
#if INCLUDE_SHENANDOAHGC
"ShenandoahCompareAndSwapN", "ShenandoahCompareAndSwapP", "ShenandoahWeakCompareAndSwapP", "ShenandoahWeakCompareAndSwapN", "ShenandoahCompareAndExchangeP", "ShenandoahCompareAndExchangeN",
#endif
- "StoreCM",
"GetAndSetB", "GetAndSetS", "GetAndAddI", "GetAndSetI", "GetAndSetP",
"GetAndAddB", "GetAndAddS", "GetAndAddL", "GetAndSetL", "GetAndSetN",
"ClearArray"
@@ -3958,15 +3957,15 @@ void MatchNode::count_commutative_op(int& count) {
"AndI","AndL",
"MaxI","MinI","MaxF","MinF","MaxD","MinD",
"MulI","MulL","MulF","MulD",
- "OrI","OrL",
- "XorI","XorL"
+ "OrI","OrL", "XorI","XorL",
+ "UMax","UMin"
};
static const char *commut_vector_op_list[] = {
"AddVB", "AddVS", "AddVI", "AddVL", "AddVF", "AddVD",
"MulVB", "MulVS", "MulVI", "MulVL", "MulVF", "MulVD",
"AndV", "OrV", "XorV",
- "MaxV", "MinV"
+ "MaxV", "MinV", "UMax","UMin"
};
if (_lChild && _rChild && (_lChild->_lChild || _rChild->_lChild)) {
@@ -4340,7 +4339,7 @@ bool MatchRule::is_vector() const {
"NegVF","NegVD","NegVI","NegVL",
"SqrtVD","SqrtVF",
"AndV" ,"XorV" ,"OrV",
- "MaxV", "MinV",
+ "MaxV", "MinV", "UMinV", "UMaxV",
"CompressV", "ExpandV", "CompressM", "CompressBitsV", "ExpandBitsV",
"AddReductionVI", "AddReductionVL",
"AddReductionVF", "AddReductionVD",
@@ -4356,14 +4355,14 @@ bool MatchRule::is_vector() const {
"Replicate","ReverseV","ReverseBytesV",
"RoundDoubleModeV","RotateLeftV" , "RotateRightV", "LoadVector","StoreVector",
"LoadVectorGather", "StoreVectorScatter", "LoadVectorGatherMasked", "StoreVectorScatterMasked",
- "VectorTest", "VectorLoadMask", "VectorStoreMask", "VectorBlend", "VectorInsert",
+ "SelectFromTwoVector", "VectorTest", "VectorLoadMask", "VectorStoreMask", "VectorBlend", "VectorInsert",
"VectorRearrange", "VectorLoadShuffle", "VectorLoadConst",
"VectorCastB2X", "VectorCastS2X", "VectorCastI2X",
"VectorCastL2X", "VectorCastF2X", "VectorCastD2X", "VectorCastF2HF", "VectorCastHF2F",
"VectorUCastB2X", "VectorUCastS2X", "VectorUCastI2X",
"VectorMaskWrapper","VectorMaskCmp","VectorReinterpret","LoadVectorMasked","StoreVectorMasked",
"FmaVD","FmaVF","PopCountVI","PopCountVL","PopulateIndex","VectorLongToMask",
- "CountLeadingZerosV", "CountTrailingZerosV", "SignumVF", "SignumVD",
+ "CountLeadingZerosV", "CountTrailingZerosV", "SignumVF", "SignumVD", "SaturatingAddV", "SaturatingSubV",
// Next are vector mask ops.
"MaskAll", "AndVMask", "OrVMask", "XorVMask", "VectorMaskCast",
"RoundVF", "RoundVD",
diff --git a/src/hotspot/share/c1/c1_LIR.hpp b/src/hotspot/share/c1/c1_LIR.hpp
index d9c3e9d3cb68f..c568caeca4b30 100644
--- a/src/hotspot/share/c1/c1_LIR.hpp
+++ b/src/hotspot/share/c1/c1_LIR.hpp
@@ -29,6 +29,7 @@
#include "c1/c1_ValueType.hpp"
#include "oops/method.hpp"
#include "utilities/globalDefinitions.hpp"
+#include "utilities/macros.hpp"
class BlockBegin;
class BlockList;
@@ -1122,7 +1123,7 @@ class LIR_Op: public CompilationResourceObj {
}
#endif
- virtual const char * name() const PRODUCT_RETURN0;
+ virtual const char * name() const PRODUCT_RETURN_NULL;
virtual void visit(LIR_OpVisitState* state);
int id() const { return _id; }
@@ -1400,7 +1401,7 @@ class LIR_Op1: public LIR_Op {
virtual bool is_patching() { return _patch != lir_patch_none; }
virtual void emit_code(LIR_Assembler* masm);
virtual LIR_Op1* as_Op1() { return this; }
- virtual const char * name() const PRODUCT_RETURN0;
+ virtual const char * name() const PRODUCT_RETURN_NULL;
void set_in_opr(LIR_Opr opr) { _opr = opr; }
diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp
index 915f00f77c523..3ebd1f42f366d 100644
--- a/src/hotspot/share/c1/c1_Runtime1.cpp
+++ b/src/hotspot/share/c1/c1_Runtime1.cpp
@@ -887,7 +887,7 @@ static Klass* resolve_field_return_klass(const methodHandle& caller, int bci, TR
// movl reg, [reg1 + ] (for field offsets)
// jmp continue
//
-// patch_stub: jmp Runtim1::patch_code (through a runtime stub)
+// patch_stub: jmp Runtime1::patch_code (through a runtime stub)
// jmp patch_site
//
// If the class is being initialized the patch body is rewritten and
@@ -1103,7 +1103,7 @@ JRT_ENTRY(void, Runtime1::patch_code(JavaThread* current, C1StubId stub_id ))
// Now copy code back
{
- MutexLocker ml_patch (current, Patching_lock, Mutex::_no_safepoint_check_flag);
+ MutexLocker ml_code (current, CodeCache_lock, Mutex::_no_safepoint_check_flag);
//
// Deoptimization may have happened while we waited for the lock.
// In that case we don't bother to do any patching we just return
@@ -1268,12 +1268,8 @@ JRT_ENTRY(void, Runtime1::patch_code(JavaThread* current, C1StubId stub_id ))
}
}
}
- }
-
- // If we are patching in a non-perm oop, make sure the nmethod
- // is on the right list.
- {
- MutexLocker ml_code (current, CodeCache_lock, Mutex::_no_safepoint_check_flag);
+ // If we are patching in a non-perm oop, make sure the nmethod
+ // is on the right list.
nmethod* nm = CodeCache::find_nmethod(caller_frame.pc());
guarantee(nm != nullptr, "only nmethods can contain non-perm oops");
diff --git a/src/hotspot/share/cds/archiveHeapLoader.cpp b/src/hotspot/share/cds/archiveHeapLoader.cpp
index 6325fb6f49d73..01831cf0f3e4b 100644
--- a/src/hotspot/share/cds/archiveHeapLoader.cpp
+++ b/src/hotspot/share/cds/archiveHeapLoader.cpp
@@ -142,15 +142,22 @@ class PatchCompressedEmbeddedPointersQuick: public BitMapClosure {
class PatchUncompressedEmbeddedPointers: public BitMapClosure {
oop* _start;
+ intptr_t _delta;
public:
- PatchUncompressedEmbeddedPointers(oop* start) : _start(start) {}
+ PatchUncompressedEmbeddedPointers(oop* start, intx runtime_offset) :
+ _start(start),
+ _delta(runtime_offset) {}
+
+ PatchUncompressedEmbeddedPointers(oop* start) :
+ _start(start),
+ _delta(ArchiveHeapLoader::mapped_heap_delta()) {}
bool do_bit(size_t offset) {
oop* p = _start + offset;
intptr_t dumptime_oop = (intptr_t)((void*)*p);
assert(dumptime_oop != 0, "null oops should have been filtered out at dump time");
- intptr_t runtime_oop = dumptime_oop + ArchiveHeapLoader::mapped_heap_delta();
+ intptr_t runtime_oop = dumptime_oop + _delta;
RawAccess::oop_store(p, cast_to_oop(runtime_oop));
return true;
}
@@ -221,10 +228,6 @@ void ArchiveHeapLoader::init_loaded_heap_relocation(LoadedArchiveHeapRegion* loa
}
bool ArchiveHeapLoader::can_load() {
- if (!UseCompressedOops) {
- // Pointer relocation for uncompressed oops is unimplemented.
- return false;
- }
return Universe::heap()->can_load_archived_objects();
}
@@ -312,13 +315,18 @@ bool ArchiveHeapLoader::load_heap_region_impl(FileMapInfo* mapinfo, LoadedArchiv
uintptr_t oopmap = bitmap_base + r->oopmap_offset();
BitMapView bm((BitMap::bm_word_t*)oopmap, r->oopmap_size_in_bits());
- PatchLoadedRegionPointers patcher((narrowOop*)load_address + FileMapInfo::current_info()->heap_oopmap_start_pos(), loaded_region);
- bm.iterate(&patcher);
+ if (UseCompressedOops) {
+ PatchLoadedRegionPointers patcher((narrowOop*)load_address + FileMapInfo::current_info()->heap_oopmap_start_pos(), loaded_region);
+ bm.iterate(&patcher);
+ } else {
+ PatchUncompressedEmbeddedPointers patcher((oop*)load_address + FileMapInfo::current_info()->heap_oopmap_start_pos(), loaded_region->_runtime_offset);
+ bm.iterate(&patcher);
+ }
return true;
}
bool ArchiveHeapLoader::load_heap_region(FileMapInfo* mapinfo) {
- assert(UseCompressedOops, "loaded heap for !UseCompressedOops is unimplemented");
+ assert(can_load(), "loaded heap for must be supported");
init_narrow_oop_decoding(mapinfo->narrow_oop_base(), mapinfo->narrow_oop_shift());
LoadedArchiveHeapRegion loaded_region;
@@ -358,8 +366,12 @@ class VerifyLoadedHeapEmbeddedPointers: public BasicOopIterateClosure {
}
}
virtual void do_oop(oop* p) {
- // Uncompressed oops are not supported by loaded heaps.
- Unimplemented();
+ oop v = *p;
+ if(v != nullptr) {
+ uintptr_t u = cast_from_oop(v);
+ ArchiveHeapLoader::assert_in_loaded_heap(u);
+ guarantee(_table->contains(u), "must point to beginning of object in loaded archived region");
+ }
}
};
@@ -376,13 +388,12 @@ void ArchiveHeapLoader::finish_initialization() {
intptr_t bottom = is_loaded() ? _loaded_heap_bottom : _mapped_heap_bottom;
// The heap roots are stored in one or more segments that are laid out consecutively.
- // The byte size of each segment (except for the last one) is max_size.
+ // The size of each segment (except for the last one) is max_size_in_{elems,bytes}.
HeapRootSegments segments = FileMapInfo::current_info()->heap_root_segments();
- int max_size = segments.max_size_in_bytes();
- HeapShared::init_root_segment_sizes(max_size);
+ HeapShared::init_root_segment_sizes(segments.max_size_in_elems());
intptr_t first_segment_addr = bottom + segments.base_offset();
for (size_t c = 0; c < segments.count(); c++) {
- oop segment_oop = cast_to_oop(first_segment_addr + (c * max_size));
+ oop segment_oop = cast_to_oop(first_segment_addr + (c * segments.max_size_in_bytes()));
assert(segment_oop->is_objArray(), "Must be");
HeapShared::add_root_segment((objArrayOop)segment_oop);
}
diff --git a/src/hotspot/share/cds/archiveHeapLoader.hpp b/src/hotspot/share/cds/archiveHeapLoader.hpp
index 700135a38166c..8b9fab91aa3fd 100644
--- a/src/hotspot/share/cds/archiveHeapLoader.hpp
+++ b/src/hotspot/share/cds/archiveHeapLoader.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -146,6 +146,7 @@ class ArchiveHeapLoader : AllStatic {
inline static oop decode_from_archive_impl(narrowOop v) NOT_CDS_JAVA_HEAP_RETURN_(nullptr);
class PatchLoadedRegionPointers;
+ class PatchUncompressedLoadedRegionPointers;
public:
diff --git a/src/hotspot/share/cds/archiveHeapWriter.cpp b/src/hotspot/share/cds/archiveHeapWriter.cpp
index 853d459c691dd..710e693bfdb14 100644
--- a/src/hotspot/share/cds/archiveHeapWriter.cpp
+++ b/src/hotspot/share/cds/archiveHeapWriter.cpp
@@ -223,6 +223,7 @@ void ArchiveHeapWriter::copy_roots_to_buffer(GrowableArrayCHeapat(root_index++));
@@ -245,6 +245,8 @@ void ArchiveHeapWriter::copy_roots_to_buffer(GrowableArrayCHeaplength(), "Post-condition: All roots are handled");
+
_heap_root_segments = segments;
}
diff --git a/src/hotspot/share/cds/archiveUtils.hpp b/src/hotspot/share/cds/archiveUtils.hpp
index 2e361ab0c4650..5a78bc26ee627 100644
--- a/src/hotspot/share/cds/archiveUtils.hpp
+++ b/src/hotspot/share/cds/archiveUtils.hpp
@@ -277,7 +277,6 @@ class HeapRootSegments {
memset(this, 0, sizeof(*this));
}
HeapRootSegments(size_t base_offset, int roots_count, int max_size_in_bytes, int max_size_in_elems) {
- assert(is_power_of_2(max_size_in_bytes), "must be");
memset(this, 0, sizeof(*this));
_base_offset = base_offset;
_count = (roots_count + max_size_in_elems - 1) / max_size_in_elems;
diff --git a/src/hotspot/share/cds/cdsConfig.cpp b/src/hotspot/share/cds/cdsConfig.cpp
index a0a562eca21a0..9e8a46e105e19 100644
--- a/src/hotspot/share/cds/cdsConfig.cpp
+++ b/src/hotspot/share/cds/cdsConfig.cpp
@@ -236,7 +236,9 @@ void CDSConfig::init_shared_archive_paths() {
}
void CDSConfig::check_internal_module_property(const char* key, const char* value) {
- if (Arguments::is_internal_module_property(key)) {
+ if (Arguments::is_internal_module_property(key) &&
+ !Arguments::is_module_path_property(key) &&
+ !Arguments::is_add_modules_property(key)) {
stop_using_optimized_module_handling();
log_info(cds)("optimized module handling: disabled due to incompatible property: %s=%s", key, value);
}
diff --git a/src/hotspot/share/cds/cdsProtectionDomain.cpp b/src/hotspot/share/cds/cdsProtectionDomain.cpp
index a4ffeea26dc4a..2eb47ff2788d9 100644
--- a/src/hotspot/share/cds/cdsProtectionDomain.cpp
+++ b/src/hotspot/share/cds/cdsProtectionDomain.cpp
@@ -199,17 +199,9 @@ Handle CDSProtectionDomain::get_shared_jar_manifest(int shared_path_index, TRAPS
Handle CDSProtectionDomain::get_shared_jar_url(int shared_path_index, TRAPS) {
Handle url_h;
if (shared_jar_url(shared_path_index) == nullptr) {
- JavaValue result(T_OBJECT);
const char* path = FileMapInfo::shared_path_name(shared_path_index);
- Handle path_string = java_lang_String::create_from_str(path, CHECK_(url_h));
- Klass* classLoaders_klass =
- vmClasses::jdk_internal_loader_ClassLoaders_klass();
- JavaCalls::call_static(&result, classLoaders_klass,
- vmSymbols::toFileURL_name(),
- vmSymbols::toFileURL_signature(),
- path_string, CHECK_(url_h));
-
- atomic_set_shared_jar_url(shared_path_index, result.get_oop());
+ oop result_oop = to_file_URL(path, url_h, CHECK_(url_h));
+ atomic_set_shared_jar_url(shared_path_index, result_oop);
}
url_h = Handle(THREAD, shared_jar_url(shared_path_index));
@@ -217,6 +209,17 @@ Handle CDSProtectionDomain::get_shared_jar_url(int shared_path_index, TRAPS) {
return url_h;
}
+oop CDSProtectionDomain::to_file_URL(const char* path, Handle url_h, TRAPS) {
+ JavaValue result(T_OBJECT);
+ Handle path_string = java_lang_String::create_from_str(path, CHECK_NULL);
+ JavaCalls::call_static(&result,
+ vmClasses::jdk_internal_loader_ClassLoaders_klass(),
+ vmSymbols::toFileURL_name(),
+ vmSymbols::toFileURL_signature(),
+ path_string, CHECK_NULL);
+ return result.get_oop();
+}
+
// Get the ProtectionDomain associated with the CodeSource from the classloader.
Handle CDSProtectionDomain::get_protection_domain_from_classloader(Handle class_loader,
Handle url, TRAPS) {
diff --git a/src/hotspot/share/cds/cdsProtectionDomain.hpp b/src/hotspot/share/cds/cdsProtectionDomain.hpp
index 0e688fcfa00e8..baab4ab0e728a 100644
--- a/src/hotspot/share/cds/cdsProtectionDomain.hpp
+++ b/src/hotspot/share/cds/cdsProtectionDomain.hpp
@@ -80,6 +80,7 @@ class CDSProtectionDomain : AllStatic {
static Handle create_jar_manifest(const char* man, size_t size, TRAPS);
static Handle get_shared_jar_manifest(int shared_path_index, TRAPS);
static Handle get_shared_jar_url(int shared_path_index, TRAPS);
+ static oop to_file_URL(const char* path, Handle url_h, TRAPS);
static Handle get_protection_domain_from_classloader(Handle class_loader,
Handle url, TRAPS);
static Handle get_shared_protection_domain(Handle class_loader,
diff --git a/src/hotspot/share/cds/filemap.cpp b/src/hotspot/share/cds/filemap.cpp
index 78613ae4b36c4..33a81a81da058 100644
--- a/src/hotspot/share/cds/filemap.cpp
+++ b/src/hotspot/share/cds/filemap.cpp
@@ -781,12 +781,12 @@ bool FileMapInfo::check_paths(int shared_path_start_idx, int num_paths, Growable
assert(strlen(rp_array->at(i)) > (size_t)runtime_prefix_len, "sanity");
const char* runtime_path = rp_array->at(i) + runtime_prefix_len;
if (!os::same_files(dumptime_path, runtime_path)) {
- return true;
+ return false;
}
i++;
j++;
}
- return false;
+ return true;
}
bool FileMapInfo::validate_boot_class_paths() {
@@ -810,7 +810,7 @@ bool FileMapInfo::validate_boot_class_paths() {
char* rp = skip_first_path_entry(runtime_boot_path);
assert(shared_path(0)->is_modules_image(), "first shared_path must be the modules image");
int dp_len = header()->app_class_paths_start_index() - 1; // ignore the first path to the module image
- bool mismatch = false;
+ bool match = true;
bool relaxed_check = !header()->has_platform_or_app_classes();
if (dp_len == 0 && rp == nullptr) {
@@ -823,7 +823,7 @@ bool FileMapInfo::validate_boot_class_paths() {
if (check_paths_existence(rp)) {
// If a path exists in the runtime boot paths, it is considered a mismatch
// since there's no boot path specified during dump time.
- mismatch = true;
+ match = false;
}
}
} else if (dp_len > 0 && rp != nullptr) {
@@ -840,16 +840,16 @@ bool FileMapInfo::validate_boot_class_paths() {
// check the full runtime boot path, must match with dump time
num = rp_len;
}
- mismatch = check_paths(1, num, rp_array, 0, 0);
+ match = check_paths(1, num, rp_array, 0, 0);
} else {
// create_path_array() ignores non-existing paths. Although the dump time and runtime boot classpath lengths
// are the same initially, after the call to create_path_array(), the runtime boot classpath length could become
// shorter. We consider boot classpath mismatch in this case.
- mismatch = true;
+ match = false;
}
}
- if (mismatch) {
+ if (!match) {
// The paths are different
return classpath_failure("[BOOT classpath mismatch, actual =", runtime_boot_path);
}
@@ -860,7 +860,7 @@ bool FileMapInfo::validate_app_class_paths(int shared_app_paths_len) {
const char *appcp = Arguments::get_appclasspath();
assert(appcp != nullptr, "null app classpath");
int rp_len = num_paths(appcp);
- bool mismatch = false;
+ bool match = false;
if (rp_len < shared_app_paths_len) {
return classpath_failure("Run time APP classpath is shorter than the one at dump time: ", appcp);
}
@@ -889,8 +889,8 @@ bool FileMapInfo::validate_app_class_paths(int shared_app_paths_len) {
// run 2: -cp x.jar:NE4:b.jar -> x.jar:b.jar -> mismatched
int j = header()->app_class_paths_start_index();
- mismatch = check_paths(j, shared_app_paths_len, rp_array, 0, 0);
- if (mismatch) {
+ match = check_paths(j, shared_app_paths_len, rp_array, 0, 0);
+ if (!match) {
// To facilitate app deployment, we allow the JAR files to be moved *together* to
// a different location, as long as they are still stored under the same directory
// structure. E.g., the following is OK.
@@ -901,10 +901,10 @@ bool FileMapInfo::validate_app_class_paths(int shared_app_paths_len) {
if (dumptime_prefix_len != 0 || runtime_prefix_len != 0) {
log_info(class, path)("LCP length for app classpath (dumptime: %u, runtime: %u)",
dumptime_prefix_len, runtime_prefix_len);
- mismatch = check_paths(j, shared_app_paths_len, rp_array,
+ match = check_paths(j, shared_app_paths_len, rp_array,
dumptime_prefix_len, runtime_prefix_len);
}
- if (mismatch) {
+ if (!match) {
return classpath_failure("[APP classpath mismatch, actual: -Djava.class.path=", appcp);
}
}
@@ -926,15 +926,35 @@ void FileMapInfo::log_paths(const char* msg, int start_idx, int end_idx) {
}
}
+void FileMapInfo::extract_module_paths(const char* runtime_path, GrowableArray* module_paths) {
+ GrowableArray* path_array = create_path_array(runtime_path);
+ int num_paths = path_array->length();
+ for (int i = 0; i < num_paths; i++) {
+ const char* name = path_array->at(i);
+ ClassLoaderExt::extract_jar_files_from_path(name, module_paths);
+ }
+ // module paths are stored in sorted order in the CDS archive.
+ module_paths->sort(ClassLoaderExt::compare_module_names);
+}
+
bool FileMapInfo::check_module_paths() {
- const char* rp = Arguments::get_property("jdk.module.path");
- int num_paths = CDSConfig::num_archives(rp);
- if (num_paths != header()->num_module_paths()) {
+ const char* runtime_path = Arguments::get_property("jdk.module.path");
+ int archived_num_module_paths = header()->num_module_paths();
+ if (runtime_path == nullptr && archived_num_module_paths == 0) {
+ return true;
+ }
+ if ((runtime_path == nullptr && archived_num_module_paths > 0) ||
+ (runtime_path != nullptr && archived_num_module_paths == 0)) {
return false;
}
ResourceMark rm;
- GrowableArray* rp_array = create_path_array(rp);
- return check_paths(header()->app_module_paths_start_index(), num_paths, rp_array, 0, 0);
+ GrowableArray* module_paths = new GrowableArray(3);
+ extract_module_paths(runtime_path, module_paths);
+ int num_paths = module_paths->length();
+ if (num_paths != archived_num_module_paths) {
+ return false;
+ }
+ return check_paths(header()->app_module_paths_start_index(), num_paths, module_paths, 0, 0);
}
bool FileMapInfo::validate_shared_path_table() {
@@ -944,6 +964,16 @@ bool FileMapInfo::validate_shared_path_table() {
// Load the shared path table info from the archive header
_shared_path_table = header()->shared_path_table();
+
+ bool matched_module_paths = true;
+ if (CDSConfig::is_dumping_dynamic_archive() || header()->has_full_module_graph()) {
+ matched_module_paths = check_module_paths();
+ }
+ if (header()->has_full_module_graph() && !matched_module_paths) {
+ CDSConfig::stop_using_optimized_module_handling();
+ log_info(cds)("optimized module handling: disabled because of mismatched module paths");
+ }
+
if (CDSConfig::is_dumping_dynamic_archive()) {
// Only support dynamic dumping with the usage of the default CDS archive
// or a simple base archive.
@@ -959,7 +989,7 @@ bool FileMapInfo::validate_shared_path_table() {
"Dynamic archiving is disabled because base layer archive has appended boot classpath");
}
if (header()->num_module_paths() > 0) {
- if (!check_module_paths()) {
+ if (!matched_module_paths) {
CDSConfig::disable_dumping_dynamic_archive();
log_warning(cds)(
"Dynamic archiving is disabled because base layer archive has a different module path");
@@ -2024,8 +2054,7 @@ void FileMapInfo::map_or_load_heap_region() {
success = ArchiveHeapLoader::load_heap_region(this);
} else {
if (!UseCompressedOops && !ArchiveHeapLoader::can_map()) {
- // TODO - remove implicit knowledge of G1
- log_info(cds)("Cannot use CDS heap data. UseG1GC is required for -XX:-UseCompressedOops");
+ log_info(cds)("Cannot use CDS heap data. Selected GC not compatible -XX:-UseCompressedOops");
} else {
log_info(cds)("Cannot use CDS heap data. UseEpsilonGC, UseG1GC, UseSerialGC, UseParallelGC, or UseShenandoahGC are required.");
}
@@ -2105,7 +2134,7 @@ address FileMapInfo::heap_region_requested_address() {
assert(CDSConfig::is_using_archive(), "runtime only");
FileMapRegion* r = region_at(MetaspaceShared::hp);
assert(is_aligned(r->mapping_offset(), sizeof(HeapWord)), "must be");
- assert(ArchiveHeapLoader::can_map(), "cannot be used by ArchiveHeapLoader::can_load() mode");
+ assert(ArchiveHeapLoader::can_use(), "GC must support mapping or loading");
if (UseCompressedOops) {
// We can avoid relocation if each region's offset from the runtime CompressedOops::base()
// is the same as its offset from the CompressedOops::base() during dumptime.
diff --git a/src/hotspot/share/cds/filemap.hpp b/src/hotspot/share/cds/filemap.hpp
index 1bf2510a3351c..6650f52440881 100644
--- a/src/hotspot/share/cds/filemap.hpp
+++ b/src/hotspot/share/cds/filemap.hpp
@@ -271,6 +271,7 @@ class FileMapHeader: private CDSFileMapHeaderBase {
bool compressed_oops() const { return _compressed_oops; }
bool compressed_class_pointers() const { return _compressed_class_ptrs; }
HeapRootSegments heap_root_segments() const { return _heap_root_segments; }
+ bool has_full_module_graph() const { return _has_full_module_graph; }
size_t heap_oopmap_start_pos() const { return _heap_oopmap_start_pos; }
size_t heap_ptrmap_start_pos() const { return _heap_ptrmap_start_pos; }
size_t rw_ptrmap_start_pos() const { return _rw_ptrmap_start_pos; }
@@ -554,6 +555,7 @@ class FileMapInfo : public CHeapObj {
GrowableArray* rp_array,
unsigned int dumptime_prefix_len,
unsigned int runtime_prefix_len) NOT_CDS_RETURN_(false);
+ void extract_module_paths(const char* runtime_path, GrowableArray* module_paths);
bool validate_boot_class_paths() NOT_CDS_RETURN_(false);
bool validate_app_class_paths(int shared_app_paths_len) NOT_CDS_RETURN_(false);
bool map_heap_region_impl() NOT_CDS_JAVA_HEAP_RETURN_(false);
diff --git a/src/hotspot/share/cds/heapShared.cpp b/src/hotspot/share/cds/heapShared.cpp
index 1fddcb0d81f8a..22040448770d5 100644
--- a/src/hotspot/share/cds/heapShared.cpp
+++ b/src/hotspot/share/cds/heapShared.cpp
@@ -33,6 +33,7 @@
#include "cds/heapShared.hpp"
#include "cds/metaspaceShared.hpp"
#include "classfile/classLoaderData.hpp"
+#include "classfile/classLoaderExt.hpp"
#include "classfile/javaClasses.inline.hpp"
#include "classfile/modules.hpp"
#include "classfile/stringTable.hpp"
@@ -55,6 +56,7 @@
#include "oops/oop.inline.hpp"
#include "oops/typeArrayOop.inline.hpp"
#include "prims/jvmtiExport.hpp"
+#include "runtime/arguments.hpp"
#include "runtime/fieldDescriptor.inline.hpp"
#include "runtime/init.hpp"
#include "runtime/javaCalls.hpp"
@@ -134,8 +136,7 @@ static ArchivableStaticFieldInfo fmg_archive_subgraph_entry_fields[] = {
KlassSubGraphInfo* HeapShared::_default_subgraph_info;
GrowableArrayCHeap* HeapShared::_pending_roots = nullptr;
GrowableArrayCHeap* HeapShared::_root_segments;
-int HeapShared::_root_segment_max_size_shift;
-int HeapShared::_root_segment_max_size_mask;
+int HeapShared::_root_segment_max_size_elems;
OopHandle HeapShared::_scratch_basic_type_mirrors[T_VOID+1];
MetaspaceObjToOopHandleTable* HeapShared::_scratch_java_mirror_table = nullptr;
MetaspaceObjToOopHandleTable* HeapShared::_scratch_references_table = nullptr;
@@ -242,15 +243,29 @@ objArrayOop HeapShared::root_segment(int segment_idx) {
return segment;
}
+void HeapShared::get_segment_indexes(int idx, int& seg_idx, int& int_idx) {
+ assert(_root_segment_max_size_elems > 0, "sanity");
+
+ // Try to avoid divisions for the common case.
+ if (idx < _root_segment_max_size_elems) {
+ seg_idx = 0;
+ int_idx = idx;
+ } else {
+ seg_idx = idx / _root_segment_max_size_elems;
+ int_idx = idx % _root_segment_max_size_elems;
+ }
+
+ assert(idx == seg_idx * _root_segment_max_size_elems + int_idx,
+ "sanity: %d index maps to %d segment and %d internal", idx, seg_idx, int_idx);
+}
+
// Returns an objArray that contains all the roots of the archived objects
oop HeapShared::get_root(int index, bool clear) {
- assert(_root_segment_max_size_shift > 0, "sanity");
- assert(_root_segment_max_size_mask > 0, "sanity");
assert(index >= 0, "sanity");
assert(!CDSConfig::is_dumping_heap() && CDSConfig::is_using_archive(), "runtime only");
assert(!_root_segments->is_empty(), "must have loaded shared heap");
- int seg_idx = index >> _root_segment_max_size_shift;
- int int_idx = index & _root_segment_max_size_mask;
+ int seg_idx, int_idx;
+ get_segment_indexes(index, seg_idx, int_idx);
oop result = root_segment(seg_idx)->obj_at(int_idx);
if (clear) {
clear_root(index);
@@ -262,10 +277,8 @@ void HeapShared::clear_root(int index) {
assert(index >= 0, "sanity");
assert(CDSConfig::is_using_archive(), "must be");
if (ArchiveHeapLoader::is_in_use()) {
- assert(_root_segment_max_size_shift > 0, "sanity");
- assert(_root_segment_max_size_mask > 0, "sanity");
- int seg_idx = index >> _root_segment_max_size_shift;
- int int_idx = index & _root_segment_max_size_mask;
+ int seg_idx, int_idx;
+ get_segment_indexes(index, seg_idx, int_idx);
if (log_is_enabled(Debug, cds, heap)) {
oop old = root_segment(seg_idx)->obj_at(int_idx);
log_debug(cds, heap)("Clearing root %d: was " PTR_FORMAT, index, p2i(old));
@@ -373,6 +386,13 @@ void HeapShared::set_scratch_java_mirror(Klass* k, oop mirror) {
}
void HeapShared::remove_scratch_objects(Klass* k) {
+ // Klass is being deallocated. Java mirror can still be alive, and it should not
+ // point to dead klass. We need to break the link from mirror to the Klass.
+ // See how InstanceKlass::deallocate_contents does it for normal mirrors.
+ oop mirror = _scratch_java_mirror_table->get_oop(k);
+ if (mirror != nullptr) {
+ java_lang_Class::set_klass(mirror, nullptr);
+ }
_scratch_java_mirror_table->remove_oop(k);
if (k->is_instance_klass()) {
_scratch_references_table->remove(InstanceKlass::cast(k)->constants());
@@ -785,10 +805,8 @@ void HeapShared::add_root_segment(objArrayOop segment_oop) {
_root_segments->push(OopHandle(Universe::vm_global(), segment_oop));
}
-void HeapShared::init_root_segment_sizes(int max_size) {
- assert(is_power_of_2(max_size), "must be");
- _root_segment_max_size_shift = log2i_exact(max_size);
- _root_segment_max_size_mask = max_size - 1;
+void HeapShared::init_root_segment_sizes(int max_size_elems) {
+ _root_segment_max_size_elems = max_size_elems;
}
void HeapShared::serialize_tables(SerializeClosure* soc) {
@@ -875,6 +893,17 @@ void HeapShared::initialize_from_archived_subgraph(JavaThread* current, Klass* k
return; // nothing to do
}
+ if (k->name()->equals("jdk/internal/module/ArchivedModuleGraph") &&
+ !CDSConfig::is_using_optimized_module_handling() &&
+ // archive was created with --module-path
+ ClassLoaderExt::num_module_paths() > 0) {
+ // ArchivedModuleGraph was created with a --module-path that's different than the runtime --module-path.
+ // Thus, it might contain references to modules that do not exist at runtime. We cannot use it.
+ log_info(cds, heap)("Skip initializing ArchivedModuleGraph subgraph: is_using_optimized_module_handling=%s num_module_paths=%d",
+ BOOL_TO_STR(CDSConfig::is_using_optimized_module_handling()), ClassLoaderExt::num_module_paths());
+ return;
+ }
+
ExceptionMark em(THREAD);
const ArchivedKlassSubGraphInfoRecord* record =
resolve_or_init_classes_for_subgraph_of(k, /*do_init=*/true, THREAD);
@@ -1123,6 +1152,13 @@ bool HeapShared::archive_reachable_objects_from(int level,
// these objects that are referenced (directly or indirectly) by static fields.
ResourceMark rm;
log_error(cds, heap)("Cannot archive object of class %s", orig_obj->klass()->external_name());
+ if (log_is_enabled(Trace, cds, heap)) {
+ WalkOopAndArchiveClosure* walker = WalkOopAndArchiveClosure::current();
+ if (walker != nullptr) {
+ LogStream ls(Log(cds, heap)::trace());
+ CDSHeapVerifier::trace_to_root(&ls, walker->referencing_obj());
+ }
+ }
MetaspaceShared::unrecoverable_writing_error();
}
diff --git a/src/hotspot/share/cds/heapShared.hpp b/src/hotspot/share/cds/heapShared.hpp
index 01610ebe64e15..317b791f5d3a3 100644
--- a/src/hotspot/share/cds/heapShared.hpp
+++ b/src/hotspot/share/cds/heapShared.hpp
@@ -143,13 +143,18 @@ class HeapShared: AllStatic {
friend class VerifySharedOopClosure;
public:
- // Can this VM write a heap region into the CDS archive? Currently only {G1|Parallel|Serial}+compressed_cp
+ // Can this VM write a heap region into the CDS archive?
static bool can_write() {
CDS_JAVA_HEAP_ONLY(
if (_disable_writing) {
return false;
}
- return (UseG1GC || UseParallelGC || UseSerialGC) && UseCompressedClassPointers;
+ // Need compressed class pointers for heap region dump.
+ if (!UseCompressedClassPointers) {
+ return false;
+ }
+ // Almost all GCs support heap region dump, except ZGC (so far).
+ return !UseZGC;
)
NOT_CDS_JAVA_HEAP(return false;)
}
@@ -291,8 +296,7 @@ class HeapShared: AllStatic {
static GrowableArrayCHeap* _pending_roots;
static GrowableArrayCHeap* _root_segments;
- static int _root_segment_max_size_shift;
- static int _root_segment_max_size_mask;
+ static int _root_segment_max_size_elems;
static OopHandle _scratch_basic_type_mirrors[T_VOID+1];
static MetaspaceObjToOopHandleTable* _scratch_java_mirror_table;
static MetaspaceObjToOopHandleTable* _scratch_references_table;
@@ -407,6 +411,8 @@ class HeapShared: AllStatic {
// Run-time only
static void clear_root(int index);
+ static void get_segment_indexes(int index, int& segment_index, int& internal_index);
+
static void setup_test_class(const char* test_class_name) PRODUCT_RETURN;
#endif // INCLUDE_CDS_JAVA_HEAP
@@ -425,7 +431,7 @@ class HeapShared: AllStatic {
static void init_for_dumping(TRAPS) NOT_CDS_JAVA_HEAP_RETURN;
static void write_subgraph_info_table() NOT_CDS_JAVA_HEAP_RETURN;
static void add_root_segment(objArrayOop segment_oop) NOT_CDS_JAVA_HEAP_RETURN;
- static void init_root_segment_sizes(int max_size) NOT_CDS_JAVA_HEAP_RETURN;
+ static void init_root_segment_sizes(int max_size_elems) NOT_CDS_JAVA_HEAP_RETURN;
static void serialize_tables(SerializeClosure* soc) NOT_CDS_JAVA_HEAP_RETURN;
#ifndef PRODUCT
diff --git a/src/hotspot/share/cds/metaspaceShared.cpp b/src/hotspot/share/cds/metaspaceShared.cpp
index c66398cefac5a..2a43fd2dd4f2a 100644
--- a/src/hotspot/share/cds/metaspaceShared.cpp
+++ b/src/hotspot/share/cds/metaspaceShared.cpp
@@ -77,6 +77,7 @@
#include "runtime/globals.hpp"
#include "runtime/globals_extension.hpp"
#include "runtime/handles.inline.hpp"
+#include "runtime/javaCalls.hpp"
#include "runtime/os.inline.hpp"
#include "runtime/safepointVerifiers.hpp"
#include "runtime/sharedRuntime.hpp"
@@ -300,6 +301,7 @@ void MetaspaceShared::post_initialize(TRAPS) {
}
ClassLoaderExt::init_paths_start_index(info->app_class_paths_start_index());
ClassLoaderExt::init_app_module_paths_start_index(info->app_module_paths_start_index());
+ ClassLoaderExt::init_num_module_paths(info->header()->num_module_paths());
}
}
}
@@ -401,6 +403,7 @@ void MetaspaceShared::serialize(SerializeClosure* soc) {
soc->do_tag(--tag);
CDS_JAVA_HEAP_ONLY(Modules::serialize(soc);)
+ CDS_JAVA_HEAP_ONLY(Modules::serialize_addmods_names(soc);)
CDS_JAVA_HEAP_ONLY(ClassLoaderDataShared::serialize(soc);)
LambdaFormInvokers::serialize(soc);
@@ -500,6 +503,8 @@ char* VM_PopulateDumpSharedSpace::dump_read_only_tables() {
LambdaFormInvokers::dump_static_archive_invokers();
// Write module name into archive
CDS_JAVA_HEAP_ONLY(Modules::dump_main_module_name();)
+ // Write module names from --add-modules into archive
+ CDS_JAVA_HEAP_ONLY(Modules::dump_addmods_names();)
// Write the other data to the output array.
DumpRegion* ro_region = ArchiveBuilder::current()->ro_region();
char* start = ro_region->top();
@@ -749,12 +754,21 @@ void MetaspaceShared::preload_classes(TRAPS) {
}
}
- // Exercise the manifest processing code to ensure classes used by CDS at runtime
- // are always archived
+ // Some classes are used at CDS runtime but are not loaded, and therefore archived, at
+ // dumptime. We can perform dummmy calls to these classes at dumptime to ensure they
+ // are archived.
+ exercise_runtime_cds_code(CHECK);
+
+ log_info(cds)("Loading classes to share: done.");
+}
+
+void MetaspaceShared::exercise_runtime_cds_code(TRAPS) {
+ // Exercise the manifest processing code
const char* dummy = "Manifest-Version: 1.0\n";
CDSProtectionDomain::create_jar_manifest(dummy, strlen(dummy), CHECK);
- log_info(cds)("Loading classes to share: done.");
+ // Exercise FileSystem and URL code
+ CDSProtectionDomain::to_file_URL("dummy.jar", Handle(), CHECK);
}
void MetaspaceShared::preload_and_dump_impl(StaticArchiveBuilder& builder, TRAPS) {
@@ -791,6 +805,9 @@ void MetaspaceShared::preload_and_dump_impl(StaticArchiveBuilder& builder, TRAPS
// Do this at the very end, when no Java code will be executed. Otherwise
// some new strings may be added to the intern table.
StringTable::allocate_shared_strings_array(CHECK);
+ } else {
+ log_info(cds)("Not dumping heap, reset CDSConfig::_is_using_optimized_module_handling");
+ CDSConfig::stop_using_optimized_module_handling();
}
#endif
diff --git a/src/hotspot/share/cds/metaspaceShared.hpp b/src/hotspot/share/cds/metaspaceShared.hpp
index f26af21676a83..ecc158cc3efcf 100644
--- a/src/hotspot/share/cds/metaspaceShared.hpp
+++ b/src/hotspot/share/cds/metaspaceShared.hpp
@@ -75,6 +75,7 @@ class MetaspaceShared : AllStatic {
#endif
private:
+ static void exercise_runtime_cds_code(TRAPS) NOT_CDS_RETURN;
static void preload_and_dump_impl(StaticArchiveBuilder& builder, TRAPS) NOT_CDS_RETURN;
static void preload_classes(TRAPS) NOT_CDS_RETURN;
diff --git a/src/hotspot/share/ci/ciEnv.cpp b/src/hotspot/share/ci/ciEnv.cpp
index 155ce032400e8..6bf8f44553d6e 100644
--- a/src/hotspot/share/ci/ciEnv.cpp
+++ b/src/hotspot/share/ci/ciEnv.cpp
@@ -1597,6 +1597,8 @@ void ciEnv::dump_replay_data_helper(outputStream* out) {
NoSafepointVerifier no_safepoint;
ResourceMark rm;
+ assert(this->task() != nullptr, "task must not be null");
+
dump_replay_data_version(out);
#if INCLUDE_JVMTI
out->print_cr("JvmtiExport can_access_local_variables %d", _jvmti_can_access_local_variables);
@@ -1617,9 +1619,7 @@ void ciEnv::dump_replay_data_helper(outputStream* out) {
objects->at(i)->dump_replay_data(out);
}
- if (this->task() != nullptr) {
- dump_compile_data(out);
- }
+ dump_compile_data(out);
out->flush();
}
diff --git a/src/hotspot/share/ci/ciMethod.cpp b/src/hotspot/share/ci/ciMethod.cpp
index 94b405cdbfacd..a74a812c6a23b 100644
--- a/src/hotspot/share/ci/ciMethod.cpp
+++ b/src/hotspot/share/ci/ciMethod.cpp
@@ -1249,7 +1249,6 @@ bool ciMethod::has_jsrs () const { FETCH_FLAG_FROM_VM(has_jsrs);
bool ciMethod::is_getter () const { FETCH_FLAG_FROM_VM(is_getter); }
bool ciMethod::is_setter () const { FETCH_FLAG_FROM_VM(is_setter); }
bool ciMethod::is_accessor () const { FETCH_FLAG_FROM_VM(is_accessor); }
-bool ciMethod::is_initializer () const { FETCH_FLAG_FROM_VM(is_initializer); }
bool ciMethod::is_empty () const { FETCH_FLAG_FROM_VM(is_empty_method); }
bool ciMethod::is_boxing_method() const {
diff --git a/src/hotspot/share/ci/ciMethod.hpp b/src/hotspot/share/ci/ciMethod.hpp
index 5cb63204d0b72..cc524930192cd 100644
--- a/src/hotspot/share/ci/ciMethod.hpp
+++ b/src/hotspot/share/ci/ciMethod.hpp
@@ -352,7 +352,6 @@ class ciMethod : public ciMetadata {
bool is_getter () const;
bool is_setter () const;
bool is_accessor () const;
- bool is_initializer () const;
bool is_empty () const;
bool can_be_statically_bound() const { return _can_be_statically_bound; }
bool has_reserved_stack_access() const { return _has_reserved_stack_access; }
diff --git a/src/hotspot/share/ci/ciTypeFlow.cpp b/src/hotspot/share/ci/ciTypeFlow.cpp
index 520eeeb10d568..2d80a18f16bc6 100644
--- a/src/hotspot/share/ci/ciTypeFlow.cpp
+++ b/src/hotspot/share/ci/ciTypeFlow.cpp
@@ -720,12 +720,18 @@ void ciTypeFlow::StateVector::do_jsr(ciBytecodeStream* str) {
void ciTypeFlow::StateVector::do_ldc(ciBytecodeStream* str) {
if (str->is_in_error()) {
trap(str, nullptr, Deoptimization::make_trap_request(Deoptimization::Reason_unhandled,
- Deoptimization::Action_none));
+ Deoptimization::Action_none));
return;
}
ciConstant con = str->get_constant();
if (con.is_valid()) {
int cp_index = str->get_constant_pool_index();
+ if (!con.is_loaded()) {
+ trap(str, nullptr, Deoptimization::make_trap_request(Deoptimization::Reason_unloaded,
+ Deoptimization::Action_reinterpret,
+ cp_index));
+ return;
+ }
BasicType basic_type = str->get_basic_type_for_constant_at(cp_index);
if (is_reference_type(basic_type)) {
ciObject* obj = con.as_object();
@@ -2207,11 +2213,10 @@ bool ciTypeFlow::can_trap(ciBytecodeStream& str) {
if (!Bytecodes::can_trap(str.cur_bc())) return false;
switch (str.cur_bc()) {
- // %%% FIXME: ldc of Class can generate an exception
case Bytecodes::_ldc:
case Bytecodes::_ldc_w:
case Bytecodes::_ldc2_w:
- return str.is_in_error();
+ return str.is_in_error() || !str.get_constant().is_loaded();
case Bytecodes::_aload_0:
// These bytecodes can trap for rewriting. We need to assume that
diff --git a/src/hotspot/share/classfile/classFileParser.cpp b/src/hotspot/share/classfile/classFileParser.cpp
index c8e95149b7c1a..93dd0af65e731 100644
--- a/src/hotspot/share/classfile/classFileParser.cpp
+++ b/src/hotspot/share/classfile/classFileParser.cpp
@@ -411,23 +411,6 @@ static inline Symbol* check_symbol_at(const ConstantPool* cp, int index) {
return nullptr;
}
-#ifdef ASSERT
-PRAGMA_DIAG_PUSH
-PRAGMA_FORMAT_NONLITERAL_IGNORED
-void ClassFileParser::report_assert_property_failure(const char* msg, TRAPS) const {
- ResourceMark rm(THREAD);
- fatal(msg, _class_name->as_C_string());
-}
-
-void ClassFileParser::report_assert_property_failure(const char* msg,
- int index,
- TRAPS) const {
- ResourceMark rm(THREAD);
- fatal(msg, index, _class_name->as_C_string());
-}
-PRAGMA_DIAG_POP
-#endif
-
void ClassFileParser::parse_constant_pool(const ClassFileStream* const stream,
ConstantPool* const cp,
const int length,
@@ -462,10 +445,10 @@ void ClassFileParser::parse_constant_pool(const ClassFileStream* const stream,
if (!_need_verify) break;
const int klass_ref_index = cp->uncached_klass_ref_index_at(index);
const int name_and_type_ref_index = cp->uncached_name_and_type_ref_index_at(index);
- check_property(valid_klass_reference_at(klass_ref_index),
+ guarantee_property(valid_klass_reference_at(klass_ref_index),
"Invalid constant pool index %u in class file %s",
klass_ref_index, CHECK);
- check_property(valid_cp_range(name_and_type_ref_index, length) &&
+ guarantee_property(valid_cp_range(name_and_type_ref_index, length) &&
cp->tag_at(name_and_type_ref_index).is_name_and_type(),
"Invalid constant pool index %u in class file %s",
name_and_type_ref_index, CHECK);
@@ -482,7 +465,7 @@ void ClassFileParser::parse_constant_pool(const ClassFileStream* const stream,
case JVM_CONSTANT_Long:
case JVM_CONSTANT_Double: {
index++;
- check_property(
+ guarantee_property(
(index < length && cp->tag_at(index).is_invalid()),
"Improper constant pool long/double index %u in class file %s",
index, CHECK);
@@ -492,10 +475,10 @@ void ClassFileParser::parse_constant_pool(const ClassFileStream* const stream,
if (!_need_verify) break;
const int name_ref_index = cp->name_ref_index_at(index);
const int signature_ref_index = cp->signature_ref_index_at(index);
- check_property(valid_symbol_at(name_ref_index),
+ guarantee_property(valid_symbol_at(name_ref_index),
"Invalid constant pool index %u in class file %s",
name_ref_index, CHECK);
- check_property(valid_symbol_at(signature_ref_index),
+ guarantee_property(valid_symbol_at(signature_ref_index),
"Invalid constant pool index %u in class file %s",
signature_ref_index, CHECK);
break;
@@ -509,7 +492,7 @@ void ClassFileParser::parse_constant_pool(const ClassFileStream* const stream,
}
case JVM_CONSTANT_ClassIndex: {
const int class_index = cp->klass_index_at(index);
- check_property(valid_symbol_at(class_index),
+ guarantee_property(valid_symbol_at(class_index),
"Invalid constant pool index %u in class file %s",
class_index, CHECK);
cp->unresolved_klass_at_put(index, class_index, num_klasses++);
@@ -517,7 +500,7 @@ void ClassFileParser::parse_constant_pool(const ClassFileStream* const stream,
}
case JVM_CONSTANT_StringIndex: {
const int string_index = cp->string_index_at(index);
- check_property(valid_symbol_at(string_index),
+ guarantee_property(valid_symbol_at(string_index),
"Invalid constant pool index %u in class file %s",
string_index, CHECK);
Symbol* const sym = cp->symbol_at(string_index);
@@ -526,7 +509,7 @@ void ClassFileParser::parse_constant_pool(const ClassFileStream* const stream,
}
case JVM_CONSTANT_MethodHandle: {
const int ref_index = cp->method_handle_index_at(index);
- check_property(valid_cp_range(ref_index, length),
+ guarantee_property(valid_cp_range(ref_index, length),
"Invalid constant pool index %u in class file %s",
ref_index, CHECK);
const constantTag tag = cp->tag_at(ref_index);
@@ -537,7 +520,7 @@ void ClassFileParser::parse_constant_pool(const ClassFileStream* const stream,
case JVM_REF_getStatic:
case JVM_REF_putField:
case JVM_REF_putStatic: {
- check_property(
+ guarantee_property(
tag.is_field(),
"Invalid constant pool index %u in class file %s (not a field)",
ref_index, CHECK);
@@ -545,7 +528,7 @@ void ClassFileParser::parse_constant_pool(const ClassFileStream* const stream,
}
case JVM_REF_invokeVirtual:
case JVM_REF_newInvokeSpecial: {
- check_property(
+ guarantee_property(
tag.is_method(),
"Invalid constant pool index %u in class file %s (not a method)",
ref_index, CHECK);
@@ -553,7 +536,7 @@ void ClassFileParser::parse_constant_pool(const ClassFileStream* const stream,
}
case JVM_REF_invokeStatic:
case JVM_REF_invokeSpecial: {
- check_property(
+ guarantee_property(
tag.is_method() ||
((_major_version >= JAVA_8_VERSION) && tag.is_interface_method()),
"Invalid constant pool index %u in class file %s (not a method)",
@@ -561,7 +544,7 @@ void ClassFileParser::parse_constant_pool(const ClassFileStream* const stream,
break;
}
case JVM_REF_invokeInterface: {
- check_property(
+ guarantee_property(
tag.is_interface_method(),
"Invalid constant pool index %u in class file %s (not an interface method)",
ref_index, CHECK);
@@ -579,7 +562,7 @@ void ClassFileParser::parse_constant_pool(const ClassFileStream* const stream,
} // case MethodHandle
case JVM_CONSTANT_MethodType: {
const int ref_index = cp->method_type_index_at(index);
- check_property(valid_symbol_at(ref_index),
+ guarantee_property(valid_symbol_at(ref_index),
"Invalid constant pool index %u in class file %s",
ref_index, CHECK);
break;
@@ -588,7 +571,7 @@ void ClassFileParser::parse_constant_pool(const ClassFileStream* const stream,
const int name_and_type_ref_index =
cp->bootstrap_name_and_type_ref_index_at(index);
- check_property(valid_cp_range(name_and_type_ref_index, length) &&
+ guarantee_property(valid_cp_range(name_and_type_ref_index, length) &&
cp->tag_at(name_and_type_ref_index).is_name_and_type(),
"Invalid constant pool index %u in class file %s",
name_and_type_ref_index, CHECK);
@@ -603,7 +586,7 @@ void ClassFileParser::parse_constant_pool(const ClassFileStream* const stream,
const int name_and_type_ref_index =
cp->bootstrap_name_and_type_ref_index_at(index);
- check_property(valid_cp_range(name_and_type_ref_index, length) &&
+ guarantee_property(valid_cp_range(name_and_type_ref_index, length) &&
cp->tag_at(name_and_type_ref_index).is_name_and_type(),
"Invalid constant pool index %u in class file %s",
name_and_type_ref_index, CHECK);
@@ -821,7 +804,7 @@ void ClassFileParser::parse_interfaces(const ClassFileStream* const stream,
for (index = 0; index < itfs_len; index++) {
const u2 interface_index = stream->get_u2(CHECK);
Klass* interf;
- check_property(
+ guarantee_property(
valid_klass_reference_at(interface_index),
"Interface name has bad constant pool index %u in class file %s",
interface_index, CHECK);
@@ -943,6 +926,7 @@ class AnnotationCollector : public ResourceObj{
_method_ForceInline,
_method_DontInline,
_method_ChangesCurrentThread,
+ _method_JvmtiHideEvents,
_method_JvmtiMountTransition,
_method_InjectedProfile,
_method_LambdaForm_Compiled,
@@ -1255,10 +1239,10 @@ void ClassFileParser::parse_field_attributes(const ClassFileStream* const cfs,
cfs->guarantee_more(6, CHECK); // attribute_name_index, attribute_length
const u2 attribute_name_index = cfs->get_u2_fast();
const u4 attribute_length = cfs->get_u4_fast();
- check_property(valid_symbol_at(attribute_name_index),
- "Invalid field attribute index %u in class file %s",
- attribute_name_index,
- CHECK);
+ guarantee_property(valid_symbol_at(attribute_name_index),
+ "Invalid field attribute index %u in class file %s",
+ attribute_name_index,
+ CHECK);
const Symbol* const attribute_name = cp->symbol_at(attribute_name_index);
if (is_static && attribute_name == vmSymbols::tag_constant_value()) {
@@ -1267,7 +1251,7 @@ void ClassFileParser::parse_field_attributes(const ClassFileStream* const cfs,
classfile_parse_error("Duplicate ConstantValue attribute in class file %s", THREAD);
return;
}
- check_property(
+ guarantee_property(
attribute_length == 2,
"Invalid ConstantValue field attribute length %u in class file %s",
attribute_length, CHECK);
@@ -1413,14 +1397,14 @@ void ClassFileParser::parse_fields(const ClassFileStream* const cfs,
FieldInfo::FieldFlags fieldFlags(0);
const u2 name_index = cfs->get_u2_fast();
- check_property(valid_symbol_at(name_index),
+ guarantee_property(valid_symbol_at(name_index),
"Invalid constant pool index %u for field name in class file %s",
name_index, CHECK);
const Symbol* const name = cp->symbol_at(name_index);
verify_legal_field_name(name, CHECK);
const u2 signature_index = cfs->get_u2_fast();
- check_property(valid_symbol_at(signature_index),
+ guarantee_property(valid_symbol_at(signature_index),
"Invalid constant pool index %u for field signature in class file %s",
signature_index, CHECK);
const Symbol* const sig = cp->symbol_at(signature_index);
@@ -1598,7 +1582,7 @@ void ClassFileParser::parse_linenumber_table(u4 code_attribute_length,
const unsigned int length_in_bytes = num_entries * (sizeof(u2) * 2);
// Verify line number attribute and table length
- check_property(
+ guarantee_property(
code_attribute_length == sizeof(u2) + length_in_bytes,
"LineNumberTable attribute has wrong length in class file %s", CHECK);
@@ -1788,7 +1772,7 @@ const ClassFileParser::unsafe_u2* ClassFileParser::parse_checked_exceptions(cons
cfs->guarantee_more(2 * len, CHECK_NULL);
for (int i = 0; i < len; i++) {
checked_exception = cfs->get_u2_fast();
- check_property(
+ guarantee_property(
valid_klass_reference_at(checked_exception),
"Exception name has bad type at constant pool %u in class file %s",
checked_exception, CHECK_NULL);
@@ -1847,6 +1831,11 @@ AnnotationCollector::annotation_index(const ClassLoaderData* loader_data,
if (!privileged) break; // only allow in privileged code
return _method_ChangesCurrentThread;
}
+ case VM_SYMBOL_ENUM_NAME(jdk_internal_vm_annotation_JvmtiHideEvents_signature): {
+ if (_location != _in_method) break; // only allow for methods
+ if (!privileged) break; // only allow in privileged code
+ return _method_JvmtiHideEvents;
+ }
case VM_SYMBOL_ENUM_NAME(jdk_internal_vm_annotation_JvmtiMountTransition_signature): {
if (_location != _in_method) break; // only allow for methods
if (!privileged) break; // only allow in privileged code
@@ -1934,6 +1923,8 @@ void MethodAnnotationCollector::apply_to(const methodHandle& m) {
m->set_dont_inline();
if (has_annotation(_method_ChangesCurrentThread))
m->set_changes_current_thread();
+ if (has_annotation(_method_JvmtiHideEvents))
+ m->set_jvmti_hide_events();
if (has_annotation(_method_JvmtiMountTransition))
m->set_jvmti_mount_transition();
if (has_annotation(_method_InjectedProfile))
@@ -2137,7 +2128,7 @@ Method* ClassFileParser::parse_method(const ClassFileStream* const cfs,
int flags = cfs->get_u2_fast();
const u2 name_index = cfs->get_u2_fast();
const int cp_size = cp->length();
- check_property(
+ guarantee_property(
valid_symbol_at(name_index),
"Illegal constant pool index %u for method name in class file %s",
name_index, CHECK_NULL);
@@ -2235,7 +2226,7 @@ Method* ClassFileParser::parse_method(const ClassFileStream* const cfs,
cfs->guarantee_more(6, CHECK_NULL); // method_attribute_name_index, method_attribute_length
const u2 method_attribute_name_index = cfs->get_u2_fast();
const u4 method_attribute_length = cfs->get_u4_fast();
- check_property(
+ guarantee_property(
valid_symbol_at(method_attribute_name_index),
"Invalid method attribute name index %u in class file %s",
method_attribute_name_index, CHECK_NULL);
@@ -2310,10 +2301,10 @@ Method* ClassFileParser::parse_method(const ClassFileStream* const cfs,
calculated_attribute_length += code_attribute_length +
(unsigned)sizeof(code_attribute_name_index) +
(unsigned)sizeof(code_attribute_length);
- check_property(valid_symbol_at(code_attribute_name_index),
- "Invalid code attribute name index %u in class file %s",
- code_attribute_name_index,
- CHECK_NULL);
+ guarantee_property(valid_symbol_at(code_attribute_name_index),
+ "Invalid code attribute name index %u in class file %s",
+ code_attribute_name_index,
+ CHECK_NULL);
if (LoadLineNumberTables &&
cp->symbol_at(code_attribute_name_index) == vmSymbols::tag_line_number_table()) {
// Parse and compress line number table
@@ -2798,7 +2789,7 @@ u2 ClassFileParser::parse_generic_signature_attribute(const ClassFileStream* con
cfs->guarantee_more(2, CHECK_0); // generic_signature_index
const u2 generic_signature_index = cfs->get_u2_fast();
- check_property(
+ guarantee_property(
valid_symbol_at(generic_signature_index),
"Invalid Signature attribute at constant pool index %u in class file %s",
generic_signature_index, CHECK_0);
@@ -2812,7 +2803,7 @@ void ClassFileParser::parse_classfile_sourcefile_attribute(const ClassFileStream
cfs->guarantee_more(2, CHECK); // sourcefile_index
const u2 sourcefile_index = cfs->get_u2_fast();
- check_property(
+ guarantee_property(
valid_symbol_at(sourcefile_index),
"Invalid SourceFile attribute at constant pool index %u in class file %s",
sourcefile_index, CHECK);
@@ -2959,13 +2950,13 @@ u2 ClassFileParser::parse_classfile_inner_classes_attribute(const ClassFileStrea
for (int n = 0; n < length; n++) {
// Inner class index
const u2 inner_class_info_index = cfs->get_u2_fast();
- check_property(
+ guarantee_property(
valid_klass_reference_at(inner_class_info_index),
"inner_class_info_index %u has bad constant type in class file %s",
inner_class_info_index, CHECK_0);
// Outer class index
const u2 outer_class_info_index = cfs->get_u2_fast();
- check_property(
+ guarantee_property(
outer_class_info_index == 0 ||
valid_klass_reference_at(outer_class_info_index),
"outer_class_info_index %u has bad constant type in class file %s",
@@ -2979,7 +2970,7 @@ u2 ClassFileParser::parse_classfile_inner_classes_attribute(const ClassFileStrea
}
// Inner class name
const u2 inner_name_index = cfs->get_u2_fast();
- check_property(
+ guarantee_property(
inner_name_index == 0 || valid_symbol_at(inner_name_index),
"inner_name_index %u has bad constant type in class file %s",
inner_name_index, CHECK_0);
@@ -3055,7 +3046,7 @@ u2 ClassFileParser::parse_classfile_nest_members_attribute(const ClassFileStream
cfs->guarantee_more(2 * length, CHECK_0);
for (int n = 0; n < length; n++) {
const u2 class_info_index = cfs->get_u2_fast();
- check_property(
+ guarantee_property(
valid_klass_reference_at(class_info_index),
"Nest member class_info_index %u has bad constant type in class file %s",
class_info_index, CHECK_0);
@@ -3088,7 +3079,7 @@ u2 ClassFileParser::parse_classfile_permitted_subclasses_attribute(const ClassFi
cfs->guarantee_more(2 * length, CHECK_0);
for (int n = 0; n < length; n++) {
const u2 class_info_index = cfs->get_u2_fast();
- check_property(
+ guarantee_property(
valid_klass_reference_at(class_info_index),
"Permitted subclass class_info_index %u has bad constant type in class file %s",
class_info_index, CHECK_0);
@@ -3137,14 +3128,14 @@ u4 ClassFileParser::parse_classfile_record_attribute(const ClassFileStream* cons
cfs->guarantee_more(6, CHECK_0); // name_index, descriptor_index, attributes_count
const u2 name_index = cfs->get_u2_fast();
- check_property(valid_symbol_at(name_index),
+ guarantee_property(valid_symbol_at(name_index),
"Invalid constant pool index %u for name in Record attribute in class file %s",
name_index, CHECK_0);
const Symbol* const name = cp->symbol_at(name_index);
verify_legal_field_name(name, CHECK_0);
const u2 descriptor_index = cfs->get_u2_fast();
- check_property(valid_symbol_at(descriptor_index),
+ guarantee_property(valid_symbol_at(descriptor_index),
"Invalid constant pool index %u for descriptor in Record attribute in class file %s",
descriptor_index, CHECK_0);
const Symbol* const descr = cp->symbol_at(descriptor_index);
@@ -3167,7 +3158,7 @@ u4 ClassFileParser::parse_classfile_record_attribute(const ClassFileStream* cons
const u2 attribute_name_index = cfs->get_u2_fast();
const u4 attribute_length = cfs->get_u4_fast();
calculate_attr_size += 6;
- check_property(
+ guarantee_property(
valid_symbol_at(attribute_name_index),
"Invalid Record attribute name index %u in class file %s",
attribute_name_index, CHECK_0);
@@ -3265,7 +3256,7 @@ void ClassFileParser::parse_classfile_signature_attribute(const ClassFileStream*
assert(cfs != nullptr, "invariant");
const u2 signature_index = cfs->get_u2(CHECK);
- check_property(
+ guarantee_property(
valid_symbol_at(signature_index),
"Invalid constant pool index %u in Signature attribute in class file %s",
signature_index, CHECK);
@@ -3323,7 +3314,7 @@ void ClassFileParser::parse_classfile_bootstrap_methods_attribute(const ClassFil
cfs->guarantee_more(sizeof(u2) * 2, CHECK); // bsm, argc
const u2 bootstrap_method_index = cfs->get_u2_fast();
const u2 argument_count = cfs->get_u2_fast();
- check_property(
+ guarantee_property(
valid_cp_range(bootstrap_method_index, cp_size) &&
cp->tag_at(bootstrap_method_index).is_method_handle(),
"bootstrap_method_index %u has bad constant type in class file %s",
@@ -3340,7 +3331,7 @@ void ClassFileParser::parse_classfile_bootstrap_methods_attribute(const ClassFil
cfs->guarantee_more(sizeof(u2) * argument_count, CHECK); // argv[argc]
for (int j = 0; j < argument_count; j++) {
const u2 argument_index = cfs->get_u2_fast();
- check_property(
+ guarantee_property(
valid_cp_range(argument_index, cp_size) &&
cp->tag_at(argument_index).is_loadable_constant(),
"argument_index %u has bad constant type in class file %s",
@@ -3401,7 +3392,7 @@ void ClassFileParser::parse_classfile_attributes(const ClassFileStream* const cf
cfs->guarantee_more(6, CHECK); // attribute_name_index, attribute_length
const u2 attribute_name_index = cfs->get_u2_fast();
const u4 attribute_length = cfs->get_u4_fast();
- check_property(
+ guarantee_property(
valid_symbol_at(attribute_name_index),
"Attribute name has bad constant pool index %u in class file %s",
attribute_name_index, CHECK);
@@ -3513,7 +3504,7 @@ void ClassFileParser::parse_classfile_attributes(const ClassFileStream* const cf
return;
}
// Validate the constant pool indices and types
- check_property(valid_klass_reference_at(enclosing_method_class_index),
+ guarantee_property(valid_klass_reference_at(enclosing_method_class_index),
"Invalid or out-of-bounds class index in EnclosingMethod attribute in class file %s", CHECK);
if (enclosing_method_method_index != 0 &&
(!cp->is_within_bounds(enclosing_method_method_index) ||
@@ -3580,7 +3571,7 @@ void ClassFileParser::parse_classfile_attributes(const ClassFileStream* const cf
}
cfs->guarantee_more(2, CHECK);
u2 class_info_index = cfs->get_u2_fast();
- check_property(
+ guarantee_property(
valid_klass_reference_at(class_info_index),
"Nest-host class_info_index %u has bad constant type in class file %s",
class_info_index, CHECK);
@@ -3789,15 +3780,15 @@ const InstanceKlass* ClassFileParser::parse_super_class(ConstantPool* const cp,
const InstanceKlass* super_klass = nullptr;
if (super_class_index == 0) {
- check_property(_class_name == vmSymbols::java_lang_Object(),
- "Invalid superclass index %u in class file %s",
- super_class_index,
- CHECK_NULL);
+ guarantee_property(_class_name == vmSymbols::java_lang_Object(),
+ "Invalid superclass index %u in class file %s",
+ super_class_index,
+ CHECK_NULL);
} else {
- check_property(valid_klass_reference_at(super_class_index),
- "Invalid superclass index %u in class file %s",
- super_class_index,
- CHECK_NULL);
+ guarantee_property(valid_klass_reference_at(super_class_index),
+ "Invalid superclass index %u in class file %s",
+ super_class_index,
+ CHECK_NULL);
// The class name should be legal because it is checked when parsing constant pool.
// However, make sure it is not an array type.
bool is_array = false;
@@ -4076,26 +4067,6 @@ void ClassFileParser::check_super_class_access(const InstanceKlass* this_klass,
return;
}
- // If the loader is not the boot loader then throw an exception if its
- // superclass is in package jdk.internal.reflect and its loader is not a
- // special reflection class loader
- if (!this_klass->class_loader_data()->is_the_null_class_loader_data()) {
- PackageEntry* super_package = super->package();
- if (super_package != nullptr &&
- super_package->name()->fast_compare(vmSymbols::jdk_internal_reflect()) == 0 &&
- !java_lang_ClassLoader::is_reflection_class_loader(this_klass->class_loader())) {
- ResourceMark rm(THREAD);
- Exceptions::fthrow(
- THREAD_AND_LOCATION,
- vmSymbols::java_lang_IllegalAccessError(),
- "class %s loaded by %s cannot access jdk/internal/reflect superclass %s",
- this_klass->external_name(),
- this_klass->class_loader_data()->loader_name_and_id(),
- super->external_name());
- return;
- }
- }
-
Reflection::VerifyClassAccessResults vca_result =
Reflection::verify_class_access(this_klass, InstanceKlass::cast(super), false);
if (vca_result != Reflection::ACCESS_OK) {
@@ -5106,7 +5077,7 @@ void ClassFileParser::fill_instance_klass(InstanceKlass* ik,
// Set PackageEntry for this_klass
oop cl = ik->class_loader();
- Handle clh = Handle(THREAD, java_lang_ClassLoader::non_reflection_class_loader(cl));
+ Handle clh = Handle(THREAD, cl);
ClassLoaderData* cld = ClassLoaderData::class_loader_data_or_null(clh());
ik->set_package(cld, nullptr, CHECK);
@@ -5543,7 +5514,7 @@ void ClassFileParser::parse_stream(const ClassFileStream* const stream,
// This class and superclass
_this_class_index = stream->get_u2_fast();
- check_property(
+ guarantee_property(
valid_cp_range(_this_class_index, cp_size) &&
cp->tag_at(_this_class_index).is_unresolved_klass(),
"Invalid this class index %u in constant pool in class file %s",
@@ -5724,9 +5695,9 @@ void ClassFileParser::post_process_parsed_stream(const ClassFileStream* const st
assert(_loader_data != nullptr, "invariant");
if (_class_name == vmSymbols::java_lang_Object()) {
- check_property(_local_interfaces == Universe::the_empty_instance_klass_array(),
- "java.lang.Object cannot implement an interface in class file %s",
- CHECK);
+ guarantee_property(_local_interfaces == Universe::the_empty_instance_klass_array(),
+ "java.lang.Object cannot implement an interface in class file %s",
+ CHECK);
}
// We check super class after class file is parsed and format is checked
if (_super_class_index > 0 && nullptr == _super_klass) {
diff --git a/src/hotspot/share/classfile/classFileParser.hpp b/src/hotspot/share/classfile/classFileParser.hpp
index d32dd6d5f78b5..a8a388f8ded9e 100644
--- a/src/hotspot/share/classfile/classFileParser.hpp
+++ b/src/hotspot/share/classfile/classFileParser.hpp
@@ -378,44 +378,6 @@ class ClassFileParser {
if (!b) { classfile_parse_error(msg, THREAD); return; }
}
- void report_assert_property_failure(const char* msg, TRAPS) const PRODUCT_RETURN;
- void report_assert_property_failure(const char* msg, int index, TRAPS) const PRODUCT_RETURN;
-
- inline void assert_property(bool b, const char* msg, TRAPS) const {
-#ifdef ASSERT
- if (!b) {
- report_assert_property_failure(msg, THREAD);
- }
-#endif
- }
-
- inline void assert_property(bool b, const char* msg, int index, TRAPS) const {
-#ifdef ASSERT
- if (!b) {
- report_assert_property_failure(msg, index, THREAD);
- }
-#endif
- }
-
- inline void check_property(bool property,
- const char* msg,
- int index,
- TRAPS) const {
- if (_need_verify) {
- guarantee_property(property, msg, index, CHECK);
- } else {
- assert_property(property, msg, index, CHECK);
- }
- }
-
- inline void check_property(bool property, const char* msg, TRAPS) const {
- if (_need_verify) {
- guarantee_property(property, msg, CHECK);
- } else {
- assert_property(property, msg, CHECK);
- }
- }
-
inline void guarantee_property(bool b,
const char* msg,
int index,
diff --git a/src/hotspot/share/classfile/classLoader.cpp b/src/hotspot/share/classfile/classLoader.cpp
index 487d69a6f4fc9..9a68e2640443f 100644
--- a/src/hotspot/share/classfile/classLoader.cpp
+++ b/src/hotspot/share/classfile/classLoader.cpp
@@ -582,6 +582,8 @@ void ClassLoader::setup_module_search_path(JavaThread* current, const char* path
new_entry = create_class_path_entry(current, path, &st,
false /*is_boot_append */, false /* from_class_path_attr */);
if (new_entry != nullptr) {
+ // ClassLoaderExt::process_module_table() filters out non-jar entries before calling this function.
+ assert(new_entry->is_jar_file(), "module path entry %s is not a jar file", new_entry->name());
add_to_module_path_entries(path, new_entry);
}
}
diff --git a/src/hotspot/share/classfile/classLoaderData.cpp b/src/hotspot/share/classfile/classLoaderData.cpp
index fab012456d922..de0b16a907afb 100644
--- a/src/hotspot/share/classfile/classLoaderData.cpp
+++ b/src/hotspot/share/classfile/classLoaderData.cpp
@@ -644,8 +644,6 @@ Dictionary* ClassLoaderData::create_dictionary() {
int size;
if (_the_null_class_loader_data == nullptr) {
size = _boot_loader_dictionary_size;
- } else if (class_loader()->is_a(vmClasses::reflect_DelegatingClassLoader_klass())) {
- size = 1; // there's only one class in relection class loader and no initiated classes
} else if (is_system_class_loader_data()) {
size = _boot_loader_dictionary_size;
} else {
@@ -815,8 +813,6 @@ ClassLoaderMetaspace* ClassLoaderData::metaspace_non_null() {
metaspace = new ClassLoaderMetaspace(_metaspace_lock, Metaspace::BootMetaspaceType);
} else if (has_class_mirror_holder()) {
metaspace = new ClassLoaderMetaspace(_metaspace_lock, Metaspace::ClassMirrorHolderMetaspaceType);
- } else if (class_loader()->is_a(vmClasses::reflect_DelegatingClassLoader_klass())) {
- metaspace = new ClassLoaderMetaspace(_metaspace_lock, Metaspace::ReflectionMetaspaceType);
} else {
metaspace = new ClassLoaderMetaspace(_metaspace_lock, Metaspace::StandardMetaspaceType);
}
diff --git a/src/hotspot/share/classfile/classLoaderExt.cpp b/src/hotspot/share/classfile/classLoaderExt.cpp
index 997f9d76676fa..b9e420899c2d6 100644
--- a/src/hotspot/share/classfile/classLoaderExt.cpp
+++ b/src/hotspot/share/classfile/classLoaderExt.cpp
@@ -55,6 +55,7 @@
jshort ClassLoaderExt::_app_class_paths_start_index = ClassLoaderExt::max_classpath_index;
jshort ClassLoaderExt::_app_module_paths_start_index = ClassLoaderExt::max_classpath_index;
jshort ClassLoaderExt::_max_used_path_index = 0;
+int ClassLoaderExt::_num_module_paths = 0;
bool ClassLoaderExt::_has_app_classes = false;
bool ClassLoaderExt::_has_platform_classes = false;
bool ClassLoaderExt::_has_non_jar_in_classpath = false;
@@ -89,21 +90,25 @@ void ClassLoaderExt::setup_app_search_path(JavaThread* current) {
os::free(app_class_path);
}
+int ClassLoaderExt::compare_module_names(const char** p1, const char** p2) {
+ return strcmp(*p1, *p2);
+}
+
void ClassLoaderExt::process_module_table(JavaThread* current, ModuleEntryTable* met) {
ResourceMark rm(current);
- GrowableArray* module_paths = new GrowableArray(5);
+ GrowableArray* module_paths = new GrowableArray(5);
class ModulePathsGatherer : public ModuleClosure {
JavaThread* _current;
- GrowableArray* _module_paths;
+ GrowableArray* _module_paths;
public:
- ModulePathsGatherer(JavaThread* current, GrowableArray* module_paths) :
+ ModulePathsGatherer(JavaThread* current, GrowableArray* module_paths) :
_current(current), _module_paths(module_paths) {}
void do_module(ModuleEntry* m) {
char* uri = m->location()->as_C_string();
if (strncmp(uri, "file:", 5) == 0) {
char* path = ClassLoader::uri_to_path(uri);
- _module_paths->append(path);
+ extract_jar_files_from_path(path, _module_paths);
}
}
};
@@ -114,6 +119,10 @@ void ClassLoaderExt::process_module_table(JavaThread* current, ModuleEntryTable*
met->modules_do(&gatherer);
}
+ // Sort the module paths before storing into CDS archive for simpler
+ // checking at runtime.
+ module_paths->sort(compare_module_names);
+
for (int i = 0; i < module_paths->length(); i++) {
ClassLoader::setup_module_search_path(current, module_paths->at(i));
}
@@ -129,6 +138,38 @@ void ClassLoaderExt::setup_module_paths(JavaThread* current) {
process_module_table(current, met);
}
+bool ClassLoaderExt::has_jar_suffix(const char* filename) {
+ // In jdk.internal.module.ModulePath.readModule(), it checks for the ".jar" suffix.
+ // Performing the same check here.
+ const char* dot = strrchr(filename, '.');
+ if (dot != nullptr && strcmp(dot + 1, "jar") == 0) {
+ return true;
+ }
+ return false;
+}
+
+void ClassLoaderExt::extract_jar_files_from_path(const char* path, GrowableArray* module_paths) {
+ DIR* dirp = os::opendir(path);
+ if (dirp == nullptr && errno == ENOTDIR && has_jar_suffix(path)) {
+ module_paths->append(path);
+ } else {
+ if (dirp != nullptr) {
+ struct dirent* dentry;
+ while ((dentry = os::readdir(dirp)) != nullptr) {
+ const char* file_name = dentry->d_name;
+ if (has_jar_suffix(file_name)) {
+ size_t full_name_len = strlen(path) + strlen(file_name) + strlen(os::file_separator()) + 1;
+ char* full_name = NEW_RESOURCE_ARRAY(char, full_name_len);
+ int n = os::snprintf(full_name, full_name_len, "%s%s%s", path, os::file_separator(), file_name);
+ assert((size_t)n == full_name_len - 1, "Unexpected number of characters in string");
+ module_paths->append(full_name);
+ }
+ }
+ os::closedir(dirp);
+ }
+ }
+}
+
char* ClassLoaderExt::read_manifest(JavaThread* current, ClassPathEntry* entry,
jint *manifest_size, bool clean_text) {
const char* name = "META-INF/MANIFEST.MF";
diff --git a/src/hotspot/share/classfile/classLoaderExt.hpp b/src/hotspot/share/classfile/classLoaderExt.hpp
index b76ce3ff33a32..1f1b38cd3121c 100644
--- a/src/hotspot/share/classfile/classLoaderExt.hpp
+++ b/src/hotspot/share/classfile/classLoaderExt.hpp
@@ -53,12 +53,15 @@ class ClassLoaderExt: public ClassLoader { // AllStatic
static jshort _app_module_paths_start_index;
// the largest path index being used during CDS dump time
static jshort _max_used_path_index;
+ // number of module paths
+ static int _num_module_paths;
static bool _has_app_classes;
static bool _has_platform_classes;
static bool _has_non_jar_in_classpath;
static char* read_manifest(JavaThread* current, ClassPathEntry* entry, jint *manifest_size, bool clean_text);
+ static bool has_jar_suffix(const char* filename);
public:
static void process_jar_manifest(JavaThread* current, ClassPathEntry* entry);
@@ -68,6 +71,8 @@ class ClassLoaderExt: public ClassLoader { // AllStatic
static void setup_search_paths(JavaThread* current);
static void setup_module_paths(JavaThread* current);
+ static void extract_jar_files_from_path(const char* path, GrowableArray* module_paths);
+ static int compare_module_names(const char** p1, const char** p2);
static char* read_manifest(JavaThread* current, ClassPathEntry* entry, jint *manifest_size) {
// Remove all the new-line continuations (which wrap long lines at 72 characters, see
@@ -87,6 +92,8 @@ class ClassLoaderExt: public ClassLoader { // AllStatic
static jshort max_used_path_index() { return _max_used_path_index; }
+ static int num_module_paths() { return _num_module_paths; }
+
static void set_max_used_path_index(jshort used_index) {
_max_used_path_index = used_index;
}
@@ -99,6 +106,10 @@ class ClassLoaderExt: public ClassLoader { // AllStatic
_app_module_paths_start_index = module_start;
}
+ static void init_num_module_paths(int num_module_paths) {
+ _num_module_paths = num_module_paths;
+ }
+
static bool is_boot_classpath(int classpath_index) {
return classpath_index < _app_class_paths_start_index;
}
diff --git a/src/hotspot/share/classfile/fieldLayoutBuilder.hpp b/src/hotspot/share/classfile/fieldLayoutBuilder.hpp
index cda64788acffa..9b0d80b2a5583 100644
--- a/src/hotspot/share/classfile/fieldLayoutBuilder.hpp
+++ b/src/hotspot/share/classfile/fieldLayoutBuilder.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -101,17 +101,13 @@ class LayoutRawBlock : public ResourceObj {
// sort fields in decreasing order.
// Note: with line types, the comparison should include alignment constraint if sizes are equals
static int compare_size_inverted(LayoutRawBlock** x, LayoutRawBlock** y) {
-#ifdef _WINDOWS
- // qsort() on Windows reverse the order of fields with the same size
- // the extension of the comparison function below preserves this order
int diff = (*y)->size() - (*x)->size();
+ // qsort() may reverse the order of fields with the same size.
+ // The extension is to ensure stable sort.
if (diff == 0) {
diff = (*x)->field_index() - (*y)->field_index();
}
return diff;
-#else
- return (*y)->size() - (*x)->size();
-#endif // _WINDOWS
}
};
diff --git a/src/hotspot/share/classfile/javaClasses.cpp b/src/hotspot/share/classfile/javaClasses.cpp
index 0ad36cd21dbf3..3dce1a343d896 100644
--- a/src/hotspot/share/classfile/javaClasses.cpp
+++ b/src/hotspot/share/classfile/javaClasses.cpp
@@ -4763,9 +4763,6 @@ bool java_lang_ClassLoader::parallelCapable(oop class_loader) {
}
bool java_lang_ClassLoader::is_trusted_loader(oop loader) {
- // Fix for 4474172; see evaluation for more details
- loader = non_reflection_class_loader(loader);
-
oop cl = SystemDictionary::java_system_loader();
while(cl != nullptr) {
if (cl == loader) return true;
@@ -4774,29 +4771,6 @@ bool java_lang_ClassLoader::is_trusted_loader(oop loader) {
return false;
}
-// Return true if this is one of the class loaders associated with
-// the generated bytecodes for serialization constructor returned
-// by sun.reflect.ReflectionFactory::newConstructorForSerialization
-bool java_lang_ClassLoader::is_reflection_class_loader(oop loader) {
- if (loader != nullptr) {
- Klass* delegating_cl_class = vmClasses::reflect_DelegatingClassLoader_klass();
- // This might be null in non-1.4 JDKs
- return (delegating_cl_class != nullptr && loader->is_a(delegating_cl_class));
- }
- return false;
-}
-
-oop java_lang_ClassLoader::non_reflection_class_loader(oop loader) {
- // See whether this is one of the class loaders associated with
- // the generated bytecodes for reflection, and if so, "magically"
- // delegate to its parent to prevent class loading from occurring
- // in places where applications using reflection didn't expect it.
- if (is_reflection_class_loader(loader)) {
- return parent(loader);
- }
- return loader;
-}
-
oop java_lang_ClassLoader::unnamedModule(oop loader) {
assert(is_instance(loader), "loader must be oop");
return loader->obj_field(_unnamedModule_offset);
diff --git a/src/hotspot/share/classfile/javaClasses.hpp b/src/hotspot/share/classfile/javaClasses.hpp
index 6f82a75e8ffeb..6ab73b161f4f0 100644
--- a/src/hotspot/share/classfile/javaClasses.hpp
+++ b/src/hotspot/share/classfile/javaClasses.hpp
@@ -1502,14 +1502,6 @@ class java_lang_ClassLoader : AllStatic {
static bool is_trusted_loader(oop loader);
- // Return true if this is one of the class loaders associated with
- // the generated bytecodes for serialization constructor returned
- // by sun.reflect.ReflectionFactory::newConstructorForSerialization
- static bool is_reflection_class_loader(oop loader);
-
- // Fix for 4474172
- static oop non_reflection_class_loader(oop loader);
-
// Testers
static bool is_subclass(Klass* klass) {
return klass->is_subclass_of(vmClasses::ClassLoader_klass());
diff --git a/src/hotspot/share/classfile/modules.cpp b/src/hotspot/share/classfile/modules.cpp
index 14e730f7a3359..94e407d045fb5 100644
--- a/src/hotspot/share/classfile/modules.cpp
+++ b/src/hotspot/share/classfile/modules.cpp
@@ -30,6 +30,7 @@
#include "classfile/classLoader.hpp"
#include "classfile/classLoaderData.inline.hpp"
#include "classfile/classLoaderDataShared.hpp"
+#include "classfile/classLoaderExt.hpp"
#include "classfile/javaAssertions.hpp"
#include "classfile/javaClasses.hpp"
#include "classfile/javaClasses.inline.hpp"
@@ -309,11 +310,6 @@ void Modules::define_module(Handle module, jboolean is_open, jstring version,
}
oop loader = java_lang_Module::loader(module());
- // Make sure loader is not the jdk.internal.reflect.DelegatingClassLoader.
- if (loader != java_lang_ClassLoader::non_reflection_class_loader(loader)) {
- THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(),
- "Class loader is an invalid delegating class loader");
- }
Handle h_loader = Handle(THREAD, loader);
// define_module can be called during start-up, before the class loader's ClassLoaderData
// has been created. SystemDictionary::register_loader ensures creation, if needed.
@@ -565,6 +561,7 @@ void Modules::verify_archived_modules() {
}
char* Modules::_archived_main_module_name = nullptr;
+char* Modules::_archived_addmods_names = nullptr;
void Modules::dump_main_module_name() {
const char* module_name = Arguments::get_property("jdk.module.main");
@@ -605,6 +602,100 @@ void Modules::serialize(SerializeClosure* soc) {
}
}
+void Modules::dump_addmods_names() {
+ unsigned int count = Arguments::addmods_count();
+ const char* addmods_names = get_addmods_names_as_sorted_string();
+ if (addmods_names != nullptr) {
+ _archived_addmods_names = ArchiveBuilder::current()->ro_strdup(addmods_names);
+ }
+ ArchivePtrMarker::mark_pointer(&_archived_addmods_names);
+}
+
+void Modules::serialize_addmods_names(SerializeClosure* soc) {
+ soc->do_ptr(&_archived_addmods_names);
+ if (soc->reading()) {
+ bool disable = false;
+ if (_archived_addmods_names[0] != '\0') {
+ if (Arguments::addmods_count() == 0) {
+ log_info(cds)("--add-modules module name(s) found in archive but not specified during runtime: %s",
+ _archived_addmods_names);
+ disable = true;
+ } else {
+ const char* addmods_names = get_addmods_names_as_sorted_string();
+ if (strcmp((const char*)_archived_addmods_names, addmods_names) != 0) {
+ log_info(cds)("Mismatched --add-modules module name(s).");
+ log_info(cds)(" dump time: %s runtime: %s", _archived_addmods_names, addmods_names);
+ disable = true;
+ }
+ }
+ } else {
+ if (Arguments::addmods_count() > 0) {
+ log_info(cds)("--add-modules module name(s) specified during runtime but not found in archive: %s",
+ get_addmods_names_as_sorted_string());
+ disable = true;
+ }
+ }
+ if (disable) {
+ log_info(cds)("Disabling optimized module handling");
+ CDSConfig::stop_using_optimized_module_handling();
+ }
+ log_info(cds)("optimized module handling: %s", CDSConfig::is_using_optimized_module_handling() ? "enabled" : "disabled");
+ log_info(cds)("full module graph: %s", CDSConfig::is_using_full_module_graph() ? "enabled" : "disabled");
+ }
+}
+
+const char* Modules::get_addmods_names_as_sorted_string() {
+ ResourceMark rm;
+ const int max_digits = 3;
+ const int extra_symbols_count = 2; // includes '.', '\0'
+ size_t prop_len = strlen("jdk.module.addmods") + max_digits + extra_symbols_count;
+ char* prop_name = resource_allocate_bytes(prop_len);
+ GrowableArray list;
+ for (unsigned int i = 0; i < Arguments::addmods_count(); i++) {
+ jio_snprintf(prop_name, prop_len, "jdk.module.addmods.%d", i);
+ const char* prop_value = Arguments::get_property(prop_name);
+ char* p = resource_allocate_bytes(strlen(prop_value) + 1);
+ strcpy(p, prop_value);
+ while (*p == ',') p++; // skip leading commas
+ while (*p) {
+ char* next = strchr(p, ',');
+ if (next == nullptr) {
+ // no more commas, p is the last element
+ list.append(p);
+ break;
+ } else {
+ *next = 0;
+ list.append(p);
+ p = next + 1;
+ }
+ }
+ }
+
+ // Example:
+ // --add-modules=java.compiler --add-modules=java.base,java.base,,
+ //
+ // list[0] = "java.compiler"
+ // list[1] = "java.base"
+ // list[2] = "java.base"
+ // list[3] = ""
+ // list[4] = ""
+ list.sort(ClassLoaderExt::compare_module_names);
+
+ const char* prefix = "";
+ stringStream st;
+ const char* last_string = ""; // This also filters out all empty strings
+ for (int i = 0; i < list.length(); i++) {
+ const char* m = list.at(i);
+ if (strcmp(m, last_string) != 0) { // filter out duplicates
+ st.print("%s%s", prefix, m);
+ last_string = m;
+ prefix = "\n";
+ }
+ }
+
+ return (const char*)os::strdup(st.as_string()); // Example: "java.base,java.compiler"
+}
+
void Modules::define_archived_modules(Handle h_platform_loader, Handle h_system_loader, TRAPS) {
assert(CDSConfig::is_using_full_module_graph(), "must be");
diff --git a/src/hotspot/share/classfile/modules.hpp b/src/hotspot/share/classfile/modules.hpp
index 3866f0d6f9b82..3ef6f57a57b4c 100644
--- a/src/hotspot/share/classfile/modules.hpp
+++ b/src/hotspot/share/classfile/modules.hpp
@@ -61,9 +61,13 @@ class Modules : AllStatic {
static void verify_archived_modules() NOT_CDS_JAVA_HEAP_RETURN;
static void dump_main_module_name() NOT_CDS_JAVA_HEAP_RETURN;
static void serialize(SerializeClosure* soc) NOT_CDS_JAVA_HEAP_RETURN;
+ static void dump_addmods_names() NOT_CDS_JAVA_HEAP_RETURN;
+ static void serialize_addmods_names(SerializeClosure* soc) NOT_CDS_JAVA_HEAP_RETURN;
+ static const char* get_addmods_names_as_sorted_string() NOT_CDS_JAVA_HEAP_RETURN_(nullptr);
#if INCLUDE_CDS_JAVA_HEAP
static char* _archived_main_module_name;
+ static char* _archived_addmods_names;
#endif
// Provides the java.lang.Module for the unnamed module defined
diff --git a/src/hotspot/share/classfile/systemDictionary.cpp b/src/hotspot/share/classfile/systemDictionary.cpp
index 7b307a0b8a37c..2338a179324b6 100644
--- a/src/hotspot/share/classfile/systemDictionary.cpp
+++ b/src/hotspot/share/classfile/systemDictionary.cpp
@@ -598,8 +598,6 @@ InstanceKlass* SystemDictionary::resolve_instance_class_or_null(Symbol* name,
HandleMark hm(THREAD);
- // Fix for 4474172; see evaluation for more details
- class_loader = Handle(THREAD, java_lang_ClassLoader::non_reflection_class_loader(class_loader()));
ClassLoaderData* loader_data = register_loader(class_loader);
Dictionary* dictionary = loader_data->dictionary();
@@ -765,12 +763,7 @@ InstanceKlass* SystemDictionary::find_instance_klass(Thread* current,
Handle class_loader,
Handle protection_domain) {
- // The result of this call should be consistent with the result
- // of the call to resolve_instance_class_or_null().
- // See evaluation 6790209 and 4474172 for more details.
- oop class_loader_oop = java_lang_ClassLoader::non_reflection_class_loader(class_loader());
- ClassLoaderData* loader_data = ClassLoaderData::class_loader_data_or_null(class_loader_oop);
-
+ ClassLoaderData* loader_data = ClassLoaderData::class_loader_data_or_null(class_loader());
if (loader_data == nullptr) {
// If the ClassLoaderData has not been setup,
// then the class loader has no entries in the dictionary.
diff --git a/src/hotspot/share/classfile/systemDictionaryShared.cpp b/src/hotspot/share/classfile/systemDictionaryShared.cpp
index ce7f862a7b6c2..05f6ace9b8a95 100644
--- a/src/hotspot/share/classfile/systemDictionaryShared.cpp
+++ b/src/hotspot/share/classfile/systemDictionaryShared.cpp
@@ -403,9 +403,6 @@ InstanceKlass* SystemDictionaryShared::find_or_load_shared_class(
if (SystemDictionary::is_system_class_loader(class_loader()) ||
SystemDictionary::is_platform_class_loader(class_loader())) {
- // Fix for 4474172; see evaluation for more details
- class_loader = Handle(
- THREAD, java_lang_ClassLoader::non_reflection_class_loader(class_loader()));
ClassLoaderData *loader_data = register_loader(class_loader);
Dictionary* dictionary = loader_data->dictionary();
diff --git a/src/hotspot/share/classfile/verifier.cpp b/src/hotspot/share/classfile/verifier.cpp
index a66fbf645f55f..b34cde4c63ab6 100644
--- a/src/hotspot/share/classfile/verifier.cpp
+++ b/src/hotspot/share/classfile/verifier.cpp
@@ -32,6 +32,7 @@
#include "classfile/stackMapTableFormat.hpp"
#include "classfile/symbolTable.hpp"
#include "classfile/systemDictionary.hpp"
+#include "classfile/systemDictionaryShared.hpp"
#include "classfile/verifier.hpp"
#include "classfile/vmClasses.hpp"
#include "classfile/vmSymbols.hpp"
@@ -212,6 +213,11 @@ bool Verifier::verify(InstanceKlass* klass, bool should_verify_class, TRAPS) {
exception_name == vmSymbols::java_lang_ClassFormatError())) {
log_info(verification)("Fail over class verification to old verifier for: %s", klass->external_name());
log_info(class, init)("Fail over class verification to old verifier for: %s", klass->external_name());
+ // Exclude any classes that fail over during dynamic dumping
+ if (CDSConfig::is_dumping_dynamic_archive()) {
+ SystemDictionaryShared::warn_excluded(klass, "Failed over class verification while dynamic dumping");
+ SystemDictionaryShared::set_excluded(klass);
+ }
message_buffer = NEW_RESOURCE_ARRAY(char, message_buffer_len);
exception_message = message_buffer;
exception_name = inference_verify(
@@ -269,10 +275,6 @@ bool Verifier::verify(InstanceKlass* klass, bool should_verify_class, TRAPS) {
bool Verifier::is_eligible_for_verification(InstanceKlass* klass, bool should_verify_class) {
Symbol* name = klass->name();
- Klass* refl_serialization_ctor_klass = vmClasses::reflect_SerializationConstructorAccessorImpl_klass();
-
- bool is_reflect_accessor = refl_serialization_ctor_klass != nullptr &&
- klass->is_subtype_of(refl_serialization_ctor_klass);
return (should_verify_for(klass->class_loader(), should_verify_class) &&
// return if the class is a bootstrapping class
@@ -289,12 +291,7 @@ bool Verifier::is_eligible_for_verification(InstanceKlass* klass, bool should_ve
// Shared classes shouldn't have stackmaps either.
// However, bytecodes for shared old classes can be verified because
// they have not been rewritten.
- !(klass->is_shared() && klass->is_rewritten()) &&
-
- // As of the fix for 4486457 we disable verification for all of the
- // dynamically-generated bytecodes associated with
- // jdk/internal/reflect/SerializationConstructorAccessor.
- (!is_reflect_accessor));
+ !(klass->is_shared() && klass->is_rewritten()));
}
Symbol* Verifier::inference_verify(
diff --git a/src/hotspot/share/classfile/vmClassMacros.hpp b/src/hotspot/share/classfile/vmClassMacros.hpp
index 10fa89007957c..395e718c55d83 100644
--- a/src/hotspot/share/classfile/vmClassMacros.hpp
+++ b/src/hotspot/share/classfile/vmClassMacros.hpp
@@ -107,11 +107,9 @@
do_klass(StackChunk_klass, jdk_internal_vm_StackChunk ) \
\
do_klass(reflect_MethodAccessorImpl_klass, reflect_MethodAccessorImpl ) \
- do_klass(reflect_DelegatingClassLoader_klass, reflect_DelegatingClassLoader ) \
do_klass(reflect_ConstantPool_klass, reflect_ConstantPool ) \
do_klass(reflect_CallerSensitive_klass, reflect_CallerSensitive ) \
do_klass(reflect_DirectConstructorHandleAccessor_NativeAccessor_klass, reflect_DirectConstructorHandleAccessor_NativeAccessor) \
- do_klass(reflect_SerializationConstructorAccessorImpl_klass, reflect_SerializationConstructorAccessorImpl ) \
\
/* support for dynamic typing */ \
do_klass(DirectMethodHandle_klass, java_lang_invoke_DirectMethodHandle ) \
diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp
index b6ce21797a618..68121c56c328f 100644
--- a/src/hotspot/share/classfile/vmIntrinsics.hpp
+++ b/src/hotspot/share/classfile/vmIntrinsics.hpp
@@ -469,6 +469,8 @@ class methodHandle;
do_intrinsic(_Reference_get, java_lang_ref_Reference, get_name, void_object_signature, F_R) \
do_intrinsic(_Reference_refersTo0, java_lang_ref_Reference, refersTo0_name, object_boolean_signature, F_RN) \
do_intrinsic(_PhantomReference_refersTo0, java_lang_ref_PhantomReference, refersTo0_name, object_boolean_signature, F_RN) \
+ do_intrinsic(_Reference_clear0, java_lang_ref_Reference, clear0_name, void_method_signature, F_RN) \
+ do_intrinsic(_PhantomReference_clear0, java_lang_ref_PhantomReference, clear0_name, void_method_signature, F_RN) \
\
/* support for com.sun.crypto.provider.AESCrypt and some of its callers */ \
do_class(com_sun_crypto_provider_aescrypt, "com/sun/crypto/provider/AESCrypt") \
@@ -609,7 +611,6 @@ class methodHandle;
do_intrinsic(_notifyJvmtiVThreadEnd, java_lang_VirtualThread, notifyJvmtiEnd_name, void_method_signature, F_RN) \
do_intrinsic(_notifyJvmtiVThreadMount, java_lang_VirtualThread, notifyJvmtiMount_name, bool_void_signature, F_RN) \
do_intrinsic(_notifyJvmtiVThreadUnmount, java_lang_VirtualThread, notifyJvmtiUnmount_name, bool_void_signature, F_RN) \
- do_intrinsic(_notifyJvmtiVThreadHideFrames, java_lang_VirtualThread, notifyJvmtiHideFrames_name, bool_void_signature, F_SN) \
do_intrinsic(_notifyJvmtiVThreadDisableSuspend, java_lang_VirtualThread, notifyJvmtiDisableSuspend_name, bool_void_signature, F_SN) \
\
/* support for UnsafeConstants */ \
@@ -979,6 +980,17 @@ class methodHandle;
"Ljdk/internal/vm/vector/VectorSupport$Vector;") \
do_name(vector_ternary_op_name, "ternaryOp") \
\
+ do_intrinsic(_VectorSelectFromTwoVectorOp, jdk_internal_vm_vector_VectorSupport, vector_select_from_op_name, vector_select_from_op_sig, F_S) \
+ do_signature(vector_select_from_op_sig, "(Ljava/lang/Class;" \
+ "Ljava/lang/Class;" \
+ "I" \
+ "Ljdk/internal/vm/vector/VectorSupport$Vector;" \
+ "Ljdk/internal/vm/vector/VectorSupport$Vector;" \
+ "Ljdk/internal/vm/vector/VectorSupport$Vector;" \
+ "Ljdk/internal/vm/vector/VectorSupport$SelectFromTwoVector;)" \
+ "Ljdk/internal/vm/vector/VectorSupport$Vector;") \
+ do_name(vector_select_from_op_name, "selectFromTwoVectorOp") \
+ \
do_intrinsic(_VectorFromBitsCoerced, jdk_internal_vm_vector_VectorSupport, vector_frombits_coerced_name, vector_frombits_coerced_sig, F_S) \
do_signature(vector_frombits_coerced_sig, "(Ljava/lang/Class;" \
"Ljava/lang/Class;" \
diff --git a/src/hotspot/share/classfile/vmSymbols.hpp b/src/hotspot/share/classfile/vmSymbols.hpp
index a65ab86fa0a8d..d8018cd0c8af2 100644
--- a/src/hotspot/share/classfile/vmSymbols.hpp
+++ b/src/hotspot/share/classfile/vmSymbols.hpp
@@ -262,12 +262,10 @@ class SerializeClosure;
\
template(jdk_internal_reflect, "jdk/internal/reflect") \
template(reflect_MethodAccessorImpl, "jdk/internal/reflect/MethodAccessorImpl") \
- template(reflect_DelegatingClassLoader, "jdk/internal/reflect/DelegatingClassLoader") \
template(reflect_Reflection, "jdk/internal/reflect/Reflection") \
template(reflect_CallerSensitive, "jdk/internal/reflect/CallerSensitive") \
template(reflect_CallerSensitive_signature, "Ljdk/internal/reflect/CallerSensitive;") \
template(reflect_DirectConstructorHandleAccessor_NativeAccessor, "jdk/internal/reflect/DirectConstructorHandleAccessor$NativeAccessor") \
- template(reflect_SerializationConstructorAccessorImpl, "jdk/internal/reflect/SerializationConstructorAccessorImpl") \
template(clazz_name, "clazz") \
template(exceptionTypes_name, "exceptionTypes") \
template(modifiers_name, "modifiers") \
@@ -308,6 +306,7 @@ class SerializeClosure;
template(jdk_internal_vm_annotation_Stable_signature, "Ljdk/internal/vm/annotation/Stable;") \
\
template(jdk_internal_vm_annotation_ChangesCurrentThread_signature, "Ljdk/internal/vm/annotation/ChangesCurrentThread;") \
+ template(jdk_internal_vm_annotation_JvmtiHideEvents_signature, "Ljdk/internal/vm/annotation/JvmtiHideEvents;") \
template(jdk_internal_vm_annotation_JvmtiMountTransition_signature, "Ljdk/internal/vm/annotation/JvmtiMountTransition;") \
\
/* Support for JSR 292 & invokedynamic (JDK 1.7 and above) */ \
@@ -399,7 +398,6 @@ class SerializeClosure;
template(notifyJvmtiEnd_name, "notifyJvmtiEnd") \
template(notifyJvmtiMount_name, "notifyJvmtiMount") \
template(notifyJvmtiUnmount_name, "notifyJvmtiUnmount") \
- template(notifyJvmtiHideFrames_name, "notifyJvmtiHideFrames") \
template(notifyJvmtiDisableSuspend_name, "notifyJvmtiDisableSuspend") \
template(doYield_name, "doYield") \
template(enter_name, "enter") \
@@ -426,6 +424,7 @@ class SerializeClosure;
template(cs_name, "cs") \
template(get_name, "get") \
template(refersTo0_name, "refersTo0") \
+ template(clear0_name, "clear0") \
template(put_name, "put") \
template(type_name, "type") \
template(findNative_name, "findNative") \
diff --git a/src/hotspot/share/code/codeBlob.cpp b/src/hotspot/share/code/codeBlob.cpp
index 81c4d001078cb..23f621ffec832 100644
--- a/src/hotspot/share/code/codeBlob.cpp
+++ b/src/hotspot/share/code/codeBlob.cpp
@@ -41,7 +41,7 @@
#include "runtime/handles.inline.hpp"
#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/javaFrameAnchor.hpp"
-#include "runtime/jniHandles.hpp"
+#include "runtime/jniHandles.inline.hpp"
#include "runtime/mutexLocker.hpp"
#include "runtime/safepoint.hpp"
#include "runtime/sharedRuntime.hpp"
@@ -623,7 +623,7 @@ UpcallStub* UpcallStub::create(const char* name, CodeBuffer* cb, jobject receive
// Track memory usage statistic after releasing CodeCache_lock
MemoryService::track_code_cache_memory_usage();
- trace_new_stub(blob, "UpcallStub");
+ trace_new_stub(blob, "UpcallStub - ", name);
return blob;
}
@@ -772,6 +772,10 @@ void UpcallStub::verify() {
void UpcallStub::print_on(outputStream* st) const {
RuntimeBlob::print_on(st);
print_value_on(st);
+ st->print_cr("Frame data offset: %d", (int) _frame_data_offset);
+ oop recv = JNIHandles::resolve(_receiver);
+ st->print("Receiver MH=");
+ recv->print_on(st);
Disassembler::decode((RuntimeBlob*)this, st);
}
diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp
index 0bab731ac565e..7fb72997749dc 100644
--- a/src/hotspot/share/code/nmethod.cpp
+++ b/src/hotspot/share/code/nmethod.cpp
@@ -2048,7 +2048,7 @@ bool nmethod::make_not_entrant() {
} // leave critical region under NMethodState_lock
#if INCLUDE_JVMCI
- // Invalidate can't occur while holding the Patching lock
+ // Invalidate can't occur while holding the NMethodState_lock
JVMCINMethodData* nmethod_data = jvmci_nmethod_data();
if (nmethod_data != nullptr) {
nmethod_data->invalidate_nmethod_mirror(this);
diff --git a/src/hotspot/share/compiler/compilerThread.cpp b/src/hotspot/share/compiler/compilerThread.cpp
index e212200a47c65..e6329f3e65579 100644
--- a/src/hotspot/share/compiler/compilerThread.cpp
+++ b/src/hotspot/share/compiler/compilerThread.cpp
@@ -55,8 +55,11 @@ CompilerThread::~CompilerThread() {
}
void CompilerThread::set_compiler(AbstractCompiler* c) {
- // Only jvmci compiler threads can call Java
- _can_call_java = c != nullptr && c->is_jvmci();
+ /*
+ * Compiler threads need to make Java upcalls to the jargraal compiler.
+ * Java upcalls are also needed by the InterpreterRuntime when using jargraal.
+ */
+ _can_call_java = c != nullptr && c->is_jvmci() JVMCI_ONLY(&& !UseJVMCINativeLibrary);
_compiler = c;
}
diff --git a/src/hotspot/share/gc/epsilon/epsilonHeap.hpp b/src/hotspot/share/gc/epsilon/epsilonHeap.hpp
index f22d1c22ac8a7..b2b522c1435e6 100644
--- a/src/hotspot/share/gc/epsilon/epsilonHeap.hpp
+++ b/src/hotspot/share/gc/epsilon/epsilonHeap.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2022, Red Hat, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -128,7 +128,7 @@ class EpsilonHeap : public CollectedHeap {
bool is_in_reserved(const void* addr) const { return _reserved.contains(addr); }
// Support for loading objects from CDS archive into the heap
- bool can_load_archived_objects() const override { return UseCompressedOops; }
+ bool can_load_archived_objects() const override { return true; }
HeapWord* allocate_loaded_archive_space(size_t size) override;
void print_on(outputStream* st) const override;
diff --git a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp
index 13b993546cde4..edbf0e902392b 100644
--- a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp
+++ b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp
@@ -24,49 +24,32 @@
#include "precompiled.hpp"
#include "classfile/javaClasses.hpp"
+#include "code/vmreg.inline.hpp"
#include "gc/g1/c2/g1BarrierSetC2.hpp"
#include "gc/g1/g1BarrierSet.hpp"
+#include "gc/g1/g1BarrierSetAssembler.hpp"
#include "gc/g1/g1BarrierSetRuntime.hpp"
#include "gc/g1/g1CardTable.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
#include "gc/g1/g1HeapRegion.hpp"
#include "opto/arraycopynode.hpp"
+#include "opto/block.hpp"
#include "opto/compile.hpp"
#include "opto/escape.hpp"
#include "opto/graphKit.hpp"
#include "opto/idealKit.hpp"
+#include "opto/machnode.hpp"
#include "opto/macro.hpp"
+#include "opto/memnode.hpp"
+#include "opto/node.hpp"
+#include "opto/output.hpp"
+#include "opto/regalloc.hpp"
#include "opto/rootnode.hpp"
+#include "opto/runtime.hpp"
#include "opto/type.hpp"
+#include "utilities/growableArray.hpp"
#include "utilities/macros.hpp"
-const TypeFunc *G1BarrierSetC2::write_ref_field_pre_entry_Type() {
- const Type **fields = TypeTuple::fields(2);
- fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
- fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // thread
- const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
-
- // create result type (range)
- fields = TypeTuple::fields(0);
- const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
-
- return TypeFunc::make(domain, range);
-}
-
-const TypeFunc *G1BarrierSetC2::write_ref_field_post_entry_Type() {
- const Type **fields = TypeTuple::fields(2);
- fields[TypeFunc::Parms+0] = TypeRawPtr::NOTNULL; // Card addr
- fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // thread
- const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
-
- // create result type (range)
- fields = TypeTuple::fields(0);
- const TypeTuple *range = TypeTuple::make(TypeFunc::Parms, fields);
-
- return TypeFunc::make(domain, range);
-}
-
-#define __ ideal.
/*
* Determine if the G1 pre-barrier can be removed. The pre-barrier is
* required by SATB to make sure all objects live at the start of the
@@ -84,8 +67,6 @@ const TypeFunc *G1BarrierSetC2::write_ref_field_post_entry_Type() {
* The compiler needs to determine that the object in which a field is about
* to be written is newly allocated, and that no prior store to the same field
* has happened since the allocation.
- *
- * Returns true if the pre-barrier can be removed
*/
bool G1BarrierSetC2::g1_can_remove_pre_barrier(GraphKit* kit,
PhaseValues* phase,
@@ -97,34 +78,28 @@ bool G1BarrierSetC2::g1_can_remove_pre_barrier(GraphKit* kit,
AllocateNode* alloc = AllocateNode::Ideal_allocation(base);
if (offset == Type::OffsetBot) {
- return false; // cannot unalias unless there are precise offsets
+ return false; // Cannot unalias unless there are precise offsets.
}
-
if (alloc == nullptr) {
- return false; // No allocation found
+ return false; // No allocation found.
}
intptr_t size_in_bytes = type2aelembytes(bt);
-
- Node* mem = kit->memory(adr_idx); // start searching here...
+ Node* mem = kit->memory(adr_idx); // Start searching here.
for (int cnt = 0; cnt < 50; cnt++) {
-
if (mem->is_Store()) {
-
Node* st_adr = mem->in(MemNode::Address);
intptr_t st_offset = 0;
Node* st_base = AddPNode::Ideal_base_and_offset(st_adr, phase, st_offset);
if (st_base == nullptr) {
- break; // inscrutable pointer
+ break; // Inscrutable pointer.
}
-
- // Break we have found a store with same base and offset as ours so break
if (st_base == base && st_offset == offset) {
+ // We have found a store with same base and offset as ours.
break;
}
-
if (st_offset != offset && st_offset != Type::OffsetBot) {
const int MAX_STORE = BytesPerLong;
if (st_offset >= offset + size_in_bytes ||
@@ -136,20 +111,18 @@ bool G1BarrierSetC2::g1_can_remove_pre_barrier(GraphKit* kit,
// in the same sequence of RawMem effects. We sometimes initialize
// a whole 'tile' of array elements with a single jint or jlong.)
mem = mem->in(MemNode::Memory);
- continue; // advance through independent store memory
+ continue; // Advance through independent store memory.
}
}
-
if (st_base != base
&& MemNode::detect_ptr_independence(base, alloc, st_base,
AllocateNode::Ideal_allocation(st_base),
phase)) {
- // Success: The bases are provably independent.
+ // Success: the bases are provably independent.
mem = mem->in(MemNode::Memory);
- continue; // advance through independent store memory
+ continue; // Advance through independent store memory.
}
} else if (mem->is_Proj() && mem->in(0)->is_Initialize()) {
-
InitializeNode* st_init = mem->in(0)->as_Initialize();
AllocateNode* st_alloc = st_init->allocation();
@@ -157,7 +130,7 @@ bool G1BarrierSetC2::g1_can_remove_pre_barrier(GraphKit* kit,
// The alloc variable is guaranteed to not be null here from earlier check.
if (alloc == st_alloc) {
// Check that the initialization is storing null so that no previous store
- // has been moved up and directly write a reference
+ // has been moved up and directly write a reference.
Node* captured_store = st_init->find_captured_store(offset,
type2aelembytes(T_OBJECT),
phase);
@@ -166,164 +139,55 @@ bool G1BarrierSetC2::g1_can_remove_pre_barrier(GraphKit* kit,
}
}
}
-
// Unless there is an explicit 'continue', we must bail out here,
// because 'mem' is an inscrutable memory state (e.g., a call).
break;
}
-
return false;
}
-// G1 pre/post barriers
-void G1BarrierSetC2::pre_barrier(GraphKit* kit,
- bool do_load,
- Node* ctl,
- Node* obj,
- Node* adr,
- uint alias_idx,
- Node* val,
- const TypeOopPtr* val_type,
- Node* pre_val,
- BasicType bt) const {
- // Some sanity checks
- // Note: val is unused in this routine.
-
- if (do_load) {
- // We need to generate the load of the previous value
- assert(obj != nullptr, "must have a base");
- assert(adr != nullptr, "where are loading from?");
- assert(pre_val == nullptr, "loaded already?");
- assert(val_type != nullptr, "need a type");
-
- if (use_ReduceInitialCardMarks()
- && g1_can_remove_pre_barrier(kit, &kit->gvn(), adr, bt, alias_idx)) {
- return;
- }
-
- } else {
- // In this case both val_type and alias_idx are unused.
- assert(pre_val != nullptr, "must be loaded already");
- // Nothing to be done if pre_val is null.
- if (pre_val->bottom_type() == TypePtr::NULL_PTR) return;
- assert(pre_val->bottom_type()->basic_type() == T_OBJECT, "or we shouldn't be here");
- }
- assert(bt == T_OBJECT, "or we shouldn't be here");
-
- IdealKit ideal(kit, true);
-
- Node* tls = __ thread(); // ThreadLocalStorage
-
- Node* no_base = __ top();
- Node* zero = __ ConI(0);
- Node* zeroX = __ ConX(0);
-
- float likely = PROB_LIKELY(0.999);
- float unlikely = PROB_UNLIKELY(0.999);
-
- BasicType active_type = in_bytes(SATBMarkQueue::byte_width_of_active()) == 4 ? T_INT : T_BYTE;
- assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 4 || in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "flag width");
-
- // Offsets into the thread
- const int marking_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
- const int index_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset());
- const int buffer_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset());
-
- // Now the actual pointers into the thread
- Node* marking_adr = __ AddP(no_base, tls, __ ConX(marking_offset));
- Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset));
- Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset));
-
- // Now some of the values
- Node* marking = __ load(__ ctrl(), marking_adr, TypeInt::INT, active_type, Compile::AliasIdxRaw);
-
- // if (!marking)
- __ if_then(marking, BoolTest::ne, zero, unlikely); {
- BasicType index_bt = TypeX_X->basic_type();
- assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size.");
- Node* index = __ load(__ ctrl(), index_adr, TypeX_X, index_bt, Compile::AliasIdxRaw);
-
- if (do_load) {
- // load original value
- pre_val = __ load(__ ctrl(), adr, val_type, bt, alias_idx, false, MemNode::unordered, LoadNode::Pinned);
- }
-
- // if (pre_val != nullptr)
- __ if_then(pre_val, BoolTest::ne, kit->null()); {
- Node* buffer = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
-
- // is the queue for this thread full?
- __ if_then(index, BoolTest::ne, zeroX, likely); {
-
- // decrement the index
- Node* next_index = kit->gvn().transform(new SubXNode(index, __ ConX(sizeof(intptr_t))));
-
- // Now get the buffer location we will log the previous value into and store it
- Node *log_addr = __ AddP(no_base, buffer, next_index);
- __ store(__ ctrl(), log_addr, pre_val, T_OBJECT, Compile::AliasIdxRaw, MemNode::unordered);
- // update the index
- __ store(__ ctrl(), index_adr, next_index, index_bt, Compile::AliasIdxRaw, MemNode::unordered);
-
- } __ else_(); {
-
- // logging buffer is full, call the runtime
- const TypeFunc *tf = write_ref_field_pre_entry_Type();
- __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), "write_ref_field_pre_entry", pre_val, tls);
- } __ end_if(); // (!index)
- } __ end_if(); // (pre_val != nullptr)
- } __ end_if(); // (!marking)
-
- // Final sync IdealKit and GraphKit.
- kit->final_sync(ideal);
-}
-
/*
- * G1 similar to any GC with a Young Generation requires a way to keep track of
- * references from Old Generation to Young Generation to make sure all live
+ * G1, similar to any GC with a Young Generation, requires a way to keep track
+ * of references from Old Generation to Young Generation to make sure all live
* objects are found. G1 also requires to keep track of object references
* between different regions to enable evacuation of old regions, which is done
- * as part of mixed collections. References are tracked in remembered sets and
- * is continuously updated as reference are written to with the help of the
- * post-barrier.
+ * as part of mixed collections. References are tracked in remembered sets,
+ * which are continuously updated as references are written to with the help of
+ * the post-barrier.
*
- * To reduce the number of updates to the remembered set the post-barrier
- * filters updates to fields in objects located in the Young Generation,
- * the same region as the reference, when the null is being written or
- * if the card is already marked as dirty by an earlier write.
+ * To reduce the number of updates to the remembered set, the post-barrier
+ * filters out updates to fields in objects located in the Young Generation, the
+ * same region as the reference, when null is being written, or if the card is
+ * already marked as dirty by an earlier write.
*
* Under certain circumstances it is possible to avoid generating the
- * post-barrier completely if it is possible during compile time to prove
- * the object is newly allocated and that no safepoint exists between the
- * allocation and the store.
- *
- * In the case of slow allocation the allocation code must handle the barrier
- * as part of the allocation in the case the allocated object is not located
- * in the nursery; this would happen for humongous objects.
+ * post-barrier completely, if it is possible during compile time to prove the
+ * object is newly allocated and that no safepoint exists between the allocation
+ * and the store. This can be seen as a compile-time version of the
+ * above-mentioned Young Generation filter.
*
- * Returns true if the post barrier can be removed
+ * In the case of a slow allocation, the allocation code must handle the barrier
+ * as part of the allocation if the allocated object is not located in the
+ * nursery; this would happen for humongous objects.
*/
bool G1BarrierSetC2::g1_can_remove_post_barrier(GraphKit* kit,
- PhaseValues* phase, Node* store,
+ PhaseValues* phase, Node* store_ctrl,
Node* adr) const {
intptr_t offset = 0;
Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
AllocateNode* alloc = AllocateNode::Ideal_allocation(base);
if (offset == Type::OffsetBot) {
- return false; // cannot unalias unless there are precise offsets
+ return false; // Cannot unalias unless there are precise offsets.
}
-
if (alloc == nullptr) {
- return false; // No allocation found
+ return false; // No allocation found.
}
- // Start search from Store node
- Node* mem = store->in(MemNode::Control);
+ Node* mem = store_ctrl; // Start search from Store node.
if (mem->is_Proj() && mem->in(0)->is_Initialize()) {
-
InitializeNode* st_init = mem->in(0)->as_Initialize();
AllocateNode* st_alloc = st_init->allocation();
-
// Make sure we are looking at the same allocation
if (alloc == st_alloc) {
return true;
@@ -333,725 +197,367 @@ bool G1BarrierSetC2::g1_can_remove_post_barrier(GraphKit* kit,
return false;
}
-//
-// Update the card table and add card address to the queue
-//
-void G1BarrierSetC2::g1_mark_card(GraphKit* kit,
- IdealKit& ideal,
- Node* card_adr,
- Node* oop_store,
- uint oop_alias_idx,
- Node* index,
- Node* index_adr,
- Node* buffer,
- const TypeFunc* tf) const {
- Node* zero = __ ConI(0);
- Node* zeroX = __ ConX(0);
- Node* no_base = __ top();
- BasicType card_bt = T_BYTE;
- // Smash zero into card. MUST BE ORDERED WRT TO STORE
- __ storeCM(__ ctrl(), card_adr, zero, oop_store, oop_alias_idx, card_bt, Compile::AliasIdxRaw);
-
- // Now do the queue work
- __ if_then(index, BoolTest::ne, zeroX); {
-
- Node* next_index = kit->gvn().transform(new SubXNode(index, __ ConX(sizeof(intptr_t))));
- Node* log_addr = __ AddP(no_base, buffer, next_index);
-
- // Order, see storeCM.
- __ store(__ ctrl(), log_addr, card_adr, T_ADDRESS, Compile::AliasIdxRaw, MemNode::unordered);
- __ store(__ ctrl(), index_adr, next_index, TypeX_X->basic_type(), Compile::AliasIdxRaw, MemNode::unordered);
-
- } __ else_(); {
- __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), "write_ref_field_post_entry", card_adr, __ thread());
- } __ end_if();
-
+Node* G1BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
+ DecoratorSet decorators = access.decorators();
+ bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
+ bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
+ bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0;
+ // If we are reading the value of the referent field of a Reference object, we
+ // need to record the referent in an SATB log buffer using the pre-barrier
+ // mechanism. Also we need to add a memory barrier to prevent commoning reads
+ // from this field across safepoints, since GC can change its value.
+ bool need_read_barrier = ((on_weak || on_phantom) && !no_keepalive);
+ if (access.is_oop() && need_read_barrier) {
+ access.set_barrier_data(G1C2BarrierPre);
+ }
+ return CardTableBarrierSetC2::load_at_resolved(access, val_type);
}
-void G1BarrierSetC2::post_barrier(GraphKit* kit,
- Node* ctl,
- Node* oop_store,
- Node* obj,
- Node* adr,
- uint alias_idx,
- Node* val,
- BasicType bt,
- bool use_precise) const {
- // If we are writing a null then we need no post barrier
+void G1BarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
+ eliminate_gc_barrier_data(node);
+}
- if (val != nullptr && val->is_Con() && val->bottom_type() == TypePtr::NULL_PTR) {
- // Must be null
- const Type* t = val->bottom_type();
- assert(t == Type::TOP || t == TypePtr::NULL_PTR, "must be null");
- // No post barrier if writing null
- return;
+void G1BarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
+ if (node->is_LoadStore()) {
+ LoadStoreNode* loadstore = node->as_LoadStore();
+ loadstore->set_barrier_data(0);
+ } else if (node->is_Mem()) {
+ MemNode* mem = node->as_Mem();
+ mem->set_barrier_data(0);
}
+}
- if (use_ReduceInitialCardMarks() && obj == kit->just_allocated_object(kit->control())) {
- // We can skip marks on a freshly-allocated object in Eden.
- // Keep this code in sync with CardTableBarrierSet::on_slowpath_allocation_exit.
- // That routine informs GC to take appropriate compensating steps,
- // upon a slow-path allocation, so as to make this card-mark
- // elision safe.
+static void refine_barrier_by_new_val_type(const Node* n) {
+ if (n->Opcode() != Op_StoreP &&
+ n->Opcode() != Op_StoreN) {
return;
}
-
- if (use_ReduceInitialCardMarks()
- && g1_can_remove_post_barrier(kit, &kit->gvn(), oop_store, adr)) {
+ MemNode* store = n->as_Mem();
+ const Node* newval = n->in(MemNode::ValueIn);
+ assert(newval != nullptr, "");
+ const Type* newval_bottom = newval->bottom_type();
+ TypePtr::PTR newval_type = newval_bottom->make_ptr()->ptr();
+ uint8_t barrier_data = store->barrier_data();
+ if (!newval_bottom->isa_oopptr() &&
+ !newval_bottom->isa_narrowoop() &&
+ newval_type != TypePtr::Null) {
+ // newval is neither an OOP nor null, so there is no barrier to refine.
+ assert(barrier_data == 0, "non-OOP stores should have no barrier data");
return;
}
-
- if (!use_precise) {
- // All card marks for a (non-array) instance are in one place:
- adr = obj;
+ if (barrier_data == 0) {
+ // No barrier to refine.
+ return;
}
- // (Else it's an array (or unknown), and we want more precise card marks.)
- assert(adr != nullptr, "");
-
- IdealKit ideal(kit, true);
-
- Node* tls = __ thread(); // ThreadLocalStorage
-
- Node* no_base = __ top();
- float likely = PROB_LIKELY_MAG(3);
- float unlikely = PROB_UNLIKELY_MAG(3);
- Node* young_card = __ ConI((jint)G1CardTable::g1_young_card_val());
- Node* dirty_card = __ ConI((jint)G1CardTable::dirty_card_val());
- Node* zeroX = __ ConX(0);
-
- const TypeFunc *tf = write_ref_field_post_entry_Type();
-
- // Offsets into the thread
- const int index_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset());
- const int buffer_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset());
-
- // Pointers into the thread
-
- Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset));
- Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset));
-
- // Now some values
- // Use ctrl to avoid hoisting these values past a safepoint, which could
- // potentially reset these fields in the JavaThread.
- Node* index = __ load(__ ctrl(), index_adr, TypeX_X, TypeX_X->basic_type(), Compile::AliasIdxRaw);
- Node* buffer = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
-
- // Convert the store obj pointer to an int prior to doing math on it
- // Must use ctrl to prevent "integerized oop" existing across safepoint
- Node* cast = __ CastPX(__ ctrl(), adr);
-
- // Divide pointer by card size
- Node* card_offset = __ URShiftX( cast, __ ConI(CardTable::card_shift()) );
-
- // Combine card table base and card offset
- Node* card_adr = __ AddP(no_base, byte_map_base_node(kit), card_offset );
-
- // If we know the value being stored does it cross regions?
-
- if (val != nullptr) {
- // Does the store cause us to cross regions?
-
- // Should be able to do an unsigned compare of region_size instead of
- // and extra shift. Do we have an unsigned compare??
- // Node* region_size = __ ConI(1 << G1HeapRegion::LogOfHRGrainBytes);
- Node* xor_res = __ URShiftX ( __ XorX( cast, __ CastPX(__ ctrl(), val)), __ ConI(checked_cast(G1HeapRegion::LogOfHRGrainBytes)));
-
- // if (xor_res == 0) same region so skip
- __ if_then(xor_res, BoolTest::ne, zeroX, likely); {
-
- // No barrier if we are storing a null.
- __ if_then(val, BoolTest::ne, kit->null(), likely); {
-
- // Ok must mark the card if not already dirty
-
- // load the original value of the card
- Node* card_val = __ load(__ ctrl(), card_adr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw);
-
- __ if_then(card_val, BoolTest::ne, young_card, unlikely); {
- kit->sync_kit(ideal);
- kit->insert_mem_bar(Op_MemBarVolatile, oop_store);
- __ sync_kit(kit);
-
- Node* card_val_reload = __ load(__ ctrl(), card_adr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw);
- __ if_then(card_val_reload, BoolTest::ne, dirty_card); {
- g1_mark_card(kit, ideal, card_adr, oop_store, alias_idx, index, index_adr, buffer, tf);
- } __ end_if();
- } __ end_if();
- } __ end_if();
- } __ end_if();
- } else {
- // The Object.clone() intrinsic uses this path if !ReduceInitialCardMarks.
- // We don't need a barrier here if the destination is a newly allocated object
- // in Eden. Otherwise, GC verification breaks because we assume that cards in Eden
- // are set to 'g1_young_gen' (see G1CardTable::verify_g1_young_region()).
- assert(!use_ReduceInitialCardMarks(), "can only happen with card marking");
- Node* card_val = __ load(__ ctrl(), card_adr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw);
- __ if_then(card_val, BoolTest::ne, young_card); {
- g1_mark_card(kit, ideal, card_adr, oop_store, alias_idx, index, index_adr, buffer, tf);
- } __ end_if();
+ if (newval_type == TypePtr::Null) {
+ // Simply elide post-barrier if writing null.
+ barrier_data &= ~G1C2BarrierPost;
+ barrier_data &= ~G1C2BarrierPostNotNull;
+ } else if (((barrier_data & G1C2BarrierPost) != 0) &&
+ newval_type == TypePtr::NotNull) {
+ // If the post-barrier has not been elided yet (e.g. due to newval being
+ // freshly allocated), mark it as not-null (simplifies barrier tests and
+ // compressed OOPs logic).
+ barrier_data |= G1C2BarrierPostNotNull;
}
-
- // Final sync IdealKit and GraphKit.
- kit->final_sync(ideal);
+ store->set_barrier_data(barrier_data);
+ return;
}
-// Helper that guards and inserts a pre-barrier.
-void G1BarrierSetC2::insert_pre_barrier(GraphKit* kit, Node* base_oop, Node* offset,
- Node* pre_val, bool need_mem_bar) const {
- // We could be accessing the referent field of a reference object. If so, when G1
- // is enabled, we need to log the value in the referent field in an SATB buffer.
- // This routine performs some compile time filters and generates suitable
- // runtime filters that guard the pre-barrier code.
- // Also add memory barrier for non volatile load from the referent field
- // to prevent commoning of loads across safepoint.
-
- // Some compile time checks.
-
- // If offset is a constant, is it java_lang_ref_Reference::_reference_offset?
- const TypeX* otype = offset->find_intptr_t_type();
- if (otype != nullptr && otype->is_con() &&
- otype->get_con() != java_lang_ref_Reference::referent_offset()) {
- // Constant offset but not the reference_offset so just return
- return;
- }
-
- // We only need to generate the runtime guards for instances.
- const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr();
- if (btype != nullptr) {
- if (btype->isa_aryptr()) {
- // Array type so nothing to do
- return;
+// Refine (not really expand) G1 barriers by looking at the new value type
+// (whether it is necessarily null or necessarily non-null).
+bool G1BarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
+ ResourceMark rm;
+ VectorSet visited;
+ Node_List worklist;
+ worklist.push(C->root());
+ while (worklist.size() > 0) {
+ Node* n = worklist.pop();
+ if (visited.test_set(n->_idx)) {
+ continue;
}
-
- const TypeInstPtr* itype = btype->isa_instptr();
- if (itype != nullptr) {
- // Can the klass of base_oop be statically determined to be
- // _not_ a sub-class of Reference and _not_ Object?
- ciKlass* klass = itype->instance_klass();
- if (klass->is_loaded() &&
- !klass->is_subtype_of(kit->env()->Reference_klass()) &&
- !kit->env()->Object_klass()->is_subtype_of(klass)) {
- return;
+ refine_barrier_by_new_val_type(n);
+ for (uint j = 0; j < n->req(); j++) {
+ Node* in = n->in(j);
+ if (in != nullptr) {
+ worklist.push(in);
}
}
}
+ return false;
+}
- // The compile time filters did not reject base_oop/offset so
- // we need to generate the following runtime filters
- //
- // if (offset == java_lang_ref_Reference::_reference_offset) {
- // if (instance_of(base, java.lang.ref.Reference)) {
- // pre_barrier(_, pre_val, ...);
+uint G1BarrierSetC2::estimated_barrier_size(const Node* node) const {
+ // These Ideal node counts are extracted from the pre-matching Ideal graph
+ // generated when compiling the following method with early barrier expansion:
+ // static void write(MyObject obj1, Object o) {
+ // obj1.o1 = o;
// }
- // }
-
- float likely = PROB_LIKELY( 0.999);
- float unlikely = PROB_UNLIKELY(0.999);
-
- IdealKit ideal(kit);
-
- Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset());
-
- __ if_then(offset, BoolTest::eq, referent_off, unlikely); {
- // Update graphKit memory and control from IdealKit.
- kit->sync_kit(ideal);
-
- Node* ref_klass_con = kit->makecon(TypeKlassPtr::make(kit->env()->Reference_klass()));
- Node* is_instof = kit->gen_instanceof(base_oop, ref_klass_con);
-
- // Update IdealKit memory and control from graphKit.
- __ sync_kit(kit);
-
- Node* one = __ ConI(1);
- // is_instof == 0 if base_oop == nullptr
- __ if_then(is_instof, BoolTest::eq, one, unlikely); {
-
- // Update graphKit from IdeakKit.
- kit->sync_kit(ideal);
-
- // Use the pre-barrier to record the value in the referent field
- pre_barrier(kit, false /* do_load */,
- __ ctrl(),
- nullptr /* obj */, nullptr /* adr */, max_juint /* alias_idx */, nullptr /* val */, nullptr /* val_type */,
- pre_val /* pre_val */,
- T_OBJECT);
- if (need_mem_bar) {
- // Add memory barrier to prevent commoning reads from this field
- // across safepoint since GC can change its value.
- kit->insert_mem_bar(Op_MemBarCPUOrder);
- }
- // Update IdealKit from graphKit.
- __ sync_kit(kit);
-
- } __ end_if(); // _ref_type != ref_none
- } __ end_if(); // offset == referent_offset
+ uint8_t barrier_data = MemNode::barrier_data(node);
+ uint nodes = 0;
+ if ((barrier_data & G1C2BarrierPre) != 0) {
+ nodes += 50;
+ }
+ if ((barrier_data & G1C2BarrierPost) != 0) {
+ nodes += 60;
+ }
+ return nodes;
+}
- // Final sync IdealKit and GraphKit.
- kit->final_sync(ideal);
+bool G1BarrierSetC2::can_initialize_object(const StoreNode* store) const {
+ assert(store->Opcode() == Op_StoreP || store->Opcode() == Op_StoreN, "OOP store expected");
+ // It is OK to move the store across the object initialization boundary only
+ // if it does not have any barrier, or if it has barriers that can be safely
+ // elided (because of the compensation steps taken on the allocation slow path
+ // when ReduceInitialCardMarks is enabled).
+ return (MemNode::barrier_data(store) == 0) || use_ReduceInitialCardMarks();
}
-#undef __
+void G1BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
+ if (ac->is_clone_inst() && !use_ReduceInitialCardMarks()) {
+ clone_in_runtime(phase, ac, G1BarrierSetRuntime::clone_addr(), "G1BarrierSetRuntime::clone");
+ return;
+ }
+ BarrierSetC2::clone_at_expansion(phase, ac);
+}
-Node* G1BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
+Node* G1BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
DecoratorSet decorators = access.decorators();
- Node* adr = access.addr().node();
- Node* obj = access.base();
-
- bool anonymous = (decorators & C2_UNSAFE_ACCESS) != 0;
- bool mismatched = (decorators & C2_MISMATCHED) != 0;
- bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0;
+ bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
bool in_heap = (decorators & IN_HEAP) != 0;
- bool in_native = (decorators & IN_NATIVE) != 0;
- bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
- bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
- bool is_unordered = (decorators & MO_UNORDERED) != 0;
+ bool tightly_coupled_alloc = (decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0;
+ bool need_store_barrier = !(tightly_coupled_alloc && use_ReduceInitialCardMarks()) && (in_heap || anonymous);
bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0;
- bool is_mixed = !in_heap && !in_native;
- bool need_cpu_mem_bar = !is_unordered || mismatched || is_mixed;
-
- Node* top = Compile::current()->top();
- Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top;
-
- // If we are reading the value of the referent field of a Reference
- // object (either by using Unsafe directly or through reflection)
- // then, if G1 is enabled, we need to record the referent in an
- // SATB log buffer using the pre-barrier mechanism.
- // Also we need to add memory barrier to prevent commoning reads
- // from this field across safepoint since GC can change its value.
- bool need_read_barrier = (((on_weak || on_phantom) && !no_keepalive) ||
- (in_heap && unknown && offset != top && obj != top));
+ if (access.is_oop() && need_store_barrier) {
+ access.set_barrier_data(get_store_barrier(access));
+ if (tightly_coupled_alloc) {
+ assert(!use_ReduceInitialCardMarks(),
+ "post-barriers are only needed for tightly-coupled initialization stores when ReduceInitialCardMarks is disabled");
+ // Pre-barriers are unnecessary for tightly-coupled initialization stores.
+ access.set_barrier_data(access.barrier_data() & ~G1C2BarrierPre);
+ }
+ }
+ if (no_keepalive) {
+ // No keep-alive means no need for the pre-barrier.
+ access.set_barrier_data(access.barrier_data() & ~G1C2BarrierPre);
+ }
+ return BarrierSetC2::store_at_resolved(access, val);
+}
- if (!access.is_oop() || !need_read_barrier) {
- return CardTableBarrierSetC2::load_at_resolved(access, val_type);
+Node* G1BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
+ Node* new_val, const Type* value_type) const {
+ GraphKit* kit = access.kit();
+ if (!access.is_oop()) {
+ return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
}
+ access.set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
+ return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
+}
- assert(access.is_parse_access(), "entry not supported at optimization time");
+Node* G1BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
+ Node* new_val, const Type* value_type) const {
+ GraphKit* kit = access.kit();
+ if (!access.is_oop()) {
+ return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
+ }
+ access.set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
+ return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
+}
- C2ParseAccess& parse_access = static_cast(access);
- GraphKit* kit = parse_access.kit();
- Node* load;
+Node* G1BarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
+ GraphKit* kit = access.kit();
+ if (!access.is_oop()) {
+ return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, value_type);
+ }
+ access.set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
+ return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, value_type);
+}
- Node* control = kit->control();
- const TypePtr* adr_type = access.addr().type();
- MemNode::MemOrd mo = access.mem_node_mo();
- bool requires_atomic_access = (decorators & MO_UNORDERED) == 0;
- bool unaligned = (decorators & C2_UNALIGNED) != 0;
- bool unsafe = (decorators & C2_UNSAFE_ACCESS) != 0;
- // Pinned control dependency is the strictest. So it's ok to substitute it for any other.
- load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo,
- LoadNode::Pinned, requires_atomic_access, unaligned, mismatched, unsafe,
- access.barrier_data());
+class G1BarrierSetC2State : public BarrierSetC2State {
+private:
+ GrowableArray* _stubs;
+public:
+ G1BarrierSetC2State(Arena* arena)
+ : BarrierSetC2State(arena),
+ _stubs(new (arena) GrowableArray(arena, 8, 0, nullptr)) {}
- if (on_weak || on_phantom) {
- // Use the pre-barrier to record the value in the referent field
- pre_barrier(kit, false /* do_load */,
- kit->control(),
- nullptr /* obj */, nullptr /* adr */, max_juint /* alias_idx */, nullptr /* val */, nullptr /* val_type */,
- load /* pre_val */, T_OBJECT);
- // Add memory barrier to prevent commoning reads from this field
- // across safepoint since GC can change its value.
- kit->insert_mem_bar(Op_MemBarCPUOrder);
- } else if (unknown) {
- // We do not require a mem bar inside pre_barrier if need_mem_bar
- // is set: the barriers would be emitted by us.
- insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);
+ GrowableArray* stubs() {
+ return _stubs;
}
- return load;
-}
-
-bool G1BarrierSetC2::is_gc_barrier_node(Node* node) const {
- if (CardTableBarrierSetC2::is_gc_barrier_node(node)) {
- return true;
+ bool needs_liveness_data(const MachNode* mach) const {
+ return G1PreBarrierStubC2::needs_barrier(mach) ||
+ G1PostBarrierStubC2::needs_barrier(mach);
}
- if (node->Opcode() != Op_CallLeaf) {
- return false;
- }
- CallLeafNode *call = node->as_CallLeaf();
- if (call->_name == nullptr) {
+
+ bool needs_livein_data() const {
return false;
}
+};
- return strcmp(call->_name, "write_ref_field_pre_entry") == 0 || strcmp(call->_name, "write_ref_field_post_entry") == 0;
+static G1BarrierSetC2State* barrier_set_state() {
+ return reinterpret_cast(Compile::current()->barrier_set_state());
}
-bool G1BarrierSetC2::is_g1_pre_val_load(Node* n) {
- if (n->is_Load() && n->as_Load()->has_pinned_control_dependency()) {
- // Make sure the only users of it are: CmpP, StoreP, and a call to write_ref_field_pre_entry
+G1BarrierStubC2::G1BarrierStubC2(const MachNode* node) : BarrierStubC2(node) {}
- // Skip possible decode
- if (n->outcnt() == 1 && n->unique_out()->is_DecodeN()) {
- n = n->unique_out();
- }
+G1PreBarrierStubC2::G1PreBarrierStubC2(const MachNode* node) : G1BarrierStubC2(node) {}
- if (n->outcnt() == 3) {
- int found = 0;
- for (SimpleDUIterator iter(n); iter.has_next(); iter.next()) {
- Node* use = iter.get();
- if (use->is_Cmp() || use->is_Store()) {
- ++found;
- } else if (use->is_CallLeaf()) {
- CallLeafNode* call = use->as_CallLeaf();
- if (strcmp(call->_name, "write_ref_field_pre_entry") == 0) {
- ++found;
- }
- }
- }
- if (found == 3) {
- return true;
- }
- }
+bool G1PreBarrierStubC2::needs_barrier(const MachNode* node) {
+ return (node->barrier_data() & G1C2BarrierPre) != 0;
+}
+
+G1PreBarrierStubC2* G1PreBarrierStubC2::create(const MachNode* node) {
+ G1PreBarrierStubC2* const stub = new (Compile::current()->comp_arena()) G1PreBarrierStubC2(node);
+ if (!Compile::current()->output()->in_scratch_emit_size()) {
+ barrier_set_state()->stubs()->append(stub);
}
- return false;
+ return stub;
}
-bool G1BarrierSetC2::is_gc_pre_barrier_node(Node *node) const {
- return is_g1_pre_val_load(node);
+void G1PreBarrierStubC2::initialize_registers(Register obj, Register pre_val, Register thread, Register tmp1, Register tmp2) {
+ _obj = obj;
+ _pre_val = pre_val;
+ _thread = thread;
+ _tmp1 = tmp1;
+ _tmp2 = tmp2;
}
-void G1BarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
- if (is_g1_pre_val_load(node)) {
- macro->replace_node(node, macro->zerocon(node->as_Load()->bottom_type()->basic_type()));
- } else {
- assert(node->Opcode() == Op_CastP2X, "ConvP2XNode required");
- assert(node->outcnt() <= 2, "expects 1 or 2 users: Xor and URShift nodes");
- // It could be only one user, URShift node, in Object.clone() intrinsic
- // but the new allocation is passed to arraycopy stub and it could not
- // be scalar replaced. So we don't check the case.
+Register G1PreBarrierStubC2::obj() const {
+ return _obj;
+}
- // An other case of only one user (Xor) is when the value check for null
- // in G1 post barrier is folded after CCP so the code which used URShift
- // is removed.
+Register G1PreBarrierStubC2::pre_val() const {
+ return _pre_val;
+}
- // Take Region node before eliminating post barrier since it also
- // eliminates CastP2X node when it has only one user.
- Node* this_region = node->in(0);
- assert(this_region != nullptr, "");
+Register G1PreBarrierStubC2::thread() const {
+ return _thread;
+}
- // Remove G1 post barrier.
+Register G1PreBarrierStubC2::tmp1() const {
+ return _tmp1;
+}
+
+Register G1PreBarrierStubC2::tmp2() const {
+ return _tmp2;
+}
- // Search for CastP2X->Xor->URShift->Cmp path which
- // checks if the store done to a different from the value's region.
- // And replace Cmp with #0 (false) to collapse G1 post barrier.
- Node* xorx = node->find_out_with(Op_XorX);
- if (xorx != nullptr) {
- Node* shift = xorx->unique_out();
- Node* cmpx = shift->unique_out();
- assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
- cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
- "missing region check in G1 post barrier");
- macro->replace_node(cmpx, macro->makecon(TypeInt::CC_EQ));
+void G1PreBarrierStubC2::emit_code(MacroAssembler& masm) {
+ G1BarrierSetAssembler* bs = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
+ bs->generate_c2_pre_barrier_stub(&masm, this);
+}
- // Remove G1 pre barrier.
+G1PostBarrierStubC2::G1PostBarrierStubC2(const MachNode* node) : G1BarrierStubC2(node) {}
- // Search "if (marking != 0)" check and set it to "false".
- // There is no G1 pre barrier if previous stored value is null
- // (for example, after initialization).
- if (this_region->is_Region() && this_region->req() == 3) {
- int ind = 1;
- if (!this_region->in(ind)->is_IfFalse()) {
- ind = 2;
- }
- if (this_region->in(ind)->is_IfFalse() &&
- this_region->in(ind)->in(0)->Opcode() == Op_If) {
- Node* bol = this_region->in(ind)->in(0)->in(1);
- assert(bol->is_Bool(), "");
- cmpx = bol->in(1);
- if (bol->as_Bool()->_test._test == BoolTest::ne &&
- cmpx->is_Cmp() && cmpx->in(2) == macro->intcon(0) &&
- cmpx->in(1)->is_Load()) {
- Node* adr = cmpx->in(1)->as_Load()->in(MemNode::Address);
- const int marking_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
- if (adr->is_AddP() && adr->in(AddPNode::Base) == macro->top() &&
- adr->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
- adr->in(AddPNode::Offset) == macro->MakeConX(marking_offset)) {
- macro->replace_node(cmpx, macro->makecon(TypeInt::CC_EQ));
- }
- }
- }
- }
- } else {
- assert(!use_ReduceInitialCardMarks(), "can only happen with card marking");
- // This is a G1 post barrier emitted by the Object.clone() intrinsic.
- // Search for the CastP2X->URShiftX->AddP->LoadB->Cmp path which checks if the card
- // is marked as young_gen and replace the Cmp with 0 (false) to collapse the barrier.
- Node* shift = node->find_out_with(Op_URShiftX);
- assert(shift != nullptr, "missing G1 post barrier");
- Node* addp = shift->unique_out();
- Node* load = addp->find_out_with(Op_LoadB);
- assert(load != nullptr, "missing G1 post barrier");
- Node* cmpx = load->unique_out();
- assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
- cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
- "missing card value check in G1 post barrier");
- macro->replace_node(cmpx, macro->makecon(TypeInt::CC_EQ));
- // There is no G1 pre barrier in this case
- }
- // Now CastP2X can be removed since it is used only on dead path
- // which currently still alive until igvn optimize it.
- assert(node->outcnt() == 0 || node->unique_out()->Opcode() == Op_URShiftX, "");
- macro->replace_node(node, macro->top());
- }
+bool G1PostBarrierStubC2::needs_barrier(const MachNode* node) {
+ return (node->barrier_data() & G1C2BarrierPost) != 0;
}
-Node* G1BarrierSetC2::step_over_gc_barrier(Node* c) const {
- if (!use_ReduceInitialCardMarks() &&
- c != nullptr && c->is_Region() && c->req() == 3) {
- for (uint i = 1; i < c->req(); i++) {
- if (c->in(i) != nullptr && c->in(i)->is_Region() &&
- c->in(i)->req() == 3) {
- Node* r = c->in(i);
- for (uint j = 1; j < r->req(); j++) {
- if (r->in(j) != nullptr && r->in(j)->is_Proj() &&
- r->in(j)->in(0) != nullptr &&
- r->in(j)->in(0)->Opcode() == Op_CallLeaf &&
- r->in(j)->in(0)->as_Call()->entry_point() == CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry)) {
- Node* call = r->in(j)->in(0);
- c = c->in(i == 1 ? 2 : 1);
- if (c != nullptr && c->Opcode() != Op_Parm) {
- c = c->in(0);
- if (c != nullptr) {
- c = c->in(0);
- assert(call->in(0) == nullptr ||
- call->in(0)->in(0) == nullptr ||
- call->in(0)->in(0)->in(0) == nullptr ||
- call->in(0)->in(0)->in(0)->in(0) == nullptr ||
- call->in(0)->in(0)->in(0)->in(0)->in(0) == nullptr ||
- c == call->in(0)->in(0)->in(0)->in(0)->in(0), "bad barrier shape");
- return c;
- }
- }
- }
- }
- }
- }
+G1PostBarrierStubC2* G1PostBarrierStubC2::create(const MachNode* node) {
+ G1PostBarrierStubC2* const stub = new (Compile::current()->comp_arena()) G1PostBarrierStubC2(node);
+ if (!Compile::current()->output()->in_scratch_emit_size()) {
+ barrier_set_state()->stubs()->append(stub);
}
- return c;
+ return stub;
}
-#ifdef ASSERT
-bool G1BarrierSetC2::has_cas_in_use_chain(Node *n) const {
- Unique_Node_List visited;
- Node_List worklist;
- worklist.push(n);
- while (worklist.size() > 0) {
- Node* x = worklist.pop();
- if (visited.member(x)) {
- continue;
- } else {
- visited.push(x);
- }
+void G1PostBarrierStubC2::initialize_registers(Register thread, Register tmp1, Register tmp2, Register tmp3) {
+ _thread = thread;
+ _tmp1 = tmp1;
+ _tmp2 = tmp2;
+ _tmp3 = tmp3;
+}
- if (x->is_LoadStore()) {
- int op = x->Opcode();
- if (op == Op_CompareAndExchangeP || op == Op_CompareAndExchangeN ||
- op == Op_CompareAndSwapP || op == Op_CompareAndSwapN ||
- op == Op_WeakCompareAndSwapP || op == Op_WeakCompareAndSwapN) {
- return true;
- }
- }
- if (!x->is_CFG()) {
- for (SimpleDUIterator iter(x); iter.has_next(); iter.next()) {
- Node* use = iter.get();
- worklist.push(use);
- }
- }
- }
- return false;
+Register G1PostBarrierStubC2::thread() const {
+ return _thread;
}
-void G1BarrierSetC2::verify_pre_load(Node* marking_if, Unique_Node_List& loads /*output*/) const {
- assert(loads.size() == 0, "Loads list should be empty");
- Node* pre_val_if = marking_if->find_out_with(Op_IfTrue)->find_out_with(Op_If);
- if (pre_val_if != nullptr) {
- Unique_Node_List visited;
- Node_List worklist;
- Node* pre_val = pre_val_if->in(1)->in(1)->in(1);
+Register G1PostBarrierStubC2::tmp1() const {
+ return _tmp1;
+}
- worklist.push(pre_val);
- while (worklist.size() > 0) {
- Node* x = worklist.pop();
- if (visited.member(x)) {
- continue;
- } else {
- visited.push(x);
- }
+Register G1PostBarrierStubC2::tmp2() const {
+ return _tmp2;
+}
- if (has_cas_in_use_chain(x)) {
- loads.clear();
- return;
- }
+Register G1PostBarrierStubC2::tmp3() const {
+ return _tmp3;
+}
- if (x->is_Con()) {
- continue;
- }
- if (x->is_EncodeP() || x->is_DecodeN()) {
- worklist.push(x->in(1));
- continue;
- }
- if (x->is_Load() || x->is_LoadStore()) {
- assert(x->in(0) != nullptr, "Pre-val load has to have a control");
- loads.push(x);
- continue;
- }
- if (x->is_Phi()) {
- for (uint i = 1; i < x->req(); i++) {
- worklist.push(x->in(i));
- }
- continue;
- }
- assert(false, "Pre-val anomaly");
- }
- }
+void G1PostBarrierStubC2::emit_code(MacroAssembler& masm) {
+ G1BarrierSetAssembler* bs = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
+ bs->generate_c2_post_barrier_stub(&masm, this);
}
-void G1BarrierSetC2::verify_no_safepoints(Compile* compile, Node* marking_check_if, const Unique_Node_List& loads) const {
- if (loads.size() == 0) {
- return;
- }
+void* G1BarrierSetC2::create_barrier_state(Arena* comp_arena) const {
+ return new (comp_arena) G1BarrierSetC2State(comp_arena);
+}
- if (loads.size() == 1) { // Handle the typical situation when there a single pre-value load
- // that is dominated by the marking_check_if, that's true when the
- // barrier itself does the pre-val load.
- Node *pre_val = loads.at(0);
- if (pre_val->in(0)->in(0) == marking_check_if) { // IfTrue->If
- return;
- }
+int G1BarrierSetC2::get_store_barrier(C2Access& access) const {
+ if (!access.is_parse_access()) {
+ // Only support for eliding barriers at parse time for now.
+ return G1C2BarrierPre | G1C2BarrierPost;
}
-
- // All other cases are when pre-value loads dominate the marking check.
- Unique_Node_List controls;
- for (uint i = 0; i < loads.size(); i++) {
- Node *c = loads.at(i)->in(0);
- controls.push(c);
+ GraphKit* kit = (static_cast(access)).kit();
+ Node* ctl = kit->control();
+ Node* adr = access.addr().node();
+ uint adr_idx = kit->C->get_alias_index(access.addr().type());
+ assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory");
+
+ bool can_remove_pre_barrier = g1_can_remove_pre_barrier(kit, &kit->gvn(), adr, access.type(), adr_idx);
+
+ // We can skip marks on a freshly-allocated object in Eden. Keep this code in
+ // sync with CardTableBarrierSet::on_slowpath_allocation_exit. That routine
+ // informs GC to take appropriate compensating steps, upon a slow-path
+ // allocation, so as to make this card-mark elision safe.
+ // The post-barrier can also be removed if null is written. This case is
+ // handled by G1BarrierSetC2::expand_barriers, which runs at the end of C2's
+ // platform-independent optimizations to exploit stronger type information.
+ bool can_remove_post_barrier = use_ReduceInitialCardMarks() &&
+ ((access.base() == kit->just_allocated_object(ctl)) ||
+ g1_can_remove_post_barrier(kit, &kit->gvn(), ctl, adr));
+
+ int barriers = 0;
+ if (!can_remove_pre_barrier) {
+ barriers |= G1C2BarrierPre;
+ }
+ if (!can_remove_post_barrier) {
+ barriers |= G1C2BarrierPost;
}
- Unique_Node_List visited;
- Unique_Node_List safepoints;
- Node_List worklist;
- uint found = 0;
+ return barriers;
+}
- worklist.push(marking_check_if);
- while (worklist.size() > 0 && found < controls.size()) {
- Node* x = worklist.pop();
- if (x == nullptr || x == compile->top()) continue;
- if (visited.member(x)) {
- continue;
- } else {
- visited.push(x);
- }
+void G1BarrierSetC2::late_barrier_analysis() const {
+ compute_liveness_at_stubs();
+}
- if (controls.member(x)) {
- found++;
- }
- if (x->is_Region()) {
- for (uint i = 1; i < x->req(); i++) {
- worklist.push(x->in(i));
- }
- } else {
- if (!x->is_SafePoint()) {
- worklist.push(x->in(0));
- } else {
- safepoints.push(x);
- }
+void G1BarrierSetC2::emit_stubs(CodeBuffer& cb) const {
+ MacroAssembler masm(&cb);
+ GrowableArray* const stubs = barrier_set_state()->stubs();
+ for (int i = 0; i < stubs->length(); i++) {
+ // Make sure there is enough space in the code buffer
+ if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == nullptr) {
+ ciEnv::current()->record_failure("CodeCache is full");
+ return;
}
+ stubs->at(i)->emit_code(masm);
}
- assert(found == controls.size(), "Pre-barrier structure anomaly or possible safepoint");
+ masm.flush();
}
-void G1BarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
- if (phase != BarrierSetC2::BeforeCodeGen) {
- return;
+#ifndef PRODUCT
+void G1BarrierSetC2::dump_barrier_data(const MachNode* mach, outputStream* st) const {
+ if ((mach->barrier_data() & G1C2BarrierPre) != 0) {
+ st->print("pre ");
}
- // Verify G1 pre-barriers
- const int marking_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
-
- Unique_Node_List visited;
- Node_List worklist;
- // We're going to walk control flow backwards starting from the Root
- worklist.push(compile->root());
- while (worklist.size() > 0) {
- Node* x = worklist.pop();
- if (x == nullptr || x == compile->top()) continue;
- if (visited.member(x)) {
- continue;
- } else {
- visited.push(x);
- }
-
- if (x->is_Region()) {
- for (uint i = 1; i < x->req(); i++) {
- worklist.push(x->in(i));
- }
- } else {
- worklist.push(x->in(0));
- // We are looking for the pattern:
- // /->ThreadLocal
- // If->Bool->CmpI->LoadB->AddP->ConL(marking_offset)
- // \->ConI(0)
- // We want to verify that the If and the LoadB have the same control
- // See GraphKit::g1_write_barrier_pre()
- if (x->is_If()) {
- IfNode *iff = x->as_If();
- if (iff->in(1)->is_Bool() && iff->in(1)->in(1)->is_Cmp()) {
- CmpNode *cmp = iff->in(1)->in(1)->as_Cmp();
- if (cmp->Opcode() == Op_CmpI && cmp->in(2)->is_Con() && cmp->in(2)->bottom_type()->is_int()->get_con() == 0
- && cmp->in(1)->is_Load()) {
- LoadNode* load = cmp->in(1)->as_Load();
- if (load->Opcode() == Op_LoadB && load->in(2)->is_AddP() && load->in(2)->in(2)->Opcode() == Op_ThreadLocal
- && load->in(2)->in(3)->is_Con()
- && load->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == marking_offset) {
-
- Node* if_ctrl = iff->in(0);
- Node* load_ctrl = load->in(0);
-
- if (if_ctrl != load_ctrl) {
- // Skip possible CProj->NeverBranch in infinite loops
- if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
- && if_ctrl->in(0)->is_NeverBranch()) {
- if_ctrl = if_ctrl->in(0)->in(0);
- }
- }
- assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
-
- Unique_Node_List loads;
- verify_pre_load(iff, loads);
- verify_no_safepoints(compile, iff, loads);
- }
- }
- }
- }
- }
+ if ((mach->barrier_data() & G1C2BarrierPost) != 0) {
+ st->print("post ");
}
-}
-#endif
-
-bool G1BarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const {
- if (opcode == Op_StoreP) {
- Node* adr = n->in(MemNode::Address);
- const Type* adr_type = gvn->type(adr);
- // Pointer stores in G1 barriers looks like unsafe access.
- // Ignore such stores to be able scalar replace non-escaping
- // allocations.
- if (adr_type->isa_rawptr() && adr->is_AddP()) {
- Node* base = conn_graph->get_addp_base(adr);
- if (base->Opcode() == Op_LoadP &&
- base->in(MemNode::Address)->is_AddP()) {
- adr = base->in(MemNode::Address);
- Node* tls = conn_graph->get_addp_base(adr);
- if (tls->Opcode() == Op_ThreadLocal) {
- int offs = (int) gvn->find_intptr_t_con(adr->in(AddPNode::Offset), Type::OffsetBot);
- const int buf_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset());
- if (offs == buf_offset) {
- return true; // G1 pre barrier previous oop value store.
- }
- if (offs == in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset())) {
- return true; // G1 post barrier card address store.
- }
- }
- }
- }
+ if ((mach->barrier_data() & G1C2BarrierPostNotNull) != 0) {
+ st->print("notnull ");
}
- return false;
}
+#endif // !PRODUCT
diff --git a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.hpp b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.hpp
index c445a87d2e46d..dc333d8c33174 100644
--- a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.hpp
+++ b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.hpp
@@ -31,29 +31,62 @@ class PhaseTransform;
class Type;
class TypeFunc;
-class G1BarrierSetC2: public CardTableBarrierSetC2 {
+const int G1C2BarrierPre = 1;
+const int G1C2BarrierPost = 2;
+const int G1C2BarrierPostNotNull = 4;
+
+class G1BarrierStubC2 : public BarrierStubC2 {
+public:
+ G1BarrierStubC2(const MachNode* node);
+ virtual void emit_code(MacroAssembler& masm) = 0;
+};
+
+class G1PreBarrierStubC2 : public G1BarrierStubC2 {
+private:
+ Register _obj;
+ Register _pre_val;
+ Register _thread;
+ Register _tmp1;
+ Register _tmp2;
+
+protected:
+ G1PreBarrierStubC2(const MachNode* node);
+
+public:
+ static bool needs_barrier(const MachNode* node);
+ static G1PreBarrierStubC2* create(const MachNode* node);
+ void initialize_registers(Register obj, Register pre_val, Register thread, Register tmp1 = noreg, Register tmp2 = noreg);
+ Register obj() const;
+ Register pre_val() const;
+ Register thread() const;
+ Register tmp1() const;
+ Register tmp2() const;
+ virtual void emit_code(MacroAssembler& masm);
+};
+
+class G1PostBarrierStubC2 : public G1BarrierStubC2 {
+private:
+ Register _thread;
+ Register _tmp1;
+ Register _tmp2;
+ Register _tmp3;
+
protected:
- virtual void pre_barrier(GraphKit* kit,
- bool do_load,
- Node* ctl,
- Node* obj,
- Node* adr,
- uint adr_idx,
- Node* val,
- const TypeOopPtr* val_type,
- Node* pre_val,
- BasicType bt) const;
-
- virtual void post_barrier(GraphKit* kit,
- Node* ctl,
- Node* store,
- Node* obj,
- Node* adr,
- uint adr_idx,
- Node* val,
- BasicType bt,
- bool use_precise) const;
+ G1PostBarrierStubC2(const MachNode* node);
+public:
+ static bool needs_barrier(const MachNode* node);
+ static G1PostBarrierStubC2* create(const MachNode* node);
+ void initialize_registers(Register thread, Register tmp1 = noreg, Register tmp2 = noreg, Register tmp3 = noreg);
+ Register thread() const;
+ Register tmp1() const;
+ Register tmp2() const;
+ Register tmp3() const;
+ virtual void emit_code(MacroAssembler& masm);
+};
+
+class G1BarrierSetC2: public CardTableBarrierSetC2 {
+protected:
bool g1_can_remove_pre_barrier(GraphKit* kit,
PhaseValues* phase,
Node* adr,
@@ -64,44 +97,31 @@ class G1BarrierSetC2: public CardTableBarrierSetC2 {
PhaseValues* phase, Node* store,
Node* adr) const;
- void g1_mark_card(GraphKit* kit,
- IdealKit& ideal,
- Node* card_adr,
- Node* oop_store,
- uint oop_alias_idx,
- Node* index,
- Node* index_adr,
- Node* buffer,
- const TypeFunc* tf) const;
-
- // Helper for unsafe accesses, that may or may not be on the referent field.
- // Generates the guards that check whether the result of
- // Unsafe.getReference should be recorded in an SATB log buffer.
- void insert_pre_barrier(GraphKit* kit, Node* base_oop, Node* offset, Node* pre_val, bool need_mem_bar) const;
-
- static const TypeFunc* write_ref_field_pre_entry_Type();
- static const TypeFunc* write_ref_field_post_entry_Type();
+ int get_store_barrier(C2Access& access) const;
virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const;
+ virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const;
+ virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
+ Node* new_val, const Type* value_type) const;
+ virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
+ Node* new_val, const Type* value_type) const;
+ virtual Node* atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const;
-#ifdef ASSERT
- bool has_cas_in_use_chain(Node* x) const;
- void verify_pre_load(Node* marking_check_if, Unique_Node_List& loads /*output*/) const;
- void verify_no_safepoints(Compile* compile, Node* marking_load, const Unique_Node_List& loads) const;
-#endif
-
- static bool is_g1_pre_val_load(Node* n);
public:
- virtual bool is_gc_pre_barrier_node(Node* node) const;
- virtual bool is_gc_barrier_node(Node* node) const;
virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const;
- virtual Node* step_over_gc_barrier(Node* c) const;
-
-#ifdef ASSERT
- virtual void verify_gc_barriers(Compile* compile, CompilePhase phase) const;
+ virtual void eliminate_gc_barrier_data(Node* node) const;
+ virtual bool expand_barriers(Compile* C, PhaseIterGVN& igvn) const;
+ virtual uint estimated_barrier_size(const Node* node) const;
+ virtual bool can_initialize_object(const StoreNode* store) const;
+ virtual void clone_at_expansion(PhaseMacroExpand* phase,
+ ArrayCopyNode* ac) const;
+ virtual void* create_barrier_state(Arena* comp_arena) const;
+ virtual void emit_stubs(CodeBuffer& cb) const;
+ virtual void late_barrier_analysis() const;
+
+#ifndef PRODUCT
+ virtual void dump_barrier_data(const MachNode* mach, outputStream* st) const;
#endif
-
- virtual bool escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const;
};
#endif // SHARE_GC_G1_C2_G1BARRIERSETC2_HPP
diff --git a/src/hotspot/share/gc/g1/g1BarrierSetRuntime.cpp b/src/hotspot/share/gc/g1/g1BarrierSetRuntime.cpp
index a0fce437807f4..2e247f46c93d8 100644
--- a/src/hotspot/share/gc/g1/g1BarrierSetRuntime.cpp
+++ b/src/hotspot/share/gc/g1/g1BarrierSetRuntime.cpp
@@ -61,3 +61,11 @@ JRT_LEAF(void, G1BarrierSetRuntime::write_ref_field_post_entry(volatile G1CardTa
G1DirtyCardQueue& queue = G1ThreadLocalData::dirty_card_queue(thread);
G1BarrierSet::dirty_card_queue_set().enqueue(queue, card_addr);
JRT_END
+
+JRT_LEAF(void, G1BarrierSetRuntime::clone(oopDesc* src, oopDesc* dst, size_t size))
+ HeapAccess<>::clone(src, dst, size);
+JRT_END
+
+address G1BarrierSetRuntime::clone_addr() {
+ return reinterpret_cast(clone);
+}
diff --git a/src/hotspot/share/gc/g1/g1BarrierSetRuntime.hpp b/src/hotspot/share/gc/g1/g1BarrierSetRuntime.hpp
index 366679f032ba9..f98e94096e72d 100644
--- a/src/hotspot/share/gc/g1/g1BarrierSetRuntime.hpp
+++ b/src/hotspot/share/gc/g1/g1BarrierSetRuntime.hpp
@@ -35,6 +35,8 @@ class oopDesc;
class JavaThread;
class G1BarrierSetRuntime: public AllStatic {
+private:
+ static void clone(oopDesc* src, oopDesc* dst, size_t size);
public:
using CardValue = G1CardTable::CardValue;
@@ -46,6 +48,8 @@ class G1BarrierSetRuntime: public AllStatic {
// C2 slow-path runtime calls.
static void write_ref_field_pre_entry(oopDesc* orig, JavaThread *thread);
static void write_ref_field_post_entry(volatile CardValue* card_addr, JavaThread* thread);
+
+ static address clone_addr();
};
#endif // SHARE_GC_G1_G1BARRIERSETRUNTIME_HPP
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
index fd73b725a1289..57236d6f6db93 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
@@ -2595,8 +2595,9 @@ void G1CollectedHeap::set_young_gen_card_set_stats(const G1MonotonicArenaMemoryS
}
void G1CollectedHeap::record_obj_copy_mem_stats() {
+ size_t total_old_allocated = _old_evac_stats.allocated() + _old_evac_stats.direct_allocated();
policy()->old_gen_alloc_tracker()->
- add_allocated_bytes_since_last_gc(_old_evac_stats.allocated() * HeapWordSize);
+ add_allocated_bytes_since_last_gc(total_old_allocated * HeapWordSize);
_gc_tracer_stw->report_evacuation_statistics(create_g1_evac_summary(&_survivor_evac_stats),
create_g1_evac_summary(&_old_evac_stats));
diff --git a/src/hotspot/share/gc/g1/g1CollectionSet.cpp b/src/hotspot/share/gc/g1/g1CollectionSet.cpp
index d315497268f99..ec90fd377503d 100644
--- a/src/hotspot/share/gc/g1/g1CollectionSet.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectionSet.cpp
@@ -26,7 +26,7 @@
#include "gc/g1/g1Analytics.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1CollectionSet.hpp"
-#include "gc/g1/g1CollectionSetCandidates.hpp"
+#include "gc/g1/g1CollectionSetCandidates.inline.hpp"
#include "gc/g1/g1CollectorState.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
@@ -346,20 +346,16 @@ void G1CollectionSet::finalize_old_part(double time_remaining_ms) {
G1CollectionCandidateRegionList pinned_retained_regions;
if (collector_state()->in_mixed_phase()) {
- time_remaining_ms = _policy->select_candidates_from_marking(&candidates()->marking_regions(),
- time_remaining_ms,
- &initial_old_regions,
- &_optional_old_regions,
- &pinned_marking_regions);
+ time_remaining_ms = select_candidates_from_marking(time_remaining_ms,
+ &initial_old_regions,
+ &pinned_marking_regions);
} else {
log_debug(gc, ergo, cset)("Do not add marking candidates to collection set due to pause type.");
}
- _policy->select_candidates_from_retained(&candidates()->retained_regions(),
- time_remaining_ms,
- &initial_old_regions,
- &_optional_old_regions,
- &pinned_retained_regions);
+ select_candidates_from_retained(time_remaining_ms,
+ &initial_old_regions,
+ &pinned_retained_regions);
// Move initially selected old regions to collection set directly.
move_candidates_to_collection_set(&initial_old_regions);
@@ -394,6 +390,215 @@ void G1CollectionSet::move_candidates_to_collection_set(G1CollectionCandidateReg
candidates()->remove(regions);
}
+static void print_finish_message(const char* reason, bool from_marking) {
+ log_debug(gc, ergo, cset)("Finish adding %s candidates to collection set (%s).",
+ from_marking ? "marking" : "retained", reason);
+}
+
+double G1CollectionSet::select_candidates_from_marking(double time_remaining_ms,
+ G1CollectionCandidateRegionList* initial_old_regions,
+ G1CollectionCandidateRegionList* pinned_old_regions) {
+ uint num_expensive_regions = 0;
+
+ uint num_initial_regions_selected = 0;
+ uint num_optional_regions_selected = 0;
+ uint num_pinned_regions = 0;
+
+ double predicted_initial_time_ms = 0.0;
+ double predicted_optional_time_ms = 0.0;
+
+ double optional_threshold_ms = time_remaining_ms * _policy->optional_prediction_fraction();
+
+ const uint min_old_cset_length = _policy->calc_min_old_cset_length(candidates()->last_marking_candidates_length());
+ const uint max_old_cset_length = MAX2(min_old_cset_length, _policy->calc_max_old_cset_length());
+ const uint max_optional_regions = max_old_cset_length - min_old_cset_length;
+ bool check_time_remaining = _policy->use_adaptive_young_list_length();
+
+ G1CollectionCandidateList* marking_list = &candidates()->marking_regions();
+ assert(marking_list != nullptr, "must be");
+
+ log_debug(gc, ergo, cset)("Start adding marking candidates to collection set. "
+ "Min %u regions, max %u regions, available %u regions"
+ "time remaining %1.2fms, optional threshold %1.2fms",
+ min_old_cset_length, max_old_cset_length, marking_list->length(), time_remaining_ms, optional_threshold_ms);
+
+ G1CollectionCandidateListIterator iter = marking_list->begin();
+ for (; iter != marking_list->end(); ++iter) {
+ if (num_initial_regions_selected + num_optional_regions_selected >= max_old_cset_length) {
+ // Added maximum number of old regions to the CSet.
+ print_finish_message("Maximum number of regions reached", true);
+ break;
+ }
+ G1HeapRegion* hr = (*iter)->_r;
+ // Skip evacuating pinned marking regions because we are not getting any free
+ // space from them (and we expect to get free space from marking candidates).
+ // Also prepare to move them to retained regions to be evacuated optionally later
+ // to not impact the mixed phase too much.
+ if (hr->has_pinned_objects()) {
+ num_pinned_regions++;
+ (*iter)->update_num_unreclaimed();
+ log_trace(gc, ergo, cset)("Marking candidate %u can not be reclaimed currently. Skipping.", hr->hrm_index());
+ pinned_old_regions->append(hr);
+ continue;
+ }
+ double predicted_time_ms = _policy->predict_region_total_time_ms(hr, false);
+ time_remaining_ms = MAX2(time_remaining_ms - predicted_time_ms, 0.0);
+ // Add regions to old set until we reach the minimum amount
+ if (initial_old_regions->length() < min_old_cset_length) {
+ initial_old_regions->append(hr);
+ num_initial_regions_selected++;
+ predicted_initial_time_ms += predicted_time_ms;
+ // Record the number of regions added with no time remaining
+ if (time_remaining_ms == 0.0) {
+ num_expensive_regions++;
+ }
+ } else if (!check_time_remaining) {
+ // In the non-auto-tuning case, we'll finish adding regions
+ // to the CSet if we reach the minimum.
+ print_finish_message("Region amount reached min", true);
+ break;
+ } else {
+ // Keep adding regions to old set until we reach the optional threshold
+ if (time_remaining_ms > optional_threshold_ms) {
+ predicted_initial_time_ms += predicted_time_ms;
+ initial_old_regions->append(hr);
+ num_initial_regions_selected++;
+ } else if (time_remaining_ms > 0) {
+ // Keep adding optional regions until time is up.
+ assert(_optional_old_regions.length() < max_optional_regions, "Should not be possible.");
+ predicted_optional_time_ms += predicted_time_ms;
+ _optional_old_regions.append(hr);
+ num_optional_regions_selected++;
+ } else {
+ print_finish_message("Predicted time too high", true);
+ break;
+ }
+ }
+ }
+ if (iter == marking_list->end()) {
+ log_debug(gc, ergo, cset)("Marking candidates exhausted.");
+ }
+
+ if (num_expensive_regions > 0) {
+ log_debug(gc, ergo, cset)("Added %u marking candidates to collection set although the predicted time was too high.",
+ num_expensive_regions);
+ }
+
+ log_debug(gc, ergo, cset)("Finish adding marking candidates to collection set. Initial: %u, optional: %u, pinned: %u, "
+ "predicted initial time: %1.2fms, predicted optional time: %1.2fms, time remaining: %1.2fms",
+ num_initial_regions_selected, num_optional_regions_selected, num_pinned_regions,
+ predicted_initial_time_ms, predicted_optional_time_ms, time_remaining_ms);
+
+ assert(initial_old_regions->length() == num_initial_regions_selected, "must be");
+ assert(_optional_old_regions.length() == num_optional_regions_selected, "must be");
+ return time_remaining_ms;
+}
+
+void G1CollectionSet::select_candidates_from_retained(double time_remaining_ms,
+ G1CollectionCandidateRegionList* initial_old_regions,
+ G1CollectionCandidateRegionList* pinned_old_regions) {
+ uint num_initial_regions_selected = 0;
+ uint num_optional_regions_selected = 0;
+ uint num_expensive_regions_selected = 0;
+ uint num_pinned_regions = 0;
+
+ double predicted_initial_time_ms = 0.0;
+ double predicted_optional_time_ms = 0.0;
+
+ uint const min_regions = _policy->min_retained_old_cset_length();
+ // We want to make sure that on the one hand we process the retained regions asap,
+ // but on the other hand do not take too many of them as optional regions.
+ // So we split the time budget into budget we will unconditionally take into the
+ // initial old regions, and budget for taking optional regions from the retained
+ // list.
+ double optional_time_remaining_ms = _policy->max_time_for_retaining();
+ time_remaining_ms = MIN2(time_remaining_ms, optional_time_remaining_ms);
+
+ G1CollectionCandidateList* retained_list = &candidates()->retained_regions();
+
+ log_debug(gc, ergo, cset)("Start adding retained candidates to collection set. "
+ "Min %u regions, available %u, "
+ "time remaining %1.2fms, optional remaining %1.2fms",
+ min_regions, retained_list->length(), time_remaining_ms, optional_time_remaining_ms);
+
+ for (G1CollectionSetCandidateInfo* ci : *retained_list) {
+ G1HeapRegion* r = ci->_r;
+ double predicted_time_ms = _policy->predict_region_total_time_ms(r, collector_state()->in_young_only_phase());
+ bool fits_in_remaining_time = predicted_time_ms <= time_remaining_ms;
+ // If we can't reclaim that region ignore it for now.
+ if (r->has_pinned_objects()) {
+ num_pinned_regions++;
+ if (ci->update_num_unreclaimed()) {
+ log_trace(gc, ergo, cset)("Retained candidate %u can not be reclaimed currently. Skipping.", r->hrm_index());
+ } else {
+ log_trace(gc, ergo, cset)("Retained candidate %u can not be reclaimed currently. Dropping.", r->hrm_index());
+ pinned_old_regions->append(r);
+ }
+ continue;
+ }
+
+ if (fits_in_remaining_time || (num_expensive_regions_selected < min_regions)) {
+ predicted_initial_time_ms += predicted_time_ms;
+ if (!fits_in_remaining_time) {
+ num_expensive_regions_selected++;
+ }
+ initial_old_regions->append(r);
+ num_initial_regions_selected++;
+ } else if (predicted_time_ms <= optional_time_remaining_ms) {
+ predicted_optional_time_ms += predicted_time_ms;
+ _optional_old_regions.append(r);
+ num_optional_regions_selected++;
+ } else {
+ // Fits neither initial nor optional time limit. Exit.
+ break;
+ }
+ time_remaining_ms = MAX2(0.0, time_remaining_ms - predicted_time_ms);
+ optional_time_remaining_ms = MAX2(0.0, optional_time_remaining_ms - predicted_time_ms);
+ }
+
+ uint num_regions_selected = num_initial_regions_selected + num_optional_regions_selected;
+ if (num_regions_selected == retained_list->length()) {
+ log_debug(gc, ergo, cset)("Retained candidates exhausted.");
+ }
+ if (num_expensive_regions_selected > 0) {
+ log_debug(gc, ergo, cset)("Added %u retained candidates to collection set although the predicted time was too high.",
+ num_expensive_regions_selected);
+ }
+
+ log_debug(gc, ergo, cset)("Finish adding retained candidates to collection set. Initial: %u, optional: %u, pinned: %u, "
+ "predicted initial time: %1.2fms, predicted optional time: %1.2fms, "
+ "time remaining: %1.2fms optional time remaining %1.2fms",
+ num_initial_regions_selected, num_optional_regions_selected, num_pinned_regions,
+ predicted_initial_time_ms, predicted_optional_time_ms, time_remaining_ms, optional_time_remaining_ms);
+}
+
+void G1CollectionSet::select_candidates_from_optional_regions(double time_remaining_ms,
+ G1CollectionCandidateRegionList* selected_regions) {
+ assert(optional_region_length() > 0,
+ "Should only be called when there are optional regions");
+
+ double total_prediction_ms = 0.0;
+
+ for (G1HeapRegion* r : _optional_old_regions) {
+ double prediction_ms = _policy->predict_region_total_time_ms(r, false);
+
+ if (prediction_ms > time_remaining_ms) {
+ log_debug(gc, ergo, cset)("Prediction %.3fms for region %u does not fit remaining time: %.3fms.",
+ prediction_ms, r->hrm_index(), time_remaining_ms);
+ break;
+ }
+ // This region will be included in the next optional evacuation.
+
+ total_prediction_ms += prediction_ms;
+ time_remaining_ms -= prediction_ms;
+
+ selected_regions->append(r);
+ }
+
+ log_debug(gc, ergo, cset)("Prepared %u regions out of %u for optional evacuation. Total predicted time: %.3fms",
+ selected_regions->length(), _optional_old_regions.length(), total_prediction_ms);
+}
+
void G1CollectionSet::prepare_optional_regions(G1CollectionCandidateRegionList* regions){
uint cur_index = 0;
for (G1HeapRegion* r : *regions) {
@@ -441,9 +646,8 @@ bool G1CollectionSet::finalize_optional_for_evacuation(double remaining_pause_ti
update_incremental_marker();
G1CollectionCandidateRegionList selected_regions;
- _policy->calculate_optional_collection_set_regions(&_optional_old_regions,
- remaining_pause_time,
- &selected_regions);
+ select_candidates_from_optional_regions(remaining_pause_time,
+ &selected_regions);
move_candidates_to_collection_set(&selected_regions);
diff --git a/src/hotspot/share/gc/g1/g1CollectionSet.hpp b/src/hotspot/share/gc/g1/g1CollectionSet.hpp
index e569d3ee966c3..5280ba7d0fd6c 100644
--- a/src/hotspot/share/gc/g1/g1CollectionSet.hpp
+++ b/src/hotspot/share/gc/g1/g1CollectionSet.hpp
@@ -196,6 +196,22 @@ class G1CollectionSet {
// and retained collection set candidates.
void finalize_old_part(double time_remaining_ms);
+ // Calculate and fill in the initial, optional and pinned old gen candidate regions from
+ // the given candidate list and the remaining time.
+ // Returns the remaining time.
+ double select_candidates_from_marking(double time_remaining_ms,
+ G1CollectionCandidateRegionList* initial_old_regions,
+ G1CollectionCandidateRegionList* pinned_old_regions);
+
+ void select_candidates_from_retained(double time_remaining_ms,
+ G1CollectionCandidateRegionList* initial_old_regions,
+ G1CollectionCandidateRegionList* pinned_old_regions);
+
+ // Calculate the number of optional regions from the given collection set candidates,
+ // the remaining time and the maximum number of these regions.
+ void select_candidates_from_optional_regions(double time_remaining_ms,
+ G1CollectionCandidateRegionList* selected);
+
// Iterate the part of the collection set given by the offset and length applying the given
// G1HeapRegionClosure. The worker_id will determine where in the part to start the iteration
// to allow for more efficient parallel iteration.
diff --git a/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp b/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp
index 2369a0f7812ea..9fb56f7c58fb2 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp
@@ -66,7 +66,7 @@ G1HeapRegionManager::G1HeapRegionManager() :
_bot_mapper(nullptr),
_cardtable_mapper(nullptr),
_committed_map(),
- _allocated_heapregions_length(0),
+ _next_highest_used_hrm_index(0),
_regions(), _heap_mapper(nullptr),
_bitmap_mapper(nullptr),
_free_list("Free list", new G1MasterFreeRegionListChecker())
@@ -76,7 +76,7 @@ void G1HeapRegionManager::initialize(G1RegionToSpaceMapper* heap_storage,
G1RegionToSpaceMapper* bitmap,
G1RegionToSpaceMapper* bot,
G1RegionToSpaceMapper* cardtable) {
- _allocated_heapregions_length = 0;
+ _next_highest_used_hrm_index = 0;
_heap_mapper = heap_storage;
@@ -169,7 +169,7 @@ void G1HeapRegionManager::expand(uint start, uint num_regions, WorkerThreads* pr
hr = new_heap_region(i);
OrderAccess::storestore();
_regions.set_by_index(i, hr);
- _allocated_heapregions_length = MAX2(_allocated_heapregions_length, i + 1);
+ _next_highest_used_hrm_index = MAX2(_next_highest_used_hrm_index, i + 1);
}
G1HeapRegionPrinter::commit(hr);
}
@@ -489,7 +489,7 @@ uint G1HeapRegionManager::find_contiguous_allow_expand(uint num_regions) {
G1HeapRegion* G1HeapRegionManager::next_region_in_heap(const G1HeapRegion* r) const {
guarantee(r != nullptr, "Start region must be a valid region");
guarantee(is_available(r->hrm_index()), "Trying to iterate starting from region %u which is not in the heap", r->hrm_index());
- for (uint i = r->hrm_index() + 1; i < _allocated_heapregions_length; i++) {
+ for (uint i = r->hrm_index() + 1; i < _next_highest_used_hrm_index; i++) {
G1HeapRegion* hr = _regions.get_by_index(i);
if (is_available(i)) {
return hr;
@@ -583,8 +583,8 @@ void G1HeapRegionManager::par_iterate(G1HeapRegionClosure* blk, G1HeapRegionClai
uint G1HeapRegionManager::shrink_by(uint num_regions_to_remove) {
assert(length() > 0, "the region sequence should not be empty");
- assert(length() <= _allocated_heapregions_length, "invariant");
- assert(_allocated_heapregions_length > 0, "we should have at least one region committed");
+ assert(length() <= _next_highest_used_hrm_index, "invariant");
+ assert(_next_highest_used_hrm_index > 0, "we should have at least one region committed");
assert(num_regions_to_remove < length(), "We should never remove all regions");
if (num_regions_to_remove == 0) {
@@ -592,7 +592,7 @@ uint G1HeapRegionManager::shrink_by(uint num_regions_to_remove) {
}
uint removed = 0;
- uint cur = _allocated_heapregions_length;
+ uint cur = _next_highest_used_hrm_index;
uint idx_last_found = 0;
uint num_last_found = 0;
@@ -624,7 +624,7 @@ void G1HeapRegionManager::shrink_at(uint index, size_t num_regions) {
}
uint G1HeapRegionManager::find_empty_from_idx_reverse(uint start_idx, uint* res_idx) const {
- guarantee(start_idx <= _allocated_heapregions_length, "checking");
+ guarantee(start_idx <= _next_highest_used_hrm_index, "checking");
guarantee(res_idx != nullptr, "checking");
auto is_available_and_empty = [&] (uint index) {
@@ -658,12 +658,12 @@ uint G1HeapRegionManager::find_empty_from_idx_reverse(uint start_idx, uint* res_
}
void G1HeapRegionManager::verify() {
- guarantee(length() <= _allocated_heapregions_length,
- "invariant: _length: %u _allocated_length: %u",
- length(), _allocated_heapregions_length);
- guarantee(_allocated_heapregions_length <= reserved_length(),
- "invariant: _allocated_length: %u _max_length: %u",
- _allocated_heapregions_length, reserved_length());
+ guarantee(length() <= _next_highest_used_hrm_index,
+ "invariant: _length: %u _next_highest_used_hrm_index: %u",
+ length(), _next_highest_used_hrm_index);
+ guarantee(_next_highest_used_hrm_index <= reserved_length(),
+ "invariant: _next_highest_used_hrm_index: %u _max_length: %u",
+ _next_highest_used_hrm_index, reserved_length());
guarantee(length() <= max_length(),
"invariant: committed regions: %u max_regions: %u",
length(), max_length());
@@ -671,7 +671,7 @@ void G1HeapRegionManager::verify() {
bool prev_committed = true;
uint num_committed = 0;
HeapWord* prev_end = heap_bottom();
- for (uint i = 0; i < _allocated_heapregions_length; i++) {
+ for (uint i = 0; i < _next_highest_used_hrm_index; i++) {
if (!is_available(i)) {
prev_committed = false;
continue;
@@ -693,7 +693,7 @@ void G1HeapRegionManager::verify() {
prev_committed = true;
prev_end = hr->end();
}
- for (uint i = _allocated_heapregions_length; i < reserved_length(); i++) {
+ for (uint i = _next_highest_used_hrm_index; i < reserved_length(); i++) {
guarantee(_regions.get_by_index(i) == nullptr, "invariant i: %u", i);
}
@@ -708,7 +708,7 @@ void G1HeapRegionManager::verify_optional() {
#endif // PRODUCT
G1HeapRegionClaimer::G1HeapRegionClaimer(uint n_workers) :
- _n_workers(n_workers), _n_regions(G1CollectedHeap::heap()->_hrm._allocated_heapregions_length), _claims(nullptr) {
+ _n_workers(n_workers), _n_regions(G1CollectedHeap::heap()->_hrm._next_highest_used_hrm_index), _claims(nullptr) {
uint* new_claims = NEW_C_HEAP_ARRAY(uint, _n_regions, mtGC);
memset(new_claims, Unclaimed, sizeof(*_claims) * _n_regions);
_claims = new_claims;
diff --git a/src/hotspot/share/gc/g1/g1HeapRegionManager.hpp b/src/hotspot/share/gc/g1/g1HeapRegionManager.hpp
index 81bca4ce6381c..563140acf5b8f 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegionManager.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionManager.hpp
@@ -64,8 +64,8 @@ class G1HeapRegionTable : public G1BiasedMappedArray {
//
// * _num_committed (returned by length()) is the number of currently
// committed regions. These may not be contiguous.
-// * _allocated_heapregions_length (not exposed outside this class) is the
-// number of regions+1 for which we have G1HeapRegions.
+// * _next_highest_used_hrm_index (not exposed outside this class) is the
+// highest heap region index +1 for which we have G1HeapRegions.
// * max_length() returns the maximum number of regions the heap may commit.
// * reserved_length() returns the maximum number of regions the heap has reserved.
//
@@ -81,8 +81,8 @@ class G1HeapRegionManager: public CHeapObj {
// can either be active (ready for use) or inactive (ready for uncommit).
G1CommittedRegionMap _committed_map;
- // Internal only. The highest heap region +1 we allocated a G1HeapRegion instance for.
- uint _allocated_heapregions_length;
+ // Internal only. The highest heap region index +1 we allocated a G1HeapRegion instance for.
+ uint _next_highest_used_hrm_index;
HeapWord* heap_bottom() const { return _regions.bottom_address_mapped(); }
HeapWord* heap_end() const {return _regions.end_address_mapped(); }
diff --git a/src/hotspot/share/gc/g1/g1HeapRegionPrinter.hpp b/src/hotspot/share/gc/g1/g1HeapRegionPrinter.hpp
index d7b1a6da92c17..577a8552091f6 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegionPrinter.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionPrinter.hpp
@@ -35,8 +35,8 @@ class G1HeapRegionPrinter : public AllStatic {
// Print an action event.
static void print(const char* action, G1HeapRegion* hr) {
- log_trace(gc, region)("G1HR %s(%s) [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT "]",
- action, hr->get_type_str(), p2i(hr->bottom()), p2i(hr->top()), p2i(hr->end()));
+ log_trace(gc, region)("G1HR %4u %s(%s) [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT "]",
+ hr->hrm_index(), action, hr->get_type_str(), p2i(hr->bottom()), p2i(hr->top()), p2i(hr->end()));
}
public:
diff --git a/src/hotspot/share/gc/g1/g1IHOPControl.cpp b/src/hotspot/share/gc/g1/g1IHOPControl.cpp
index 25915cdd3566c..d1bfcf0d095f9 100644
--- a/src/hotspot/share/gc/g1/g1IHOPControl.cpp
+++ b/src/hotspot/share/gc/g1/g1IHOPControl.cpp
@@ -151,7 +151,7 @@ double G1AdaptiveIHOPControl::last_mutator_period_old_allocation_rate() const {
assert(_last_allocation_time_s > 0, "This should not be called when the last GC is full");
return _old_gen_alloc_tracker->last_period_old_gen_growth() / _last_allocation_time_s;
- }
+}
void G1AdaptiveIHOPControl::update_allocation_info(double allocation_time_s,
size_t additional_buffer_size) {
diff --git a/src/hotspot/share/gc/g1/g1OldGenAllocationTracker.hpp b/src/hotspot/share/gc/g1/g1OldGenAllocationTracker.hpp
index eb86b81d2abbd..265c7029e14da 100644
--- a/src/hotspot/share/gc/g1/g1OldGenAllocationTracker.hpp
+++ b/src/hotspot/share/gc/g1/g1OldGenAllocationTracker.hpp
@@ -32,17 +32,17 @@ class G1AdaptiveIHOPControl;
// Track allocation details in the old generation.
class G1OldGenAllocationTracker : public CHeapObj {
- // Total number of bytes allocated in the old generation during
- // last mutator period.
+ // Total number of bytes allocated in the old generation at the end
+ // of the last gc.
size_t _last_period_old_gen_bytes;
- // Total growth of the old geneneration for last mutator period,
- // taking eager reclaim into consideration.
+ // Total growth of the old geneneration since the last gc,
+ // taking eager-reclaim into consideration.
size_t _last_period_old_gen_growth;
// Total size of humongous objects for last gc.
size_t _humongous_bytes_after_last_gc;
- // Non-humongous old generation allocations during last mutator period.
+ // Non-humongous old generation allocations since the last gc.
size_t _allocated_bytes_since_last_gc;
// Humongous allocations during last mutator period.
size_t _allocated_humongous_bytes_since_last_gc;
diff --git a/src/hotspot/share/gc/g1/g1Policy.cpp b/src/hotspot/share/gc/g1/g1Policy.cpp
index e7e57c962c734..1b71901f0fe05 100644
--- a/src/hotspot/share/gc/g1/g1Policy.cpp
+++ b/src/hotspot/share/gc/g1/g1Policy.cpp
@@ -53,7 +53,7 @@
#include "gc/shared/gcTraceTime.inline.hpp"
G1Policy::G1Policy(STWGCTimer* gc_timer) :
- _predictor(G1ConfidencePercent / 100.0),
+ _predictor((100 - G1ConfidencePercent) / 100.0),
_analytics(new G1Analytics(&_predictor)),
_remset_tracker(),
_mmu_tracker(new G1MMUTracker(GCPauseIntervalMillis / 1000.0, MaxGCPauseMillis / 1000.0)),
@@ -1467,219 +1467,6 @@ uint G1Policy::calc_max_old_cset_length() const {
return (uint)ceil(result);
}
-static void print_finish_message(const char* reason, bool from_marking) {
- log_debug(gc, ergo, cset)("Finish adding %s candidates to collection set (%s).",
- from_marking ? "marking" : "retained", reason);
-}
-
-double G1Policy::select_candidates_from_marking(G1CollectionCandidateList* marking_list,
- double time_remaining_ms,
- G1CollectionCandidateRegionList* initial_old_regions,
- G1CollectionCandidateRegionList* optional_old_regions,
- G1CollectionCandidateRegionList* pinned_old_regions) {
- assert(marking_list != nullptr, "must be");
-
- uint num_expensive_regions = 0;
-
- uint num_initial_regions_selected = 0;
- uint num_optional_regions_selected = 0;
- uint num_pinned_regions = 0;
-
- double predicted_initial_time_ms = 0.0;
- double predicted_optional_time_ms = 0.0;
-
- double optional_threshold_ms = time_remaining_ms * optional_prediction_fraction();
-
- const uint min_old_cset_length = calc_min_old_cset_length(candidates()->last_marking_candidates_length());
- const uint max_old_cset_length = MAX2(min_old_cset_length, calc_max_old_cset_length());
- const uint max_optional_regions = max_old_cset_length - min_old_cset_length;
- bool check_time_remaining = use_adaptive_young_list_length();
-
- log_debug(gc, ergo, cset)("Start adding marking candidates to collection set. "
- "Min %u regions, max %u regions, available %u regions"
- "time remaining %1.2fms, optional threshold %1.2fms",
- min_old_cset_length, max_old_cset_length, marking_list->length(), time_remaining_ms, optional_threshold_ms);
-
- G1CollectionCandidateListIterator iter = marking_list->begin();
- for (; iter != marking_list->end(); ++iter) {
- if (num_initial_regions_selected + num_optional_regions_selected >= max_old_cset_length) {
- // Added maximum number of old regions to the CSet.
- print_finish_message("Maximum number of regions reached", true);
- break;
- }
- G1HeapRegion* hr = (*iter)->_r;
- // Skip evacuating pinned marking regions because we are not getting any free
- // space from them (and we expect to get free space from marking candidates).
- // Also prepare to move them to retained regions to be evacuated optionally later
- // to not impact the mixed phase too much.
- if (hr->has_pinned_objects()) {
- num_pinned_regions++;
- (*iter)->update_num_unreclaimed();
- log_trace(gc, ergo, cset)("Marking candidate %u can not be reclaimed currently. Skipping.", hr->hrm_index());
- pinned_old_regions->append(hr);
- continue;
- }
- double predicted_time_ms = predict_region_total_time_ms(hr, false);
- time_remaining_ms = MAX2(time_remaining_ms - predicted_time_ms, 0.0);
- // Add regions to old set until we reach the minimum amount
- if (initial_old_regions->length() < min_old_cset_length) {
- initial_old_regions->append(hr);
- num_initial_regions_selected++;
- predicted_initial_time_ms += predicted_time_ms;
- // Record the number of regions added with no time remaining
- if (time_remaining_ms == 0.0) {
- num_expensive_regions++;
- }
- } else if (!check_time_remaining) {
- // In the non-auto-tuning case, we'll finish adding regions
- // to the CSet if we reach the minimum.
- print_finish_message("Region amount reached min", true);
- break;
- } else {
- // Keep adding regions to old set until we reach the optional threshold
- if (time_remaining_ms > optional_threshold_ms) {
- predicted_initial_time_ms += predicted_time_ms;
- initial_old_regions->append(hr);
- num_initial_regions_selected++;
- } else if (time_remaining_ms > 0) {
- // Keep adding optional regions until time is up.
- assert(optional_old_regions->length() < max_optional_regions, "Should not be possible.");
- predicted_optional_time_ms += predicted_time_ms;
- optional_old_regions->append(hr);
- num_optional_regions_selected++;
- } else {
- print_finish_message("Predicted time too high", true);
- break;
- }
- }
- }
- if (iter == marking_list->end()) {
- log_debug(gc, ergo, cset)("Marking candidates exhausted.");
- }
-
- if (num_expensive_regions > 0) {
- log_debug(gc, ergo, cset)("Added %u marking candidates to collection set although the predicted time was too high.",
- num_expensive_regions);
- }
-
- log_debug(gc, ergo, cset)("Finish adding marking candidates to collection set. Initial: %u, optional: %u, pinned: %u, "
- "predicted initial time: %1.2fms, predicted optional time: %1.2fms, time remaining: %1.2fms",
- num_initial_regions_selected, num_optional_regions_selected, num_pinned_regions,
- predicted_initial_time_ms, predicted_optional_time_ms, time_remaining_ms);
-
- assert(initial_old_regions->length() == num_initial_regions_selected, "must be");
- assert(optional_old_regions->length() == num_optional_regions_selected, "must be");
- return time_remaining_ms;
-}
-
-void G1Policy::select_candidates_from_retained(G1CollectionCandidateList* retained_list,
- double time_remaining_ms,
- G1CollectionCandidateRegionList* initial_old_regions,
- G1CollectionCandidateRegionList* optional_old_regions,
- G1CollectionCandidateRegionList* pinned_old_regions) {
-
- uint const min_regions = min_retained_old_cset_length();
-
- uint num_initial_regions_selected = 0;
- uint num_optional_regions_selected = 0;
- uint num_expensive_regions_selected = 0;
- uint num_pinned_regions = 0;
-
- double predicted_initial_time_ms = 0.0;
- double predicted_optional_time_ms = 0.0;
-
- // We want to make sure that on the one hand we process the retained regions asap,
- // but on the other hand do not take too many of them as optional regions.
- // So we split the time budget into budget we will unconditionally take into the
- // initial old regions, and budget for taking optional regions from the retained
- // list.
- double optional_time_remaining_ms = max_time_for_retaining();
- time_remaining_ms = MIN2(time_remaining_ms, optional_time_remaining_ms);
-
- log_debug(gc, ergo, cset)("Start adding retained candidates to collection set. "
- "Min %u regions, available %u, "
- "time remaining %1.2fms, optional remaining %1.2fms",
- min_regions, retained_list->length(), time_remaining_ms, optional_time_remaining_ms);
-
- for (G1CollectionSetCandidateInfo* ci : *retained_list) {
- G1HeapRegion* r = ci->_r;
- double predicted_time_ms = predict_region_total_time_ms(r, collector_state()->in_young_only_phase());
- bool fits_in_remaining_time = predicted_time_ms <= time_remaining_ms;
- // If we can't reclaim that region ignore it for now.
- if (r->has_pinned_objects()) {
- num_pinned_regions++;
- if (ci->update_num_unreclaimed()) {
- log_trace(gc, ergo, cset)("Retained candidate %u can not be reclaimed currently. Skipping.", r->hrm_index());
- } else {
- log_trace(gc, ergo, cset)("Retained candidate %u can not be reclaimed currently. Dropping.", r->hrm_index());
- pinned_old_regions->append(r);
- }
- continue;
- }
-
- if (fits_in_remaining_time || (num_expensive_regions_selected < min_regions)) {
- predicted_initial_time_ms += predicted_time_ms;
- if (!fits_in_remaining_time) {
- num_expensive_regions_selected++;
- }
- initial_old_regions->append(r);
- num_initial_regions_selected++;
- } else if (predicted_time_ms <= optional_time_remaining_ms) {
- predicted_optional_time_ms += predicted_time_ms;
- optional_old_regions->append(r);
- num_optional_regions_selected++;
- } else {
- // Fits neither initial nor optional time limit. Exit.
- break;
- }
- time_remaining_ms = MAX2(0.0, time_remaining_ms - predicted_time_ms);
- optional_time_remaining_ms = MAX2(0.0, optional_time_remaining_ms - predicted_time_ms);
- }
-
- uint num_regions_selected = num_initial_regions_selected + num_optional_regions_selected;
- if (num_regions_selected == retained_list->length()) {
- log_debug(gc, ergo, cset)("Retained candidates exhausted.");
- }
- if (num_expensive_regions_selected > 0) {
- log_debug(gc, ergo, cset)("Added %u retained candidates to collection set although the predicted time was too high.",
- num_expensive_regions_selected);
- }
-
- log_debug(gc, ergo, cset)("Finish adding retained candidates to collection set. Initial: %u, optional: %u, pinned: %u, "
- "predicted initial time: %1.2fms, predicted optional time: %1.2fms, "
- "time remaining: %1.2fms optional time remaining %1.2fms",
- num_initial_regions_selected, num_optional_regions_selected, num_pinned_regions,
- predicted_initial_time_ms, predicted_optional_time_ms, time_remaining_ms, optional_time_remaining_ms);
-}
-
-void G1Policy::calculate_optional_collection_set_regions(G1CollectionCandidateRegionList* optional_regions,
- double time_remaining_ms,
- G1CollectionCandidateRegionList* selected_regions) {
- assert(_collection_set->optional_region_length() > 0,
- "Should only be called when there are optional regions");
-
- double total_prediction_ms = 0.0;
-
- for (G1HeapRegion* r : *optional_regions) {
- double prediction_ms = predict_region_total_time_ms(r, false);
-
- if (prediction_ms > time_remaining_ms) {
- log_debug(gc, ergo, cset)("Prediction %.3fms for region %u does not fit remaining time: %.3fms.",
- prediction_ms, r->hrm_index(), time_remaining_ms);
- break;
- }
- // This region will be included in the next optional evacuation.
-
- total_prediction_ms += prediction_ms;
- time_remaining_ms -= prediction_ms;
-
- selected_regions->append(r);
- }
-
- log_debug(gc, ergo, cset)("Prepared %u regions out of %u for optional evacuation. Total predicted time: %.3fms",
- selected_regions->length(), optional_regions->length(), total_prediction_ms);
-}
-
void G1Policy::transfer_survivors_to_cset(const G1SurvivorRegions* survivors) {
start_adding_survivor_regions();
diff --git a/src/hotspot/share/gc/g1/g1Policy.hpp b/src/hotspot/share/gc/g1/g1Policy.hpp
index 98d444084678c..9a6ffb570be70 100644
--- a/src/hotspot/share/gc/g1/g1Policy.hpp
+++ b/src/hotspot/share/gc/g1/g1Policy.hpp
@@ -335,27 +335,7 @@ class G1Policy: public CHeapObj {
// Amount of allowed waste in bytes in the collection set.
size_t allowed_waste_in_collection_set() const;
- // Calculate and fill in the initial, optional and pinned old gen candidate regions from
- // the given candidate list and the remaining time.
- // Returns the remaining time.
- double select_candidates_from_marking(G1CollectionCandidateList* marking_list,
- double time_remaining_ms,
- G1CollectionCandidateRegionList* initial_old_regions,
- G1CollectionCandidateRegionList* optional_old_regions,
- G1CollectionCandidateRegionList* pinned_old_regions);
-
- void select_candidates_from_retained(G1CollectionCandidateList* retained_list,
- double time_remaining_ms,
- G1CollectionCandidateRegionList* initial_old_regions,
- G1CollectionCandidateRegionList* optional_old_regions,
- G1CollectionCandidateRegionList* pinned_old_regions);
-
- // Calculate the number of optional regions from the given collection set candidates,
- // the remaining time and the maximum number of these regions and return the number
- // of actually selected regions in num_optional_regions.
- void calculate_optional_collection_set_regions(G1CollectionCandidateRegionList* optional_old_regions,
- double time_remaining_ms,
- G1CollectionCandidateRegionList* selected);
+
private:
@@ -423,12 +403,12 @@ class G1Policy: public CHeapObj {
size_t desired_survivor_size(uint max_regions) const;
+public:
// Fraction used when predicting how many optional regions to include in
// the CSet. This fraction of the available time is used for optional regions,
// the rest is used to add old regions to the normal CSet.
double optional_prediction_fraction() const { return 0.2; }
-public:
// Fraction used when evacuating the optional regions. This fraction of the
// remaining time is used to choose what regions to include in the evacuation.
double optional_evacuation_fraction() const { return 0.75; }
diff --git a/src/hotspot/share/gc/g1/g1Predictions.hpp b/src/hotspot/share/gc/g1/g1Predictions.hpp
index 510f296a9f3ac..ae2a8f418802a 100644
--- a/src/hotspot/share/gc/g1/g1Predictions.hpp
+++ b/src/hotspot/share/gc/g1/g1Predictions.hpp
@@ -29,8 +29,9 @@
// Utility class containing various helper methods for prediction.
class G1Predictions {
- private:
- double _sigma;
+private:
+ // Scale factor indicating to which degree stddev should be taking into account in predictions.
+ double _stddev_scale;
// This function is used to estimate the stddev of sample sets. There is some
// special consideration of small sample sets: the actual stddev for them is
@@ -46,16 +47,14 @@ class G1Predictions {
}
return estimate;
}
- public:
- G1Predictions(double sigma) : _sigma(sigma) {
- assert(sigma >= 0.0, "Confidence must be larger than or equal to zero");
- }
- // Confidence factor.
- double sigma() const { return _sigma; }
+public:
+ G1Predictions(double stddev_scale) : _stddev_scale(stddev_scale) {
+ assert(stddev_scale >= 0.0, "must be");
+ }
double predict(TruncatedSeq const* seq) const {
- return seq->davg() + _sigma * stddev_estimate(seq);
+ return seq->davg() + _stddev_scale * stddev_estimate(seq);
}
double predict_in_unit_interval(TruncatedSeq const* seq) const {
diff --git a/src/hotspot/share/gc/g1/g1RemSet.cpp b/src/hotspot/share/gc/g1/g1RemSet.cpp
index f5f65cf1c48aa..bb5ac5036fe47 100644
--- a/src/hotspot/share/gc/g1/g1RemSet.cpp
+++ b/src/hotspot/share/gc/g1/g1RemSet.cpp
@@ -967,6 +967,10 @@ class G1MergeHeapRootsTask : public WorkerTask {
_merged[G1GCPhaseTimes::MergeRSCards] += increment;
}
+ void dec_remset_cards(size_t decrement) {
+ _merged[G1GCPhaseTimes::MergeRSCards] -= decrement;
+ }
+
size_t merged(uint i) const { return _merged[i]; }
};
@@ -1091,6 +1095,11 @@ class G1MergeHeapRootsTask : public WorkerTask {
G1MergeCardSetStats stats() {
_merge_card_set_cache.flush();
+ // Compensation for the dummy cards that were initially pushed into the
+ // card cache.
+ // We do not need to compensate for the other counters because the dummy
+ // card mark will never update another counter because it is initally "dirty".
+ _stats.dec_remset_cards(G1MergeCardSetCache::CacheSize);
return _stats;
}
};
diff --git a/src/hotspot/share/gc/g1/g1_globals.hpp b/src/hotspot/share/gc/g1/g1_globals.hpp
index c8016ddc0ddf5..ed02ba2dc5cad 100644
--- a/src/hotspot/share/gc/g1/g1_globals.hpp
+++ b/src/hotspot/share/gc/g1/g1_globals.hpp
@@ -111,7 +111,8 @@
range(1, max_intx) \
\
product(uint, G1ConfidencePercent, 50, \
- "Confidence level for MMU/pause predictions") \
+ "Confidence level for MMU/pause predictions. A higher value " \
+ "means that G1 will use less safety margin for its predictions.") \
range(1, 100) \
\
product(uintx, G1SummarizeRSetStatsPeriod, 0, DIAGNOSTIC, \
diff --git a/src/hotspot/share/gc/parallel/mutableNUMASpace.hpp b/src/hotspot/share/gc/parallel/mutableNUMASpace.hpp
index 76653f6c5e641..10ca66e354992 100644
--- a/src/hotspot/share/gc/parallel/mutableNUMASpace.hpp
+++ b/src/hotspot/share/gc/parallel/mutableNUMASpace.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2006, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -53,12 +53,10 @@
* bytes that can be moved during the adaptation phase.
* Chunks may contain pages from a wrong locality group. The page-scanner has
* been introduced to address the problem. Remote pages typically appear due to
- * the memory shortage in the target locality group. Besides Solaris would
- * allocate a large page from the remote locality group even if there are small
- * local pages available. The page-scanner scans the pages right after the
- * collection and frees remote pages in hope that subsequent reallocation would
- * be more successful. This approach proved to be useful on systems with high
- * load where multiple processes are competing for the memory.
+ * the memory shortage in the target locality group. The page-scanner scans the pages
+ * right after the collection and frees remote pages in hope that subsequent
+ * reallocation would be more successful. This approach proved to be useful on systems
+ * with high load where multiple processes are competing for the memory.
*/
class MutableNUMASpace : public MutableSpace {
diff --git a/src/hotspot/share/gc/parallel/mutableSpace.cpp b/src/hotspot/share/gc/parallel/mutableSpace.cpp
index d90121b0eaf2c..01d4e6bb04d6f 100644
--- a/src/hotspot/share/gc/parallel/mutableSpace.cpp
+++ b/src/hotspot/share/gc/parallel/mutableSpace.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
#include "precompiled.hpp"
#include "gc/parallel/mutableSpace.hpp"
#include "gc/shared/pretouchTask.hpp"
+#include "gc/shared/spaceDecorator.hpp"
#include "memory/iterator.inline.hpp"
#include "memory/universe.hpp"
#include "oops/oop.inline.hpp"
diff --git a/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp b/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp
index a6601611c9be2..22d1296507d13 100644
--- a/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp
+++ b/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp
@@ -246,7 +246,7 @@ class ParallelScavengeHeap : public CollectedHeap {
}
// Support for loading objects from CDS archive into the heap
- bool can_load_archived_objects() const override { return UseCompressedOops; }
+ bool can_load_archived_objects() const override { return true; }
HeapWord* allocate_loaded_archive_space(size_t size) override;
void complete_loaded_archive_space(MemRegion archive_space) override;
diff --git a/src/hotspot/share/gc/parallel/psOldGen.cpp b/src/hotspot/share/gc/parallel/psOldGen.cpp
index 2715ab90768e4..52ea4dc042cbb 100644
--- a/src/hotspot/share/gc/parallel/psOldGen.cpp
+++ b/src/hotspot/share/gc/parallel/psOldGen.cpp
@@ -31,6 +31,7 @@
#include "gc/parallel/psOldGen.hpp"
#include "gc/shared/cardTableBarrierSet.hpp"
#include "gc/shared/gcLocker.hpp"
+#include "gc/shared/spaceDecorator.hpp"
#include "logging/log.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/java.hpp"
diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.cpp b/src/hotspot/share/gc/parallel/psParallelCompact.cpp
index 1ab7b2af7ed74..66ce20295f549 100644
--- a/src/hotspot/share/gc/parallel/psParallelCompact.cpp
+++ b/src/hotspot/share/gc/parallel/psParallelCompact.cpp
@@ -60,6 +60,7 @@
#include "gc/shared/referencePolicy.hpp"
#include "gc/shared/referenceProcessor.hpp"
#include "gc/shared/referenceProcessorPhaseTimes.hpp"
+#include "gc/shared/spaceDecorator.hpp"
#include "gc/shared/strongRootsScope.hpp"
#include "gc/shared/taskTerminator.hpp"
#include "gc/shared/weakProcessor.inline.hpp"
diff --git a/src/hotspot/share/gc/parallel/psScavenge.cpp b/src/hotspot/share/gc/parallel/psScavenge.cpp
index 7df2143ccbb94..7701cea313b6a 100644
--- a/src/hotspot/share/gc/parallel/psScavenge.cpp
+++ b/src/hotspot/share/gc/parallel/psScavenge.cpp
@@ -51,6 +51,7 @@
#include "gc/shared/referenceProcessor.hpp"
#include "gc/shared/referenceProcessorPhaseTimes.hpp"
#include "gc/shared/scavengableNMethods.hpp"
+#include "gc/shared/spaceDecorator.hpp"
#include "gc/shared/strongRootsScope.hpp"
#include "gc/shared/taskTerminator.hpp"
#include "gc/shared/weakProcessor.inline.hpp"
diff --git a/src/hotspot/share/gc/parallel/psYoungGen.cpp b/src/hotspot/share/gc/parallel/psYoungGen.cpp
index 7b0d3b215075e..dd9619e454656 100644
--- a/src/hotspot/share/gc/parallel/psYoungGen.cpp
+++ b/src/hotspot/share/gc/parallel/psYoungGen.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -29,6 +29,7 @@
#include "gc/parallel/psYoungGen.hpp"
#include "gc/shared/gcUtil.hpp"
#include "gc/shared/genArguments.hpp"
+#include "gc/shared/spaceDecorator.hpp"
#include "logging/log.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/java.hpp"
diff --git a/src/hotspot/share/gc/serial/serialHeap.hpp b/src/hotspot/share/gc/serial/serialHeap.hpp
index 750bb322b2ac4..d787d216e37ab 100644
--- a/src/hotspot/share/gc/serial/serialHeap.hpp
+++ b/src/hotspot/share/gc/serial/serialHeap.hpp
@@ -291,7 +291,7 @@ class SerialHeap : public CollectedHeap {
void safepoint_synchronize_end() override;
// Support for loading objects from CDS archive into the heap
- bool can_load_archived_objects() const override { return UseCompressedOops; }
+ bool can_load_archived_objects() const override { return true; }
HeapWord* allocate_loaded_archive_space(size_t size) override;
void complete_loaded_archive_space(MemRegion archive_space) override;
diff --git a/src/hotspot/share/gc/shared/barrierSetConfig.hpp b/src/hotspot/share/gc/shared/barrierSetConfig.hpp
index 76681aa898687..368312af06b21 100644
--- a/src/hotspot/share/gc/shared/barrierSetConfig.hpp
+++ b/src/hotspot/share/gc/shared/barrierSetConfig.hpp
@@ -33,7 +33,6 @@
EPSILONGC_ONLY(f(EpsilonBarrierSet)) \
G1GC_ONLY(f(G1BarrierSet)) \
SHENANDOAHGC_ONLY(f(ShenandoahBarrierSet)) \
- ZGC_ONLY(f(XBarrierSet)) \
ZGC_ONLY(f(ZBarrierSet))
#define FOR_EACH_ABSTRACT_BARRIER_SET_DO(f) \
diff --git a/src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp b/src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp
index 9523428821b7e..001b5b00372bc 100644
--- a/src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp
+++ b/src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp
@@ -40,7 +40,6 @@
#include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
#endif
#if INCLUDE_ZGC
-#include "gc/x/xBarrierSet.inline.hpp"
#include "gc/z/zBarrierSet.inline.hpp"
#endif
diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp
index 59e0245204441..643a7936b9b17 100644
--- a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp
+++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp
@@ -109,6 +109,10 @@ Label* BarrierStubC2::continuation() {
return &_continuation;
}
+uint8_t BarrierStubC2::barrier_data() const {
+ return _node->barrier_data();
+}
+
void BarrierStubC2::preserve(Register r) {
const VMReg vm_reg = r->as_VMReg();
assert(vm_reg->is_Register(), "r must be a general-purpose register");
diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp
index c1485c069c83c..00fbf1f2c9f8b 100644
--- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp
+++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp
@@ -254,6 +254,8 @@ class BarrierStubC2 : public ArenaObj {
Label* entry();
// Return point from the stub (typically end of barrier).
Label* continuation();
+ // High-level, GC-specific barrier flags.
+ uint8_t barrier_data() const;
// Preserve the value in reg across runtime calls in this barrier.
void preserve(Register reg);
@@ -340,6 +342,8 @@ class BarrierSetC2: public CHeapObj {
// Estimated size of the node barrier in number of C2 Ideal nodes.
// This is used to guide heuristics in C2, e.g. whether to unroll a loop.
virtual uint estimated_barrier_size(const Node* node) const { return 0; }
+ // Whether the given store can be used to initialize a newly allocated object.
+ virtual bool can_initialize_object(const StoreNode* store) const { return true; }
enum CompilePhase {
BeforeOptimize,
diff --git a/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.cpp b/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.cpp
index 87bb9f3cd5170..11b742156a831 100644
--- a/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.cpp
+++ b/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.cpp
@@ -125,39 +125,10 @@ void CardTableBarrierSetC2::post_barrier(GraphKit* kit,
kit->final_sync(ideal);
}
-void CardTableBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const {
- BarrierSetC2::clone(kit, src, dst, size, is_array);
- const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
-
- // If necessary, emit some card marks afterwards. (Non-arrays only.)
- bool card_mark = !is_array && !use_ReduceInitialCardMarks();
- if (card_mark) {
- assert(!is_array, "");
- // Put in store barrier for any and all oops we are sticking
- // into this object. (We could avoid this if we could prove
- // that the object type contains no oop fields at all.)
- Node* no_particular_value = nullptr;
- Node* no_particular_field = nullptr;
- int raw_adr_idx = Compile::AliasIdxRaw;
- post_barrier(kit, kit->control(),
- kit->memory(raw_adr_type),
- dst,
- no_particular_field,
- raw_adr_idx,
- no_particular_value,
- T_OBJECT,
- false);
- }
-}
-
bool CardTableBarrierSetC2::use_ReduceInitialCardMarks() const {
return ReduceInitialCardMarks;
}
-bool CardTableBarrierSetC2::is_gc_barrier_node(Node* node) const {
- return ModRefBarrierSetC2::is_gc_barrier_node(node) || node->Opcode() == Op_StoreCM;
-}
-
void CardTableBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
assert(node->Opcode() == Op_CastP2X, "ConvP2XNode required");
Node *shift = node->unique_out();
diff --git a/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.hpp
index 9512f09ff8a6d..3bbf14892d3ef 100644
--- a/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.hpp
+++ b/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.hpp
@@ -42,8 +42,6 @@ class CardTableBarrierSetC2: public ModRefBarrierSetC2 {
Node* byte_map_base_node(GraphKit* kit) const;
public:
- virtual void clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const;
- virtual bool is_gc_barrier_node(Node* node) const;
virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const;
virtual bool array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, bool is_clone_instance, ArrayCopyPhase phase) const;
diff --git a/src/hotspot/share/gc/shared/gcArguments.cpp b/src/hotspot/share/gc/shared/gcArguments.cpp
index 9736c0f7fdcab..2522925746be1 100644
--- a/src/hotspot/share/gc/shared/gcArguments.cpp
+++ b/src/hotspot/share/gc/shared/gcArguments.cpp
@@ -30,6 +30,7 @@
#include "runtime/arguments.hpp"
#include "runtime/globals.hpp"
#include "runtime/globals_extension.hpp"
+#include "utilities/formatBuffer.hpp"
#include "utilities/macros.hpp"
size_t HeapAlignment = 0;
@@ -166,6 +167,13 @@ void GCArguments::initialize_heap_flags_and_sizes() {
FLAG_SET_ERGO(MinHeapDeltaBytes, align_up(MinHeapDeltaBytes, SpaceAlignment));
+ if (checked_cast(ObjectAlignmentInBytes) > GCCardSizeInBytes) {
+ err_msg message("ObjectAlignmentInBytes %u is larger than GCCardSizeInBytes %u",
+ ObjectAlignmentInBytes, GCCardSizeInBytes);
+ vm_exit_during_initialization("Invalid combination of GCCardSizeInBytes and ObjectAlignmentInBytes",
+ message);
+ }
+
DEBUG_ONLY(assert_flags();)
}
diff --git a/src/hotspot/share/gc/shared/gcConfig.cpp b/src/hotspot/share/gc/shared/gcConfig.cpp
index 506b368d6cf05..8eb265b54d939 100644
--- a/src/hotspot/share/gc/shared/gcConfig.cpp
+++ b/src/hotspot/share/gc/shared/gcConfig.cpp
@@ -44,7 +44,7 @@
#include "gc/shenandoah/shenandoahArguments.hpp"
#endif
#if INCLUDE_ZGC
-#include "gc/z/shared/zSharedArguments.hpp"
+#include "gc/z/zArguments.hpp"
#endif
struct IncludedGC {
@@ -62,7 +62,7 @@ struct IncludedGC {
PARALLELGC_ONLY(static ParallelArguments parallelArguments;)
SERIALGC_ONLY(static SerialArguments serialArguments;)
SHENANDOAHGC_ONLY(static ShenandoahArguments shenandoahArguments;)
- ZGC_ONLY(static ZSharedArguments zArguments;)
+ ZGC_ONLY(static ZArguments zArguments;)
// Table of included GCs, for translating between command
// line flag, CollectedHeap::Name and GCArguments instance.
diff --git a/src/hotspot/share/gc/shared/gcConfiguration.cpp b/src/hotspot/share/gc/shared/gcConfiguration.cpp
index 2e8d3eb2a515a..824e119e69649 100644
--- a/src/hotspot/share/gc/shared/gcConfiguration.cpp
+++ b/src/hotspot/share/gc/shared/gcConfiguration.cpp
@@ -43,11 +43,7 @@ GCName GCConfiguration::young_collector() const {
}
if (UseZGC) {
- if (ZGenerational) {
- return ZMinor;
- } else {
- return NA;
- }
+ return ZMinor;
}
if (UseShenandoahGC) {
@@ -66,12 +62,8 @@ GCName GCConfiguration::old_collector() const {
return ParallelOld;
}
- if (UseZGC) {
- if (ZGenerational) {
- return ZMajor;
- } else {
- return Z;
- }
+if (UseZGC) {
+ return ZMajor;
}
if (UseShenandoahGC) {
diff --git a/src/hotspot/share/gc/shared/gcName.hpp b/src/hotspot/share/gc/shared/gcName.hpp
index 3d2dd350ac10e..b9b87c231ca91 100644
--- a/src/hotspot/share/gc/shared/gcName.hpp
+++ b/src/hotspot/share/gc/shared/gcName.hpp
@@ -37,7 +37,6 @@ enum GCName {
G1Full,
ZMinor,
ZMajor,
- Z, // Support for the legacy, single-gen mode
Shenandoah,
NA,
GCNameEndSentinel
@@ -56,7 +55,6 @@ class GCNameHelper {
case G1Full: return "G1Full";
case ZMinor: return "ZGC Minor";
case ZMajor: return "ZGC Major";
- case Z: return "Z";
case Shenandoah: return "Shenandoah";
case NA: return "N/A";
default: ShouldNotReachHere(); return nullptr;
diff --git a/src/hotspot/share/gc/shared/gc_globals.hpp b/src/hotspot/share/gc/shared/gc_globals.hpp
index 34bc638c9baca..9086c25ee48fb 100644
--- a/src/hotspot/share/gc/shared/gc_globals.hpp
+++ b/src/hotspot/share/gc/shared/gc_globals.hpp
@@ -43,7 +43,7 @@
#include "gc/shenandoah/shenandoah_globals.hpp"
#endif
#if INCLUDE_ZGC
-#include "gc/z/shared/z_shared_globals.hpp"
+#include "gc/z/z_globals.hpp"
#endif
#define GC_FLAGS(develop, \
@@ -93,7 +93,7 @@
range, \
constraint)) \
\
- ZGC_ONLY(GC_Z_SHARED_FLAGS( \
+ ZGC_ONLY(GC_Z_FLAGS( \
develop, \
develop_pd, \
product, \
@@ -118,9 +118,6 @@
product(bool, UseZGC, false, \
"Use the Z garbage collector") \
\
- product(bool, ZGenerational, true, \
- "Use the generational version of ZGC") \
- \
product(bool, UseShenandoahGC, false, \
"Use the Shenandoah garbage collector") \
\
diff --git a/src/hotspot/share/gc/shared/oopStorage.cpp b/src/hotspot/share/gc/shared/oopStorage.cpp
index 568888ac7d97e..2373d6b1d93a8 100644
--- a/src/hotspot/share/gc/shared/oopStorage.cpp
+++ b/src/hotspot/share/gc/shared/oopStorage.cpp
@@ -300,7 +300,12 @@ void OopStorage::Block::set_active_index(size_t index) {
size_t OopStorage::Block::active_index_safe(const Block* block) {
STATIC_ASSERT(sizeof(intptr_t) == sizeof(block->_active_index));
- return SafeFetchN((intptr_t*)&block->_active_index, 0);
+ // Be careful, because block could be a false positive from block_for_ptr.
+ assert(block != nullptr, "precondition");
+ uintptr_t block_addr = reinterpret_cast(block);
+ uintptr_t index_loc = block_addr + offset_of(Block, _active_index);
+ static_assert(sizeof(size_t) == sizeof(intptr_t), "assumption");
+ return static_cast(SafeFetchN(reinterpret_cast(index_loc), 0));
}
unsigned OopStorage::Block::get_index(const oop* ptr) const {
@@ -366,21 +371,23 @@ void OopStorage::Block::delete_block(const Block& block) {
OopStorage::Block*
OopStorage::Block::block_for_ptr(const OopStorage* owner, const oop* ptr) {
STATIC_ASSERT(_data_pos == 0);
- // Const-ness of ptr is not related to const-ness of containing block.
+ assert(ptr != nullptr, "precondition");
// Blocks are allocated section-aligned, so get the containing section.
- oop* section_start = align_down(const_cast(ptr), block_alignment);
+ uintptr_t section_start = align_down(reinterpret_cast(ptr), block_alignment);
// Start with a guess that the containing section is the last section,
// so the block starts section_count-1 sections earlier.
- oop* section = section_start - (section_size * (section_count - 1));
+ size_t section_size_in_bytes = sizeof(oop) * section_size;
+ uintptr_t section = section_start - (section_size_in_bytes * (section_count - 1));
// Walk up through the potential block start positions, looking for
// the owner in the expected location. If we're below the actual block
// start position, the value at the owner position will be some oop
// (possibly null), which can never match the owner.
intptr_t owner_addr = reinterpret_cast(owner);
- for (unsigned i = 0; i < section_count; ++i, section += section_size) {
- Block* candidate = reinterpret_cast(section);
- if (SafeFetchN(&candidate->_owner_address, 0) == owner_addr) {
- return candidate;
+ for (unsigned i = 0; i < section_count; ++i, section += section_size_in_bytes) {
+ uintptr_t owner_loc = section + offset_of(Block, _owner_address);
+ static_assert(sizeof(OopStorage*) == sizeof(intptr_t), "assumption");
+ if (SafeFetchN(reinterpret_cast(owner_loc), 0) == owner_addr) {
+ return reinterpret_cast(section);
}
}
return nullptr;
@@ -643,8 +650,7 @@ class OopStorage::WithActiveArray : public StackObj {
}
};
-OopStorage::Block* OopStorage::find_block_or_null(const oop* ptr) const {
- assert(ptr != nullptr, "precondition");
+OopStorage::Block* OopStorage::block_for_ptr(const oop* ptr) const {
return Block::block_for_ptr(this, ptr);
}
@@ -771,7 +777,7 @@ static inline void check_release_entry(const oop* entry) {
void OopStorage::release(const oop* ptr) {
check_release_entry(ptr);
- Block* block = find_block_or_null(ptr);
+ Block* block = block_for_ptr(ptr);
assert(block != nullptr, "%s: invalid release " PTR_FORMAT, name(), p2i(ptr));
log_trace(oopstorage, ref)("%s: releasing " PTR_FORMAT, name(), p2i(ptr));
block->release_entries(block->bitmask_for_entry(ptr), this);
@@ -782,7 +788,7 @@ void OopStorage::release(const oop* const* ptrs, size_t size) {
size_t i = 0;
while (i < size) {
check_release_entry(ptrs[i]);
- Block* block = find_block_or_null(ptrs[i]);
+ Block* block = block_for_ptr(ptrs[i]);
assert(block != nullptr, "%s: invalid release " PTR_FORMAT, name(), p2i(ptrs[i]));
size_t count = 0;
uintx releasing = 0;
@@ -989,7 +995,8 @@ bool OopStorage::delete_empty_blocks() {
}
OopStorage::EntryStatus OopStorage::allocation_status(const oop* ptr) const {
- const Block* block = find_block_or_null(ptr);
+ if (ptr == nullptr) return INVALID_ENTRY;
+ const Block* block = block_for_ptr(ptr);
if (block != nullptr) {
// Prevent block deletion and _active_array modification.
MutexLocker ml(_allocation_mutex, Mutex::_no_safepoint_check_flag);
@@ -1137,7 +1144,7 @@ const char* OopStorage::name() const { return _name; }
bool OopStorage::print_containing(const oop* addr, outputStream* st) {
if (addr != nullptr) {
- Block* block = find_block_or_null(addr);
+ Block* block = block_for_ptr(addr);
if (block != nullptr && block->print_containing(addr, st)) {
st->print(" in oop storage \"%s\"", name());
return true;
diff --git a/src/hotspot/share/gc/shared/oopStorage.hpp b/src/hotspot/share/gc/shared/oopStorage.hpp
index 96cc5a23d6a91..34c980a058659 100644
--- a/src/hotspot/share/gc/shared/oopStorage.hpp
+++ b/src/hotspot/share/gc/shared/oopStorage.hpp
@@ -288,7 +288,7 @@ class OopStorage : public CHeapObjBase {
Block* block_for_allocation();
void log_block_transition(Block* block, const char* new_state) const;
- Block* find_block_or_null(const oop* ptr) const;
+ Block* block_for_ptr(const oop* ptr) const;
void delete_empty_block(const Block& block);
bool reduce_deferred_updates();
void record_needs_cleanup();
diff --git a/src/hotspot/share/gc/shared/oopStorage.inline.hpp b/src/hotspot/share/gc/shared/oopStorage.inline.hpp
index 545da0be0a76e..da0926a20b6e2 100644
--- a/src/hotspot/share/gc/shared/oopStorage.inline.hpp
+++ b/src/hotspot/share/gc/shared/oopStorage.inline.hpp
@@ -184,7 +184,10 @@ class OopStorage::Block /* No base class, to avoid messing up alignment. */ {
void set_active_index(size_t index);
static size_t active_index_safe(const Block* block); // Returns 0 if access fails.
- // Returns null if ptr is not in a block or not allocated in that block.
+ // Return block of owner containing ptr, if ptr is a valid entry of owner.
+ // If ptr is not a valid entry of owner then returns either null or a "false
+ // positive" pointer; see allocation_status.
+ // precondition: ptr != nullptr
static Block* block_for_ptr(const OopStorage* owner, const oop* ptr);
oop* allocate();
diff --git a/src/hotspot/share/gc/shared/vmStructs_gc.hpp b/src/hotspot/share/gc/shared/vmStructs_gc.hpp
index c4acb0b1ed6ca..bba9c9e099fc9 100644
--- a/src/hotspot/share/gc/shared/vmStructs_gc.hpp
+++ b/src/hotspot/share/gc/shared/vmStructs_gc.hpp
@@ -46,7 +46,7 @@
#include "gc/shenandoah/vmStructs_shenandoah.hpp"
#endif
#if INCLUDE_ZGC
-#include "gc/z/shared/vmStructs_z_shared.hpp"
+#include "gc/z/vmStructs_z.hpp"
#endif
#define VM_STRUCTS_GC(nonstatic_field, \
@@ -69,7 +69,7 @@
SHENANDOAHGC_ONLY(VM_STRUCTS_SHENANDOAH(nonstatic_field, \
volatile_nonstatic_field, \
static_field)) \
- ZGC_ONLY(VM_STRUCTS_Z_SHARED(nonstatic_field, \
+ ZGC_ONLY(VM_STRUCTS_Z(nonstatic_field, \
volatile_nonstatic_field, \
static_field)) \
\
@@ -91,6 +91,7 @@
nonstatic_field(CardTableBarrierSet, _defer_initial_card_mark, bool) \
nonstatic_field(CardTableBarrierSet, _card_table, CardTable*) \
\
+ static_field(CollectedHeap, _lab_alignment_reserve, size_t) \
nonstatic_field(CollectedHeap, _reserved, MemRegion) \
nonstatic_field(CollectedHeap, _is_stw_gc_active, bool) \
nonstatic_field(CollectedHeap, _total_collections, unsigned int) \
@@ -120,7 +121,7 @@
SHENANDOAHGC_ONLY(VM_TYPES_SHENANDOAH(declare_type, \
declare_toplevel_type, \
declare_integer_type)) \
- ZGC_ONLY(VM_TYPES_Z_SHARED(declare_type, \
+ ZGC_ONLY(VM_TYPES_Z(declare_type, \
declare_toplevel_type, \
declare_integer_type)) \
\
@@ -174,7 +175,7 @@
declare_constant_with_value)) \
SHENANDOAHGC_ONLY(VM_INT_CONSTANTS_SHENANDOAH(declare_constant, \
declare_constant_with_value)) \
- ZGC_ONLY(VM_INT_CONSTANTS_Z_SHARED(declare_constant, \
+ ZGC_ONLY(VM_INT_CONSTANTS_Z(declare_constant, \
declare_constant_with_value)) \
\
/********************************************/ \
@@ -198,6 +199,6 @@
declare_constant(CollectedHeap::G1) \
#define VM_LONG_CONSTANTS_GC(declare_constant) \
- ZGC_ONLY(VM_LONG_CONSTANTS_Z_SHARED(declare_constant))
+ ZGC_ONLY(VM_LONG_CONSTANTS_Z(declare_constant))
#endif // SHARE_GC_SHARED_VMSTRUCTS_GC_HPP
diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
index 7ac9dcc2e8134..691c78cd02486 100644
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
@@ -480,10 +480,17 @@ Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue&
const TypePtr* adr_type = access.addr().type();
Node* adr = access.addr().node();
+ bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0;
+
if (!access.is_oop()) {
return BarrierSetC2::store_at_resolved(access, val);
}
+ if (no_keepalive) {
+ // No keep-alive means no need for the pre-barrier.
+ return BarrierSetC2::store_at_resolved(access, val);
+ }
+
if (access.is_parse_access()) {
C2ParseAccess& parse_access = static_cast(access);
GraphKit* kit = parse_access.kit();
diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
index efa0ced603cda..10dd650917170 100644
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
@@ -1041,6 +1041,7 @@ void ShenandoahBarrierC2Support::fix_ctrl(Node* barrier, Node* region, const Mem
Node* u = ctrl->fast_out(i);
if (u->_idx < last &&
u != barrier &&
+ !u->depends_only_on_test() && // preserve dependency on test
!uses_to_ignore.member(u) &&
(u->in(0) != ctrl || (!u->is_Region() && !u->is_Phi())) &&
(ctrl->Opcode() != Op_CatchProj || u->Opcode() != Op_CreateEx)) {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp
index 51903a60fb4fd..58527e808e43c 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp
@@ -25,18 +25,78 @@
#define SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_HPP
#include "code/nmethod.hpp"
+#include "gc/shared/stringdedup/stringDedup.hpp"
+#include "gc/shenandoah/shenandoahGenerationType.hpp"
+#include "gc/shenandoah/shenandoahTaskqueue.hpp"
#include "memory/iterator.hpp"
-#include "oops/accessDecorators.hpp"
-#include "runtime/handshake.hpp"
+#include "runtime/javaThread.hpp"
class BarrierSetNMethod;
class ShenandoahBarrierSet;
class ShenandoahHeap;
class ShenandoahMarkingContext;
-class ShenandoahHeapRegionSet;
-class Thread;
+class ShenandoahReferenceProcessor;
-class ShenandoahForwardedIsAliveClosure: public BoolObjectClosure {
+//
+// ========= Super
+//
+
+class ShenandoahSuperClosure : public MetadataVisitingOopIterateClosure {
+protected:
+ ShenandoahHeap* const _heap;
+
+public:
+ inline ShenandoahSuperClosure();
+ inline ShenandoahSuperClosure(ShenandoahReferenceProcessor* rp);
+ inline void do_nmethod(nmethod* nm);
+};
+
+//
+// ========= Marking
+//
+
+class ShenandoahMarkRefsSuperClosure : public ShenandoahSuperClosure {
+private:
+ ShenandoahObjToScanQueue* _queue;
+ ShenandoahMarkingContext* const _mark_context;
+ bool _weak;
+
+protected:
+ template
+ void work(T *p);
+
+public:
+ inline ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp);
+
+ bool is_weak() const {
+ return _weak;
+ }
+
+ void set_weak(bool weak) {
+ _weak = weak;
+ }
+
+ virtual void do_nmethod(nmethod* nm) {
+ assert(!is_weak(), "Can't handle weak marking of nmethods");
+ ShenandoahSuperClosure::do_nmethod(nm);
+ }
+};
+
+template
+class ShenandoahMarkRefsClosure : public ShenandoahMarkRefsSuperClosure {
+private:
+ template
+ inline void do_oop_work(T* p) { work(p); }
+
+public:
+ ShenandoahMarkRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) :
+ ShenandoahMarkRefsSuperClosure(q, rp) {};
+
+ virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+ virtual void do_oop(oop* p) { do_oop_work(p); }
+};
+
+class ShenandoahForwardedIsAliveClosure : public BoolObjectClosure {
private:
ShenandoahMarkingContext* const _mark_context;
public:
@@ -44,7 +104,7 @@ class ShenandoahForwardedIsAliveClosure: public BoolObjectClosure {
inline bool do_object_b(oop obj);
};
-class ShenandoahIsAliveClosure: public BoolObjectClosure {
+class ShenandoahIsAliveClosure : public BoolObjectClosure {
private:
ShenandoahMarkingContext* const _mark_context;
public:
@@ -63,39 +123,29 @@ class ShenandoahIsAliveSelector : public StackObj {
class ShenandoahKeepAliveClosure : public OopClosure {
private:
ShenandoahBarrierSet* const _bs;
-public:
- inline ShenandoahKeepAliveClosure();
- inline void do_oop(oop* p);
- inline void do_oop(narrowOop* p);
-private:
template
void do_oop_work(T* p);
-};
-class ShenandoahOopClosureBase : public MetadataVisitingOopIterateClosure {
public:
- inline void do_nmethod(nmethod* nm);
+ inline ShenandoahKeepAliveClosure();
+ inline void do_oop(oop* p) { do_oop_work(p); }
+ inline void do_oop(narrowOop* p) { do_oop_work(p); }
};
-class ShenandoahUpdateRefsClosure: public ShenandoahOopClosureBase {
-private:
- ShenandoahHeap* _heap;
-public:
- inline ShenandoahUpdateRefsClosure();
- inline void do_oop(oop* p);
- inline void do_oop(narrowOop* p);
-private:
- template
- inline void do_oop_work(T* p);
-};
-template
-class ShenandoahEvacuateUpdateRootClosureBase : public ShenandoahOopClosureBase {
+//
+// ========= Evacuating + Roots
+//
+
+template
+class ShenandoahEvacuateUpdateRootClosureBase : public ShenandoahSuperClosure {
protected:
- ShenandoahHeap* const _heap;
Thread* const _thread;
public:
- inline ShenandoahEvacuateUpdateRootClosureBase();
+ inline ShenandoahEvacuateUpdateRootClosureBase() :
+ ShenandoahSuperClosure(),
+ _thread(STABLE_THREAD ? Thread::current() : nullptr) {}
+
inline void do_oop(oop* p);
inline void do_oop(narrowOop* p);
protected:
@@ -103,10 +153,11 @@ class ShenandoahEvacuateUpdateRootClosureBase : public ShenandoahOopClosureBase
inline void do_oop_work(T* p);
};
-using ShenandoahEvacuateUpdateMetadataClosure = ShenandoahEvacuateUpdateRootClosureBase;
-using ShenandoahEvacuateUpdateRootsClosure = ShenandoahEvacuateUpdateRootClosureBase;
+using ShenandoahEvacuateUpdateMetadataClosure = ShenandoahEvacuateUpdateRootClosureBase;
+using ShenandoahEvacuateUpdateRootsClosure = ShenandoahEvacuateUpdateRootClosureBase;
using ShenandoahContextEvacuateUpdateRootsClosure = ShenandoahEvacuateUpdateRootClosureBase;
+
template
class ShenandoahCleanUpdateWeakOopsClosure : public OopClosure {
private:
@@ -119,7 +170,7 @@ class ShenandoahCleanUpdateWeakOopsClosure : public OopClosure {
inline void do_oop(narrowOop* p);
};
-class ShenandoahNMethodAndDisarmClosure: public NMethodToOopClosure {
+class ShenandoahNMethodAndDisarmClosure : public NMethodToOopClosure {
private:
BarrierSetNMethod* const _bs;
@@ -128,6 +179,51 @@ class ShenandoahNMethodAndDisarmClosure: public NMethodToOopClosure {
inline void do_nmethod(nmethod* nm);
};
+
+//
+// ========= Update References
+//
+
+template
+class ShenandoahMarkUpdateRefsClosure : public ShenandoahMarkRefsSuperClosure {
+private:
+ template
+ inline void work(T* p);
+
+public:
+ ShenandoahMarkUpdateRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp);
+
+ virtual void do_oop(narrowOop* p) { work(p); }
+ virtual void do_oop(oop* p) { work(p); }
+};
+
+class ShenandoahUpdateRefsSuperClosure : public ShenandoahSuperClosure {};
+
+class ShenandoahNonConcUpdateRefsClosure : public ShenandoahUpdateRefsSuperClosure {
+private:
+ template
+ inline void work(T* p);
+
+public:
+ virtual void do_oop(narrowOop* p) { work(p); }
+ virtual void do_oop(oop* p) { work(p); }
+};
+
+class ShenandoahConcUpdateRefsClosure : public ShenandoahUpdateRefsSuperClosure {
+private:
+ template
+ inline void work(T* p);
+
+public:
+ virtual void do_oop(narrowOop* p) { work(p); }
+ virtual void do_oop(oop* p) { work(p); }
+};
+
+
+//
+// ========= Utilities
+//
+
#ifdef ASSERT
class ShenandoahAssertNotForwardedClosure : public OopClosure {
private:
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp
index befc0d7c1f121..edfb62b40466a 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp
@@ -33,15 +33,46 @@
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahNMethod.inline.hpp"
+#include "gc/shenandoah/shenandoahMark.inline.hpp"
+#include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
+#include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
#include "memory/iterator.inline.hpp"
#include "oops/compressedOops.inline.hpp"
#include "runtime/atomic.hpp"
#include "runtime/javaThread.hpp"
-ShenandoahForwardedIsAliveClosure::ShenandoahForwardedIsAliveClosure() :
- _mark_context(ShenandoahHeap::heap()->marking_context()) {
+//
+// ========= Super
+//
+
+ShenandoahSuperClosure::ShenandoahSuperClosure() :
+ MetadataVisitingOopIterateClosure(), _heap(ShenandoahHeap::heap()) {}
+
+ShenandoahSuperClosure::ShenandoahSuperClosure(ShenandoahReferenceProcessor* rp) :
+ MetadataVisitingOopIterateClosure(rp), _heap(ShenandoahHeap::heap()) {}
+
+void ShenandoahSuperClosure::do_nmethod(nmethod* nm) {
+ nm->run_nmethod_entry_barrier();
}
+//
+// ========= Marking
+//
+
+ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) :
+ ShenandoahSuperClosure(rp),
+ _queue(q),
+ _mark_context(ShenandoahHeap::heap()->marking_context()),
+ _weak(false) {}
+
+template
+inline void ShenandoahMarkRefsSuperClosure::work(T* p) {
+ ShenandoahMark::mark_through_ref(p, _queue, _mark_context, _weak);
+}
+
+ShenandoahForwardedIsAliveClosure::ShenandoahForwardedIsAliveClosure() :
+ _mark_context(ShenandoahHeap::heap()->marking_context()) {}
+
bool ShenandoahForwardedIsAliveClosure::do_object_b(oop obj) {
if (CompressedOops::is_null(obj)) {
return false;
@@ -52,8 +83,7 @@ bool ShenandoahForwardedIsAliveClosure::do_object_b(oop obj) {
}
ShenandoahIsAliveClosure::ShenandoahIsAliveClosure() :
- _mark_context(ShenandoahHeap::heap()->marking_context()) {
-}
+ _mark_context(ShenandoahHeap::heap()->marking_context()) {}
bool ShenandoahIsAliveClosure::do_object_b(oop obj) {
if (CompressedOops::is_null(obj)) {
@@ -69,21 +99,8 @@ BoolObjectClosure* ShenandoahIsAliveSelector::is_alive_closure() {
reinterpret_cast(&_alive_cl);
}
-void ShenandoahOopClosureBase::do_nmethod(nmethod* nm) {
- nm->run_nmethod_entry_barrier();
-}
-
ShenandoahKeepAliveClosure::ShenandoahKeepAliveClosure() :
- _bs(ShenandoahBarrierSet::barrier_set()) {
-}
-
-void ShenandoahKeepAliveClosure::do_oop(oop* p) {
- do_oop_work(p);
-}
-
-void ShenandoahKeepAliveClosure::do_oop(narrowOop* p) {
- do_oop_work(p);
-}
+ _bs(ShenandoahBarrierSet::barrier_set()) {}
template
void ShenandoahKeepAliveClosure::do_oop_work(T* p) {
@@ -97,26 +114,14 @@ void ShenandoahKeepAliveClosure::do_oop_work(T* p) {
}
}
-ShenandoahUpdateRefsClosure::ShenandoahUpdateRefsClosure() :
- _heap(ShenandoahHeap::heap()) {
-}
-
-template
-void ShenandoahUpdateRefsClosure::do_oop_work(T* p) {
- _heap->update_with_forwarded(p);
-}
-
-void ShenandoahUpdateRefsClosure::do_oop(oop* p) { do_oop_work(p); }
-void ShenandoahUpdateRefsClosure::do_oop(narrowOop* p) { do_oop_work(p); }
-template
-ShenandoahEvacuateUpdateRootClosureBase::ShenandoahEvacuateUpdateRootClosureBase() :
- _heap(ShenandoahHeap::heap()), _thread(stable_thread ? Thread::current() : nullptr) {
-}
+//
+// ========= Evacuating + Roots
+//
-template
-void ShenandoahEvacuateUpdateRootClosureBase::do_oop(oop* p) {
- if (concurrent) {
+template
+void ShenandoahEvacuateUpdateRootClosureBase::do_oop(oop* p) {
+ if (CONCURRENT) {
ShenandoahEvacOOMScope scope;
do_oop_work(p);
} else {
@@ -124,9 +129,9 @@ void ShenandoahEvacuateUpdateRootClosureBase::do_oop(
}
}
-template
-void ShenandoahEvacuateUpdateRootClosureBase::do_oop(narrowOop* p) {
- if (concurrent) {
+template
+void ShenandoahEvacuateUpdateRootClosureBase::do_oop(narrowOop* p) {
+ if (CONCURRENT) {
ShenandoahEvacOOMScope scope;
do_oop_work(p);
} else {
@@ -134,9 +139,9 @@ void ShenandoahEvacuateUpdateRootClosureBase::do_oop(
}
}
-template
+template
template
-void ShenandoahEvacuateUpdateRootClosureBase::do_oop_work(T* p) {
+void ShenandoahEvacuateUpdateRootClosureBase::do_oop_work(T* p) {
assert(_heap->is_concurrent_weak_root_in_progress() ||
_heap->is_concurrent_strong_root_in_progress(),
"Only do this in root processing phase");
@@ -149,12 +154,12 @@ void ShenandoahEvacuateUpdateRootClosureBase::do_oop_work
shenandoah_assert_marked(p, obj);
oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
if (resolved == obj) {
- Thread* thr = stable_thread ? _thread : Thread::current();
+ Thread* thr = STABLE_THREAD ? _thread : Thread::current();
assert(thr == Thread::current(), "Wrong thread");
resolved = _heap->evacuate_object(obj, thr);
}
- if (atomic) {
+ if (CONCURRENT) {
ShenandoahHeap::atomic_update_oop(resolved, p, o);
} else {
RawAccess::oop_store(p, resolved);
@@ -204,6 +209,42 @@ void ShenandoahNMethodAndDisarmClosure::do_nmethod(nmethod* nm) {
_bs->disarm(nm);
}
+
+//
+// ========= Update References
+//
+
+template
+ShenandoahMarkUpdateRefsClosure::ShenandoahMarkUpdateRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) :
+ ShenandoahMarkRefsSuperClosure(q, rp) {
+ assert(_heap->is_stw_gc_in_progress(), "Can only be used for STW GC");
+}
+
+template
+template
+inline void ShenandoahMarkUpdateRefsClosure::work(T* p) {
+ // Update the location
+ _heap->non_conc_update_with_forwarded(p);
+
+ // ...then do the usual thing
+ ShenandoahMarkRefsSuperClosure::work(p);
+}
+
+template
+inline void ShenandoahNonConcUpdateRefsClosure::work(T* p) {
+ _heap->non_conc_update_with_forwarded(p);
+}
+
+template
+inline void ShenandoahConcUpdateRefsClosure::work(T* p) {
+ _heap->conc_update_with_forwarded(p);
+}
+
+
+//
+// ========= Utilities
+//
+
#ifdef ASSERT
template
void ShenandoahAssertNotForwardedClosure::do_oop_work(T* p) {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp
index ed40fecfc1db8..d801dda372eb4 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp
@@ -29,13 +29,13 @@
#include "gc/shared/collectorCounters.hpp"
#include "gc/shared/continuationGCSupport.inline.hpp"
#include "gc/shenandoah/shenandoahBreakpoint.hpp"
+#include "gc/shenandoah/shenandoahClosures.inline.hpp"
#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc/shenandoah/shenandoahConcurrentGC.hpp"
#include "gc/shenandoah/shenandoahFreeSet.hpp"
#include "gc/shenandoah/shenandoahLock.hpp"
#include "gc/shenandoah/shenandoahMark.inline.hpp"
#include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
-#include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
#include "gc/shenandoah/shenandoahPhaseTimings.hpp"
#include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
#include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
@@ -931,7 +931,10 @@ void ShenandoahConcurrentGC::op_updaterefs() {
class ShenandoahUpdateThreadClosure : public HandshakeClosure {
private:
- ShenandoahUpdateRefsClosure _cl;
+ // This closure runs when thread is stopped for handshake, which means
+ // we can use non-concurrent closure here, as long as it only updates
+ // locations modified by the thread itself, i.e. stack locations.
+ ShenandoahNonConcUpdateRefsClosure _cl;
public:
ShenandoahUpdateThreadClosure();
void do_thread(Thread* thread);
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp
index 75cdb99e177d1..0e2ef4144adc9 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp
@@ -34,7 +34,6 @@
#include "gc/shenandoah/shenandoahMark.inline.hpp"
#include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
#include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
-#include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
#include "gc/shenandoah/shenandoahPhaseTimings.hpp"
#include "gc/shenandoah/shenandoahStringDedup.hpp"
#include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp
index 8a4ef63b8e38b..df2d6d092e630 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp
@@ -177,10 +177,13 @@ void ShenandoahControlThread::run_service() {
// it is a normal completion, or the abort.
heap->free_set()->log_status_under_lock();
- // Notify Universe about new heap usage. This has implications for
- // global soft refs policy, and we better report it every time heap
- // usage goes down.
- heap->update_capacity_and_used_at_gc();
+ {
+ // Notify Universe about new heap usage. This has implications for
+ // global soft refs policy, and we better report it every time heap
+ // usage goes down.
+ ShenandoahHeapLocker locker(heap->lock());
+ heap->update_capacity_and_used_at_gc();
+ }
// Signal that we have completed a visit to all live objects.
heap->record_whole_heap_examined_timestamp();
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp
index 81d07a414cd95..2249c38455f1a 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp
@@ -32,7 +32,6 @@
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahMetrics.hpp"
#include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
-#include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
#include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
#include "gc/shenandoah/shenandoahSTWMark.hpp"
#include "gc/shenandoah/shenandoahUtils.hpp"
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp
index 03e19a3af5e30..310cd5b8061eb 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp
@@ -910,7 +910,6 @@ void ShenandoahFreeSet::try_recycle_trashed(ShenandoahHeapRegion* r) {
void ShenandoahFreeSet::recycle_trash() {
// lock is not reentrable, check we don't have it
shenandoah_assert_not_heaplocked();
-
size_t count = 0;
for (size_t i = 0; i < _heap->num_regions(); i++) {
ShenandoahHeapRegion* r = _heap->get_region(i);
@@ -919,16 +918,45 @@ void ShenandoahFreeSet::recycle_trash() {
}
}
- // Relinquish the lock after this much time passed.
- static constexpr jlong deadline_ns = 30000; // 30 us
+ size_t total_batches = 0;
+ jlong batch_start_time = 0;
+ jlong recycle_trash_start_time = os::javaTimeNanos(); // This value will be treated as the initial batch_start_time
+ jlong batch_end_time = recycle_trash_start_time;
+ // Process as many batches as can be processed within 10 us.
+ static constexpr jlong deadline_ns = 10000; // 10 us
size_t idx = 0;
+ jlong predicted_next_batch_end_time;
+ jlong batch_process_time_estimate = 0;
while (idx < count) {
- os::naked_yield(); // Yield to allow allocators to take the lock
- ShenandoahHeapLocker locker(_heap->lock());
- const jlong deadline = os::javaTimeNanos() + deadline_ns;
- while (idx < count && os::javaTimeNanos() < deadline) {
- try_recycle_trashed(_trash_regions[idx++]);
+ if (idx > 0) {
+ os::naked_yield(); // Yield to allow allocators to take the lock, except on the first iteration
}
+ // Avoid another call to javaTimeNanos() if we already know time at which last batch ended
+ batch_start_time = batch_end_time;
+ const jlong deadline = batch_start_time + deadline_ns;
+
+ ShenandoahHeapLocker locker(_heap->lock());
+ do {
+ // Measurements on typical 2024 hardware suggest it typically requires between 1400 and 2000 ns to process a batch of
+ // 32 regions, assuming low contention with other threads. Sometimes this goes higher, when mutator threads
+ // are contending for CPU cores and/or the heap lock. On this hardware with a 10 us deadline, we expect 3-6 batches
+ // to be processed between yields most of the time.
+ //
+ // Note that deadline is enforced since the end of previous batch. In the case that yield() or acquisition of heap lock
+ // takes a "long time", we will have less time to process regions, but we will always process at least one batch between
+ // yields. Yielding more frequently when there is heavy contention for the heap lock or for CPU cores is considered the
+ // right thing to do.
+ const size_t REGIONS_PER_BATCH = 32;
+ size_t max_idx = MIN2(count, idx + REGIONS_PER_BATCH);
+ while (idx < max_idx) {
+ try_recycle_trashed(_trash_regions[idx++]);
+ }
+ total_batches++;
+ batch_end_time = os::javaTimeNanos();
+ // Estimate includes historic combination of yield times and heap lock acquisition times.
+ batch_process_time_estimate = (batch_end_time - recycle_trash_start_time) / total_batches;
+ predicted_next_batch_end_time = batch_end_time + batch_process_time_estimate;
+ } while ((idx < count) && (predicted_next_batch_end_time < deadline));
}
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
index de7d81d0f43cd..c2d94353d54bb 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
@@ -31,6 +31,7 @@
#include "gc/shared/tlab_globals.hpp"
#include "gc/shared/workerThread.hpp"
#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/shenandoahClosures.inline.hpp"
#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc/shenandoah/shenandoahConcurrentGC.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.hpp"
@@ -44,7 +45,6 @@
#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
#include "gc/shenandoah/shenandoahMetrics.hpp"
-#include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
#include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
#include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
#include "gc/shenandoah/shenandoahSTWMark.hpp"
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp
index 922f54edf3c0c..719abde4b165d 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp
@@ -66,13 +66,13 @@ class ShenandoahUpdateRootsTask : public WorkerTask {
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
ShenandoahParallelWorkerSession worker_session(worker_id);
- ShenandoahUpdateRefsClosure cl;
+ ShenandoahNonConcUpdateRefsClosure cl;
if (_check_alive) {
ShenandoahForwardedIsAliveClosure is_alive;
- _root_updater->roots_do(worker_id, &is_alive, &cl);
+ _root_updater->roots_do(worker_id, &is_alive, &cl);
} else {
- AlwaysTrueClosure always_true;;
- _root_updater->roots_do(worker_id, &always_true, &cl);
+ AlwaysTrueClosure always_true;
+ _root_updater->roots_do(worker_id, &always_true, &cl);
}
}
};
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
index 7ae4a1cf8b3dc..07914947ead27 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
@@ -53,7 +53,6 @@
#include "gc/shenandoah/shenandoahMemoryPool.hpp"
#include "gc/shenandoah/shenandoahMetrics.hpp"
#include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
-#include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
#include "gc/shenandoah/shenandoahPacer.inline.hpp"
#include "gc/shenandoah/shenandoahPadding.hpp"
#include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
@@ -2012,8 +2011,8 @@ void ShenandoahHeap::stw_process_weak_roots(bool full_gc) {
// Cleanup weak roots
if (has_forwarded_objects()) {
ShenandoahForwardedIsAliveClosure is_alive;
- ShenandoahUpdateRefsClosure keep_alive;
- ShenandoahParallelWeakRootsCleaningTask
+ ShenandoahNonConcUpdateRefsClosure keep_alive;
+ ShenandoahParallelWeakRootsCleaningTask
cleaning_task(timing_phase, &is_alive, &keep_alive, num_workers);
_workers->run_task(&cleaning_task);
} else {
@@ -2197,7 +2196,7 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask {
do_work(worker_id);
} else {
ShenandoahParallelWorkerSession worker_session(worker_id);
- do_work(worker_id);
+ do_work(worker_id);
}
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
index 1ee1f9dfc88be..7e616f925d035 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
@@ -543,7 +543,7 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
// ---------- CDS archive support
- bool can_load_archived_objects() const override { return UseCompressedOops; }
+ bool can_load_archived_objects() const override { return true; }
HeapWord* allocate_loaded_archive_space(size_t size) override;
void complete_loaded_archive_space(MemRegion archive_space) override;
@@ -660,7 +660,7 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
inline void conc_update_with_forwarded(T* p);
template
- inline void update_with_forwarded(T* p);
+ inline void non_conc_update_with_forwarded(T* p);
static inline void atomic_update_oop(oop update, oop* addr, oop compare);
static inline void atomic_update_oop(oop update, narrowOop* addr, oop compare);
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
index f2cc602d8cbc0..cd2cc2eb20930 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
@@ -108,7 +108,7 @@ inline void ShenandoahHeap::leave_evacuation(Thread* t) {
}
template
-inline void ShenandoahHeap::update_with_forwarded(T* p) {
+inline void ShenandoahHeap::non_conc_update_with_forwarded(T* p) {
T o = RawAccess<>::oop_load(p);
if (!CompressedOops::is_null(o)) {
oop obj = CompressedOops::decode_not_null(o);
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp
index 4c45f4a7659b8..775b84a8966c1 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp
@@ -28,19 +28,11 @@
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
#include "gc/shenandoah/shenandoahClosures.inline.hpp"
#include "gc/shenandoah/shenandoahMark.inline.hpp"
-#include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
#include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
#include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
#include "gc/shenandoah/shenandoahUtils.hpp"
#include "gc/shenandoah/shenandoahVerifier.hpp"
-ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) :
- MetadataVisitingOopIterateClosure(rp),
- _queue(q),
- _mark_context(ShenandoahHeap::heap()->marking_context()),
- _weak(false)
-{ }
-
ShenandoahMark::ShenandoahMark() :
_task_queues(ShenandoahHeap::heap()->marking_context()->task_queues()) {
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp
index 99995c469eb64..2eca17bde275d 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp
@@ -30,9 +30,9 @@
#include "gc/shared/continuationGCSupport.inline.hpp"
#include "gc/shenandoah/shenandoahAsserts.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
+#include "gc/shenandoah/shenandoahClosures.inline.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
-#include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
#include "gc/shenandoah/shenandoahStringDedup.inline.hpp"
#include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
#include "gc/shenandoah/shenandoahUtils.hpp"
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp
index 5eb0b277b5ef8..75687302ca573 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp
@@ -28,7 +28,6 @@
#include "gc/shenandoah/shenandoahClosures.inline.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahNMethod.inline.hpp"
-#include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/continuation.hpp"
#include "runtime/safepointVerifiers.hpp"
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp
deleted file mode 100644
index e433c2910c8d8..0000000000000
--- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Copyright (c) 2015, 2022, Red Hat, Inc. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHOOPCLOSURES_HPP
-#define SHARE_GC_SHENANDOAH_SHENANDOAHOOPCLOSURES_HPP
-
-#include "gc/shared/stringdedup/stringDedup.hpp"
-#include "gc/shenandoah/shenandoahClosures.inline.hpp"
-#include "gc/shenandoah/shenandoahGenerationType.hpp"
-#include "gc/shenandoah/shenandoahHeap.inline.hpp"
-#include "gc/shenandoah/shenandoahTaskqueue.hpp"
-#include "gc/shenandoah/shenandoahUtils.hpp"
-#include "memory/iterator.hpp"
-#include "runtime/javaThread.hpp"
-
-class ShenandoahMarkRefsSuperClosure : public MetadataVisitingOopIterateClosure {
-private:
- ShenandoahObjToScanQueue* _queue;
- ShenandoahMarkingContext* const _mark_context;
- bool _weak;
-
-protected:
- template
- void work(T *p);
-
-public:
- ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp);
-
- bool is_weak() const {
- return _weak;
- }
-
- void set_weak(bool weak) {
- _weak = weak;
- }
-
- virtual void do_nmethod(nmethod* nm) {
- assert(!is_weak(), "Can't handle weak marking of nmethods");
- nm->run_nmethod_entry_barrier();
- }
-};
-
-class ShenandoahMarkUpdateRefsSuperClosure : public ShenandoahMarkRefsSuperClosure {
-protected:
- ShenandoahHeap* const _heap;
-
- template
- inline void work(T* p);
-
-public:
- ShenandoahMarkUpdateRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) :
- ShenandoahMarkRefsSuperClosure(q, rp),
- _heap(ShenandoahHeap::heap()) {
- assert(_heap->is_stw_gc_in_progress(), "Can only be used for STW GC");
- };
-};
-
-template
-class ShenandoahMarkUpdateRefsClosure : public ShenandoahMarkUpdateRefsSuperClosure {
-private:
- template