diff --git a/.github/scripts/ci-matrix-result-check.py b/.github/scripts/ci-matrix-result-check.py index b029aabf..aa5856af 100644 --- a/.github/scripts/ci-matrix-result-check.py +++ b/.github/scripts/ci-matrix-result-check.py @@ -36,11 +36,11 @@ def read_in_plans(): value = m.group(1) else: raise ValueError(f"Cannot find a plan string in {prop}") - + # Store the value in the dictionary key = chr(97+i) results[key] = value - + return results def read_in_actual_results(line, plan_dict): @@ -144,7 +144,7 @@ def print_log(directory, search_string): if expected[plan] == "ignore": print(f"Result for {plan} is ignored") continue - + if expected[plan] != actual[plan]: error_no = 1 if expected[plan] == "pass": diff --git a/.github/scripts/ci-test-assertions.sh b/.github/scripts/ci-test-assertions.sh index 1b13f6ed..2be901ca 100755 --- a/.github/scripts/ci-test-assertions.sh +++ b/.github/scripts/ci-test-assertions.sh @@ -56,4 +56,5 @@ sudo sysctl -w vm.max_map_count=655300 export MMTK_PLAN=PageProtect build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop -build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex +# Note: Disable compressed pointers for luindex as it does not work well with GC plans that uses virtual memory excessively. +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex diff --git a/.github/scripts/ci-test-malloc-mark-sweep.sh b/.github/scripts/ci-test-malloc-mark-sweep.sh index 93a62ec6..e6b2d89e 100755 --- a/.github/scripts/ci-test-malloc-mark-sweep.sh +++ b/.github/scripts/ci-test-malloc-mark-sweep.sh @@ -16,7 +16,7 @@ run_test() { # Malloc marksweep is horribly slow. We just run fop. # build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar antlr - build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms50M -Xmx50M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop + build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms50M -Xmx50M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop # build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex # build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar pmd # build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar hsqldb diff --git a/.github/scripts/ci-test-only-normal-no-compressed-oops.sh b/.github/scripts/ci-test-only-normal-no-compressed-oops.sh new file mode 100755 index 00000000..e3d0e1bf --- /dev/null +++ b/.github/scripts/ci-test-only-normal-no-compressed-oops.sh @@ -0,0 +1,164 @@ +set -xe + +. $(dirname "$0")/common.sh + +unset JAVA_TOOL_OPTIONS +cd $OPENJDK_PATH + +# --- SemiSpace --- +export MMTK_PLAN=SemiSpace + +# Test - the benchmarks that are commented out do not work yet +# Note: the command line options are necessary for now to ensure the benchmarks work. We may later change the options if we do not have these many constraints. +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar antlr +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar bloat - does not work for stock build +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar jython - does not work for stock build +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar lusearch - validation failed +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar pmd +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar xalan - mmtk-core gets stuck in slowdebug build + +# These benchmarks take 40s+ for slowdebug build, we may consider removing them from the CI +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar hsqldb +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar eclipse + +# Test heap resizing +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms20M -Xmx100M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop + +# --- Immix --- +export MMTK_PLAN=Immix + +# Test - the benchmarks that are commented out do not work yet +# Note: the command line options are necessary for now to ensure the benchmarks work. We may later change the options if we do not have these many constraints. +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar antlr +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar bloat - does not work for stock build +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar jython - does not work for stock build +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar lusearch - validation failed +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar pmd +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar xalan - mmtk-core gets stuck in slowdebug build + +# These benchmarks take 40s+ for slowdebug build, we may consider removing them from the CI +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar hsqldb +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar eclipse + +# Test heap resizing +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms20M -Xmx100M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop + +# --- Immix --- +export MMTK_PLAN=GenImmix + +# Test - the benchmarks that are commented out do not work yet +# Note: the command line options are necessary for now to ensure the benchmarks work. We may later change the options if we do not have these many constraints. +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar antlr +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar bloat - does not work for stock build +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar jython - does not work for stock build +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar lusearch - validation failed +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar pmd +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar xalan - mmtk-core gets stuck in slowdebug build + +# These benchmarks take 40s+ for slowdebug build, we may consider removing them from the CI +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar hsqldb +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar eclipse + +# Test heap resizing +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms20M -Xmx100M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop + +# --- StickyImmix --- +export MMTK_PLAN=StickyImmix + +# Test - the benchmarks that are commented out do not work yet +# Note: the command line options are necessary for now to ensure the benchmarks work. We may later change the options if we do not have these many constraints. +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms75M -Xmx75M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar antlr +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms75M -Xmx75M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms75M -Xmx75M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms75M -Xmx75M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar lusearch +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms75M -Xmx75M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar pmd +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms75M -Xmx75M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar xalan + +# These benchmarks take 40s+ for slowdebug build, we may consider removing them from the CI +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms200M -Xmx200M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar hsqldb +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms200M -Xmx200M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar eclipse + +# --- GenCopy --- +export MMTK_PLAN=GenCopy + +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar antlr +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex +# Fail non-deterministically +# build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar hsqldb + +# Fail with OOM (code 137) +# build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar pmd + +# This passes, but it takes horribly long time to run. We exclude it. +# build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar eclipse + +# --- NoGC --- + +# Build +export MMTK_PLAN=NoGC + +# Test - the benchmarks that are commented out do not work yet +# Note: We could increase heap size when mmtk core can work for larger heap. We may get more benchmarks running. + +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms1G -Xmx1G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar antlr +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms1G -Xmx1G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar bloat - does not work for stock build +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms1G -Xmx1G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar eclipes - OOM +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms1G -Xmx1G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms1G -Xmx1G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar hsqldb - OOM +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms1G -Xmx1G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar jython - does not work for stock build +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms1G -Xmx1G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms1G -Xmx1G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar lusearch - OOM +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms1G -Xmx1G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar pmd - OOM +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms1G -Xmx1G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar xalan - OOM + +# --- MarkCompact --- +export MMTK_PLAN=MarkCompact +# Test - the benchmarks that are commented out do not work yet +# Note: the command line options are necessary for now to ensure the benchmarks work. We may later change the options if we do not have these many constraints. +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar antlr +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar bloat - does not work for stock build +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar jython - does not work for stock build +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar lusearch - validation failed +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar pmd +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar xalan - mmtk-core gets stuck in slowdebug build + +# These benchmarks take 40s+ for slowdebug build, we may consider removing them from the CI +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:TieredStopAtLevel=1 -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar hsqldb +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:TieredStopAtLevel=1 -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar eclipse + +# --- MarkSweep --- +export MMTK_PLAN=MarkSweep +# Test - the benchmarks that are commented out do not work yet +# Note: the command line options are necessary for now to ensure the benchmarks work. We may later change the options if we do not have these many constraints. +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar antlr +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar bloat - does not work for stock build +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar jython - does not work for stock build +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar lusearch - validation failed +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar pmd +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar xalan - mmtk-core gets stuck in slowdebug build + +# These benchmarks take 40s+ for slowdebug build, we may consider removing them from the CI +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:TieredStopAtLevel=1 -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar hsqldb +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:TieredStopAtLevel=1 -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar eclipse + +# --- PageProtect --- +# Make sure this runs last in our tests unless we want to set it back to the default limit. +sudo sysctl -w vm.max_map_count=655300 + +export MMTK_PLAN=PageProtect + +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar antlr +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex +# build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar pmd diff --git a/.github/scripts/ci-test-only-normal.sh b/.github/scripts/ci-test-only-normal.sh index 25fdea0c..0c76812e 100755 --- a/.github/scripts/ci-test-only-normal.sh +++ b/.github/scripts/ci-test-only-normal.sh @@ -151,14 +151,3 @@ build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHea # These benchmarks take 40s+ for slowdebug build, we may consider removing them from the CI build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:TieredStopAtLevel=1 -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar hsqldb build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:TieredStopAtLevel=1 -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar eclipse - -# --- PageProtect --- -# Make sure this runs last in our tests unless we want to set it back to the default limit. -sudo sysctl -w vm.max_map_count=655300 - -export MMTK_PLAN=PageProtect - -build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar antlr -build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop -build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex -# build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar pmd diff --git a/.github/scripts/ci-test.sh b/.github/scripts/ci-test.sh index 6112d1b9..07126bb2 100755 --- a/.github/scripts/ci-test.sh +++ b/.github/scripts/ci-test.sh @@ -6,6 +6,8 @@ cd $cur cd $cur ./ci-test-only-normal.sh cd $cur +./ci-test-only-normal-no-compressed-oops.sh +cd $cur ./ci-test-only-weak-ref.sh cd $cur ./ci-test-assertions.sh diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 41ca79e8..cb019d0c 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -463,9 +463,11 @@ dependencies = [ name = "mmtk_openjdk" version = "0.19.0" dependencies = [ + "atomic", "built 0.5.2", "lazy_static", "libc", + "memoffset", "mmtk", "once_cell", ] diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 7a6a0074..ba09d8c3 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -24,6 +24,8 @@ openjdk_version = "28e56ee32525c32c5a88391d0b01f24e5cd16c0f" libc = "0.2" lazy_static = "1.1" once_cell = "1.10.0" +atomic = "0.5.1" +memoffset = "0.9.0" # Be very careful to commit any changes to the following mmtk dependency, as our CI scripts (including mmtk-core CI) # rely on matching these lines to modify them: e.g. comment out the git dependency and use the local path. # These changes are safe: diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 0bd27a20..8e48292c 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -1,10 +1,14 @@ -use crate::UPCALLS; +use super::UPCALLS; +use crate::OpenJDKEdge; +use atomic::Atomic; +use atomic::Ordering; use mmtk::util::constants::*; use mmtk::util::conversions; use mmtk::util::ObjectReference; use mmtk::util::{Address, OpaquePointer}; use std::ffi::CStr; use std::fmt; +use std::sync::atomic::AtomicUsize; use std::{mem, slice}; #[repr(i32)] @@ -80,7 +84,7 @@ impl Klass { pub const LH_HEADER_SIZE_SHIFT: i32 = BITS_IN_BYTE as i32 * 2; pub const LH_HEADER_SIZE_MASK: i32 = (1 << BITS_IN_BYTE) - 1; pub unsafe fn cast<'a, T>(&self) -> &'a T { - &*(self as *const _ as usize as *const T) + &*(self as *const Self as *const T) } /// Force slow-path for instance size calculation? const fn layout_helper_needs_slow_path(lh: i32) -> bool { @@ -168,7 +172,7 @@ impl InstanceKlass { const VTABLE_START_OFFSET: usize = Self::HEADER_SIZE * BYTES_IN_WORD; fn start_of_vtable(&self) -> *const usize { - unsafe { (self as *const _ as *const u8).add(Self::VTABLE_START_OFFSET) as _ } + (Address::from_ref(self) + Self::VTABLE_START_OFFSET).to_ptr() } fn start_of_itable(&self) -> *const usize { @@ -263,24 +267,53 @@ impl InstanceRefKlass { } *DISCOVERED_OFFSET } - pub fn referent_address(oop: Oop) -> Address { - oop.get_field_address(Self::referent_offset()) + pub fn referent_address(oop: Oop) -> OpenJDKEdge { + oop.get_field_address(Self::referent_offset()).into() } - pub fn discovered_address(oop: Oop) -> Address { - oop.get_field_address(Self::discovered_offset()) + pub fn discovered_address(oop: Oop) -> OpenJDKEdge { + oop.get_field_address(Self::discovered_offset()).into() } } +#[repr(C)] +union KlassPointer { + /// uncompressed Klass pointer + klass: &'static Klass, + /// compressed Klass pointer + narrow_klass: u32, +} + #[repr(C)] pub struct OopDesc { pub mark: usize, - pub klass: &'static Klass, + klass: KlassPointer, +} + +static COMPRESSED_KLASS_BASE: Atomic
= Atomic::new(Address::ZERO); +static COMPRESSED_KLASS_SHIFT: AtomicUsize = AtomicUsize::new(0); + +/// When enabling compressed pointers, the class pointers are also compressed. +/// The c++ part of the binding should pass the compressed klass base and shift to rust binding, as object scanning will need it. +pub fn set_compressed_klass_base_and_shift(base: Address, shift: usize) { + COMPRESSED_KLASS_BASE.store(base, Ordering::Relaxed); + COMPRESSED_KLASS_SHIFT.store(shift, Ordering::Relaxed); } impl OopDesc { pub fn start(&self) -> Address { unsafe { mem::transmute(self) } } + + pub fn klass(&self) -> &'static Klass { + if COMPRESSED { + let compressed = unsafe { self.klass.narrow_klass }; + let addr = COMPRESSED_KLASS_BASE.load(Ordering::Relaxed) + + ((compressed as usize) << COMPRESSED_KLASS_SHIFT.load(Ordering::Relaxed)); + unsafe { &*addr.to_ptr::() } + } else { + unsafe { self.klass.klass } + } + } } impl fmt::Debug for OopDesc { @@ -292,8 +325,24 @@ impl fmt::Debug for OopDesc { } } +/// 32-bit compressed klass pointers +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct NarrowKlass(u32); + pub type Oop = &'static OopDesc; +/// 32-bit compressed reference pointers +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct NarrowOop(u32); + +impl NarrowOop { + pub fn slot(&self) -> Address { + Address::from_ref(self) + } +} + /// Convert ObjectReference to Oop impl From for &OopDesc { fn from(o: ObjectReference) -> Self { @@ -323,8 +372,8 @@ impl OopDesc { } /// Calculate object instance size - pub unsafe fn size(&self) -> usize { - let klass = self.klass; + pub unsafe fn size(&self) -> usize { + let klass = self.klass::(); let lh = klass.layout_helper; // The (scalar) instance size is pre-recorded in the TIB? if lh > Klass::LH_NEUTRAL_VALUE { @@ -336,7 +385,7 @@ impl OopDesc { } else if lh <= Klass::LH_NEUTRAL_VALUE { if lh < Klass::LH_NEUTRAL_VALUE { // Calculate array size - let array_length = self.as_array_oop().length(); + let array_length = self.as_array_oop().length::(); let mut size_in_bytes: usize = (array_length as usize) << Klass::layout_helper_log2_element_size(lh); size_in_bytes += Klass::layout_helper_header_size(lh) as usize; @@ -356,34 +405,57 @@ pub struct ArrayOopDesc(OopDesc); pub type ArrayOop = &'static ArrayOopDesc; impl ArrayOopDesc { - const LENGTH_OFFSET: usize = mem::size_of::(); + fn length_offset() -> usize { + let klass_offset_in_bytes = memoffset::offset_of!(OopDesc, klass); + if COMPRESSED { + klass_offset_in_bytes + mem::size_of::() + } else { + klass_offset_in_bytes + mem::size_of::() + } + } fn element_type_should_be_aligned(ty: BasicType) -> bool { ty == BasicType::T_DOUBLE || ty == BasicType::T_LONG } - fn header_size(ty: BasicType) -> usize { - let typesize_in_bytes = - conversions::raw_align_up(Self::LENGTH_OFFSET + BYTES_IN_INT, BYTES_IN_LONG); + fn header_size(ty: BasicType) -> usize { + let typesize_in_bytes = conversions::raw_align_up( + Self::length_offset::() + BYTES_IN_INT, + BYTES_IN_LONG, + ); if Self::element_type_should_be_aligned(ty) { conversions::raw_align_up(typesize_in_bytes / BYTES_IN_WORD, BYTES_IN_LONG) } else { typesize_in_bytes / BYTES_IN_WORD } } - fn length(&self) -> i32 { - unsafe { *((self as *const _ as *const u8).add(Self::LENGTH_OFFSET) as *const i32) } + fn length(&self) -> i32 { + unsafe { (Address::from_ref(self) + Self::length_offset::()).load::() } } - fn base(&self, ty: BasicType) -> Address { - let base_offset_in_bytes = Self::header_size(ty) * BYTES_IN_WORD; - Address::from_ptr(unsafe { (self as *const _ as *const u8).add(base_offset_in_bytes) }) + fn base(&self, ty: BasicType) -> Address { + let base_offset_in_bytes = Self::header_size::(ty) * BYTES_IN_WORD; + Address::from_ref(self) + base_offset_in_bytes } - // This provides an easy way to access the array data in Rust. However, the array data - // is Java types, so we have to map Java types to Rust types. The caller needs to guarantee: - // 1. matches the actual Java type - // 2. matches the argument, BasicType `ty` - pub unsafe fn data(&self, ty: BasicType) -> &[T] { - slice::from_raw_parts(self.base(ty).to_ptr(), self.length() as _) + /// This provides an easy way to access the array data in Rust. However, the array data + /// is Java types, so we have to map Java types to Rust types. The caller needs to guarantee: + /// 1. `` matches the actual Java type + /// 2. `` matches the argument, BasicType `ty` + pub unsafe fn data(&self, ty: BasicType) -> &[T] { + slice::from_raw_parts( + self.base::(ty).to_ptr(), + self.length::() as _, + ) + } + + pub unsafe fn slice( + &self, + ty: BasicType, + ) -> crate::OpenJDKEdgeRange { + let base = self.base::(ty); + let start = base; + let lshift = OpenJDKEdge::::LOG_BYTES_IN_EDGE; + let end = base + ((self.length::() as usize) << lshift); + (start..end).into() } } diff --git a/mmtk/src/active_plan.rs b/mmtk/src/active_plan.rs index 5ecaa92d..3b3f3c69 100644 --- a/mmtk/src/active_plan.rs +++ b/mmtk/src/active_plan.rs @@ -1,6 +1,5 @@ use crate::MutatorClosure; use crate::OpenJDK; -use crate::SINGLETON; use crate::UPCALLS; use mmtk::util::opaque_pointer::*; use mmtk::vm::ActivePlan; @@ -9,12 +8,12 @@ use mmtk::Plan; use std::collections::VecDeque; use std::marker::PhantomData; -struct OpenJDKMutatorIterator<'a> { - mutators: VecDeque<&'a mut Mutator>, +struct OpenJDKMutatorIterator<'a, const COMPRESSED: bool> { + mutators: VecDeque<&'a mut Mutator>>, phantom_data: PhantomData<&'a ()>, } -impl<'a> OpenJDKMutatorIterator<'a> { +impl<'a, const COMPRESSED: bool> OpenJDKMutatorIterator<'a, COMPRESSED> { fn new() -> Self { let mut mutators = VecDeque::new(); unsafe { @@ -29,8 +28,8 @@ impl<'a> OpenJDKMutatorIterator<'a> { } } -impl<'a> Iterator for OpenJDKMutatorIterator<'a> { - type Item = &'a mut Mutator; +impl<'a, const COMPRESSED: bool> Iterator for OpenJDKMutatorIterator<'a, COMPRESSED> { + type Item = &'a mut Mutator>; fn next(&mut self) -> Option { self.mutators.pop_front() @@ -39,24 +38,24 @@ impl<'a> Iterator for OpenJDKMutatorIterator<'a> { pub struct VMActivePlan {} -impl ActivePlan for VMActivePlan { - fn global() -> &'static dyn Plan { - SINGLETON.get_plan() +impl ActivePlan> for VMActivePlan { + fn global() -> &'static dyn Plan> { + crate::singleton::().get_plan() } fn is_mutator(tls: VMThread) -> bool { unsafe { ((*UPCALLS).is_mutator)(tls) } } - fn mutator(tls: VMMutatorThread) -> &'static mut Mutator { + fn mutator(tls: VMMutatorThread) -> &'static mut Mutator> { unsafe { let m = ((*UPCALLS).get_mmtk_mutator)(tls); - &mut *m + &mut *(m as *mut Mutator>) } } - fn mutators<'a>() -> Box> + 'a> { - Box::new(OpenJDKMutatorIterator::new()) + fn mutators<'a>() -> Box>> + 'a> { + Box::new(OpenJDKMutatorIterator::::new()) } fn number_of_mutators() -> usize { diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index b4c52b9f..295b4353 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -1,7 +1,7 @@ +use crate::edges::OpenJDKEdge; use crate::OpenJDK; use crate::OpenJDK_Upcalls; use crate::BUILDER; -use crate::SINGLETON; use crate::UPCALLS; use libc::c_char; use mmtk::memory_manager; @@ -9,7 +9,6 @@ use mmtk::plan::BarrierSelector; use mmtk::scheduler::GCController; use mmtk::scheduler::GCWorker; use mmtk::util::alloc::AllocatorSelector; -use mmtk::util::constants::LOG_BYTES_IN_ADDRESS; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; use mmtk::AllocationSemantics; @@ -20,6 +19,30 @@ use std::cell::RefCell; use std::ffi::{CStr, CString}; use std::sync::atomic::Ordering; +macro_rules! with_singleton { + (|$x: ident| $($expr:tt)*) => { + if crate::use_compressed_oops() { + let $x: &'static mmtk::MMTK> = &*crate::SINGLETON_COMPRESSED; + $($expr)* + } else { + let $x: &'static mmtk::MMTK> = &*crate::SINGLETON_UNCOMPRESSED; + $($expr)* + } + }; +} + +macro_rules! with_mutator { + (|$x: ident| $($expr:tt)*) => { + if crate::use_compressed_oops() { + let $x = unsafe { &mut *($x as *mut Mutator>) }; + $($expr)* + } else { + let $x = unsafe { &mut *($x as *mut Mutator>) }; + $($expr)* + } + }; +} + // Supported barriers: static NO_BARRIER: sync::Lazy = sync::Lazy::new(|| CString::new("NoBarrier").unwrap()); static OBJECT_BARRIER: sync::Lazy = @@ -32,13 +55,15 @@ pub extern "C" fn get_mmtk_version() -> *const c_char { #[no_mangle] pub extern "C" fn mmtk_active_barrier() -> *const c_char { - match SINGLETON.get_plan().constraints().barrier { - BarrierSelector::NoBarrier => NO_BARRIER.as_ptr(), - BarrierSelector::ObjectBarrier => OBJECT_BARRIER.as_ptr(), - // In case we have more barriers in mmtk-core. - #[allow(unreachable_patterns)] - _ => unimplemented!(), - } + with_singleton!(|singleton| { + match singleton.get_plan().constraints().barrier { + BarrierSelector::NoBarrier => NO_BARRIER.as_ptr(), + BarrierSelector::ObjectBarrier => OBJECT_BARRIER.as_ptr(), + // In case we have more barriers in mmtk-core. + #[allow(unreachable_patterns)] + _ => unimplemented!(), + } + }) } /// # Safety @@ -87,7 +112,11 @@ pub extern "C" fn openjdk_gc_init(calls: *const OpenJDK_Upcalls) { // Make sure that we haven't initialized MMTk (by accident) yet assert!(!crate::MMTK_INITIALIZED.load(Ordering::SeqCst)); // Make sure we initialize MMTk here - lazy_static::initialize(&SINGLETON); + if crate::use_compressed_oops() { + lazy_static::initialize(&crate::SINGLETON_COMPRESSED); + } else { + lazy_static::initialize(&crate::SINGLETON_UNCOMPRESSED); + } } #[no_mangle] @@ -108,60 +137,64 @@ pub extern "C" fn mmtk_set_heap_size(min: usize, max: usize) -> bool { } #[no_mangle] -pub extern "C" fn bind_mutator(tls: VMMutatorThread) -> *mut Mutator { - Box::into_raw(memory_manager::bind_mutator(&SINGLETON, tls)) +pub extern "C" fn bind_mutator(tls: VMMutatorThread) -> *mut libc::c_void { + with_singleton!(|singleton| { + Box::into_raw(memory_manager::bind_mutator(singleton, tls)) as *mut libc::c_void + }) } #[no_mangle] // It is fine we turn the pointer back to box, as we turned a boxed value to the raw pointer in bind_mutator() #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C" fn destroy_mutator(mutator: *mut Mutator) { - memory_manager::destroy_mutator(unsafe { &mut *mutator }) +pub extern "C" fn destroy_mutator(mutator: *mut libc::c_void) { + with_mutator!(|mutator| memory_manager::destroy_mutator(mutator)) } #[no_mangle] // We trust the mutator pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C" fn flush_mutator(mutator: *mut Mutator) { - memory_manager::flush_mutator(unsafe { &mut *mutator }) +pub extern "C" fn flush_mutator(mutator: *mut libc::c_void) { + with_mutator!(|mutator| memory_manager::flush_mutator(mutator)) } #[no_mangle] // We trust the mutator pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] pub extern "C" fn alloc( - mutator: *mut Mutator, + mutator: *mut libc::c_void, size: usize, align: usize, offset: usize, allocator: AllocationSemantics, ) -> Address { - memory_manager::alloc::(unsafe { &mut *mutator }, size, align, offset, allocator) + with_mutator!(|mutator| memory_manager::alloc(mutator, size, align, offset, allocator)) } #[no_mangle] pub extern "C" fn get_allocator_mapping(allocator: AllocationSemantics) -> AllocatorSelector { - memory_manager::get_allocator_mapping(&SINGLETON, allocator) + with_singleton!(|singleton| memory_manager::get_allocator_mapping(singleton, allocator)) } #[no_mangle] pub extern "C" fn get_max_non_los_default_alloc_bytes() -> usize { - SINGLETON - .get_plan() - .constraints() - .max_non_los_default_alloc_bytes + with_singleton!(|singleton| { + singleton + .get_plan() + .constraints() + .max_non_los_default_alloc_bytes + }) } #[no_mangle] // We trust the mutator pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] pub extern "C" fn post_alloc( - mutator: *mut Mutator, + mutator: *mut libc::c_void, refer: ObjectReference, bytes: usize, allocator: AllocationSemantics, ) { - memory_manager::post_alloc::(unsafe { &mut *mutator }, refer, bytes, allocator) + with_mutator!(|mutator| memory_manager::post_alloc(mutator, refer, bytes, allocator)) } #[no_mangle] @@ -172,56 +205,93 @@ pub extern "C" fn will_never_move(object: ObjectReference) -> bool { #[no_mangle] // We trust the gc_collector pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C" fn start_control_collector( - tls: VMWorkerThread, - gc_controller: *mut GCController, -) { - let mut gc_controller = unsafe { Box::from_raw(gc_controller) }; - memory_manager::start_control_collector(&SINGLETON, tls, &mut gc_controller); +pub extern "C" fn start_control_collector(tls: VMWorkerThread, gc_controller: *mut libc::c_void) { + if crate::use_compressed_oops() { + let mut gc_controller = + unsafe { Box::from_raw(gc_controller as *mut GCController>) }; + memory_manager::start_control_collector( + crate::singleton::(), + tls, + &mut gc_controller, + ); + } else { + let mut gc_controller = + unsafe { Box::from_raw(gc_controller as *mut GCController>) }; + memory_manager::start_control_collector( + crate::singleton::(), + tls, + &mut gc_controller, + ); + } } #[no_mangle] // We trust the worker pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C" fn start_worker(tls: VMWorkerThread, worker: *mut GCWorker) { - let mut worker = unsafe { Box::from_raw(worker) }; - memory_manager::start_worker::(&SINGLETON, tls, &mut worker) +pub extern "C" fn start_worker(tls: VMWorkerThread, worker: *mut libc::c_void) { + if crate::use_compressed_oops() { + let mut worker = unsafe { Box::from_raw(worker as *mut GCWorker>) }; + memory_manager::start_worker::>(crate::singleton::(), tls, &mut worker) + } else { + let mut worker = unsafe { Box::from_raw(worker as *mut GCWorker>) }; + memory_manager::start_worker::>( + crate::singleton::(), + tls, + &mut worker, + ) + } } #[no_mangle] pub extern "C" fn initialize_collection(tls: VMThread) { - memory_manager::initialize_collection(&SINGLETON, tls) + with_singleton!(|singleton| memory_manager::initialize_collection(singleton, tls)) } #[no_mangle] pub extern "C" fn used_bytes() -> usize { - memory_manager::used_bytes(&SINGLETON) + with_singleton!(|singleton| memory_manager::used_bytes(singleton)) } #[no_mangle] pub extern "C" fn free_bytes() -> usize { - memory_manager::free_bytes(&SINGLETON) + with_singleton!(|singleton| memory_manager::free_bytes(singleton)) } #[no_mangle] pub extern "C" fn total_bytes() -> usize { - memory_manager::total_bytes(&SINGLETON) + with_singleton!(|singleton| memory_manager::total_bytes(singleton)) } #[no_mangle] #[cfg(feature = "sanity")] pub extern "C" fn scan_region() { - memory_manager::scan_region(&SINGLETON) + with_singleton!(|singleton| memory_manager::scan_region(singleton)) } #[no_mangle] pub extern "C" fn handle_user_collection_request(tls: VMMutatorThread) { - memory_manager::handle_user_collection_request::(&SINGLETON, tls); + with_singleton!(|singleton| { + memory_manager::handle_user_collection_request(singleton, tls); + }) +} + +#[no_mangle] +pub extern "C" fn mmtk_enable_compressed_oops() { + crate::edges::enable_compressed_oops() +} + +#[no_mangle] +pub extern "C" fn mmtk_set_compressed_klass_base_and_shift(base: Address, shift: usize) { + crate::abi::set_compressed_klass_base_and_shift(base, shift) } #[no_mangle] pub extern "C" fn is_in_mmtk_spaces(object: ObjectReference) -> bool { - memory_manager::is_in_mmtk_spaces::(object) + if crate::use_compressed_oops() { + memory_manager::is_in_mmtk_spaces::>(object) + } else { + memory_manager::is_in_mmtk_spaces::>(object) + } } #[no_mangle] @@ -231,22 +301,22 @@ pub extern "C" fn is_mapped_address(addr: Address) -> bool { #[no_mangle] pub extern "C" fn modify_check(object: ObjectReference) { - memory_manager::modify_check(&SINGLETON, object) + with_singleton!(|singleton| memory_manager::modify_check(singleton, object)) } #[no_mangle] pub extern "C" fn add_weak_candidate(reff: ObjectReference) { - memory_manager::add_weak_candidate(&SINGLETON, reff) + with_singleton!(|singleton| memory_manager::add_weak_candidate(singleton, reff)) } #[no_mangle] pub extern "C" fn add_soft_candidate(reff: ObjectReference) { - memory_manager::add_soft_candidate(&SINGLETON, reff) + with_singleton!(|singleton| memory_manager::add_soft_candidate(singleton, reff)) } #[no_mangle] pub extern "C" fn add_phantom_candidate(reff: ObjectReference) { - memory_manager::add_phantom_candidate(&SINGLETON, reff) + with_singleton!(|singleton| memory_manager::add_phantom_candidate(singleton, reff)) } // The harness_begin()/end() functions are different than other API functions in terms of the thread state. @@ -263,7 +333,9 @@ pub extern "C" fn harness_begin(_id: usize) { #[no_mangle] pub extern "C" fn mmtk_harness_begin_impl() { // Pass null as tls, OpenJDK binding does not rely on the tls value to block the current thread and do a GC - memory_manager::harness_begin(&SINGLETON, VMMutatorThread(VMThread::UNINITIALIZED)); + with_singleton!(|singleton| { + memory_manager::harness_begin(singleton, VMMutatorThread(VMThread::UNINITIALIZED)); + }) } #[no_mangle] @@ -273,7 +345,7 @@ pub extern "C" fn harness_end(_id: usize) { #[no_mangle] pub extern "C" fn mmtk_harness_end_impl() { - memory_manager::harness_end(&SINGLETON); + with_singleton!(|singleton| memory_manager::harness_end(singleton)) } #[no_mangle] @@ -319,6 +391,18 @@ pub extern "C" fn process_bulk(options: *const c_char) -> bool { memory_manager::process_bulk(&mut builder, options_str.to_str().unwrap()) } +#[no_mangle] +pub extern "C" fn mmtk_narrow_oop_base() -> Address { + debug_assert!(crate::use_compressed_oops()); + crate::edges::BASE.load(Ordering::Relaxed) +} + +#[no_mangle] +pub extern "C" fn mmtk_narrow_oop_shift() -> usize { + debug_assert!(crate::use_compressed_oops()); + crate::edges::SHIFT.load(Ordering::Relaxed) +} + #[no_mangle] pub extern "C" fn starting_heap_address() -> Address { memory_manager::starting_heap_address() @@ -331,7 +415,7 @@ pub extern "C" fn last_heap_address() -> Address { #[no_mangle] pub extern "C" fn openjdk_max_capacity() -> usize { - memory_manager::total_bytes(&SINGLETON) + with_singleton!(|singleton| memory_manager::total_bytes(singleton)) } #[no_mangle] @@ -342,91 +426,108 @@ pub extern "C" fn executable() -> bool { /// Full pre barrier #[no_mangle] pub extern "C" fn mmtk_object_reference_write_pre( - mutator: &'static mut Mutator, + mutator: *mut libc::c_void, src: ObjectReference, slot: Address, target: ObjectReference, ) { - mutator - .barrier() - .object_reference_write_pre(src, slot.into(), target); + with_mutator!(|mutator| { + mutator + .barrier() + .object_reference_write_pre(src, slot.into(), target); + }) } /// Full post barrier #[no_mangle] pub extern "C" fn mmtk_object_reference_write_post( - mutator: &'static mut Mutator, + mutator: *mut libc::c_void, src: ObjectReference, slot: Address, target: ObjectReference, ) { - mutator - .barrier() - .object_reference_write_post(src, slot.into(), target); + with_mutator!(|mutator| { + mutator + .barrier() + .object_reference_write_post(src, slot.into(), target); + }) } /// Barrier slow-path call #[no_mangle] pub extern "C" fn mmtk_object_reference_write_slow( - mutator: &'static mut Mutator, + mutator: *mut libc::c_void, src: ObjectReference, slot: Address, target: ObjectReference, ) { - mutator - .barrier() - .object_reference_write_slow(src, slot.into(), target); + with_mutator!(|mutator| { + mutator + .barrier() + .object_reference_write_slow(src, slot.into(), target); + }) +} + +fn log_bytes_in_edge() -> usize { + if crate::use_compressed_oops() { + OpenJDKEdge::::LOG_BYTES_IN_EDGE + } else { + OpenJDKEdge::::LOG_BYTES_IN_EDGE + } } /// Array-copy pre-barrier #[no_mangle] pub extern "C" fn mmtk_array_copy_pre( - mutator: &'static mut Mutator, + mutator: *mut libc::c_void, src: Address, dst: Address, count: usize, ) { - let bytes = count << LOG_BYTES_IN_ADDRESS; - mutator - .barrier() - .memory_region_copy_pre((src..src + bytes).into(), (dst..dst + bytes).into()); + let bytes = count << log_bytes_in_edge(); + with_mutator!(|mutator| { + mutator + .barrier() + .memory_region_copy_pre((src..src + bytes).into(), (dst..dst + bytes).into()); + }) } /// Array-copy post-barrier #[no_mangle] pub extern "C" fn mmtk_array_copy_post( - mutator: &'static mut Mutator, + mutator: *mut libc::c_void, src: Address, dst: Address, count: usize, ) { - let bytes = count << LOG_BYTES_IN_ADDRESS; - mutator - .barrier() - .memory_region_copy_post((src..src + bytes).into(), (dst..dst + bytes).into()); + with_mutator!(|mutator| { + let bytes = count << log_bytes_in_edge(); + mutator + .barrier() + .memory_region_copy_post((src..src + bytes).into(), (dst..dst + bytes).into()); + }) } /// C2 Slowpath allocation barrier #[no_mangle] -pub extern "C" fn mmtk_object_probable_write( - mutator: &'static mut Mutator, - obj: ObjectReference, -) { - mutator.barrier().object_probable_write(obj); +pub extern "C" fn mmtk_object_probable_write(mutator: *mut libc::c_void, obj: ObjectReference) { + with_mutator!(|mutator| mutator.barrier().object_probable_write(obj)); } // finalization #[no_mangle] pub extern "C" fn add_finalizer(object: ObjectReference) { - memory_manager::add_finalizer(&SINGLETON, object); + with_singleton!(|singleton| memory_manager::add_finalizer(singleton, object)); } #[no_mangle] pub extern "C" fn get_finalized_object() -> ObjectReference { - match memory_manager::get_finalized_object(&SINGLETON) { - Some(obj) => obj, - None => ObjectReference::NULL, - } + with_singleton!(|singleton| { + match memory_manager::get_finalized_object(singleton) { + Some(obj) => obj, + None => ObjectReference::NULL, + } + }) } thread_local! { diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 776c737a..81e5bcc8 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -11,15 +11,15 @@ pub struct VMCollection {} const GC_THREAD_KIND_CONTROLLER: libc::c_int = 0; const GC_THREAD_KIND_WORKER: libc::c_int = 1; -impl Collection for VMCollection { +impl Collection> for VMCollection { fn stop_all_mutators(tls: VMWorkerThread, mut mutator_visitor: F) where - F: FnMut(&'static mut Mutator), + F: FnMut(&'static mut Mutator>), { unsafe { ((*UPCALLS).stop_all_mutators)( tls, - MutatorClosure::from_rust_closure(&mut mutator_visitor), + MutatorClosure::from_rust_closure::<_, COMPRESSED>(&mut mutator_visitor), ); } } @@ -36,7 +36,7 @@ impl Collection for VMCollection { } } - fn spawn_gc_thread(tls: VMThread, ctx: GCThreadContext) { + fn spawn_gc_thread(tls: VMThread, ctx: GCThreadContext>) { let (ctx_ptr, kind) = match ctx { GCThreadContext::Controller(c) => ( Box::into_raw(c) as *mut libc::c_void, diff --git a/mmtk/src/edges.rs b/mmtk/src/edges.rs index 44f666d1..d4f0e4aa 100644 --- a/mmtk/src/edges.rs +++ b/mmtk/src/edges.rs @@ -1,49 +1,185 @@ -use std::ops::Range; +use std::{ + ops::Range, + sync::atomic::{AtomicBool, AtomicUsize, Ordering}, +}; +use atomic::Atomic; use mmtk::{ - util::{Address, ObjectReference}, - vm::edge_shape::{AddressRangeIterator, Edge, MemorySlice}, + util::{ + constants::{LOG_BYTES_IN_INT, LOG_BYTES_IN_WORD}, + Address, ObjectReference, + }, + vm::edge_shape::{Edge, MemorySlice}, }; +static USE_COMPRESSED_OOPS: AtomicBool = AtomicBool::new(false); +pub static BASE: Atomic
= Atomic::new(Address::ZERO); +pub static SHIFT: AtomicUsize = AtomicUsize::new(0); + +/// Enables compressed oops +/// +/// This function can only be called once during MMTkHeap::initialize. +pub fn enable_compressed_oops() { + static COMPRESSED_OOPS_INITIALIZED: AtomicBool = AtomicBool::new(false); + assert!( + !COMPRESSED_OOPS_INITIALIZED.fetch_or(true, Ordering::Relaxed), + "cannot enable compressed pointers twice." + ); + if cfg!(not(target_arch = "x86_64")) { + panic!("Compressed pointer is only enable on x86_64 platforms.\ + For other RISC architectures, we need to find a way to process compressed embeded pointers in code objects first."); + } + USE_COMPRESSED_OOPS.store(true, Ordering::Relaxed) +} + +/// Check if the compressed pointer is enabled +pub fn use_compressed_oops() -> bool { + USE_COMPRESSED_OOPS.load(Ordering::Relaxed) +} + +/// Set compressed pointer base and shift based on heap range +pub fn initialize_compressed_oops_base_and_shift() { + let heap_end = mmtk::memory_manager::last_heap_address().as_usize(); + if heap_end <= (4usize << 30) { + BASE.store(Address::ZERO, Ordering::Relaxed); + SHIFT.store(0, Ordering::Relaxed); + } else if heap_end <= (32usize << 30) { + BASE.store(Address::ZERO, Ordering::Relaxed); + SHIFT.store(3, Ordering::Relaxed); + } else { + // set heap base as HEAP_START - 4096, to make sure null pointer value does not conflict with HEAP_START + BASE.store( + mmtk::memory_manager::starting_heap_address() - 4096, + Ordering::Relaxed, + ); + SHIFT.store(3, Ordering::Relaxed); + } +} + /// The type of edges in OpenJDK. /// Currently it has the same layout as `Address`, but we override its load and store methods. +/// +/// If `COMPRESSED = false`, every edge is uncompressed. +/// +/// If `COMPRESSED = true`, +/// * If this is a field of an object, the edge is compressed. +/// * If this is a root pointer: The c++ part of the binding should pass all the root pointers to +/// rust as tagged pointers. +/// * If the 63rd bit of the pointer is set to 1, the value referenced by the pointer is a +/// 32-bit compressed integer. +/// * Otherwise, it is a uncompressed root pointer. #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] #[repr(transparent)] -pub struct OpenJDKEdge { +pub struct OpenJDKEdge { pub addr: Address, } -impl From
for OpenJDKEdge { +impl From
for OpenJDKEdge { fn from(value: Address) -> Self { Self { addr: value } } } +impl OpenJDKEdge { + pub const LOG_BYTES_IN_EDGE: usize = if COMPRESSED { 2 } else { 3 }; + pub const BYTES_IN_EDGE: usize = 1 << Self::LOG_BYTES_IN_EDGE; + + const MASK: usize = 1usize << 63; + + /// Check if the pointer is tagged as "compressed" + const fn is_compressed(&self) -> bool { + self.addr.as_usize() & Self::MASK == 0 + } + + /// Get the edge address with tags stripped + const fn untagged_address(&self) -> Address { + unsafe { Address::from_usize(self.addr.as_usize() << 1 >> 1) } + } + + fn x86_read_unaligned(&self) -> T { + debug_assert!(cfg!(any(target_arch = "x86", target_arch = "x86_64"))); + // Workaround: On x86 (including x86_64), machine instructions may contain pointers as + // immediates, and they may be unaligned. It is an undefined behavior in Rust to + // dereference unaligned pointers. We have to explicitly use unaligned memory access + // methods. On x86, ordinary MOV instructions can load and store memory at unaligned + // addresses, so we expect `ptr.read_unaligned()` to have no performance penalty over + // `ptr.read()` if `ptr` is actually aligned. + unsafe { + let slot = if UNTAG { + self.untagged_address() + } else { + self.addr + }; + let ptr = slot.to_ptr::(); + ptr.read_unaligned() + } + } + + fn x86_write_unaligned(&self, v: T) { + debug_assert!(cfg!(any(target_arch = "x86", target_arch = "x86_64"))); + unsafe { + let slot = if UNTAG { + self.untagged_address() + } else { + self.addr + }; + let ptr = slot.to_mut_ptr::(); + ptr.write_unaligned(v) + } + } + + /// encode an object pointer to its u32 compressed form + fn compress(o: ObjectReference) -> u32 { + if o.is_null() { + 0u32 + } else { + ((o.to_raw_address() - BASE.load(Ordering::Relaxed)) >> SHIFT.load(Ordering::Relaxed)) + as u32 + } + } + + /// decode an object pointer from its u32 compressed form + fn decompress(v: u32) -> ObjectReference { + if v == 0 { + ObjectReference::NULL + } else { + ObjectReference::from_raw_address( + BASE.load(Ordering::Relaxed) + ((v as usize) << SHIFT.load(Ordering::Relaxed)), + ) + } + } +} -impl Edge for OpenJDKEdge { +impl Edge for OpenJDKEdge { fn load(&self) -> ObjectReference { if cfg!(any(target_arch = "x86", target_arch = "x86_64")) { - // Workaround: On x86 (including x86_64), machine instructions may contain pointers as - // immediates, and they may be unaligned. It is an undefined behavior in Rust to - // dereference unaligned pointers. We have to explicitly use unaligned memory access - // methods. On x86, ordinary MOV instructions can load and store memory at unaligned - // addresses, so we expect `ptr.read_unaligned()` to have no performance penalty over - // `ptr.read()` if `ptr` is actually aligned. - unsafe { - let ptr = self.addr.to_ptr::(); - ptr.read_unaligned() + if COMPRESSED { + if self.is_compressed() { + Self::decompress(self.x86_read_unaligned::()) + } else { + self.x86_read_unaligned::() + } + } else { + self.x86_read_unaligned::() } } else { + debug_assert!(!COMPRESSED); unsafe { self.addr.load() } } } fn store(&self, object: ObjectReference) { if cfg!(any(target_arch = "x86", target_arch = "x86_64")) { - unsafe { - let ptr = self.addr.to_mut_ptr::(); - ptr.write_unaligned(object) + if COMPRESSED { + if self.is_compressed() { + self.x86_write_unaligned::(Self::compress(object)) + } else { + self.x86_write_unaligned::(object) + } + } else { + self.x86_write_unaligned::(object) } } else { + debug_assert!(!COMPRESSED); unsafe { self.addr.store(object) } } } @@ -51,53 +187,93 @@ impl Edge for OpenJDKEdge { /// A range of OpenJDKEdge, usually used for arrays. #[derive(Clone, PartialEq, Eq, Hash, Debug)] -pub struct OpenJDKEdgeRange { - range: Range
, +pub struct OpenJDKEdgeRange { + range: Range>, } -impl From> for OpenJDKEdgeRange { +impl From> for OpenJDKEdgeRange { fn from(value: Range
) -> Self { - Self { range: value } + Self { + range: Range { + start: value.start.into(), + end: value.end.into(), + }, + } } } -pub struct OpenJDKEdgeRangeIterator { - inner: AddressRangeIterator, +pub struct OpenJDKEdgeRangeIterator { + cursor: Address, + limit: Address, } -impl Iterator for OpenJDKEdgeRangeIterator { - type Item = OpenJDKEdge; +impl Iterator for OpenJDKEdgeRangeIterator { + type Item = OpenJDKEdge; fn next(&mut self) -> Option { - self.inner.next().map(|a| a.into()) + if self.cursor >= self.limit { + None + } else { + let edge = self.cursor; + self.cursor += OpenJDKEdge::::BYTES_IN_EDGE; + Some(edge.into()) + } + } +} + +impl From> for Range
{ + fn from(value: OpenJDKEdgeRange) -> Self { + value.range.start.addr..value.range.end.addr } } // Note that we cannot implement MemorySlice for `Range` because neither // `MemorySlice` nor `Range` are defined in the `mmtk-openjdk` crate. ("orphan rule") -impl MemorySlice for OpenJDKEdgeRange { - type Edge = OpenJDKEdge; - type EdgeIterator = OpenJDKEdgeRangeIterator; +impl MemorySlice for OpenJDKEdgeRange { + type Edge = OpenJDKEdge; + type EdgeIterator = OpenJDKEdgeRangeIterator; fn iter_edges(&self) -> Self::EdgeIterator { OpenJDKEdgeRangeIterator { - inner: self.range.iter_edges(), + cursor: self.range.start.addr, + limit: self.range.end.addr, } } fn object(&self) -> Option { - self.range.object() + None } fn start(&self) -> Address { - self.range.start() + self.range.start.addr } fn bytes(&self) -> usize { - self.range.bytes() + self.range.end.addr - self.range.start.addr } fn copy(src: &Self, tgt: &Self) { - MemorySlice::copy(&src.range, &tgt.range) + debug_assert_eq!(src.bytes(), tgt.bytes()); + // Raw memory copy + if COMPRESSED { + debug_assert_eq!( + src.bytes() & ((1 << LOG_BYTES_IN_INT) - 1), + 0, + "bytes are not a multiple of 32-bit integers" + ); + unsafe { + let words = tgt.bytes() >> LOG_BYTES_IN_INT; + let src = src.start().to_ptr::(); + let tgt = tgt.start().to_mut_ptr::(); + std::ptr::copy(src, tgt, words) + } + } else { + debug_assert_eq!( + src.bytes() & ((1 << LOG_BYTES_IN_WORD) - 1), + 0, + "bytes are not a multiple of words" + ); + Range::
::copy(&src.clone().into(), &tgt.clone().into()) + } } } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index b9496410..4a33cc75 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -1,25 +1,32 @@ use std::sync::atomic::Ordering; use crate::scanning::to_edges_closure; -use crate::{OpenJDK, OpenJDKEdge, UPCALLS}; +use crate::OpenJDK; +use crate::OpenJDKEdge; +use crate::UPCALLS; use mmtk::scheduler::*; use mmtk::vm::RootsWorkFactory; +use mmtk::vm::*; use mmtk::MMTK; macro_rules! scan_roots_work { ($struct_name: ident, $func_name: ident) => { - pub struct $struct_name> { + pub struct $struct_name> { factory: F, + _p: std::marker::PhantomData, } - impl> $struct_name { + impl> $struct_name { pub fn new(factory: F) -> Self { - Self { factory } + Self { + factory, + _p: std::marker::PhantomData, + } } } - impl> GCWork for $struct_name { - fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { + impl> GCWork for $struct_name { + fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { unsafe { ((*UPCALLS).$func_name)(to_edges_closure(&mut self.factory)); } @@ -43,18 +50,27 @@ scan_roots_work!( scan_roots_work!(ScanWeakProcessorRoots, scan_weak_processor_roots); scan_roots_work!(ScanVMThreadRoots, scan_vm_thread_roots); -pub struct ScanCodeCacheRoots> { +pub struct ScanCodeCacheRoots>> +{ factory: F, } -impl> ScanCodeCacheRoots { +impl>> + ScanCodeCacheRoots +{ pub fn new(factory: F) -> Self { Self { factory } } } -impl> GCWork for ScanCodeCacheRoots { - fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { +impl>> + GCWork> for ScanCodeCacheRoots +{ + fn do_work( + &mut self, + _worker: &mut GCWorker>, + _mmtk: &'static MMTK>, + ) { // Collect all the cached roots let mut edges = Vec::with_capacity(crate::CODE_CACHE_ROOTS_SIZE.load(Ordering::Relaxed)); for roots in (*crate::CODE_CACHE_ROOTS.lock().unwrap()).values() { @@ -63,7 +79,9 @@ impl> GCWork for ScanCodeCacheRoots } } // Create work packet - self.factory.create_process_edge_roots_work(edges); + if !edges.is_empty() { + self.factory.create_process_edge_roots_work(edges); + } // Use the following code to scan CodeCache directly, instead of scanning the "remembered set". // unsafe { // ((*UPCALLS).scan_code_cache_roots)(create_process_edges_work:: as _); diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 10f7e16b..a45ad4c3 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -6,11 +6,15 @@ use std::ptr::null_mut; use std::sync::atomic::AtomicUsize; use std::sync::Mutex; +pub use edges::use_compressed_oops; use edges::{OpenJDKEdge, OpenJDKEdgeRange}; use libc::{c_char, c_void, uintptr_t}; use mmtk::util::alloc::AllocationError; -use mmtk::util::opaque_pointer::*; +use mmtk::util::constants::LOG_BYTES_IN_GBYTE; +use mmtk::util::heap::vm_layout::VMLayout; +use mmtk::util::{conversions, opaque_pointer::*}; use mmtk::util::{Address, ObjectReference}; +use mmtk::vm::edge_shape::Edge; use mmtk::vm::VMBinding; use mmtk::{MMTKBuilder, Mutator, MMTK}; @@ -36,27 +40,28 @@ pub struct NewBuffer { /// A closure for reporting mutators. The C++ code should pass `data` back as the last argument. #[repr(C)] pub struct MutatorClosure { - pub func: extern "C" fn(mutator: *mut Mutator, data: *mut libc::c_void), + pub func: extern "C" fn(mutator: *mut libc::c_void, data: *mut libc::c_void), pub data: *mut libc::c_void, } impl MutatorClosure { - fn from_rust_closure(callback: &mut F) -> Self + fn from_rust_closure(callback: &mut F) -> Self where - F: FnMut(&'static mut Mutator), + F: FnMut(&'static mut Mutator>), { Self { - func: Self::call_rust_closure::, + func: Self::call_rust_closure::, data: callback as *mut F as *mut libc::c_void, } } - extern "C" fn call_rust_closure( - mutator: *mut Mutator, + extern "C" fn call_rust_closure( + mutator: *mut libc::c_void, callback_ptr: *mut libc::c_void, ) where - F: FnMut(&'static mut Mutator), + F: FnMut(&'static mut Mutator>), { + let mutator = mutator as *mut Mutator>; let callback: &mut F = unsafe { &mut *(callback_ptr as *mut F) }; callback(unsafe { &mut *mutator }); } @@ -85,7 +90,7 @@ pub struct OpenJDK_Upcalls { pub scan_object: extern "C" fn(trace: *mut c_void, object: ObjectReference, tls: OpaquePointer), pub dump_object: extern "C" fn(object: ObjectReference), pub get_object_size: extern "C" fn(object: ObjectReference) -> usize, - pub get_mmtk_mutator: extern "C" fn(tls: VMMutatorThread) -> *mut Mutator, + pub get_mmtk_mutator: extern "C" fn(tls: VMMutatorThread) -> *mut libc::c_void, pub is_mutator: extern "C" fn(tls: VMThread) -> bool, pub harness_begin: extern "C" fn(), pub harness_end: extern "C" fn(), @@ -131,20 +136,20 @@ pub static VO_BIT_ADDRESS: uintptr_t = #[no_mangle] pub static FREE_LIST_ALLOCATOR_SIZE: uintptr_t = - std::mem::size_of::>(); + std::mem::size_of::>>(); #[derive(Default)] -pub struct OpenJDK; +pub struct OpenJDK; -impl VMBinding for OpenJDK { - type VMObjectModel = object_model::VMObjectModel; +impl VMBinding for OpenJDK { + type VMObjectModel = object_model::VMObjectModel; type VMScanning = scanning::VMScanning; type VMCollection = collection::VMCollection; type VMActivePlan = active_plan::VMActivePlan; type VMReferenceGlue = reference_glue::VMReferenceGlue; - type VMEdge = OpenJDKEdge; - type VMMemorySlice = OpenJDKEdgeRange; + type VMEdge = OpenJDKEdge; + type VMMemorySlice = OpenJDKEdgeRange; const MIN_ALIGNMENT: usize = 8; const MAX_ALIGNMENT: usize = 8; @@ -158,7 +163,18 @@ pub static MMTK_INITIALIZED: AtomicBool = AtomicBool::new(false); lazy_static! { pub static ref BUILDER: Mutex = Mutex::new(MMTKBuilder::new_no_env_vars()); - pub static ref SINGLETON: MMTK = { + pub static ref SINGLETON_COMPRESSED: MMTK> = { + assert!(use_compressed_oops()); + let mut builder = BUILDER.lock().unwrap(); + assert!(!MMTK_INITIALIZED.load(Ordering::Relaxed)); + set_compressed_pointer_vm_layout(&mut builder); + let ret = mmtk::memory_manager::mmtk_init(&builder); + MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); + edges::initialize_compressed_oops_base_and_shift(); + *ret + }; + pub static ref SINGLETON_UNCOMPRESSED: MMTK> = { + assert!(!use_compressed_oops()); let builder = BUILDER.lock().unwrap(); assert!(!MMTK_INITIALIZED.load(Ordering::Relaxed)); let ret = mmtk::memory_manager::mmtk_init(&builder); @@ -167,9 +183,23 @@ lazy_static! { }; } +fn singleton() -> &'static MMTK> { + if COMPRESSED { + unsafe { + &*(&*SINGLETON_COMPRESSED as *const MMTK> + as *const MMTK>) + } + } else { + unsafe { + &*(&*SINGLETON_UNCOMPRESSED as *const MMTK> + as *const MMTK>) + } + } +} + #[no_mangle] pub static MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES: usize = - mmtk::util::alloc::MarkCompactAllocator::::HEADER_RESERVED_IN_BYTES; + mmtk::util::alloc::MarkCompactAllocator::>::HEADER_RESERVED_IN_BYTES; lazy_static! { /// A global storage for all the cached CodeCache root pointers @@ -178,3 +208,25 @@ lazy_static! { /// A counter tracking the total size of the `CODE_CACHE_ROOTS`. static CODE_CACHE_ROOTS_SIZE: AtomicUsize = AtomicUsize::new(0); + +fn set_compressed_pointer_vm_layout(builder: &mut MMTKBuilder) { + let max_heap_size = builder.options.gc_trigger.max_heap_size(); + assert!( + max_heap_size <= (32usize << LOG_BYTES_IN_GBYTE), + "Heap size is larger than 32 GB" + ); + let start = 0x4000_0000; + let end = match start + max_heap_size { + end if end <= (4usize << 30) => 4usize << 30, + end if end <= (32usize << 30) => 32usize << 30, + _ => 0x4000_0000 + (32usize << 30), + }; + let constants = VMLayout { + log_address_space: 35, + heap_start: conversions::chunk_align_down(unsafe { Address::from_usize(start) }), + heap_end: conversions::chunk_align_up(unsafe { Address::from_usize(end) }), + log_space_extent: 31, + force_use_contiguous_spaces: false, + }; + builder.set_vm_layout(constants); +} diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 3cfe50fd..a35d6739 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -6,9 +6,9 @@ use mmtk::util::copy::*; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::*; -pub struct VMObjectModel {} +pub struct VMObjectModel {} -impl ObjectModel for VMObjectModel { +impl ObjectModel> for VMObjectModel { const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = vm_metadata::LOGGING_SIDE_METADATA_SPEC; const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec = @@ -24,9 +24,9 @@ impl ObjectModel for VMObjectModel { fn copy( from: ObjectReference, copy: CopySemantics, - copy_context: &mut GCWorkerCopyContext, + copy_context: &mut GCWorkerCopyContext>, ) -> ObjectReference { - let bytes = unsafe { Oop::from(from).size() }; + let bytes = unsafe { Oop::from(from).size::() }; let dst = copy_context.alloc_copy(from, bytes, ::std::mem::size_of::(), 0, copy); // Copy let src = from.to_raw_address(); @@ -38,7 +38,7 @@ impl ObjectModel for VMObjectModel { fn copy_to(from: ObjectReference, to: ObjectReference, region: Address) -> Address { let need_copy = from != to; - let bytes = unsafe { ((*UPCALLS).get_object_size)(from) }; + let bytes = unsafe { Oop::from(from).size::() }; if need_copy { // copy obj to target let dst = to.to_raw_address(); @@ -50,7 +50,7 @@ impl ObjectModel for VMObjectModel { } let start = Self::ref_to_object_start(to); if region != Address::ZERO { - fill_alignment_gap::(region, start); + fill_alignment_gap::>(region, start); } start + bytes } @@ -60,7 +60,7 @@ impl ObjectModel for VMObjectModel { } fn get_current_size(object: ObjectReference) -> usize { - unsafe { Oop::from(object).size() } + unsafe { Oop::from(object).size::() } } fn get_size_when_copied(object: ObjectReference) -> usize { @@ -106,6 +106,7 @@ impl ObjectModel for VMObjectModel { let oop = Oop::from(object); // It is only valid if klass.id is between 0 and 5 (see KlassID in openjdk/src/hotspot/share/oops/klass.hpp) // If oop.klass is not a valid pointer, we may segfault here. - oop.klass.id as i32 >= 0 && (oop.klass.id as i32) < 6 + let klass_id = oop.klass::().id as i32; + (0..6).contains(&klass_id) } } diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index a738a934..59d7f054 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -1,112 +1,129 @@ -use crate::SINGLETON; +use crate::OpenJDKEdge; -use crate::abi::*; -use crate::{OpenJDKEdge, UPCALLS}; -use mmtk::util::constants::*; +use super::abi::*; +use super::UPCALLS; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::EdgeVisitor; +use std::cell::UnsafeCell; use std::{mem, slice}; +type E = OpenJDKEdge; + trait OopIterate: Sized { - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor); + fn oop_iterate( + &self, + oop: Oop, + closure: &mut impl EdgeVisitor>, + ); } impl OopIterate for OopMapBlock { - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { + fn oop_iterate( + &self, + oop: Oop, + closure: &mut impl EdgeVisitor>, + ) { + let log_bytes_in_oop = if COMPRESSED { 2 } else { 3 }; let start = oop.get_field_address(self.offset); for i in 0..self.count as usize { - let edge = (start + (i << LOG_BYTES_IN_ADDRESS)).into(); + let edge = (start + (i << log_bytes_in_oop)).into(); closure.visit_edge(edge); } } } impl OopIterate for InstanceKlass { - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { + fn oop_iterate( + &self, + oop: Oop, + closure: &mut impl EdgeVisitor>, + ) { let oop_maps = self.nonstatic_oop_maps(); for map in oop_maps { - map.oop_iterate(oop, closure) + map.oop_iterate::(oop, closure) } } } impl OopIterate for InstanceMirrorKlass { - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { - self.instance_klass.oop_iterate(oop, closure); - // if (Devirtualizer::do_metadata(closure)) { - // Klass* klass = java_lang_Class::as_Klass(obj); - // // We'll get NULL for primitive mirrors. - // if (klass != NULL) { - // if (klass->is_instance_klass() && InstanceKlass::cast(klass)->is_anonymous()) { - // // An anonymous class doesn't have its own class loader, so when handling - // // the java mirror for an anonymous class we need to make sure its class - // // loader data is claimed, this is done by calling do_cld explicitly. - // // For non-anonymous classes the call to do_cld is made when the class - // // loader itself is handled. - // Devirtualizer::do_cld(closure, klass->class_loader_data()); - // } else { - // Devirtualizer::do_klass(closure, klass); - // } - // } else { - // // We would like to assert here (as below) that if klass has been NULL, then - // // this has been a mirror for a primitive type that we do not need to follow - // // as they are always strong roots. - // // However, we might get across a klass that just changed during CMS concurrent - // // marking if allocation occurred in the old generation. - // // This is benign here, as we keep alive all CLDs that were loaded during the - // // CMS concurrent phase in the class loading, i.e. they will be iterated over - // // and kept alive during remark. - // // assert(java_lang_Class::is_primitive(obj), "Sanity check"); - // } - // } + fn oop_iterate( + &self, + oop: Oop, + closure: &mut impl EdgeVisitor>, + ) { + self.instance_klass.oop_iterate::(oop, closure); // static fields - let start: *const Oop = Self::start_of_static_fields(oop).to_ptr::(); + let start = Self::start_of_static_fields(oop); let len = Self::static_oop_field_count(oop); - let slice = unsafe { slice::from_raw_parts(start, len as _) }; - for oop in slice { - closure.visit_edge(Address::from_ref(oop as &Oop).into()); + if COMPRESSED { + let start: *const NarrowOop = start.to_ptr::(); + let slice = unsafe { slice::from_raw_parts(start, len as _) }; + for narrow_oop in slice { + closure.visit_edge(narrow_oop.slot().into()); + } + } else { + let start: *const Oop = start.to_ptr::(); + let slice = unsafe { slice::from_raw_parts(start, len as _) }; + for oop in slice { + closure.visit_edge(Address::from_ref(oop as &Oop).into()); + } } } } impl OopIterate for InstanceClassLoaderKlass { - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { - self.instance_klass.oop_iterate(oop, closure); - // if (Devirtualizer::do_metadata(closure)) { - // ClassLoaderData* cld = java_lang_ClassLoader::loader_data(obj); - // // cld can be null if we have a non-registered class loader. - // if (cld != NULL) { - // Devirtualizer::do_cld(closure, cld); - // } - // } + fn oop_iterate( + &self, + oop: Oop, + closure: &mut impl EdgeVisitor>, + ) { + self.instance_klass.oop_iterate::(oop, closure); } } impl OopIterate for ObjArrayKlass { - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { + fn oop_iterate( + &self, + oop: Oop, + closure: &mut impl EdgeVisitor>, + ) { let array = unsafe { oop.as_array_oop() }; - for oop in unsafe { array.data::(BasicType::T_OBJECT) } { - closure.visit_edge(Address::from_ref(oop as &Oop).into()); + if COMPRESSED { + for narrow_oop in unsafe { array.data::(BasicType::T_OBJECT) } { + closure.visit_edge(narrow_oop.slot().into()); + } + } else { + for oop in unsafe { array.data::(BasicType::T_OBJECT) } { + closure.visit_edge(Address::from_ref(oop as &Oop).into()); + } } } } impl OopIterate for TypeArrayKlass { - fn oop_iterate(&self, _oop: Oop, _closure: &mut impl EdgeVisitor) { + fn oop_iterate( + &self, + _oop: Oop, + _closure: &mut impl EdgeVisitor>, + ) { // Performance tweak: We skip processing the klass pointer since all // TypeArrayKlasses are guaranteed processed via the null class loader. } } impl OopIterate for InstanceRefKlass { - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { + fn oop_iterate( + &self, + oop: Oop, + closure: &mut impl EdgeVisitor>, + ) { use crate::abi::*; use crate::api::{add_phantom_candidate, add_soft_candidate, add_weak_candidate}; - self.instance_klass.oop_iterate(oop, closure); + self.instance_klass.oop_iterate::(oop, closure); - if Self::should_scan_weak_refs() { + if Self::should_scan_weak_refs::() { let reference = ObjectReference::from(oop); match self.instance_klass.reference_type { ReferenceType::None => { @@ -128,27 +145,44 @@ impl OopIterate for InstanceRefKlass { } impl InstanceRefKlass { - fn should_scan_weak_refs() -> bool { - !*SINGLETON.get_options().no_reference_types + fn should_scan_weak_refs() -> bool { + !*crate::singleton::() + .get_options() + .no_reference_types } - fn process_ref_as_strong(oop: Oop, closure: &mut impl EdgeVisitor) { - let referent_addr = Self::referent_address(oop); - closure.visit_edge(referent_addr.into()); - let discovered_addr = Self::discovered_address(oop); - closure.visit_edge(discovered_addr.into()); + fn process_ref_as_strong( + oop: Oop, + closure: &mut impl EdgeVisitor>, + ) { + let referent_addr = Self::referent_address::(oop); + closure.visit_edge(referent_addr); + let discovered_addr = Self::discovered_address::(oop); + closure.visit_edge(discovered_addr); } } #[allow(unused)] -fn oop_iterate_slow(oop: Oop, closure: &mut impl EdgeVisitor, tls: OpaquePointer) { +fn oop_iterate_slow>>( + oop: Oop, + closure: &mut V, + tls: OpaquePointer, +) { unsafe { - ((*UPCALLS).scan_object)(closure as *mut _ as _, mem::transmute(oop), tls); + CLOSURE.with(|x| *x.get() = closure as *mut V as *mut u8); + ((*UPCALLS).scan_object)( + mem::transmute( + scan_object_fn:: as *const unsafe extern "C" fn(edge: Address), + ), + mem::transmute(oop), + tls, + ); } } -fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor) { - let klass_id = oop.klass.id; - debug_assert!( +fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor>) { + let klass = oop.klass::(); + let klass_id = klass.id; + assert!( klass_id as i32 >= 0 && (klass_id as i32) < 6, "Invalid klass-id: {:x} for oop: {:x}", klass_id as i32, @@ -156,41 +190,50 @@ fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor) { ); match klass_id { KlassID::Instance => { - let instance_klass = unsafe { oop.klass.cast::() }; - instance_klass.oop_iterate(oop, closure); + let instance_klass = unsafe { klass.cast::() }; + instance_klass.oop_iterate::(oop, closure); } KlassID::InstanceClassLoader => { - let instance_klass = unsafe { oop.klass.cast::() }; - instance_klass.oop_iterate(oop, closure); + let instance_klass = unsafe { klass.cast::() }; + instance_klass.oop_iterate::(oop, closure); } KlassID::InstanceMirror => { - let instance_klass = unsafe { oop.klass.cast::() }; - instance_klass.oop_iterate(oop, closure); + let instance_klass = unsafe { klass.cast::() }; + instance_klass.oop_iterate::(oop, closure); } KlassID::ObjArray => { - let array_klass = unsafe { oop.klass.cast::() }; - array_klass.oop_iterate(oop, closure); + let array_klass = unsafe { klass.cast::() }; + array_klass.oop_iterate::(oop, closure); } KlassID::TypeArray => { - let array_klass = unsafe { oop.klass.cast::() }; - array_klass.oop_iterate(oop, closure); + // Skip scanning primitive arrays as they contain no reference fields. } KlassID::InstanceRef => { - let instance_klass = unsafe { oop.klass.cast::() }; - instance_klass.oop_iterate(oop, closure); - } // _ => oop_iterate_slow(oop, closure, tls), + let instance_klass = unsafe { klass.cast::() }; + instance_klass.oop_iterate::(oop, closure); + } } } -pub fn scan_object( +thread_local! { + static CLOSURE: UnsafeCell<*mut u8> = UnsafeCell::new(std::ptr::null_mut()); +} + +pub unsafe extern "C" fn scan_object_fn< + const COMPRESSED: bool, + V: EdgeVisitor>, +>( + edge: Address, +) { + let ptr: *mut u8 = CLOSURE.with(|x| *x.get()); + let closure = &mut *(ptr as *mut V); + closure.visit_edge(edge.into()); +} + +pub fn scan_object( object: ObjectReference, - closure: &mut impl EdgeVisitor, + closure: &mut impl EdgeVisitor>, _tls: VMWorkerThread, ) { - // println!("*****scan_object(0x{:x}) -> \n 0x{:x}, 0x{:x} \n", - // object, - // unsafe { *(object.value() as *const usize) }, - // unsafe { *((object.value() + 8) as *const usize) } - // ); - unsafe { oop_iterate(mem::transmute(object), closure) } + unsafe { oop_iterate::(mem::transmute(object), closure) } } diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 966f4e92..3ecd5f1a 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -3,20 +3,21 @@ use crate::OpenJDK; use crate::UPCALLS; use mmtk::util::opaque_pointer::VMWorkerThread; use mmtk::util::ObjectReference; +use mmtk::vm::edge_shape::Edge; use mmtk::vm::ReferenceGlue; pub struct VMReferenceGlue {} -impl ReferenceGlue for VMReferenceGlue { +impl ReferenceGlue> for VMReferenceGlue { type FinalizableType = ObjectReference; fn set_referent(reff: ObjectReference, referent: ObjectReference) { let oop = Oop::from(reff); - unsafe { InstanceRefKlass::referent_address(oop).store(referent) }; + InstanceRefKlass::referent_address::(oop).store(referent); } fn get_referent(object: ObjectReference) -> ObjectReference { let oop = Oop::from(object); - unsafe { InstanceRefKlass::referent_address(oop).load::() } + InstanceRefKlass::referent_address::(oop).load() } fn enqueue_references(references: &[ObjectReference], _tls: VMWorkerThread) { unsafe { diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index d92d5dcd..5b3ee419 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -1,6 +1,7 @@ use crate::gc_work::*; +use crate::Edge; use crate::{EdgesClosure, OpenJDK}; -use crate::{NewBuffer, OpenJDKEdge, SINGLETON, UPCALLS}; +use crate::{NewBuffer, OpenJDKEdge, UPCALLS}; use mmtk::memory_manager; use mmtk::scheduler::WorkBucketStage; use mmtk::util::opaque_pointer::*; @@ -13,7 +14,7 @@ pub struct VMScanning {} const WORK_PACKET_CAPACITY: usize = 4096; -extern "C" fn report_edges_and_renew_buffer>( +extern "C" fn report_edges_and_renew_buffer>( ptr: *mut Address, length: usize, capacity: usize, @@ -22,7 +23,7 @@ extern "C" fn report_edges_and_renew_buffer>( if !ptr.is_null() { // Note: Currently OpenJDKEdge has the same layout as Address. If the layout changes, we // should fix the Rust-to-C interface. - let buf = unsafe { Vec::::from_raw_parts(ptr as _, length, capacity) }; + let buf = unsafe { Vec::::from_raw_parts(ptr as _, length, capacity) }; let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; factory.create_process_edge_roots_work(buf); } @@ -36,20 +37,20 @@ extern "C" fn report_edges_and_renew_buffer>( NewBuffer { ptr, capacity } } -pub(crate) fn to_edges_closure>(factory: &mut F) -> EdgesClosure { +pub(crate) fn to_edges_closure>(factory: &mut F) -> EdgesClosure { EdgesClosure { - func: report_edges_and_renew_buffer::, + func: report_edges_and_renew_buffer::, data: factory as *mut F as *mut libc::c_void, } } -impl Scanning for VMScanning { - fn scan_object>( +impl Scanning> for VMScanning { + fn scan_object>>( tls: VMWorkerThread, object: ObjectReference, edge_visitor: &mut EV, ) { - crate::object_scanning::scan_object(object, edge_visitor, tls) + crate::object_scanning::scan_object::(object, edge_visitor, tls); } fn notify_initial_thread_scan_complete(_partial_scan: bool, _tls: VMWorkerThread) { @@ -59,8 +60,8 @@ impl Scanning for VMScanning { fn scan_roots_in_mutator_thread( _tls: VMWorkerThread, - mutator: &'static mut Mutator, - mut factory: impl RootsWorkFactory, + mutator: &'static mut Mutator>, + mut factory: impl RootsWorkFactory>, ) { let tls = mutator.get_tls(); unsafe { @@ -68,9 +69,12 @@ impl Scanning for VMScanning { } } - fn scan_vm_specific_roots(_tls: VMWorkerThread, factory: impl RootsWorkFactory) { + fn scan_vm_specific_roots( + _tls: VMWorkerThread, + factory: impl RootsWorkFactory>, + ) { memory_manager::add_work_packets( - &SINGLETON, + crate::singleton::(), WorkBucketStage::Prepare, vec![ Box::new(ScanUniverseRoots::new(factory.clone())) as _, diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index b08937cd..f78452e2 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -188,6 +188,11 @@ extern bool openjdk_is_gc_initialized(); extern bool mmtk_set_heap_size(size_t min, size_t max); +extern bool mmtk_enable_compressed_oops(); +extern void* mmtk_narrow_oop_base(); +extern size_t mmtk_narrow_oop_shift(); +extern size_t mmtk_set_compressed_klass_base_and_shift(void* base, size_t shift); + extern size_t used_bytes(); extern void* starting_heap_address(); extern void* last_heap_address(); diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 309862b7..1f626102 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -77,12 +77,12 @@ MMTkHeap::MMTkHeap(MMTkCollectorPolicy* policy) : jint MMTkHeap::initialize() { assert(!UseTLAB , "should disable UseTLAB"); - assert(!UseCompressedOops , "should disable CompressedOops"); - assert(!UseCompressedClassPointers , "should disable UseCompressedClassPointers"); const size_t min_heap_size = collector_policy()->min_heap_byte_size(); const size_t max_heap_size = collector_policy()->max_heap_byte_size(); // printf("policy max heap size %zu, min heap size %zu\n", heap_size, collector_policy()->min_heap_byte_size()); + if (UseCompressedOops) mmtk_enable_compressed_oops(); + // Note that MMTk options may be set from several different sources, with increasing priorities: // 1. Default values defined in mmtk::util::options::Options // 2. Default values defined in ThirdPartyHeapArguments::initialize @@ -130,6 +130,10 @@ jint MMTkHeap::initialize() { _start = (HeapWord*) starting_heap_address(); _end = (HeapWord*) last_heap_address(); // printf("start: %p, end: %p\n", _start, _end); + if (UseCompressedOops) { + Universe::set_narrow_oop_base((address) mmtk_narrow_oop_base()); + Universe::set_narrow_oop_shift(mmtk_narrow_oop_shift()); + } initialize_reserved_region(_start, _end); @@ -173,6 +177,9 @@ void MMTkHeap::schedule_finalizer() { void MMTkHeap::post_initialize() { CollectedHeap::post_initialize(); + if (UseCompressedOops) { + mmtk_set_compressed_klass_base_and_shift((void*) Universe::narrow_klass_base(), (size_t) Universe::narrow_klass_shift()); + } } void MMTkHeap::enable_collection() { @@ -380,12 +387,18 @@ void MMTkHeap::print_tracing_info() const { // Registering and unregistering an nmethod (compiled code) with the heap. // Override with specific mechanism for each specialized heap type. class MMTkRegisterNMethodOopClosure: public OopClosure { - template void do_oop_work(T* p) { - mmtk_add_nmethod_oop((void*) p); + template void do_oop_work(T* p, bool narrow) { + if (UseCompressedOops && !narrow) { + guarantee((uintptr_t(p) & (1ull << 63)) == 0, "test"); + auto tagged_p = (T*) (uintptr_t(p) | (1ull << 63)); + mmtk_add_nmethod_oop((void*) tagged_p); + } else { + mmtk_add_nmethod_oop((void*) p); + } } public: - void do_oop(oop* p) { do_oop_work(p); } - void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p, false); } + void do_oop(narrowOop* p) { do_oop_work(p, true); } }; diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp index 990f6d5e..ee92b223 100644 --- a/openjdk/mmtkRootsClosure.hpp +++ b/openjdk/mmtkRootsClosure.hpp @@ -7,63 +7,24 @@ #include "oops/oop.inline.hpp" #include "utilities/globalDefinitions.hpp" -#define ROOTS_BUFFER_SIZE 4096 - class MMTkRootsClosure : public OopClosure { - void* _trace; - void* _buffer[ROOTS_BUFFER_SIZE]; - size_t _cursor; - - template - void do_oop_work(T* p) { - // T heap_oop = RawAccess<>::oop_load(p); - // if (!CompressedOops::is_null(heap_oop)) { - // oop obj = CompressedOops::decode_not_null(heap_oop); - // oop fwd = (oop) trace_root_object(_trace, obj); - // RawAccess<>::oop_store(p, fwd); - // } - _buffer[_cursor++] = (void*) p; - if (_cursor >= ROOTS_BUFFER_SIZE) { - flush(); - } - } - - NOINLINE void flush() { - // bulk_report_delayed_root_edge(_trace, _buffer, _cursor); - _cursor = 0; - } - -public: - MMTkRootsClosure(void* trace): _trace(trace), _cursor(0) {} - - ~MMTkRootsClosure() { - if (_cursor > 0) flush(); - } - - virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { - // printf("narrowoop root %p -> %d %p %p\n", (void*) p, *p, *((void**) p), (void*) oopDesc::load_decode_heap_oop(p)); - do_oop_work(p); - } -}; - -class MMTkRootsClosure2 : public OopClosure { EdgesClosure _edges_closure; void** _buffer; size_t _cap; size_t _cursor; template - void do_oop_work(T* p) { - // T heap_oop = RawAccess<>::oop_load(p); - // if (!CompressedOops::is_null(heap_oop)) { - // oop obj = CompressedOops::decode_not_null(heap_oop); - // oop fwd = (oop) trace_root_object(_trace, obj); - // RawAccess<>::oop_store(p, fwd); - // } - _buffer[_cursor++] = (void*) p; - if (_cursor >= _cap) { - flush(); + void do_oop_work(T* p, bool narrow) { + T heap_oop = RawAccess<>::oop_load(p); + if (!CompressedOops::is_null(heap_oop)) { + if (UseCompressedOops && !narrow) { + guarantee((uintptr_t(p) & (1ull << 63)) == 0, "test"); + p = (T*) (uintptr_t(p) | (1ull << 63)); + } + _buffer[_cursor++] = (void*) p; + if (_cursor >= _cap) { + flush(); + } } } @@ -77,21 +38,21 @@ class MMTkRootsClosure2 : public OopClosure { } public: - MMTkRootsClosure2(EdgesClosure edges_closure): _edges_closure(edges_closure), _cursor(0) { + MMTkRootsClosure(EdgesClosure edges_closure): _edges_closure(edges_closure), _cursor(0) { NewBuffer buf = edges_closure.invoke(NULL, 0, 0); _buffer = buf.buf; _cap = buf.cap; } - ~MMTkRootsClosure2() { + ~MMTkRootsClosure() { if (_cursor > 0) flush(); if (_buffer != NULL) { release_buffer(_buffer, _cursor, _cap); } } - virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p, false); } + virtual void do_oop(narrowOop* p) { do_oop_work(p, true); } }; class MMTkScanObjectClosure : public BasicOopIterateClosure { @@ -99,19 +60,18 @@ class MMTkScanObjectClosure : public BasicOopIterateClosure { CLDToOopClosure follow_cld_closure; template - void do_oop_work(T* p) { - // oop ref = (void*) oopDesc::decode_heap_oop(oopDesc::load_heap_oop(p)); - // process_edge(_trace, (void*) p); + void do_oop_work(T* p, bool narrow) { + if (UseCompressedOops && !narrow) { + guarantee((uintptr_t(p) & (1ull << 63)) == 0, "test"); + p = (T*) (uintptr_t(p) | (1ull << 63)); + } } public: MMTkScanObjectClosure(void* trace): _trace(trace), follow_cld_closure(this, false) {} - virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { - // printf("narrowoop edge %p -> %d %p %p\n", (void*) p, *p, *((void**) p), (void*) oopDesc::load_decode_heap_oop(p)); - do_oop_work(p); - } + virtual void do_oop(oop* p) { do_oop_work(p, false); } + virtual void do_oop(narrowOop* p) { do_oop_work(p, true); } virtual bool do_metadata() { return true; diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 68743261..af3033ae 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -184,14 +184,14 @@ static void mmtk_get_mutators(MutatorClosure closure) { } static void mmtk_scan_roots_in_all_mutator_threads(EdgesClosure closure) { - MMTkRootsClosure2 cl(closure); + MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_roots_in_all_mutator_threads(cl); } static void mmtk_scan_roots_in_mutator_thread(EdgesClosure closure, void* tls) { ResourceMark rm; JavaThread* thread = (JavaThread*) tls; - MMTkRootsClosure2 cl(closure); + MMTkRootsClosure cl(closure); thread->oops_do(&cl, NULL); } @@ -268,18 +268,18 @@ static void mmtk_schedule_finalizer() { MMTkHeap::heap()->schedule_finalizer(); } -static void mmtk_scan_universe_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_universe_roots(cl); } -static void mmtk_scan_jni_handle_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_jni_handle_roots(cl); } -static void mmtk_scan_object_synchronizer_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_object_synchronizer_roots(cl); } -static void mmtk_scan_management_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_management_roots(cl); } -static void mmtk_scan_jvmti_export_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_jvmti_export_roots(cl); } -static void mmtk_scan_aot_loader_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_aot_loader_roots(cl); } -static void mmtk_scan_system_dictionary_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_system_dictionary_roots(cl); } -static void mmtk_scan_code_cache_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_code_cache_roots(cl); } -static void mmtk_scan_string_table_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_string_table_roots(cl); } -static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl); } -static void mmtk_scan_weak_processor_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_weak_processor_roots(cl); } -static void mmtk_scan_vm_thread_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_vm_thread_roots(cl); } +static void mmtk_scan_universe_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_universe_roots(cl); } +static void mmtk_scan_jni_handle_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_jni_handle_roots(cl); } +static void mmtk_scan_object_synchronizer_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_object_synchronizer_roots(cl); } +static void mmtk_scan_management_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_management_roots(cl); } +static void mmtk_scan_jvmti_export_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_jvmti_export_roots(cl); } +static void mmtk_scan_aot_loader_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_aot_loader_roots(cl); } +static void mmtk_scan_system_dictionary_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_system_dictionary_roots(cl); } +static void mmtk_scan_code_cache_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_code_cache_roots(cl); } +static void mmtk_scan_string_table_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_string_table_roots(cl); } +static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl); } +static void mmtk_scan_weak_processor_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_weak_processor_roots(cl); } +static void mmtk_scan_vm_thread_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_vm_thread_roots(cl); } static size_t mmtk_number_of_mutators() { return Threads::number_of_threads(); diff --git a/openjdk/thirdPartyHeapArguments.cpp b/openjdk/thirdPartyHeapArguments.cpp index 4014ae71..39f5713c 100644 --- a/openjdk/thirdPartyHeapArguments.cpp +++ b/openjdk/thirdPartyHeapArguments.cpp @@ -43,8 +43,7 @@ void ThirdPartyHeapArguments::initialize() { GCArguments::initialize(); assert(UseThirdPartyHeap , "Error, should UseThirdPartyHeap"); FLAG_SET_DEFAULT(UseTLAB, false); - FLAG_SET_DEFAULT(UseCompressedOops, false); - FLAG_SET_DEFAULT(UseCompressedClassPointers, false); + FLAG_SET_DEFAULT(UseCompressedClassPointers, UseCompressedOops); FLAG_SET_DEFAULT(ParallelGCThreads, Abstract_VM_Version::parallel_worker_threads()); if (ParallelGCThreads == 0) { assert(!FLAG_IS_DEFAULT(ParallelGCThreads), "ParallelGCThreads should not be 0.");