diff --git a/substratevm/mx.substratevm/mx_substratevm_namespace.py b/substratevm/mx.substratevm/mx_substratevm_namespace.py index 3a7a688fae43..826c4019e9ea 100644 --- a/substratevm/mx.substratevm/mx_substratevm_namespace.py +++ b/substratevm/mx.substratevm/mx_substratevm_namespace.py @@ -40,7 +40,7 @@ ignore_files = {"copy_x86.hpp", "copy_aarch64.hpp", "osThread_linux.hpp"} ignore_includes = {"CPU_HEADER(copy)", "OS_HEADER(osThread)"} -files_with_cpp_guard = {"sharedGCStructs.hpp"} +files_with_cpp_guard = {"sharedGCStructs.h", "shenandoahGCStructs.h", "g1GCStructs.h"} SVM_NAMESPACE = "svm_namespace" diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index b6271d04235d..6e00c3ff12aa 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -2103,6 +2103,8 @@ "dependency:com.oracle.svm.native.jvm.posix/*", "dependency:com.oracle.svm.native.libcontainer/*", "file:debug/include", + "file:src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/include", + "file:src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/include", ], }, }, diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index dab9ee137f8a..abc534e827ec 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -153,6 +153,7 @@ public String getName() { } @Override + @Platforms(Platform.HOSTED_ONLY.class) public String getDefaultMaxHeapSize() { return String.format("%s%% of RAM", SerialAndEpsilonGCOptions.MaximumHeapSizePercent.getValue()); } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/shenandoah/ShenandoahPhysicalMemorySupport.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/shenandoah/ShenandoahPhysicalMemorySupport.java new file mode 100644 index 000000000000..2edac1e6213e --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/shenandoah/ShenandoahPhysicalMemorySupport.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.posix.linux.shenandoah; + +import org.graalvm.word.UnsignedWord; + +import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; +import com.oracle.svm.core.gc.shenandoah.ShenandoahCommittedMemoryProvider; +import com.oracle.svm.core.gc.shenandoah.UseShenandoahGC; +import com.oracle.svm.core.heap.PhysicalMemory.PhysicalMemorySupport; +import com.oracle.svm.core.posix.linux.LinuxPhysicalMemorySupportImpl; +import com.oracle.svm.core.traits.BuiltinTraits.NoLayeredCallbacks; +import com.oracle.svm.core.traits.BuiltinTraits.RuntimeAccessOnly; +import com.oracle.svm.core.traits.SingletonLayeredInstallationKind.Disallowed; +import com.oracle.svm.core.traits.SingletonTraits; + +@AutomaticallyRegisteredImageSingleton(value = PhysicalMemorySupport.class, onlyWith = UseShenandoahGC.class) +@SingletonTraits(access = RuntimeAccessOnly.class, layeredCallbacks = NoLayeredCallbacks.class, layeredInstallationKind = Disallowed.class) +public class ShenandoahPhysicalMemorySupport extends LinuxPhysicalMemorySupportImpl { + @Override + public UnsignedWord size() { + return ShenandoahCommittedMemoryProvider.getInstance().getPhysicalMemorySize(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 8978a570808d..f79a7bbe3918 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -588,12 +588,12 @@ public HostedOptionKey multi @Option(help = "Please use '--gc=*' instead. Possible values are listed with '--help'.")// public static final HostedOptionKey SupportedGCs = new HostedOptionKey<>( - ReplacingLocatableMultiOptionValue.DelimitedString.buildWithCommaDelimiter(GCOptionValue.SERIAL.getValue())) { + ReplacingLocatableMultiOptionValue.DelimitedString.buildWithCommaDelimiter(GCOptionValue.Serial.getValue())) { @Override protected void onValueUpdate(EconomicMap, Object> values, ReplacingLocatableMultiOptionValue.DelimitedString oldValue, ReplacingLocatableMultiOptionValue.DelimitedString newValue) { - if (newValue.contains(GCOptionValue.G1.getValue())) { + if (newValue.contains(GCOptionValue.G1.getValue()) || newValue.contains(GCOptionValue.Shenandoah.getValue())) { SubstrateOptions.SpawnIsolates.update(values, true); SubstrateOptions.AllowVMInternalThreads.update(values, true); SubstrateOptions.ConcealedOptions.UseDedicatedVMOperationThread.update(values, true); @@ -602,17 +602,16 @@ protected void onValueUpdate(EconomicMap, Object> values, Replacing super.onValueUpdate(values, oldValue, newValue); } - }; @Fold public static boolean useSerialGC() { - return !SubstrateOptions.SupportedGCs.hasBeenSet() || SubstrateOptions.SupportedGCs.getValue().contains(GCOptionValue.SERIAL.getValue()); + return !SubstrateOptions.SupportedGCs.hasBeenSet() || SubstrateOptions.SupportedGCs.getValue().contains(GCOptionValue.Serial.getValue()); } @Fold public static boolean useEpsilonGC() { - return SubstrateOptions.SupportedGCs.getValue().contains(GCOptionValue.EPSILON.getValue()); + return SubstrateOptions.SupportedGCs.getValue().contains(GCOptionValue.Epsilon.getValue()); } @Fold @@ -620,6 +619,11 @@ public static boolean useG1GC() { return SubstrateOptions.SupportedGCs.getValue().contains(GCOptionValue.G1.getValue()); } + @Fold + public static boolean useShenandoahGC() { + return SubstrateOptions.SupportedGCs.getValue().contains(GCOptionValue.Shenandoah.getValue()); + } + public static class DeprecatedOptions { @LayerVerifiedOption(kind = Kind.Changed, severity = Severity.Error)// @@ -629,9 +633,9 @@ public static class DeprecatedOptions { @Override protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { if (newValue) { - SubstrateOptions.SupportedGCs.update(values, ReplacingLocatableMultiOptionValue.DelimitedString.buildWithCommaDelimiter(GCOptionValue.SERIAL.getValue())); + SubstrateOptions.SupportedGCs.update(values, ReplacingLocatableMultiOptionValue.DelimitedString.buildWithCommaDelimiter(GCOptionValue.Serial.getValue())); } else if (!values.containsKey(SubstrateOptions.SupportedGCs) || - ((ReplacingLocatableMultiOptionValue.DelimitedString) values.get(SubstrateOptions.SupportedGCs)).contains(GCOptionValue.SERIAL.getValue())) { + ((ReplacingLocatableMultiOptionValue.DelimitedString) values.get(SubstrateOptions.SupportedGCs)).contains(GCOptionValue.Serial.getValue())) { SubstrateOptions.SupportedGCs.update(values, ReplacingLocatableMultiOptionValue.DelimitedString.buildWithCommaDelimiter()); } } @@ -644,8 +648,23 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o @Override protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { if (newValue) { - SubstrateOptions.SupportedGCs.update(values, ReplacingLocatableMultiOptionValue.DelimitedString.buildWithCommaDelimiter(GCOptionValue.EPSILON.getValue())); - } else if (((AccumulatingLocatableMultiOptionValue.Strings) values.get(SubstrateOptions.SupportedGCs)).contains(GCOptionValue.EPSILON.getValue())) { + SubstrateOptions.SupportedGCs.update(values, ReplacingLocatableMultiOptionValue.DelimitedString.buildWithCommaDelimiter(GCOptionValue.Epsilon.getValue())); + } else if (((AccumulatingLocatableMultiOptionValue.Strings) values.get(SubstrateOptions.SupportedGCs)).contains(GCOptionValue.Epsilon.getValue())) { + SubstrateOptions.SupportedGCs.update(values, ReplacingLocatableMultiOptionValue.DelimitedString.buildWithCommaDelimiter()); + } + } + }; + + @LayerVerifiedOption(kind = Kind.Changed, severity = Severity.Error)// + @APIOption(name = "shenandoah", group = GCGroup.class, customHelp = "Shenandoah garbage collector")// + @Option(help = "Use the Shenandoah GC", deprecated = true, deprecationMessage = "Please use '--gc=shenandoah' instead")// + public static final HostedOptionKey UseShenandoahGC = new HostedOptionKey<>(false) { + @Override + protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { + if (newValue) { + SubstrateOptions.SupportedGCs.update(values, ReplacingLocatableMultiOptionValue.DelimitedString.buildWithCommaDelimiter(GCOptionValue.Shenandoah.getValue())); + } else if (!values.containsKey(SubstrateOptions.SupportedGCs) || + ((ReplacingLocatableMultiOptionValue.DelimitedString) values.get(SubstrateOptions.SupportedGCs)).contains(GCOptionValue.Shenandoah.getValue())) { SubstrateOptions.SupportedGCs.update(values, ReplacingLocatableMultiOptionValue.DelimitedString.buildWithCommaDelimiter()); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/OWNERS.toml b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/OWNERS.toml new file mode 100644 index 000000000000..1f7d0b0a8706 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/OWNERS.toml @@ -0,0 +1,7 @@ +[[rule]] +files = "*" +all = [ + "christian.haeubl@oracle.com", +] +any = [ +] diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/InitializeReservedRegistersForPossiblyUnattachedThread.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/InitializeReservedRegistersForPossiblyUnattachedThread.java new file mode 100644 index 000000000000..4df2bc074220 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/InitializeReservedRegistersForPossiblyUnattachedThread.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +import org.graalvm.nativeimage.Isolate; +import org.graalvm.nativeimage.IsolateThread; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.c.function.CEntryPointOptions.Prologue; +import com.oracle.svm.core.graal.nodes.WriteCurrentVMThreadNode; +import com.oracle.svm.core.graal.snippets.CEntryPointSnippets; + +/** + * Prologue that only initializes the base registers so that a thread can execute SVM code. + * Unattached threads need to pass null as the {@link IsolateThread}, so that this prologue behaves + * the same as {@link InitializeReservedRegistersForUnattachedThread}. + */ +public class InitializeReservedRegistersForPossiblyUnattachedThread implements Prologue { + @Uninterruptible(reason = "prologue") + @SuppressWarnings("unused") + public static void enter(Isolate heapBase, IsolateThread thread) { + CEntryPointSnippets.initBaseRegisters(heapBase); + WriteCurrentVMThreadNode.writeCurrentVMThread(thread); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/InitializeReservedRegistersForUnattachedThread.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/InitializeReservedRegistersForUnattachedThread.java new file mode 100644 index 000000000000..6e92e5e1142c --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/InitializeReservedRegistersForUnattachedThread.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +import org.graalvm.nativeimage.Isolate; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.c.function.CEntryPointOptions.Prologue; +import com.oracle.svm.core.graal.nodes.WriteCurrentVMThreadNode; +import com.oracle.svm.core.graal.snippets.CEntryPointSnippets; + +import jdk.graal.compiler.word.Word; + +/** + * Prologue that only initializes the base registers so that an unattached thread can execute SVM + * code. + */ +public final class InitializeReservedRegistersForUnattachedThread implements Prologue { + @Uninterruptible(reason = "prologue") + @SuppressWarnings("unused") + public static void enter(Isolate heapBase) { + CEntryPointSnippets.initBaseRegisters(heapBase); + WriteCurrentVMThreadNode.writeCurrentVMThread(Word.nullPointer()); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCDebugLevel.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCDebugLevel.java new file mode 100644 index 000000000000..7d5e1f51b0d5 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCDebugLevel.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +/** Defines which debug-levels are supported for GCs such as G1. */ +public enum NativeGCDebugLevel { + Product("", 0), + FastDebug("-fastdebug", 1), + Debug("-debug", 2); + + private final String libSuffix; + private final int index; + + NativeGCDebugLevel(String libSuffix, int index) { + this.libSuffix = libSuffix; + this.index = index; + } + + public String getLibSuffix() { + return libSuffix; + } + + public int getIndex() { + return index; + } + + public static NativeGCDebugLevel fromString(String value) { + return switch (value) { + case "product" -> Product; + case "fastdebug" -> FastDebug; + case "debug" -> Debug; + default -> null; + }; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCHeaderFiles.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCHeaderFiles.java new file mode 100644 index 000000000000..6bd3ae7be7bc --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCHeaderFiles.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +import java.util.Collections; +import java.util.List; + +import org.graalvm.nativeimage.c.CContext; + +import com.oracle.svm.core.c.ProjectHeaderFile; + +/** + * Determines which header files must be included when building a native-image that uses a GC such + * as G1. + */ +public class NativeGCHeaderFiles implements CContext.Directives { + @Override + public boolean isInConfiguration() { + return UseNativeGC.get(); + } + + @Override + public List getHeaderFiles() { + return Collections.singletonList(ProjectHeaderFile.resolve("", "include/sharedGCStructs.h")); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCOptions.java new file mode 100644 index 000000000000..1e654d76498c --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCOptions.java @@ -0,0 +1,448 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +import static com.oracle.svm.core.option.RuntimeOptionKey.RuntimeOptionKeyFlag.IsolateCreationOnly; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.function.Consumer; +import java.util.function.Supplier; + +import org.graalvm.collections.UnmodifiableEconomicMap; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.option.HostedOptionKey; +import com.oracle.svm.core.option.HostedOptionValues; +import com.oracle.svm.core.option.RuntimeOptionKey; +import com.oracle.svm.core.option.RuntimeOptionValues; +import com.oracle.svm.core.option.SubstrateOptionKey; +import com.oracle.svm.core.util.UserError; +import com.oracle.svm.core.util.VMError; + +import jdk.graal.compiler.options.Option; +import jdk.graal.compiler.options.OptionKey; +import jdk.graal.compiler.options.OptionType; +import jdk.graal.compiler.options.OptionValues; +import jdk.vm.ci.code.CodeUtil; + +/** + * Defines options that are specific to GCs such as G1. Hosted options are properly validated at + * build-time. Runtime options, for which a default value is specified at build-time using + * {@code -R:...}, undergo only basic build-time validation. Comprehensive validation of these + * runtime options is performed by the C++ code during GC initialization. + *

+ * Please note that the default values below don't necessarily match the default values that are + * specified in files such as {@code gc_globals.hpp}. Some of these values are computed during + * startup as they depend on the used GC and operating system. + *

+ * All options that are relevant for the C++ code are serialized into byte arrays (see + * {@link HostedArgumentsSupplier} and {@link RuntimeArgumentsSupplier}). During VM startup, SVM + * passes those byte arrays to the C++ code. + *

+ * Important: After VM startup, only the C++ code knows which value each runtime option has. + * Option values are not synced back to SVM, so Java code must not access any of the runtime options + * below, as doing so may return outdated or incorrect values. + */ +public class NativeGCOptions { + public static final int K = 1024; + public static final int M = 1024 * K; + + /* gc_globals.hpp */ + + @Option(help = "Number of parallel threads that the GC will use.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ParallelGCThreads = new NativeGCRuntimeOptionKey<>(0, IsolateCreationOnly); + + @Option(help = "Dynamically choose the number of threads up to a maximum of ParallelGCThreads that the GC will use for garbage collection work.", type = OptionType.Expert)// + protected static final RuntimeOptionKey UseDynamicNumberOfGCThreads = new NativeGCRuntimeOptionKey<>(true, IsolateCreationOnly); + + @Option(help = "Size of heap (bytes) per GC thread used in calculating the number of GC threads.", type = OptionType.Expert)// + protected static final RuntimeOptionKey HeapSizePerGCThread = new NativeGCRuntimeOptionKey<>(42L * M, IsolateCreationOnly); + + @Option(help = "Number of concurrent threads that the GC will use.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ConcGCThreads = new NativeGCRuntimeOptionKey<>(0, IsolateCreationOnly); + + @Option(help = "Determines if System.gc() invokes a concurrent collection.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ExplicitGCInvokesConcurrent = new NativeGCRuntimeOptionKey<>(false, IsolateCreationOnly); + + @Option(help = "Wasted fraction of parallel allocation buffer.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ParallelGCBufferWastePct = new NativeGCRuntimeOptionKey<>(10, IsolateCreationOnly); + + @Option(help = "Target wasted space in last buffer as percent of overall allocation.", type = OptionType.Expert)// + protected static final RuntimeOptionKey TargetPLABWastePct = new NativeGCRuntimeOptionKey<>(10, IsolateCreationOnly); + + @Option(help = "Percentage (0-100) used to weight the current sample when computing exponentially decaying average for ResizePLAB.", type = OptionType.Expert)// + protected static final RuntimeOptionKey PLABWeight = new NativeGCRuntimeOptionKey<>(75, IsolateCreationOnly); + + @Option(help = "Dynamically resize (survivor space) promotion LAB's.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ResizePLAB = new NativeGCRuntimeOptionKey<>(true, IsolateCreationOnly); + + @Option(help = "Scan a subset of object array and push remainder, if array is bigger than this.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ParGCArrayScanChunk = new NativeGCRuntimeOptionKey<>(50, IsolateCreationOnly); + + @Option(help = "Force all freshly committed pages to be pre-touched.", type = OptionType.Expert)// + protected static final RuntimeOptionKey AlwaysPreTouch = new NativeGCRuntimeOptionKey<>(false, IsolateCreationOnly); + + @Option(help = "Per-thread chunk size for parallel memory pre-touch.", type = OptionType.Expert)// + protected static final RuntimeOptionKey PreTouchParallelChunkSize = new NativeGCRuntimeOptionKey<>(4L * M, IsolateCreationOnly); + + @Option(help = "Maximum size of marking stack in bytes.", type = OptionType.Expert)// + protected static final RuntimeOptionKey MarkStackSizeMax = new NativeGCRuntimeOptionKey<>(512L * M, IsolateCreationOnly); + + @Option(help = "Size of marking stack in bytes.", type = OptionType.Expert)// + protected static final RuntimeOptionKey MarkStackSize = new NativeGCRuntimeOptionKey<>(4L * M, IsolateCreationOnly); + + @Option(help = "Enable parallel reference processing whenever possible.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ParallelRefProcEnabled = new NativeGCRuntimeOptionKey<>(false, IsolateCreationOnly); + + @Option(help = "Enable balancing of reference processing queues.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ParallelRefProcBalancingEnabled = new NativeGCRuntimeOptionKey<>(true, IsolateCreationOnly); + + @Option(help = "The percent occupancy (IHOP) of the current old generation capacity above which a concurrent mark cycle will be initiated. Its value may change over time if adaptive IHOP is enabled, otherwise " + + "the value remains constant. In the latter case a value of 0 will result as frequent as possible concurrent marking cycles. A value of 100 disables concurrent marking. Fragmentation waste in the old generation " + + "is not considered free space in this calculation.", type = OptionType.Expert)// + protected static final RuntimeOptionKey InitiatingHeapOccupancyPercent = new NativeGCRuntimeOptionKey<>(45, IsolateCreationOnly); + + protected static final RuntimeOptionKey MaxRAM = SubstrateOptions.ConcealedOptions.MaxRAM; + + @Option(help = "Maximum ergonomically set heap size (in bytes); zero means use MaxRAM * MaxRAMPercentage / 100.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ErgoHeapSizeLimit = new NativeGCRuntimeOptionKey<>(0L, IsolateCreationOnly); + + @Option(help = "Maximum percentage of real memory used for maximum heap size.", type = OptionType.Expert)// + public static final RuntimeOptionKey MaxRAMPercentage = new NativeGCRuntimeOptionKey<>(25.0, IsolateCreationOnly); + + @Option(help = "Minimum percentage of real memory used for maximum heap size on systems with small physical memory size.", type = OptionType.Expert)// + protected static final RuntimeOptionKey MinRAMPercentage = new NativeGCRuntimeOptionKey<>(50.0, IsolateCreationOnly); + + @Option(help = "Percentage of real memory used for initial heap size.", type = OptionType.Expert)// + protected static final RuntimeOptionKey InitialRAMPercentage = new NativeGCRuntimeOptionKey<>(0.2, IsolateCreationOnly); + + protected static final RuntimeOptionKey ActiveProcessorCount = SubstrateOptions.ActiveProcessorCount; + + @Option(help = "Adaptive size policy maximum GC pause time goal in millisecond, or the maximum GC time per MMU time slice.", type = OptionType.Expert)// + protected static final RuntimeOptionKey MaxGCPauseMillis = new NativeGCRuntimeOptionKey<>(200L, IsolateCreationOnly); + + @Option(help = "Time slice for MMU specification.", type = OptionType.Expert)// + protected static final RuntimeOptionKey GCPauseIntervalMillis = new NativeGCRuntimeOptionKey<>(201L, IsolateCreationOnly); + + @Option(help = "Adaptive size policy application time to GC time ratio.", type = OptionType.Expert)// + protected static final RuntimeOptionKey GCTimeRatio = new NativeGCRuntimeOptionKey<>(12, IsolateCreationOnly); + + @Option(help = "How far ahead to prefetch destination area (<= 0 means off).", type = OptionType.Expert)// + protected static final RuntimeOptionKey PrefetchCopyIntervalInBytes = new NativeGCRuntimeOptionKey<>(-1L, IsolateCreationOnly); + + @Option(help = "How far ahead to prefetch scan area (<= 0 means off).", type = OptionType.Expert)// + protected static final RuntimeOptionKey PrefetchScanIntervalInBytes = new NativeGCRuntimeOptionKey<>(-1L, IsolateCreationOnly); + + @Option(help = "Verify memory system before GC.", type = OptionType.Debug)// + protected static final RuntimeOptionKey VerifyBeforeGC = new NativeGCRuntimeOptionKey<>(false, IsolateCreationOnly); + + @Option(help = "Verify memory system after GC.", type = OptionType.Debug)// + protected static final RuntimeOptionKey VerifyAfterGC = new NativeGCRuntimeOptionKey<>(false, IsolateCreationOnly); + + @Option(help = "Verify memory system during GC (between phases).", type = OptionType.Debug)// + protected static final RuntimeOptionKey VerifyDuringGC = new NativeGCRuntimeOptionKey<>(false, IsolateCreationOnly); + + @Option(help = "Initial heap size (in bytes); zero means use ergonomics.", type = OptionType.Expert)// + protected static final RuntimeOptionKey InitialHeapSize = new NativeGCRuntimeOptionKey<>(0L, IsolateCreationOnly); + + @Option(help = "Initial new generation size (in bytes).", type = OptionType.Expert)// + protected static final RuntimeOptionKey NewSize = new NativeGCRuntimeOptionKey<>(1L * M, IsolateCreationOnly); + + @Option(help = "Ratio of eden/survivor space size.", type = OptionType.Expert)// + protected static final RuntimeOptionKey SurvivorRatio = new NativeGCRuntimeOptionKey<>(8L, IsolateCreationOnly); + + @Option(help = "Ratio of old/new generation sizes.", type = OptionType.Expert)// + protected static final RuntimeOptionKey NewRatio = new NativeGCRuntimeOptionKey<>(2L, IsolateCreationOnly); + + @Option(help = "Number of times an allocation that queues behind a GC will retry before printing a warning.", type = OptionType.Expert)// + protected static final RuntimeOptionKey QueuedAllocationWarningCount = new NativeGCRuntimeOptionKey<>(0L, IsolateCreationOnly); + + @Option(help = "GC invoke count where +VerifyHeap kicks in.", type = OptionType.Debug)// + protected static final RuntimeOptionKey VerifyGCStartAt = new NativeGCRuntimeOptionKey<>(0L, IsolateCreationOnly); + + @Option(help = "Maximum value for tenuring threshold.", type = OptionType.Expert)// + protected static final RuntimeOptionKey MaxTenuringThreshold = new NativeGCRuntimeOptionKey<>(15, IsolateCreationOnly); + + @Option(help = "Desired percentage of survivor space used after scavenge.", type = OptionType.Expert)// + protected static final RuntimeOptionKey TargetSurvivorRatio = new NativeGCRuntimeOptionKey<>(50, IsolateCreationOnly); + + @Option(help = "Number of entries we will try to leave on the stack during gc.", type = OptionType.Expert)// + protected static final RuntimeOptionKey GCDrainStackTargetSize = new NativeGCRuntimeOptionKey<>(64, IsolateCreationOnly); + + @Option(help = "Card table entry size (in bytes).", type = OptionType.Expert)// + public static final HostedOptionKey GCCardSizeInBytes = new NativeGCHostedOptionKey<>(512, NativeGCOptions::validatePowerOfTwo); + + /* tlab_globals.hpp */ + @Option(help = "Zero out the newly created TLAB.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ZeroTLAB = new NativeGCRuntimeOptionKey<>(false, IsolateCreationOnly); + + @Option(help = "Size of young gen promotion LAB's (in HeapWords).", type = OptionType.Expert)// + protected static final RuntimeOptionKey YoungPLABSize = new NativeGCRuntimeOptionKey<>(4096L, IsolateCreationOnly); + + @Option(help = "Size of old gen promotion LAB's (in HeapWords).", type = OptionType.Expert)// + protected static final RuntimeOptionKey OldPLABSize = new NativeGCRuntimeOptionKey<>(1024L, IsolateCreationOnly); + + @Option(help = "Allocation averaging weight.", type = OptionType.Expert)// + protected static final RuntimeOptionKey TLABAllocationWeight = new NativeGCRuntimeOptionKey<>(35L, IsolateCreationOnly); + + @Option(help = "Percentage of Eden that can be wasted (half-full TLABs at GC).", type = OptionType.Expert)// + protected static final RuntimeOptionKey TLABWasteTargetPercent = new NativeGCRuntimeOptionKey<>(1L, IsolateCreationOnly); + + @Option(help = "Maximum TLAB waste at a refill (internal fragmentation).", type = OptionType.Expert)// + protected static final RuntimeOptionKey TLABRefillWasteFraction = new NativeGCRuntimeOptionKey<>(64L, IsolateCreationOnly); + + @Option(help = "Increment allowed waste at slow allocation.", type = OptionType.Expert)// + protected static final RuntimeOptionKey TLABWasteIncrement = new NativeGCRuntimeOptionKey<>(4L, IsolateCreationOnly); + + /* globals.hpp */ + + @Option(help = "The minimum percentage of heap free after GC to avoid expansion.", type = OptionType.Expert)// + protected static final RuntimeOptionKey MinHeapFreeRatio = new NativeGCRuntimeOptionKey<>(40L, IsolateCreationOnly); + + @Option(help = "Number of milliseconds per MB of free space in the heap.", type = OptionType.Expert)// + protected static final RuntimeOptionKey SoftRefLRUPolicyMSPerMB = new NativeGCRuntimeOptionKey<>(1000L, IsolateCreationOnly); + + @Option(help = "The minimum change in heap space due to GC (in bytes).", type = OptionType.Expert)// + protected static final RuntimeOptionKey MinHeapDeltaBytes = new NativeGCRuntimeOptionKey<>(168L * K, IsolateCreationOnly); + + /* This runtime option depends on a hosted option (see special handling in svmToGC.cpp). */ + protected static final RuntimeOptionKey UsePerfData = SubstrateOptions.ConcealedOptions.UsePerfData; + + /* globals_linux.hpp */ + protected static final HostedOptionKey UseContainerSupport = SubstrateOptions.UseContainerSupport; + + @Platforms(Platform.HOSTED_ONLY.class) + public static ArrayList getOptionFields(Class[] optionClasses) { + ArrayList result = new ArrayList<>(); + for (Class clazz : optionClasses) { + for (Field field : clazz.getDeclaredFields()) { + if (Modifier.isStatic(field.getModifiers()) && OptionKey.class.isAssignableFrom(field.getType())) { + field.setAccessible(true); + result.add(field); + } + } + } + return result; + } + + private static void validatePowerOfTwo(HostedOptionKey optionKey) { + int value = optionKey.getValue(); + if (!CodeUtil.isPowerOf2(value)) { + throw UserError.invalidOptionValue(optionKey, value, "The value must be a power of two"); + } + } + + private static void validatePlatformAndGC(SubstrateOptionKey optionKey) { + if (!optionKey.hasBeenSet()) { + return; + } + + if (!Platform.includedIn(Platform.LINUX_AMD64.class) && !Platform.includedIn(Platform.LINUX_AARCH64.class)) { + throw UserError.abort("The option '%s' can only be used on linux/amd64 or linux/aarch64.", optionKey.getName()); + } else if (!SubstrateOptions.useG1GC() && !SubstrateOptions.useShenandoahGC()) { + throw UserError.abort("The option '%s' can only be used with the G1 ('--gc=G1') or the Shenandoah ('--gc=shenandoah') garbage collector.", optionKey.getName()); + } + } + + public static class NativeGCHostedOptionKey extends HostedOptionKey { + private final boolean passToCpp; + + public NativeGCHostedOptionKey(T defaultValue, Consumer> validation) { + this(defaultValue, true, validation); + } + + public NativeGCHostedOptionKey(T defaultValue, boolean passToCpp, Consumer> validation) { + super(defaultValue, validation); + this.passToCpp = passToCpp; + } + + public boolean shouldPassToCpp() { + return passToCpp; + } + + @Override + public void validate() { + validatePlatformAndGC(this); + super.validate(); + } + } + + public static class NativeGCRuntimeOptionKey extends RuntimeOptionKey { + public NativeGCRuntimeOptionKey(T defaultValue, RuntimeOptionKeyFlag... flags) { + super(defaultValue, flags); + } + + @Override + public void validate() { + validatePlatformAndGC(this); + super.validate(); + } + } + + /** Serializes GC-relevant hosted options and their build-time values into a byte array. */ + @Platforms(Platform.HOSTED_ONLY.class) + public static final class HostedArgumentsSupplier implements Supplier { + private final ArrayList optionFields; + + public HostedArgumentsSupplier(ArrayList optionFields) { + this.optionFields = optionFields; + } + + @Override + public byte[] get() { + NativeGCArgumentsBuffer buffer = new NativeGCArgumentsBuffer(); + UnmodifiableEconomicMap, Object> map = HostedOptionValues.singleton().getMap(); + for (Field field : optionFields) { + try { + Class type = field.getType(); + if (HostedOptionKey.class.isAssignableFrom(type)) { + HostedOptionKey key = (HostedOptionKey) field.get(null); + if (shouldPassToCpp(key)) { + buffer.putString(key.getName()); + buffer.putPrimitive(key.getValueOrDefault(map)); + } + } + } catch (IllegalArgumentException | IllegalAccessException e) { + throw VMError.shouldNotReachHere(e); + } + } + buffer.putEnd(); + return buffer.toArray(); + } + + private static boolean shouldPassToCpp(HostedOptionKey key) { + if (key instanceof NativeGCHostedOptionKey) { + return ((NativeGCHostedOptionKey) key).shouldPassToCpp(); + } + return true; + } + } + + /** Serializes GC-relevant runtime options and their build-time values into a byte array. */ + @Platforms(Platform.HOSTED_ONLY.class) + public static final class RuntimeArgumentsSupplier implements Supplier { + private final ArrayList optionFields; + + public RuntimeArgumentsSupplier(ArrayList optionFields) { + this.optionFields = optionFields; + } + + @Override + public byte[] get() { + NativeGCArgumentsBuffer buffer = new NativeGCArgumentsBuffer(); + OptionValues optionValues = RuntimeOptionValues.singleton(); + for (Field field : optionFields) { + try { + Class type = field.getType(); + if (RuntimeOptionKey.class.isAssignableFrom(type)) { + RuntimeOptionKey key = (RuntimeOptionKey) field.get(null); + if (key.hasBeenSet(optionValues)) { + buffer.putString(key.getName()); + buffer.putPrimitive(key.getValue(optionValues)); + } + } + } catch (IllegalArgumentException | IllegalAccessException e) { + throw VMError.shouldNotReachHere(e); + } + } + buffer.putEnd(); + return buffer.toArray(); + } + } + + /** + * Writes Strings as ISO_8859_1 with zero termination. Primitive values are encoded as 64-bit + * values (regardless of their actual size and type). + */ + @Platforms(Platform.HOSTED_ONLY.class) + private static class NativeGCArgumentsBuffer { + private ByteBuffer buffer; + + NativeGCArgumentsBuffer() { + buffer = allocateBuffer(128); + } + + public void putString(String value) { + assert value != null && !value.isEmpty() : value; + byte[] latin1String = value.getBytes(StandardCharsets.ISO_8859_1); + ensureCapacity(latin1String.length + 1); + + buffer.put(latin1String); + buffer.put((byte) 0); + } + + public void putPrimitive(Object value) { + ensureCapacity(8); + + long rawLong = switch (value) { + case Boolean _ -> ((boolean) value) ? 1L : 0L; + case Byte _ -> ((byte) value) & 0xFFL; + case Character _ -> ((char) value) & 0xFFFFL; + case Short _ -> ((short) value) & 0xFFFFL; + case Integer _ -> ((int) value) & 0xFFFFFFFFL; + case Long _ -> (long) value; + case Double _ -> Double.doubleToLongBits((double) value); + default -> throw VMError.shouldNotReachHere("Unexpected type: " + value.getClass()); + }; + + buffer.putLong(rawLong); + } + + public void putEnd() { + ensureCapacity(1); + buffer.put((byte) 0); + } + + public void ensureCapacity(int requested) { + int needed = buffer.position() + requested; + if (buffer.capacity() < needed) { + int newCapacity = buffer.capacity(); + do { + newCapacity *= 2; + } while (newCapacity < needed); + + ByteBuffer newBuffer = allocateBuffer(newCapacity); + newBuffer.put(buffer.array(), 0, buffer.position()); + buffer = newBuffer; + } + } + + public byte[] toArray() { + return Arrays.copyOf(buffer.array(), buffer.position()); + } + + private static ByteBuffer allocateBuffer(int size) { + return ByteBuffer.allocate(size).order(ByteOrder.nativeOrder()); + } + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCStackWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCStackWalker.java new file mode 100644 index 000000000000..aab39e78e427 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCStackWalker.java @@ -0,0 +1,484 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +import java.util.function.BooleanSupplier; + +import org.graalvm.nativeimage.CurrentIsolate; +import org.graalvm.nativeimage.Isolate; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.StackValue; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.function.CEntryPoint.Publish; +import org.graalvm.nativeimage.c.function.CEntryPointLiteral; +import org.graalvm.nativeimage.c.function.CFunctionPointer; +import org.graalvm.nativeimage.c.function.CodePointer; +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; + +import com.oracle.svm.core.NeverInline; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.c.NonmovableArray; +import com.oracle.svm.core.c.NonmovableArrays; +import com.oracle.svm.core.c.function.CEntryPointOptions; +import com.oracle.svm.core.code.CodeInfo; +import com.oracle.svm.core.code.CodeInfoAccess; +import com.oracle.svm.core.deopt.DeoptimizedFrame; +import com.oracle.svm.core.gc.shared.NativeGCStructs.CodeInfos; +import com.oracle.svm.core.gc.shared.NativeGCStructs.CodeInfosPerThread; +import com.oracle.svm.core.gc.shared.NativeGCStructs.StackFrame; +import com.oracle.svm.core.gc.shared.NativeGCStructs.StackFrames; +import com.oracle.svm.core.gc.shared.NativeGCStructs.StackFramesPerThread; +import com.oracle.svm.core.graal.RuntimeCompilation; +import com.oracle.svm.core.heap.StoredContinuation; +import com.oracle.svm.core.heap.StoredContinuationAccess; +import com.oracle.svm.core.heap.StoredContinuationAccess.ContinuationStackFrameVisitor; +import com.oracle.svm.core.heap.StoredContinuationAccess.ContinuationStackFrameVisitorData; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; +import com.oracle.svm.core.snippets.KnownIntrinsics; +import com.oracle.svm.core.stack.JavaStackWalker; +import com.oracle.svm.core.stack.ParameterizedStackFrameVisitor; +import com.oracle.svm.core.thread.ContinuationSupport; +import com.oracle.svm.core.thread.VMOperation; +import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.util.VMError; + +import jdk.graal.compiler.word.Word; + +/** + * Whenever GC-related C++ code needs information about the stack frames, it calls into Native Image + * and does a stack walk to collect that information. Relevant information is collected in a data + * structure that is allocated in native memory. Once the C++ code doesn't need this information + * anymore, it calls into Native Image to free the native memory. + */ +public final class NativeGCStackWalker { + private static final ThreadStackFrameCollector THREAD_STACK_FRAME_COLLECTOR = new ThreadStackFrameCollector(); + private static final ContinuationStackFrameCollector CONTINUATION_STACK_FRAME_COLLECTOR = new ContinuationStackFrameCollector(); + private static final CodeInfoCollector CODE_INFO_COLLECTOR = new CodeInfoCollector(); + + public final CEntryPointLiteral funcFetchThreadStackFrames; + public final CEntryPointLiteral funcFreeThreadStackFrames; + public final CEntryPointLiteral funcFetchContinuationStackFrames; + public final CEntryPointLiteral funcFreeContinuationStackFrames; + public final CEntryPointLiteral funcFetchCodeInfos; + public final CEntryPointLiteral funcFreeCodeInfos; + + @Platforms(Platform.HOSTED_ONLY.class) + public NativeGCStackWalker() { + funcFetchThreadStackFrames = CEntryPointLiteral.create(NativeGCStackWalker.class, "fetchThreadStackFrames", Isolate.class, IsolateThread.class); + funcFreeThreadStackFrames = CEntryPointLiteral.create(NativeGCStackWalker.class, "freeThreadStackFrames", Isolate.class, IsolateThread.class, StackFramesPerThread.class); + + if (ContinuationSupport.isSupported()) { + funcFetchContinuationStackFrames = CEntryPointLiteral.create(NativeGCStackWalker.class, "fetchContinuationStackFrames", Isolate.class, PointerBase.class, Pointer.class); + funcFreeContinuationStackFrames = CEntryPointLiteral.create(NativeGCStackWalker.class, "freeContinuationStackFrames", Isolate.class, PointerBase.class, StackFrames.class); + } else { + funcFetchContinuationStackFrames = null; + funcFreeContinuationStackFrames = null; + } + + if (RuntimeCompilation.isEnabled()) { + funcFetchCodeInfos = CEntryPointLiteral.create(NativeGCStackWalker.class, "fetchCodeInfos", Isolate.class, IsolateThread.class); + funcFreeCodeInfos = CEntryPointLiteral.create(NativeGCStackWalker.class, "freeCodeInfos", Isolate.class, IsolateThread.class, CodeInfosPerThread.class); + } else { + funcFetchCodeInfos = null; + funcFreeCodeInfos = null; + } + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForPossiblyUnattachedThread.class, epilogue = CEntryPointOptions.NoEpilogue.class) + public static StackFramesPerThread fetchThreadStackFrames(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") IsolateThread thread) { + return walkStack(THREAD_STACK_FRAME_COLLECTOR); + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForPossiblyUnattachedThread.class, epilogue = CEntryPointOptions.NoEpilogue.class) + public static void freeThreadStackFrames(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") IsolateThread thread, StackFramesPerThread stackFramesPerThread) { + for (int i = 0; i < stackFramesPerThread.count(); i++) { + NullableNativeMemory.free(stackFramesPerThread.threads().addressOf(i).read()); + } + NullableNativeMemory.free(stackFramesPerThread); + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + @CEntryPoint(include = UseNativeGCAndContinuations.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForUnattachedThread.class, epilogue = CEntryPointOptions.NoEpilogue.class) + public static StackFrames fetchContinuationStackFrames(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") PointerBase heapBase, Pointer storedContinuation) { + StoredContinuation s = (StoredContinuation) storedContinuation.toObject(); + ContinuationStackFrameCollectorData data = StackValue.get(ContinuationStackFrameCollectorData.class); + ContinuationStackFrameCollector.initialize(data); + StoredContinuationAccess.walkFrames(s, CONTINUATION_STACK_FRAME_COLLECTOR, data); + return data.getStack(); + } + + @Uninterruptible(reason = "May be called by an unattached thread (during or outside of a safepoint).") + @CEntryPoint(include = UseNativeGCAndContinuations.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForUnattachedThread.class, epilogue = CEntryPointOptions.NoEpilogue.class) + public static void freeContinuationStackFrames(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") PointerBase heapBase, StackFrames stackFrames) { + NullableNativeMemory.free(stackFrames); + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForPossiblyUnattachedThread.class, epilogue = CEntryPointOptions.NoEpilogue.class) + public static CodeInfosPerThread fetchCodeInfos(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") IsolateThread thread) { + return walkStack(CODE_INFO_COLLECTOR); + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForPossiblyUnattachedThread.class, epilogue = CEntryPointOptions.NoEpilogue.class) + public static void freeCodeInfos(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") IsolateThread thread, CodeInfosPerThread codeInfos) { + for (int i = 0; i < codeInfos.count(); i++) { + NullableNativeMemory.free(codeInfos.threads().addressOf(i).read()); + } + NullableNativeMemory.free(codeInfos); + } + + @NeverInline("Stack walking.") + @Uninterruptible(reason = "GC may only call uninterruptible code.") + private static T walkStack(Collector collector) { + VMOperation.guaranteeInProgressAtSafepoint("Doing a stack walk for every thread is only possible when we are at a safepoint."); + collector.startWalking(); + + /* Walk the current thread. */ + collector.newThread(); + JavaStackWalker.walkCurrentThread(KnownIntrinsics.readCallerStackPointer(), collector, null); + collector.finishThread(); + + /* Walk all other threads. */ + for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { + if (vmThread == CurrentIsolate.getCurrentThread()) { + /* The current thread is already walked by code above. */ + continue; + } + collector.newThread(); + JavaStackWalker.walkThread(vmThread, collector, null); + collector.finishThread(); + } + return collector.finishWalking(); + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + private static V malloc(UnsignedWord size) { + V result = NullableNativeMemory.malloc(size, NmtCategory.GC); + if (result.isNull()) { + throw VMError.shouldNotReachHere("malloc returned null."); + } + return result; + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + private static V realloc(V ptr, UnsignedWord size) { + V result = NullableNativeMemory.realloc(ptr, size, NmtCategory.GC); + if (result.isNull()) { + throw VMError.shouldNotReachHere("realloc returned null."); + } + return result; + } + + @RawStructure + private interface ContinuationStackFrameCollectorData extends ContinuationStackFrameVisitorData { + @RawField + int getCapacity(); + + @RawField + void setCapacity(int value); + + @RawField + StackFrames getStack(); + + @RawField + void setStack(StackFrames frames); + } + + private static class ContinuationStackFrameCollector extends ContinuationStackFrameVisitor { + private static final int DEFAULT_NUM_STACK_FRAMES = 10; + + @Platforms(value = Platform.HOSTED_ONLY.class) + ContinuationStackFrameCollector() { + } + + @Uninterruptible(reason = "GC may only call uninterruptible code and StoredContinuation must not move.", callerMustBe = true) + public static void initialize(ContinuationStackFrameCollectorData data) { + UnsignedWord size = computeStackFramesSize(DEFAULT_NUM_STACK_FRAMES); + StackFrames frames = malloc(size); + frames.setCount(0); + + data.setStack(frames); + data.setCapacity(DEFAULT_NUM_STACK_FRAMES); + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code and StoredContinuation must not move.", callerMustBe = true) + public void visitFrame(ContinuationStackFrameVisitorData data, Pointer sp, NonmovableArray referenceMapEncoding, long referenceMapIndex, ContinuationStackFrameVisitor visitor) { + ContinuationStackFrameCollectorData d = (ContinuationStackFrameCollectorData) data; + + long index = d.getStack().count(); + if (index == d.getCapacity()) { + int newCapacity = d.getCapacity() * 2; + StackFrames newStackFrames = realloc(d.getStack(), computeStackFramesSize(newCapacity)); + d.setStack(newStackFrames); + d.setCapacity(newCapacity); + } + + StackFrames stack = d.getStack(); + StackFrame frame = stack.frames().addressOf(index); + frame.setStackPointer(sp); + frame.setEncodedReferenceMap(NonmovableArrays.getArrayBase(referenceMapEncoding)); + frame.setReferenceMapIndex(referenceMapIndex); + stack.setCount(index + 1); + } + + @Uninterruptible(reason = "GC may only execute uninterruptible code.") + private static UnsignedWord computeStackFramesSize(int frames) { + return SizeOf.unsigned(StackFrames.class).add(SizeOf.unsigned(StackFrame.class).multiply(frames)); + } + } + + private abstract static class Collector extends ParameterizedStackFrameVisitor { + protected static final int INITIALLY_RESERVED_THREADS = 5; + protected static final int INITIALLY_RESERVED_STACK_FRAMES = 10; + + protected T threads; + protected int threadsLength; + + protected U currentStack; + protected int currentStackLength; + + @Platforms(value = Platform.HOSTED_ONLY.class) + Collector() { + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public abstract void startWalking(); + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public abstract void newThread(); + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public abstract void finishThread(); + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public T finishWalking() { + assert currentStack.isNull(); + assert currentStackLength == 0; + + T result = threads; + this.threads = Word.nullPointer(); + this.threadsLength = 0; + return result; + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + protected final boolean unknownFrame(Pointer sp, CodePointer ip, Object data) { + throw JavaStackWalker.fatalErrorUnknownFrameEncountered(sp, ip); + } + } + + private static class ThreadStackFrameCollector extends Collector { + @Platforms(value = Platform.HOSTED_ONLY.class) + ThreadStackFrameCollector() { + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public void startWalking() { + assert threads.isNull() && threadsLength == 0 && currentStack.isNull() && currentStackLength == 0; + + this.threadsLength = INITIALLY_RESERVED_THREADS; + this.threads = malloc(computeStackFramesPerThreadSize()); + this.threads.setCount(0); + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public void newThread() { + assert currentStack.isNull() && currentStackLength == 0; + + this.currentStackLength = INITIALLY_RESERVED_STACK_FRAMES; + this.currentStack = malloc(computeStackFramesSize()); + this.currentStack.setCount(0); + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public void finishThread() { + long index = this.threads.count(); + if (index == threadsLength) { + this.threadsLength *= 2; + this.threads = realloc(threads, computeStackFramesPerThreadSize()); + } + + threads.threads().addressOf(index).write(currentStack); + this.threads.setCount(index + 1); + + this.currentStack = Word.nullPointer(); + this.currentStackLength = 0; + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, Object data) { + NonmovableArray referenceMapEncoding = CodeInfoAccess.getStackReferenceMapEncoding(codeInfo); + long referenceMapIndex = CodeInfoAccess.lookupStackReferenceMapIndex(codeInfo, CodeInfoAccess.relativeIP(codeInfo, ip)); + append(sp, referenceMapEncoding, referenceMapIndex); + return true; + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame, Object data) { + /* Nothing to do - the DeoptimizedFrame is an object and therefore visible to the GC. */ + return true; + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + private void append(Pointer sp, NonmovableArray referenceMapEncoding, long referenceMapIndex) { + long index = currentStack.count(); + if (index == currentStackLength) { + currentStackLength *= 2; + currentStack = realloc(currentStack, computeStackFramesSize()); + } + + StackFrame frame = currentStack.frames().addressOf(index); + frame.setStackPointer(sp); + frame.setEncodedReferenceMap(NonmovableArrays.getArrayBase(referenceMapEncoding)); + frame.setReferenceMapIndex(referenceMapIndex); + currentStack.setCount(index + 1); + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + private UnsignedWord computeStackFramesPerThreadSize() { + return SizeOf.unsigned(StackFramesPerThread.class).add(SizeOf.unsigned(StackFrames.class).multiply(threadsLength)); + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + private UnsignedWord computeStackFramesSize() { + return SizeOf.unsigned(StackFrames.class).add(SizeOf.unsigned(StackFrame.class).multiply(currentStackLength)); + } + } + + private static class CodeInfoCollector extends Collector { + @Platforms(value = Platform.HOSTED_ONLY.class) + CodeInfoCollector() { + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public void startWalking() { + assert threads.isNull() && threadsLength == 0 && currentStack.isNull() && currentStackLength == 0; + + this.threadsLength = INITIALLY_RESERVED_THREADS; + this.threads = malloc(computeThreadsInfoSize()); + this.threads.setCount(0); + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public void newThread() { + assert currentStack.isNull() && currentStackLength == 0; + + this.currentStackLength = INITIALLY_RESERVED_STACK_FRAMES; + this.currentStack = malloc(computeStackInfoSize()); + this.currentStack.setCount(0); + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public void finishThread() { + long index = this.threads.count(); + if (index == threadsLength) { + this.threadsLength *= 2; + this.threads = realloc(threads, computeThreadsInfoSize()); + } + + threads.threads().addressOf(index).write(currentStack); + this.threads.setCount(index + 1); + + this.currentStack = Word.nullPointer(); + this.currentStackLength = 0; + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, Object data) { + if (!CodeInfoAccess.isAOTImageCode(codeInfo)) { + append(codeInfo); + } + return true; + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame, Object data) { + /* Nothing to do. */ + return true; + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + private void append(CodeInfo codeInfo) { + long index = this.currentStack.count(); + if (index == currentStackLength) { + currentStackLength *= 2; + currentStack = realloc(currentStack, computeStackInfoSize()); + } + + currentStack.codeInfos().addressOf(index).write(codeInfo); + currentStack.setCount(index + 1); + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + private UnsignedWord computeThreadsInfoSize() { + return SizeOf.unsigned(StackFramesPerThread.class).add(SizeOf.unsigned(StackFrames.class).multiply(threadsLength)); + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + private UnsignedWord computeStackInfoSize() { + return SizeOf.unsigned(StackFrames.class).add(SizeOf.unsigned(StackFrame.class).multiply(currentStackLength)); + } + } + + private static class UseNativeGCAndContinuations implements BooleanSupplier { + @Platforms(Platform.HOSTED_ONLY.class) + UseNativeGCAndContinuations() { + } + + @Override + public boolean getAsBoolean() { + return UseNativeGC.get() && ContinuationSupport.isSupported(); + } + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCStructs.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCStructs.java new file mode 100644 index 000000000000..781ab3f92089 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCStructs.java @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +import org.graalvm.nativeimage.c.CContext; +import org.graalvm.nativeimage.c.struct.CField; +import org.graalvm.nativeimage.c.struct.CFieldAddress; +import org.graalvm.nativeimage.c.struct.CPointerTo; +import org.graalvm.nativeimage.c.struct.CStruct; +import org.graalvm.nativeimage.c.type.CCharPointer; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; + +import com.oracle.svm.core.code.CodeInfoPointer; + +/** Data structures imported from GC-related header files. */ +@CContext(NativeGCHeaderFiles.class) +public class NativeGCStructs { + /* Data structures for frames that are currently on the stack. */ + + @CStruct(addStructKeyword = true) + public interface StackFrame extends PointerBase { + StackFrame addressOf(long index); + + @CField("stack_pointer") + void setStackPointer(Pointer value); + + @CField("encoded_reference_map") + void setEncodedReferenceMap(CCharPointer value); + + @CField("reference_map_index") + void setReferenceMapIndex(long value); + } + + @CStruct(addStructKeyword = true) + public interface StackFrames extends PointerBase { + @CField + long count(); + + @CField("count") + void setCount(long value); + + @CFieldAddress + StackFrame frames(); + } + + @CStruct(addStructKeyword = true) + public interface StackFramesPerThread extends PointerBase { + @CField + long count(); + + @CField("count") + void setCount(long value); + + @CFieldAddress + StackFramesPointer threads(); + } + + @CPointerTo(StackFrames.class) + public interface StackFramesPointer extends PointerBase { + StackFramesPointer addressOf(long index); + + void write(StackFrames value); + + StackFrames read(); + } + + /* Data structures for JIT-compiled code that is currently on the stack. */ + + @CStruct(addStructKeyword = true) + public interface CodeInfos extends PointerBase { + @CField + long count(); + + @CField("count") + void setCount(long value); + + @CFieldAddress("code_infos") + CodeInfoPointer codeInfos(); + } + + @CStruct(addStructKeyword = true) + public interface CodeInfosPerThread extends PointerBase { + @CField + long count(); + + @CField("count") + void setCount(long value); + + @CFieldAddress + CodeInfosPointer threads(); + } + + @CPointerTo(CodeInfos.class) + public interface CodeInfosPointer extends PointerBase { + CodeInfosPointer addressOf(long index); + + void write(CodeInfos value); + + CodeInfos read(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCThreadTransitions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCThreadTransitions.java new file mode 100644 index 000000000000..a0192d9050f7 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCThreadTransitions.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.function.CEntryPoint.Publish; +import org.graalvm.nativeimage.c.function.CEntryPointLiteral; +import org.graalvm.nativeimage.c.function.CFunctionPointer; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.c.InitializeReservedRegistersPrologue; +import com.oracle.svm.core.c.function.CEntryPointOptions; +import com.oracle.svm.core.c.function.CEntryPointOptions.NoEpilogue; +import com.oracle.svm.core.thread.ThreadStatusTransition; + +/** + * The methods below are called if any GC-related C++ code needs to do a thread status transitions. + * This ensures that the GC doesn't need to know how thread status transitions work in detail. + */ +public class NativeGCThreadTransitions { + public final CEntryPointLiteral funcVMToNative; + public final CEntryPointLiteral funcFastNativeToVM; + public final CEntryPointLiteral funcSlowNativeToVM; + + @Platforms(Platform.HOSTED_ONLY.class) + public NativeGCThreadTransitions() { + funcVMToNative = CEntryPointLiteral.create(NativeGCThreadTransitions.class, "fromVMToNative", IsolateThread.class); + funcFastNativeToVM = CEntryPointLiteral.create(NativeGCThreadTransitions.class, "fastFromNativeToVM", IsolateThread.class); + funcSlowNativeToVM = CEntryPointLiteral.create(NativeGCThreadTransitions.class, "slowFromNativeToVM", IsolateThread.class); + } + + @Uninterruptible(reason = "Must not do a safepoint check after the thread status transition.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersPrologue.class, epilogue = NoEpilogue.class) + public static void fromVMToNative(@SuppressWarnings("unused") IsolateThread thread) { + ThreadStatusTransition.fromVMToNative(); + } + + @Uninterruptible(reason = "Must not do a safepoint check after the thread status transition.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersPrologue.class, epilogue = NoEpilogue.class) + public static boolean fastFromNativeToVM(@SuppressWarnings("unused") IsolateThread thread) { + return ThreadStatusTransition.tryFromNativeToVM(); + } + + @Uninterruptible(reason = "Must not do a safepoint check after the thread status transition.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersPrologue.class, epilogue = NoEpilogue.class) + public static void slowFromNativeToVM(@SuppressWarnings("unused") IsolateThread thread) { + ThreadStatusTransition.fromNativeToVM(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCVMOperationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCVMOperationSupport.java new file mode 100644 index 000000000000..ecbe8a53e780 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCVMOperationSupport.java @@ -0,0 +1,342 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; +import static com.oracle.svm.core.thread.VMOperationControl.isDedicatedVMOperationThread; + +import org.graalvm.nativeimage.Isolate; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.function.CEntryPoint.Publish; +import org.graalvm.nativeimage.c.function.CEntryPointLiteral; +import org.graalvm.nativeimage.c.function.CFunctionPointer; +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.c.function.CEntryPointOptions; +import com.oracle.svm.core.c.function.CEntryPointOptions.NoEpilogue; +import com.oracle.svm.core.heap.AbstractPinnedObjectSupport; +import com.oracle.svm.core.heap.RestrictHeapAccess; +import com.oracle.svm.core.heap.RestrictHeapAccess.Access; +import com.oracle.svm.core.heap.VMOperationInfo; +import com.oracle.svm.core.heap.VMOperationInfos; +import com.oracle.svm.core.locks.VMCondition; +import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.thread.NativeVMOperation; +import com.oracle.svm.core.thread.NativeVMOperationData; +import com.oracle.svm.core.thread.RecurringCallbackSupport; +import com.oracle.svm.core.thread.ThreadStatusTransition; +import com.oracle.svm.core.thread.VMOperation; +import com.oracle.svm.core.thread.VMOperationControl; +import com.oracle.svm.core.thread.VMThreads.StatusSupport; +import com.oracle.svm.core.util.DuplicatedInNativeCode; + +/** + * GC-related VM operations can be scheduled by any isolate thread (including the VM operation + * thread), or any unattached GC-internal thread. + * + *

+ * Executing a GC-related VM operation involves a rather complex mechanism. So, avoid changes if + * possible, or be prepared to redesign the whole mechanism (changes are prone to race conditions + * and deadlocks). + * + *

+ * If an isolate thread wants to queue a VM operation while it is in C++ code (and therefore in + * {@link StatusSupport#STATUS_IN_VM}), it needs to execute the following steps: + *

    + *
  1. The queuing thread allocates all necessary data structures (e.g., + * {@link NativeGCVMOperationData}) on the C++-side in native memory.
    2. + *
  2. The queuing thread does a status transition to {@link StatusSupport#STATUS_IN_NATIVE}.
    3. + *
  3. The queuing thread calls {@link #enqueue} (via one of its callers that is annotated with + * {@link CEntryPoint}, see usages) to queue a non-blocking VM operation that wraps the GC VM + * operation (see {@link NativeGCVMOperationWrapper}). For queuing this VM operation, the thread + * needs to acquire the VM operation mutex and might get blocked. This is fine as we already did a + * transition to {@link StatusSupport#STATUS_IN_NATIVE} before.
    4. + *
  4. After queuing the VM operation, the queuing thread immediately returns to C++ code. No thread + * transition is necessary as the thread has still {@link StatusSupport#STATUS_IN_NATIVE}.
    5. + *
  5. Back in C++ code, the queuing thread waits until the VM operation thread starts executing the + * VM operation wrapper.
    6. + *
  6. The queuing thread executes the VM operation prologue which may acquire locks on the C++ + * side. Because we explicitly blocked the VM operation thread, it is guaranteed that locks (such as + * the {@code Heap_lock}) can be acquired eventually.
    7. + *
  7. The VM operation thread requests a safepoint and executes the main part of the GC-related VM + * operation while all Java threads are frozen. After the main part was executed, the safepoint is + * released.
    8. + *
  8. The queuing thread changes its thread status to {@link StatusSupport#STATUS_IN_VM} to prevent + * unexpected safepoints. This is necessary because it must be possible to transport uninitialized + * memory from the VM operation back to the queuing thread while ensuring that the GC does not see + * the uninitialized memory.
    9. + *
  9. After that, the VM operation thread may finish executing the wrapper VM operation at any + * time.
    10. + *
  10. The queuing thread executes the VM operation epilogue.
    11. + *
  11. The queuing thread waits until the execution of the VM operation wrapper finished.
    12. + *
  12. The queuing thread frees the native data structures that were allocated as it is now + * guaranteed that the VM operation thread won't access them anymore.
    13. + *
+ * + *

+ * For unattached GC threads, the mechanism is slightly simpler as the thread state transitions are + * not necessary. Besides that, pretty much the whole mechanism is the same. + * + *

+ * In Native Image, the VM operation thread is a normal Java thread. So, it can happen that for + * example a GC is needed while some non-GC-related VM operation is already in progress. Below is a + * detailed list of steps that the VM operation thread executes in that case: + *

    + *
  1. The VM operation thread is in C++ code and has {@link StatusSupport#STATUS_IN_VM} (e.g., + * currently in the allocation slowpath).
    2. + *
  2. The VM operation thread allocates all necessary data structures (e.g., + * {@link NativeGCVMOperationData}) on the C++ side in native memory.
    3. + *
  3. The VM operation thread calls one of the VM operation {@link CEntryPoint} methods (see usages + * of {@link #enqueue}).
    4. + *
  4. The VM operation thread does a transition to {@link StatusSupport#STATUS_IN_JAVA}.
    5. + *
  5. The VM operation thread executes the VM operation prologue.
    6. + *
  6. The VM operation thread initiates a safepoint if the surrounding VM operation didn't already + * start one.
    7. + *
  7. The VM operation thread executes the main part of the VM operation.
    8. + *
  8. The VM operation thread releases the safepoint if one was initiated before.
    9. + *
  9. The VM operation thread executes the VM operation epilogue.
    10. + *
  10. The VM operation thread does a transition to {@link StatusSupport#STATUS_IN_VM}.
    11. + *
  11. The VM operation thread returns to the C++ code.
    12. + *
  12. The VM operation thread frees the native data structures that were allocated before.
    13. + *
+ */ +public class NativeGCVMOperationSupport { + private static final NativeGCVMOperationWrapper VM_OPERATION_WRAPPER = new NativeGCVMOperationWrapper(); + + public final CEntryPointLiteral funcUpdateVMOperationExecutionStatus; + public final CEntryPointLiteral funcWaitForVMOperationExecutionStatus; + public final CEntryPointLiteral funcIsVMOperationFinished; + + @Platforms(Platform.HOSTED_ONLY.class) + public NativeGCVMOperationSupport() { + funcUpdateVMOperationExecutionStatus = CEntryPointLiteral.create(NativeGCVMOperationSupport.class, "updateVMOperationExecutionStatus", + Isolate.class, IsolateThread.class, NativeGCVMOperationWrapperData.class, int.class); + funcWaitForVMOperationExecutionStatus = CEntryPointLiteral.create(NativeGCVMOperationSupport.class, "waitForVMOperationExecutionStatus", + Isolate.class, IsolateThread.class, NativeGCVMOperationWrapperData.class, int.class); + funcIsVMOperationFinished = CEntryPointLiteral.create(NativeGCVMOperationSupport.class, "isVMOperationFinished", + Isolate.class, IsolateThread.class, NativeGCVMOperationWrapperData.class); + } + + @Uninterruptible(reason = "Can be called from an unattached thread.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForPossiblyUnattachedThread.class, epilogue = NoEpilogue.class) + public static void updateVMOperationExecutionStatus(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") IsolateThread isolateThread, NativeGCVMOperationWrapperData data, + int status) { + NativeGCVMOperationWrapper.updateExecutionStatus(data, status); + } + + @Uninterruptible(reason = "Can be called from an unattached thread.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForPossiblyUnattachedThread.class, epilogue = NoEpilogue.class) + public static void waitForVMOperationExecutionStatus(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") IsolateThread isolateThread, NativeGCVMOperationWrapperData data, + int minStatus) { + NativeGCVMOperationWrapper.waitForStatusUpdate(data, minStatus); + } + + @Uninterruptible(reason = "Can be called from an unattached thread.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForPossiblyUnattachedThread.class, epilogue = NoEpilogue.class) + public static boolean isVMOperationFinished(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") IsolateThread isolateThread, NativeGCVMOperationWrapperData data) { + return data.getFinished(); + } + + @Uninterruptible(reason = "Can be called from an unattached thread.") + public static void enqueue(NativeGCVMOperation op, NativeGCVMOperationData data, NativeGCVMOperationWrapperData wrapperData) { + data.setNativeVMOperation(op); + if (VMOperationControl.isDedicatedVMOperationThread()) { + /* The VM operation thread needs to execute the VM operation directly. */ + ThreadStatusTransition.fromVMToJava(false); + executeDirectly(op, data); + ThreadStatusTransition.fromJavaToVM(); + } else { + wrapperData.setNativeVMOperation(VM_OPERATION_WRAPPER); + wrapperData.setVMOperationData(data); + VM_OPERATION_WRAPPER.enqueueFromNonJavaThread(wrapperData); + } + } + + @Uninterruptible(reason = "Bridge between uninterruptible and interruptible code.", calleeMustBe = false) + private static void executeDirectly(NativeGCVMOperation op, NativeGCVMOperationData data) { + assert StatusSupport.isStatusJava() : "Thread must be in Java state to execute interruptible code"; + assert VMOperationControl.isDedicatedVMOperationThread(); + + /* Execute the prologue in the non-blocking part of the VM operation thread. */ + if (!op.executePrologue(data)) { + return; + } + + /* Queue the blocking VM operation. */ + assert op.getCausesSafepoint(); + op.enqueue(data); + + /* Execute the epilogue in the non-blocking part. */ + op.executeEpilogue(data); + } + + /** + * A non-blocking VM operation that wraps a blocking GC-related VM operation and deals with the + * necessary synchronization between SVM and C++. + * + * This has to be a {@link NativeVMOperation} as every queued instance needs a separate state + * (see {@link NativeGCVMOperationWrapperData}). + */ + @DuplicatedInNativeCode + private static class NativeGCVMOperationWrapper extends NativeVMOperation { + private static final int BLOCK_VM_THREAD = 0; + private static final int EXECUTE_PROLOGUE = BLOCK_VM_THREAD + 1; + private static final int EXECUTE_VM_OPERATION = EXECUTE_PROLOGUE + 1; + private static final int CANCELLED = EXECUTE_VM_OPERATION + 1; + private static final int ADJUST_THREAD_STATUS = CANCELLED + 1; + private static final int FINISHED = ADJUST_THREAD_STATUS + 1; + + private static final VMMutex EXECUTION_STATUS_MUTEX = new VMMutex("vmOpExecutionStatus"); + private static final VMCondition EXECUTION_STATUS_CONDITION = new VMCondition(EXECUTION_STATUS_MUTEX); + + protected NativeGCVMOperationWrapper() { + super(VMOperationInfos.get(NativeGCVMOperationWrapper.class, "Native GC VM operation wrapper", SystemEffect.NONE)); + } + + @Override + @RestrictHeapAccess(access = Access.NO_ALLOCATION, reason = "Used to implement GC functionality.") + protected void operate(NativeVMOperationData data) { + assert RecurringCallbackSupport.isCallbackUnsupportedOrTimerSuspended(); + operate0((NativeGCVMOperationWrapperData) data); + } + + private static void operate0(NativeGCVMOperationWrapperData wrapperData) { + assert wrapperData.getExecutionStatus() == BLOCK_VM_THREAD; + assert !isDedicatedVMOperationThread(wrapperData.getQueuingThread()); + assert !VMOperation.isInProgressAtSafepoint() : "GC-related VM operation that need this kind of synchronization must start their own safepoint"; + + /* Notify C++ code that the VM operation thread is in place. */ + updateExecutionStatus(wrapperData, EXECUTE_PROLOGUE); + + /* Wait until C++ code executed the prologue. */ + waitForStatusUpdate(wrapperData, EXECUTE_VM_OPERATION); + + if (wrapperData.getExecutionStatus() != CANCELLED) { + assert wrapperData.getExecutionStatus() == EXECUTE_VM_OPERATION; + + NativeGCVMOperationData opData = wrapperData.getVMOperationData(); + NativeGCVMOperation op = (NativeGCVMOperation) opData.getNativeVMOperation(); + + /* Execute the blocking part of the VM operation. */ + assert op.getCausesSafepoint(); + op.enqueue(opData); + + /* Notify C++ code that the VM operation was executed. */ + updateExecutionStatus(wrapperData, ADJUST_THREAD_STATUS); + + /* Wait until the C++ code changed the thread status to VM. */ + waitForStatusUpdate(wrapperData, FINISHED); + } + } + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.") + static void updateExecutionStatus(NativeGCVMOperationWrapperData data, int status) { + EXECUTION_STATUS_MUTEX.lockNoTransitionUnspecifiedOwner(); + try { + data.setExecutionStatus(status); + EXECUTION_STATUS_CONDITION.broadcast(); + } finally { + EXECUTION_STATUS_MUTEX.unlockNoTransitionUnspecifiedOwner(); + } + } + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.") + static void waitForStatusUpdate(NativeGCVMOperationWrapperData data, int minStatus) { + EXECUTION_STATUS_MUTEX.lockNoTransitionUnspecifiedOwner(); + try { + while (data.getExecutionStatus() < minStatus) { + EXECUTION_STATUS_CONDITION.blockNoTransitionUnspecifiedOwner(); + } + } finally { + EXECUTION_STATUS_MUTEX.unlockNoTransitionUnspecifiedOwner(); + } + } + } + + /** + * Stack allocated on the C++ side but only accessed by Native Image. We can't stack allocate + * the structure on the Native Image side because the stack is destroyed when we return to C++ + * after queuing the VM operation. + */ + @RawStructure + public interface NativeGCVMOperationWrapperData extends NativeVMOperationData { + @RawField + int getExecutionStatus(); + + @RawField + void setExecutionStatus(int value); + + @RawField + NativeGCVMOperationData getVMOperationData(); + + @RawField + void setVMOperationData(NativeGCVMOperationData value); + } + + public abstract static class NativeGCVMOperation extends NativeVMOperation { + private final boolean isGC; + + @Platforms(Platform.HOSTED_ONLY.class) + protected NativeGCVMOperation(VMOperationInfo info, boolean isGC) { + super(info); + this.isGC = isGC; + } + + @Override + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public boolean isGC() { + return isGC; + } + + public abstract boolean executePrologue(NativeGCVMOperationData data); + + @Override + protected void operate(NativeVMOperationData data) { + /* Cleanup obsolete pinned objects before calling into C++. */ + AbstractPinnedObjectSupport.singleton().removeClosedObjectsAndGetFirstOpenObject(); + operate0((NativeGCVMOperationData) data); + } + + protected abstract void operate0(NativeGCVMOperationData data); + + public abstract void executeEpilogue(NativeGCVMOperationData data); + } + + /** + * {@link NativeGCVMOperationData} structs are always allocated on the C++ side. Note that the + * C++ code uses pointer arithmetics and cast to convert this into a C++ {@code VM_Operation}. + */ + @RawStructure + public interface NativeGCVMOperationData extends NativeVMOperationData { + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/UseNativeGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/UseNativeGC.java new file mode 100644 index 000000000000..6945f146ae13 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/UseNativeGC.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +import java.util.function.BooleanSupplier; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + +import com.oracle.svm.core.SubstrateOptions; + +@Platforms(Platform.HOSTED_ONLY.class) +public class UseNativeGC implements BooleanSupplier { + @Override + public boolean getAsBoolean() { + return get(); + } + + public static boolean get() { + return SubstrateOptions.useG1GC() || SubstrateOptions.useShenandoahGC(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/graal/NativeGCAllocationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/graal/NativeGCAllocationSupport.java new file mode 100644 index 000000000000..f932c1601a8e --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/graal/NativeGCAllocationSupport.java @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared.graal; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; +import static jdk.graal.compiler.core.common.spi.ForeignCallDescriptor.CallSideEffect.NO_SIDE_EFFECT; +import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.VERY_SLOW_PATH_PROBABILITY; +import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.probability; + +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.word.UnsignedWord; + +import com.oracle.svm.core.SubstrateGCOptions; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider; +import com.oracle.svm.core.graal.snippets.GCAllocationSupport; +import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.heap.NoAllocationVerifier; +import com.oracle.svm.core.heap.OutOfMemoryUtil; +import com.oracle.svm.core.heap.Pod; +import com.oracle.svm.core.hub.DynamicHub; +import com.oracle.svm.core.snippets.SnippetRuntime; +import com.oracle.svm.core.snippets.SnippetRuntime.SubstrateForeignCallDescriptor; +import com.oracle.svm.core.snippets.SubstrateForeignCallTarget; +import com.oracle.svm.core.stack.StackOverflowCheck; +import com.oracle.svm.core.thread.ContinuationSupport; +import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; +import com.oracle.svm.core.threadlocal.FastThreadLocalObject; + +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.core.common.spi.ForeignCallDescriptor; +import jdk.graal.compiler.word.Word; + +/** + * This class contains the {@link SubstrateForeignCallTarget}s for the allocation slow path. These + * methods are {@link Uninterruptible} to ensure that newly allocated objects are either placed in + * the young generation or that the covered cards are marked as dirty (this dirtying is done by the + * GC at a later point in time, see ReduceInitialCardMarks and DeferInitialCardMark in HotSpot). + * This allows the compiler to safely eliminate GC write barriers for initializing writes to the + * last allocated object, which reduces code size. + */ +public abstract class NativeGCAllocationSupport implements GCAllocationSupport { + private static final SubstrateForeignCallDescriptor SLOW_NEW_INSTANCE = SnippetRuntime.findForeignCall(NativeGCAllocationSupport.class, "slowPathNewInstance", NO_SIDE_EFFECT); + private static final SubstrateForeignCallDescriptor SLOW_NEW_ARRAY = SnippetRuntime.findForeignCall(NativeGCAllocationSupport.class, "slowPathNewArray", NO_SIDE_EFFECT); + private static final SubstrateForeignCallDescriptor SLOW_NEW_STORED_CONTINUATION = SnippetRuntime.findForeignCall(NativeGCAllocationSupport.class, "slowPathNewStoredContinuation", NO_SIDE_EFFECT); + private static final SubstrateForeignCallDescriptor SLOW_NEW_POD_INSTANCE = SnippetRuntime.findForeignCall(NativeGCAllocationSupport.class, "slowPathNewPodInstance", NO_SIDE_EFFECT); + private static final SubstrateForeignCallDescriptor[] UNCONDITIONAL_FOREIGN_CALLS = new SubstrateForeignCallDescriptor[]{SLOW_NEW_INSTANCE, SLOW_NEW_ARRAY}; + + /* The following thread locals may only be accessed in uninterruptible code. */ + public static final FastThreadLocalObject podReferenceMapTL = FastThreadLocalFactory.createObject(Object.class, "podReferenceMap"); + + @Fold + public static NativeGCAllocationSupport singleton() { + return ImageSingletons.lookup(NativeGCAllocationSupport.class); + } + + public static void registerForeignCalls(SubstrateForeignCallsProvider foreignCalls) { + foreignCalls.register(UNCONDITIONAL_FOREIGN_CALLS); + if (ContinuationSupport.isSupported()) { + foreignCalls.register(SLOW_NEW_STORED_CONTINUATION); + } + if (Pod.RuntimeSupport.isPresent()) { + foreignCalls.register(SLOW_NEW_POD_INSTANCE); + } + } + + @Override + public ForeignCallDescriptor getNewInstanceStub() { + return SLOW_NEW_INSTANCE; + } + + @Override + public ForeignCallDescriptor getNewArrayStub() { + return SLOW_NEW_ARRAY; + } + + @Override + public ForeignCallDescriptor getNewStoredContinuationStub() { + return SLOW_NEW_STORED_CONTINUATION; + } + + @Override + public ForeignCallDescriptor getNewPodInstanceStub() { + return SLOW_NEW_POD_INSTANCE; + } + + @Override + public boolean useTLAB() { + return SubstrateGCOptions.TlabOptions.UseTLAB.getValue(); + } + + @Override + public boolean shouldAllocateInTLAB(UnsignedWord size, boolean isArray) { + /* Always try to allocate in the TLAB (humongous objects will not fit into the TLAB). */ + return true; + } + + @SubstrateForeignCallTarget(stubCallingConvention = false) + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.") + private static Object slowPathNewInstance(Word objectHeader) { + StackOverflowCheck.singleton().makeYellowZoneAvailable(); + try { + DynamicHub hub = Heap.getHeap().getObjectHeader().dynamicHubFromObjectHeader(objectHeader); + guaranteeAllocationAllowed("allocateInstance", hub); + + Object result = NativeGCAllocationSupport.singleton().allocateInstance0(hub); + return checkForOOME(result); + } finally { + StackOverflowCheck.singleton().protectYellowZone(); + } + } + + @SubstrateForeignCallTarget(stubCallingConvention = false) + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.") + private static Object slowPathNewArray(Word objectHeader, int length) { + StackOverflowCheck.singleton().makeYellowZoneAvailable(); + try { + DynamicHub hub = Heap.getHeap().getObjectHeader().dynamicHubFromObjectHeader(objectHeader); + guaranteeAllocationAllowed("allocateArray", hub); + checkArrayLength(length); + + Object result = NativeGCAllocationSupport.singleton().allocateArray0(length, hub); + return checkForOOME(result); + } finally { + StackOverflowCheck.singleton().protectYellowZone(); + } + } + + @SubstrateForeignCallTarget(stubCallingConvention = false) + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.") + private static Object slowPathNewStoredContinuation(Word objectHeader, int length) { + StackOverflowCheck.singleton().makeYellowZoneAvailable(); + try { + DynamicHub hub = Heap.getHeap().getObjectHeader().dynamicHubFromObjectHeader(objectHeader); + guaranteeAllocationAllowed("allocateStoredContinuation", hub); + checkArrayLength(length); + + Object result = NativeGCAllocationSupport.singleton().allocateStoredContinuation0(length, hub); + return checkForOOME(result); + } finally { + StackOverflowCheck.singleton().protectYellowZone(); + } + } + + @SubstrateForeignCallTarget(stubCallingConvention = false) + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.") + private static Object slowPathNewPodInstance(Word objectHeader, int length, byte[] referenceMap) { + StackOverflowCheck.singleton().makeYellowZoneAvailable(); + try { + DynamicHub hub = Heap.getHeap().getObjectHeader().dynamicHubFromObjectHeader(objectHeader); + guaranteeAllocationAllowed("allocatePod", hub); + checkArrayLength(length); + + /* The reference map object can live in the Java heap. */ + podReferenceMapTL.set(referenceMap); + Object result = NativeGCAllocationSupport.singleton().allocatePod0(length, hub); + podReferenceMapTL.set(null); + return checkForOOME(result); + } finally { + StackOverflowCheck.singleton().protectYellowZone(); + } + } + + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.", callerMustBe = true, calleeMustBe = false) + protected abstract Object allocateInstance0(DynamicHub hub); + + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.", callerMustBe = true, calleeMustBe = false) + protected abstract Object allocateArray0(int length, DynamicHub hub); + + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.", callerMustBe = true, calleeMustBe = false) + protected abstract Object allocateStoredContinuation0(int length, DynamicHub hub); + + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.", callerMustBe = true, calleeMustBe = false) + protected abstract Object allocatePod0(int length, DynamicHub hub); + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static void checkArrayLength(int length) { + if (probability(VERY_SLOW_PATH_PROBABILITY, length < 0)) { + throwNegativeArraySizeExceptionInterruptibly(); + } + } + + @Uninterruptible(reason = "No need to be uninterruptible because no object was allocated.", calleeMustBe = false) + private static void throwNegativeArraySizeExceptionInterruptibly() { + throwNegativeArraySizeException(); + } + + private static void throwNegativeArraySizeException() { + throw new NegativeArraySizeException(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static void guaranteeAllocationAllowed(String callSite, DynamicHub hub) { + if (probability(VERY_SLOW_PATH_PROBABILITY, Heap.getHeap().isAllocationDisallowed())) { + throwAllocationNotAllowedInterruptibly(callSite, hub); + } + } + + @Uninterruptible(reason = "No need to be uninterruptible because it kills the process.", calleeMustBe = false) + private static void throwAllocationNotAllowedInterruptibly(String callSite, DynamicHub hub) { + throw NoAllocationVerifier.exit(callSite, DynamicHub.toClass(hub).getName()); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static Object checkForOOME(Object result) { + if (probability(VERY_SLOW_PATH_PROBABILITY, result == null)) { + throwHeapSizeExceededInterruptibly(); + } + return result; + } + + @Uninterruptible(reason = "No need to be uninterruptible because no object was allocated.", calleeMustBe = false) + private static void throwHeapSizeExceededInterruptibly() { + throw OutOfMemoryUtil.heapSizeExceeded(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/include/sharedGCStructs.h b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/include/sharedGCStructs.h new file mode 100644 index 000000000000..f94f3564c414 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/include/sharedGCStructs.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SVM_SHARED_GC_STRUCTS_HPP +#define SVM_SHARED_GC_STRUCTS_HPP + +#include + +#ifdef __cplusplus +namespace svm_gc { +#endif + +// forward declarations +typedef struct CodeInfo CodeInfo; + +// data structures for frames that are currently on the stack +struct StackFrame { + u_char *stack_pointer; + u_char *encoded_reference_map; + size_t reference_map_index; +}; + +struct StackFrames { + size_t count; + struct StackFrame frames[0]; // variable-sized array +}; + +struct StackFramesPerThread { + size_t count; + struct StackFrames *threads[0]; // variable-sized array +}; + +// data structures for JIT-compiled code that is currently on the stack +struct CodeInfos { + size_t count; + struct CodeInfo *code_infos[0]; // variable-sized array +}; + +struct CodeInfosPerThread { + size_t count; + struct CodeInfos *threads[0]; // variable-sized array +}; + +#ifdef __cplusplus +} // namespace svm_gc +#endif + +#endif // SVM_SHARED_GC_STRUCTS_HPP diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahCommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahCommittedMemoryProvider.java new file mode 100644 index 000000000000..772f247d138f --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahCommittedMemoryProvider.java @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; +import static com.oracle.svm.core.gc.shenandoah.ShenandoahOptions.ShenandoahRegionSize; + +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.StackValue; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.nativeimage.c.type.CCharPointerPointer; +import org.graalvm.nativeimage.c.type.WordPointer; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; + +import com.oracle.svm.core.IsolateArguments; +import com.oracle.svm.core.NeverInline; +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.UnmanagedMemoryUtil; +import com.oracle.svm.core.c.function.CEntryPointErrors; +import com.oracle.svm.core.container.Container; +import com.oracle.svm.core.container.ContainerLibrary; +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahLibrary; +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahStructs.ShenandoahHeapOptions; +import com.oracle.svm.core.graal.snippets.CEntryPointSnippets; +import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.heap.ReferenceAccess; +import com.oracle.svm.core.os.AbstractCommittedMemoryProvider; +import com.oracle.svm.core.os.AbstractImageHeapProvider; +import com.oracle.svm.core.os.CommittedMemoryProvider; +import com.oracle.svm.core.os.ImageHeapProvider; +import com.oracle.svm.core.os.VirtualMemoryProvider; +import com.oracle.svm.core.snippets.KnownIntrinsics; +import com.oracle.svm.core.util.UnsignedUtils; + +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.core.common.CompressEncoding; +import jdk.graal.compiler.word.Word; + +/** + * Reserves one contiguous block of memory in which the image heap and the collected Java heap are + * placed. The layout of this block of memory is as follows: + * + * 
+ * | null regions |  image heap   |     collected Java heap      |
+ * | (protected)  | closed | open | size determined by -Xms/-Xmx |
+ * ^
+ * heapBase
+ *
+ * + *
    + *
  • The memory right after the heap base is protected and cannot be accessed. This ensures that + * Java null values never point to valid objects.
    • + *
  • The image heap consists of closed and open regions (see {@link ShenandoahRegionType}).
    • + *
  • The size of the Java heap is determined by the min and max heap size values (-Xms, -Xmx) that + * are specified by the user. If uncompressed references are used, it is guaranteed that the image + * heap does not reduce the size of the Java heap, e.g., if the user specifies '-Xmx1g', then the + * Java heap will have a maximum size of 1g, regardless of the image heap size. However, if + * compressed references are used, the image heap and the Java heap need to coexist in the 32 GB + * address space, which can reduce the maximum size of the Java heap.
    • + *
+ */ +public class ShenandoahCommittedMemoryProvider extends AbstractCommittedMemoryProvider { + private Pointer reservedBegin; + private UnsignedWord reservedSize; + private UnsignedWord maxHeapSize; + private UnsignedWord physicalMemorySize; + + @Platforms(Platform.HOSTED_ONLY.class) + public ShenandoahCommittedMemoryProvider() { + assert SubstrateOptions.SpawnIsolates.getValue(); + } + + @Override + @Uninterruptible(reason = "Still being initialized.") + public int initialize(WordPointer heapBaseOut, IsolateArguments arguments) { + int argc = arguments.getArgc(); + CCharPointerPointer argv = arguments.getArgv(); + + UnsignedWord nullRegionSize = Word.unsigned(ShenandoahHeap.get().getImageHeapOffsetInAddressSpace()); + // The image heap size in the file may be smaller than the image heap at run-time because we + // don't fill the last image heap region completely. This reduces the file size. + UnsignedWord imageHeapSize = UnsignedUtils.roundUp(AbstractImageHeapProvider.getImageHeapSizeInFile(), Word.unsigned(getRegionSize())); + UnsignedWord heapBaseAlignment = Word.unsigned(Heap.getHeap().getHeapBaseAlignment()); + + int heapOptionStructSize = SizeOf.get(ShenandoahHeapOptions.class); + ShenandoahHeapOptions heapOptions = StackValue.get(ShenandoahHeapOptions.class); + UnmanagedMemoryUtil.fill((Pointer) heapOptions, Word.unsigned(heapOptionStructSize), (byte) 0); + + boolean isContainerized = Container.isSupported() && Container.singleton().isContainerized(); + long containerMemoryLimitInBytes = isContainerized ? ContainerLibrary.getMemoryLimitInBytes() : 0; + int containerActiveProcessorCount = isContainerized ? ContainerLibrary.getActiveProcessorCount() : 0; + + ShenandoahLibrary.parseOptions(ShenandoahLibrary.VERSION, argc, argv, ShenandoahOptions.HOSTED_ARGUMENTS.get(), ShenandoahOptions.RUNTIME_ARGUMENTS.get(), + ReferenceAccess.singleton().getMaxAddressSpaceSize(), heapBaseAlignment, nullRegionSize, imageHeapSize, + getCompressedReferenceShift(), isContainerized, containerMemoryLimitInBytes, containerActiveProcessorCount, heapOptions); + + UnsignedWord heapAddressSpaceSize = heapOptions.heapAddressSpaceSize(); + UnsignedWord newMaxHeapSize = heapOptions.maxHeapSize(); + assert heapAddressSpaceSize.belowOrEqual(ReferenceAccess.singleton().getMaxAddressSpaceSize()) : "must be"; + + if (heapAddressSpaceSize.belowThan(nullRegionSize.add(imageHeapSize))) { + return CEntryPointErrors.INSUFFICIENT_ADDRESS_SPACE; + } + + Pointer reservedMemory = reserveHeapMemory(heapAddressSpaceSize, heapBaseAlignment); + if (reservedMemory.isNull()) { + return CEntryPointErrors.RESERVE_ADDRESS_SPACE_FAILED; + } + + WordPointer imageHeapEndOut = StackValue.get(WordPointer.class); + int result = ImageHeapProvider.get().initialize(reservedMemory, heapAddressSpaceSize, heapBaseOut, imageHeapEndOut); + if (result != CEntryPointErrors.NO_ERROR) { + VirtualMemoryProvider.get().free(reservedMemory, heapAddressSpaceSize); + return result; + } + + CEntryPointSnippets.initBaseRegisters(heapBaseOut.read()); + return initialize0(reservedMemory, heapAddressSpaceSize, newMaxHeapSize, heapOptions); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE) + @NeverInline("Force loading of a new instance reference, now that the heap base is initialized.") + @SuppressWarnings("hiding") + private static int initialize0(Pointer reservedBegin, UnsignedWord reservedSize, UnsignedWord maxHeapSize, ShenandoahHeapOptions heapOptions) { + ShenandoahCommittedMemoryProvider instance = getInstance(); + instance.reservedBegin = reservedBegin; + instance.reservedSize = reservedSize; + instance.maxHeapSize = maxHeapSize; + instance.physicalMemorySize = heapOptions.physicalMemorySize(); + return CEntryPointErrors.NO_ERROR; + } + + @Uninterruptible(reason = "Still being initialized.") + private static Pointer reserveHeapMemory(UnsignedWord heapAddressSpaceSize, UnsignedWord heapBaseAlignment) { + return VirtualMemoryProvider.get().reserve(heapAddressSpaceSize, heapBaseAlignment, false); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static ShenandoahCommittedMemoryProvider getInstance() { + return (ShenandoahCommittedMemoryProvider) CommittedMemoryProvider.get(); + } + + @Override + public UnsignedWord getCollectedHeapAddressSpaceSize() { + Pointer collectedHeapStart = KnownIntrinsics.heapBase().add(getCollectedHeapOffsetInAddressSpace()); + assert collectedHeapStart.aboveOrEqual(reservedBegin); + return reservedSize.subtract(collectedHeapStart.subtract(reservedBegin)); + } + + private static UnsignedWord getCollectedHeapOffsetInAddressSpace() { + return UnsignedUtils.roundUp(ImageHeapProvider.get().getImageHeapEndOffsetInAddressSpace(), Word.unsigned(getRegionSize())); + } + + @Override + public UnsignedWord getReservedAddressSpaceSize() { + return reservedSize; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public UnsignedWord getMaxHeapSize() { + return maxHeapSize; + } + + public UnsignedWord getPhysicalMemorySize() { + return physicalMemorySize; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public int getMaxRegions() { + long result = getMaxHeapSize().unsignedDivide(getRegionSize()).rawValue(); + assert (int) result == result; + return (int) result; + } + + @Fold + static int getCompressedReferenceShift() { + return ImageSingletons.lookup(CompressEncoding.class).getShift(); + } + + @Fold + protected static int getRegionSize() { + return ShenandoahRegionSize.getValue(); + } + + @Override + @Uninterruptible(reason = "Tear-down in progress.") + public int tearDown() { + /* + * ImageHeapProvider.freeImageHeap must not be called because the ImageHeapProvider did not + * allocate any memory for the image heap. + */ + return unmapAddressSpace(KnownIntrinsics.heapBase()); + } + + @Uninterruptible(reason = "Tear-down in progress.") + private int unmapAddressSpace(Pointer heapBase) { + assert heapBase.aboveOrEqual(reservedBegin) && heapBase.belowOrEqual(reservedBegin.add(getRegionSize())); + if (VirtualMemoryProvider.get().free(reservedBegin, reservedSize) != 0) { + return CEntryPointErrors.FREE_ADDRESS_SPACE_FAILED; + } + return CEntryPointErrors.NO_ERROR; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahConstants.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahConstants.java new file mode 100644 index 000000000000..ffa4985a7fb9 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahConstants.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah; + +import static com.oracle.svm.core.gc.shenandoah.ShenandoahOptions.ShenandoahRegionSize; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + +import com.oracle.svm.core.gc.shared.NativeGCOptions; + +import jdk.graal.compiler.api.replacements.Fold; +import jdk.vm.ci.code.CodeUtil; + +/** + * Defines Shenandoah-specific constants that are used during code generation. If the value of a + * constant depends on the debug-level of the linked Shenandoah library, the constant is defined as + * an array of values (i.e., one value per debug-level). + */ +public class ShenandoahConstants { + private static final int TLAB_TOP_OFFSET = 104; + private static final int TLAB_END_OFFSET = 120; + private static final byte DIRTY_CARD_VALUE = 0; + private static final int[] JAVA_THREAD_SIZE = {280, 312, 312}; + + @Fold + public static int tlabTopOffset() { + return TLAB_TOP_OFFSET; + } + + @Fold + public static int tlabEndOffset() { + return TLAB_END_OFFSET; + } + + @Fold + public static byte dirtyCardValue() { + return DIRTY_CARD_VALUE; + } + + @Fold + public static int cardTableShift() { + return CodeUtil.log2(NativeGCOptions.GCCardSizeInBytes.getValue()); + } + + @Fold + public static int cardSize() { + return NativeGCOptions.GCCardSizeInBytes.getValue(); + } + + @Fold + public static int javaThreadSize() { + return JAVA_THREAD_SIZE[debugLevelIndex()]; + } + + @Fold + public static int logOfHeapRegionGrainBytes() { + return CodeUtil.log2(ShenandoahRegionSize.getValue()); + } + + @Platforms(Platform.HOSTED_ONLY.class) + private static int debugLevelIndex() { + return ShenandoahOptions.getDebugLevel().getIndex(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahGC.java new file mode 100644 index 000000000000..6aaca1924ce1 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahGC.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + +import com.oracle.svm.core.gc.shared.NativeGCOptions; +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahLibrary; +import com.oracle.svm.core.heap.GC; +import com.oracle.svm.core.heap.GCCause; + +public class ShenandoahGC implements GC { + @Platforms(Platform.HOSTED_ONLY.class) + public ShenandoahGC() { + } + + @Override + public void collect(GCCause cause) { + ShenandoahLibrary.collect(cause.getId()); + } + + @Override + public void collectCompletely(GCCause cause) { + ShenandoahLibrary.collect(cause.getId()); + } + + @Override + public void collectionHint(boolean fullGC) { + /* Ignore collection hints. */ + } + + @Override + public String getName() { + return "Shenandoah GC"; + } + + @Override + @Platforms(Platform.HOSTED_ONLY.class) + public String getDefaultMaxHeapSize() { + return String.format("%s%% of RAM", NativeGCOptions.MaxRAMPercentage.getValue()); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahHeap.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahHeap.java new file mode 100644 index 000000000000..5fee840bf774 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahHeap.java @@ -0,0 +1,752 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah; + +import static com.oracle.svm.core.gc.shenandoah.ShenandoahOptions.ShenandoahRegionSize; +import static com.oracle.svm.core.heap.RuntimeCodeCacheCleaner.CLASSES_ASSUMED_REACHABLE; + +import java.lang.ref.Reference; +import java.util.ArrayList; +import java.util.List; + +import org.graalvm.nativeimage.CurrentIsolate; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.c.function.CEntryPointLiteral; +import org.graalvm.nativeimage.c.function.CFunctionPointer; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.nativeimage.c.type.CCharPointer; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; + +import com.oracle.svm.core.BuildPhaseProvider.ReadyForCompilation; +import com.oracle.svm.core.StaticFieldsSupport; +import com.oracle.svm.core.SubstrateDiagnostics; +import com.oracle.svm.core.SubstrateDiagnostics.DiagnosticThunk; +import com.oracle.svm.core.SubstrateDiagnostics.DiagnosticThunkRegistry; +import com.oracle.svm.core.SubstrateDiagnostics.ErrorContext; +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.SubstrateUtil; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.UnmanagedMemoryUtil; +import com.oracle.svm.core.VMInspectionOptions; +import com.oracle.svm.core.annotate.Substitute; +import com.oracle.svm.core.annotate.TargetClass; +import com.oracle.svm.core.c.NonmovableArrays; +import com.oracle.svm.core.code.RuntimeCodeInfoMemory; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.gc.shared.NativeGCStackWalker; +import com.oracle.svm.core.gc.shared.NativeGCThreadTransitions; +import com.oracle.svm.core.gc.shared.NativeGCVMOperationSupport; +import com.oracle.svm.core.gc.shared.NativeGCVMOperationSupport.NativeGCVMOperationData; +import com.oracle.svm.core.gc.shared.NativeGCVMOperationSupport.NativeGCVMOperationWrapperData; +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahLibrary; +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahStructs.ShenandoahInitState; +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahStructs.ShenandoahInternalState; +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahStructs.ShenandoahRegionInfo; +import com.oracle.svm.core.graal.RuntimeCompilation; +import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; +import com.oracle.svm.core.heap.FillerArray; +import com.oracle.svm.core.heap.FillerObject; +import com.oracle.svm.core.heap.GC; +import com.oracle.svm.core.heap.GCCause; +import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.heap.InstanceReferenceMapEncoder; +import com.oracle.svm.core.heap.NoAllocationVerifier; +import com.oracle.svm.core.heap.ObjectHeader; +import com.oracle.svm.core.heap.ObjectVisitor; +import com.oracle.svm.core.heap.RestrictHeapAccess; +import com.oracle.svm.core.heap.RuntimeCodeInfoGCSupport; +import com.oracle.svm.core.heap.StoredContinuation; +import com.oracle.svm.core.heap.UnknownObjectField; +import com.oracle.svm.core.hub.DynamicHub; +import com.oracle.svm.core.hub.DynamicHubTypeCheckUtil; +import com.oracle.svm.core.hub.LayoutEncoding; +import com.oracle.svm.core.layeredimagesingleton.MultiLayeredImageSingleton; +import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.option.RuntimeOptionKey; +import com.oracle.svm.core.snippets.KnownIntrinsics; +import com.oracle.svm.core.thread.PlatformThreads; +import com.oracle.svm.core.thread.Safepoint; +import com.oracle.svm.core.thread.ThreadStatus; +import com.oracle.svm.core.thread.VMOperationControl; +import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.thread.VMThreads.SafepointBehavior; +import com.oracle.svm.core.threadlocal.FastThreadLocal; +import com.oracle.svm.core.threadlocal.FastThreadLocalBytes; +import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; +import com.oracle.svm.core.threadlocal.FastThreadLocalWord; +import com.oracle.svm.core.threadlocal.VMThreadLocalSupport; +import com.oracle.svm.core.util.VMError; + +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.nodes.extended.MembarNode; +import jdk.graal.compiler.replacements.ReplacementsUtil; +import jdk.graal.compiler.word.Word; +import jdk.vm.ci.meta.JavaKind; + +public final class ShenandoahHeap extends Heap { + public static final FastThreadLocalBytes javaThreadTL = FastThreadLocalFactory.createBytes(ShenandoahConstants::javaThreadSize, "ShenandoahHeap.javaThread"); + private static final FastThreadLocalWord cardTableAddressTL = FastThreadLocalFactory.createWord("ShenandoahHeap.cardTableAddress").setMaxOffset(FastThreadLocal.FIRST_CACHE_LINE); + + private final ShenandoahGC gc = new ShenandoahGC(); + private final ShenandoahImageHeapInfo imageHeapInfo = new ShenandoahImageHeapInfo(); + private final ShenandoahRuntimeCodeInfoGCSupport runtimeCodeInfoGCSupport = new ShenandoahRuntimeCodeInfoGCSupport(); + private final NativeGCStackWalker stackWalker = new NativeGCStackWalker(); + private final NativeGCThreadTransitions threadTransitions = new NativeGCThreadTransitions(); + private final NativeGCVMOperationSupport vmOperationSupport = new NativeGCVMOperationSupport(); + private final ShenandoahVMOperations vmOperations = new ShenandoahVMOperations(); + private final ShenandoahObjectHeader objectHeader = new ShenandoahObjectHeader(); + + private boolean isInitialized = false; + private List> classList; + /* The card table address is relative to the heap base and not an absolute address */ + private Word cardTableAddress; + + @UnknownObjectField(availability = ReadyForCompilation.class) private byte[] accessedFieldOffsets; + + @Platforms(Platform.HOSTED_ONLY.class) + public ShenandoahHeap() { + DiagnosticThunkRegistry.singleton().add(new DumpHeapSettingsAndGCInternalState()); + DiagnosticThunkRegistry.singleton().add(new DumpRegionInformation()); + DiagnosticThunkRegistry.singleton().add(new DumpCurrentGCThreadName()); + } + + @Fold + public static ShenandoahHeap get() { + return ImageSingletons.lookup(ShenandoahHeap.class); + } + + @Fold + public static ShenandoahImageHeapInfo getImageHeapInfo() { + return ShenandoahHeap.get().imageHeapInfo; + } + + @Uninterruptible(reason = "Called during startup.") + private void initialize(IsolateThread isolateThread) { + VMThreads.guaranteeOwnsThreadMutex("Only the first thread may initialize the heap"); + assert !isInitialized; + isInitialized = true; + + ShenandoahHeap heap = ImageSingletons.lookup(ShenandoahHeap.class); + VMThreadLocalSupport threadLocalSupport = ImageSingletons.lookup(VMThreadLocalSupport.class); + + Pointer heapBase = KnownIntrinsics.heapBase(); + assert heap.getImageHeapOffsetInAddressSpace() % ShenandoahRegionSize.getValue() == 0 : "null regions must be full regions"; + int closedImageHeapRegions = imageHeapInfo.getNumClosedRegions(); + int openImageHeapRegions = imageHeapInfo.getNumOpenRegions(); + Word imageHeapRegionTypes = Word.objectToUntrackedPointer(imageHeapInfo.getRegionTypes()); + Word imageHeapRegionFreeSpaces = Word.objectToUntrackedPointer(imageHeapInfo.getRegionFreeSpaces()); + Word dynamicHubClass = Word.objectToUntrackedPointer(DynamicHub.class); + Word fillerObjectClass = Word.objectToUntrackedPointer(FillerObject.class); + Word fillerArrayClass = Word.objectToUntrackedPointer(FillerArray.class); + Word stringClass = Word.objectToUntrackedPointer(String.class); + Word systemClass = Word.objectToUntrackedPointer(System.class); + Word staticObjectFields = Word.objectToUntrackedPointer(StaticFieldsSupport.getStaticObjectFieldsAtRuntime(MultiLayeredImageSingleton.UNKNOWN_LAYER_NUMBER)); + Word staticPrimitiveFields = Word.objectToUntrackedPointer(StaticFieldsSupport.getStaticPrimitiveFieldsAtRuntime(MultiLayeredImageSingleton.UNKNOWN_LAYER_NUMBER)); + Word vmOperationThread = Word.objectToUntrackedPointer(VMOperationControl.getDedicatedVMOperationThread()); + Word safepointMaster = Word.objectToUntrackedPointer(Safepoint.singleton()); + Word runtimeCodeInfoMemory = Word.objectToUntrackedPointer(RuntimeCodeInfoMemory.singleton()); + int referenceMapCompressedOffsetShift = InstanceReferenceMapEncoder.REFERENCE_MAP_COMPRESSED_OFFSET_SHIFT; + Word threadLocalsReferenceMap = NonmovableArrays.addressOf(threadLocalSupport.getThreadLocalsReferenceMap(), threadLocalSupport.getThreadLocalsReferenceMapIndex()); + Word classesAssumedReachableForCodeUnloading = getClassesAssumedReachableForCodeUnloading(); + boolean perfDataSupport = VMInspectionOptions.hasJvmstatSupport(); + boolean useStringInlining = false; + boolean closedTypeWorldHubLayout = SubstrateOptions.useClosedTypeWorldHubLayout(); + boolean useInterfaceHashing = SubstrateOptions.useInterfaceHashing(); + int interfaceHashingMaxId = SubstrateOptions.interfaceHashingMaxId(); + int dynamicHubHashingInterfaceMask = DynamicHubTypeCheckUtil.HASHING_INTERFACE_MASK; + int dynamicHubHashingShiftOffset = DynamicHubTypeCheckUtil.HASHING_SHIFT_OFFSET; + Word offsets = Word.objectToUntrackedPointer(accessedFieldOffsets).add(getByteArrayBaseOffset()); + int offsetsLength = accessedFieldOffsets.length; + CFunctionPointer collectForAllocationOp = getFunctionPointer(vmOperations.funcCollectForAllocation); + CFunctionPointer collectFullOp = getFunctionPointer(vmOperations.funcCollectFull); + CFunctionPointer waitForVMOperationExecutionStatus = getFunctionPointer(vmOperationSupport.funcWaitForVMOperationExecutionStatus); + CFunctionPointer updateVMOperationExecutionStatus = getFunctionPointer(vmOperationSupport.funcUpdateVMOperationExecutionStatus); + CFunctionPointer isVMOperationFinished = getFunctionPointer(vmOperationSupport.funcIsVMOperationFinished); + CFunctionPointer fetchThreadStackFrames = getFunctionPointer(stackWalker.funcFetchThreadStackFrames); + CFunctionPointer freeThreadStackFrames = getFunctionPointer(stackWalker.funcFreeThreadStackFrames); + CFunctionPointer fetchContinuationStackFrames = getFunctionPointer(stackWalker.funcFetchContinuationStackFrames); + CFunctionPointer freeContinuationStackFrames = getFunctionPointer(stackWalker.funcFreeContinuationStackFrames); + CFunctionPointer fetchCodeInfos = getFunctionPointer(stackWalker.funcFetchCodeInfos); + CFunctionPointer freeCodeInfos = getFunctionPointer(stackWalker.funcFreeCodeInfos); + CFunctionPointer cleanRuntimeCodeCache = getFunctionPointer(runtimeCodeInfoGCSupport.funcCleanCodeCache); + CFunctionPointer transitionVMToNative = getFunctionPointer(threadTransitions.funcVMToNative); + CFunctionPointer fastTransitionNativeToVM = getFunctionPointer(threadTransitions.funcFastNativeToVM); + CFunctionPointer slowTransitionNativeToVM = getFunctionPointer(threadTransitions.funcSlowNativeToVM); + + ShenandoahInitState initState = ShenandoahLibrary.create(isolateThread, heapBase, + closedImageHeapRegions, openImageHeapRegions, imageHeapRegionTypes, imageHeapRegionFreeSpaces, + dynamicHubClass, fillerObjectClass, fillerArrayClass, stringClass, systemClass, + staticObjectFields, staticPrimitiveFields, vmOperationThread, safepointMaster, runtimeCodeInfoMemory, + referenceMapCompressedOffsetShift, threadLocalsReferenceMap, + classesAssumedReachableForCodeUnloading, perfDataSupport, useStringInlining, closedTypeWorldHubLayout, + useInterfaceHashing, interfaceHashingMaxId, dynamicHubHashingInterfaceMask, dynamicHubHashingShiftOffset, + offsets, offsetsLength, + collectForAllocationOp, collectFullOp, + waitForVMOperationExecutionStatus, updateVMOperationExecutionStatus, isVMOperationFinished, + fetchThreadStackFrames, freeThreadStackFrames, + fetchContinuationStackFrames, freeContinuationStackFrames, + fetchCodeInfos, freeCodeInfos, cleanRuntimeCodeCache, + transitionVMToNative, fastTransitionNativeToVM, slowTransitionNativeToVM); + + VMError.guarantee(initState.isNonNull(), "Fatal error while initializing Shenandoah"); + validateInitState(initState); + ShenandoahHeap.get().cardTableAddress = initState.cardTableAddress(); + } + + @Uninterruptible(reason = "Called during startup.") + private static Word getClassesAssumedReachableForCodeUnloading() { + if (RuntimeCompilation.isEnabled()) { + return Word.objectToUntrackedPointer(CLASSES_ASSUMED_REACHABLE); + } + return Word.nullPointer(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static CFunctionPointer getFunctionPointer(CEntryPointLiteral f) { + if (f == null) { + return Word.nullPointer(); + } + return f.getFunctionPointer(); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void suspendAllocation() { + // Retire the TLAB so that the next allocation is forced to take the slow path. + ShenandoahLibrary.retireTlab(); + } + + @Override + public void resumeAllocation() { + // Nothing to do - the next allocation will refill the TLAB. + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Override + public boolean isAllocationDisallowed() { + return NoAllocationVerifier.isActive() || SafepointBehavior.ignoresSafepoints(); + } + + @Fold + @Override + public GC getGC() { + return gc; + } + + @Fold + @Override + public RuntimeCodeInfoGCSupport getRuntimeCodeInfoGCSupport() { + return runtimeCodeInfoGCSupport; + } + + @Override + public void walkObjects(ObjectVisitor visitor) { + walkImageHeapObjects(visitor); + walkCollectedHeapObjects(visitor); + } + + @Override + public void walkImageHeapObjects(ObjectVisitor visitor) { + if (visitor == null) { + return; + } + + byte[] regionTypes = imageHeapInfo.getRegionTypes(); + int imageHeapRegions = imageHeapInfo.getNumRegions(); + for (int region = 0; region < imageHeapRegions; region++) { + if (!ShenandoahRegionType.isContinuesHumongous(regionTypes[region])) { + Pointer cur = imageHeapInfo.getRegionStart(region); + Pointer top = imageHeapInfo.getRegionTop(region); + while (cur.belowThan(top)) { + Object o = cur.toObject(); + visitor.visitObject(o); + cur = LayoutEncoding.getImageHeapObjectEnd(o); + } + } + } + } + + @Override + public void walkCollectedHeapObjects(ObjectVisitor visitor) { + if (visitor == null) { + return; + } + ShenandoahHeapWalker.walkCollectedHeap(visitor); + } + + @Fold + @Override + public int getHeapBaseAlignment() { + int buildTimePageSize = SubstrateOptions.getPageSize(); + return Math.max(buildTimePageSize * ShenandoahConstants.cardSize(), ShenandoahRegionSize.getValue()); + } + + @Fold + @Override + public int getImageHeapAlignment() { + return ShenandoahRegionSize.getValue(); + } + + @Fold + @Override + public int getImageHeapOffsetInAddressSpace() { + int buildTimePageSize = SubstrateOptions.getPageSize(); + int result = Math.max(buildTimePageSize * ShenandoahConstants.cardSize(), ShenandoahRegionSize.getValue()); + assert result % getImageHeapAlignment() == 0 : "start of image heap must be aligned"; + return result; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Override + public boolean isInImageHeap(Object object) { + return isInPrimaryImageHeap(object); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Override + public boolean isInImageHeap(Pointer pointer) { + return isInPrimaryImageHeap(pointer); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Override + public boolean isInPrimaryImageHeap(Object object) { + Word pointer = Word.objectToUntrackedPointer(object); + return isInPrimaryImageHeap(pointer); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Override + public boolean isInPrimaryImageHeap(Pointer pointer) { + return pointer.aboveOrEqual(imageHeapInfo.getImageHeapStart()) && pointer.belowThan(imageHeapInfo.getImageHeapEnd()); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public int getClassCount() { + return imageHeapInfo.getDynamicHubCount(); + } + + @Override + protected List> getClassesInImageHeap() { + if (classList == null) { + ArrayList> classes = findAllDynamicHubs(); + /* Ensure that other threads see consistent values once the list is published. */ + MembarNode.memoryBarrier(MembarNode.FenceKind.STORE_STORE); + classList = classes; + } + return classList; + } + + private ArrayList> findAllDynamicHubs() { + byte[] regionTypes = imageHeapInfo.getRegionTypes(); + int hubCount = getClassCount(); + + /* DynamicHubs are somewhere in the closed image heap. */ + ArrayList> classes = new ArrayList<>(hubCount); + for (int region = 0; region < imageHeapInfo.getNumClosedRegions(); region++) { + if (!ShenandoahRegionType.isHumongous(regionTypes[region])) { + Pointer cur = imageHeapInfo.getRegionStart(region); + Pointer top = imageHeapInfo.getRegionTop(region); + while (cur.belowThan(top)) { + Object o = cur.toObject(); + if (o instanceof Class) { + classes.add((Class) o); + if (classes.size() == hubCount) { + return classes; + } + } + cur = LayoutEncoding.getImageHeapObjectEnd(o); + } + } + } + + throw VMError.shouldNotReachHere("Found fewer DynamicHubs in the image heap than expected."); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Override + public ObjectHeader getObjectHeader() { + return objectHeader; + } + + @Uninterruptible(reason = "Tear-down in progress.") + @Override + public boolean tearDown() { + return ShenandoahLibrary.tearDown(); + } + + @Platforms(Platform.HOSTED_ONLY.class) + public void setAccessedFieldOffsets(byte[] fieldOffsets) { + assert this.accessedFieldOffsets == null; + this.accessedFieldOffsets = fieldOffsets; + } + + @Fold + static int getByteArrayBaseOffset() { + return ConfigurationValues.getObjectLayout().getArrayBaseOffset(JavaKind.Byte); + } + + @Override + public void prepareForSafepoint() { + ShenandoahLibrary.prepareForSafepoint(); + } + + @Override + public void endSafepoint() { + ShenandoahLibrary.endSafepoint(); + } + + public static Pointer addressOfCardTableAddress() { + return (Pointer) cardTableAddressTL.getAddress(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static void validateInitState(ShenandoahInitState state) { + VMError.guarantee(ShenandoahConstants.tlabTopOffset() == state.tlabTopOffset(), "Failed while validating the Shenandoah state: tlabTopOffset"); + VMError.guarantee(ShenandoahConstants.tlabEndOffset() == state.tlabEndOffset(), "Failed while validating the Shenandoah state: tlabEndOffset"); + VMError.guarantee(ShenandoahConstants.dirtyCardValue() == state.dirtyCardValue(), "Failed while validating the Shenandoah state: dirtyCardValue"); + VMError.guarantee(ShenandoahConstants.cardTableShift() == state.cardTableShift(), "Failed while validating the Shenandoah state: cardTableShift"); + VMError.guarantee(ShenandoahConstants.logOfHeapRegionGrainBytes() == state.logOfHeapRegionGrainBytes(), "Failed while validating the Shenandoah state: logOfHeapRegionGrainBytes"); + VMError.guarantee(ShenandoahConstants.javaThreadSize() == state.javaThreadSize(), "Failed while validating the Shenandoah state: javaThreadSize"); + VMError.guarantee(SizeOf.get(NativeGCVMOperationData.class) <= state.vmOperationDataSize(), "Failed while validating the Shenandoah state: vmOperationDataSize"); + VMError.guarantee(SizeOf.get(NativeGCVMOperationWrapperData.class) <= state.vmOperationWrapperDataSize(), "Failed while validating the Shenandoah state: vmOperationWrapperDataSize"); + } + + @Uninterruptible(reason = "Called during startup.") + @Override + public void attachThread(IsolateThread isolateThread) { + if (isInitialized) { + ShenandoahLibrary.attachThread(isolateThread); + } else { + /* The thread gets attached as a side effect of the initialization. */ + initialize(isolateThread); + } + cardTableAddressTL.set(isolateThread, cardTableAddress); + } + + @Override + @Uninterruptible(reason = "Thread is detaching and holds the THREAD_MUTEX.") + public void detachThread(IsolateThread isolateThread) { + ShenandoahLibrary.detachThread(isolateThread); + + /* Use a value, so that it looks as if the card table starts at address 0. */ + long invalidCardTableAddress = KnownIntrinsics.heapBase().unsignedShiftRight(ShenandoahConstants.cardTableShift()).rawValue(); + cardTableAddressTL.set(isolateThread, Word.signed(-invalidCardTableAddress)); + } + + @Override + public void doReferenceHandling() { + /* Nothing to do, Shenandoah only supports a dedicated reference handler thread. */ + } + + @Override + public boolean hasReferencePendingList() { + return ShenandoahLibrary.hasReferencePendingList(); + } + + @Override + public void waitForReferencePendingList() throws InterruptedException { + /* + * The order is crucial here to prevent transient issues. First, we call into C++ to get the + * current wakeup count, then we check if the thread was interrupted. This ensures that the + * C++ code is able to properly detect the case where the thread is interrupted right before + * blocking in ShenandoahLibrary.waitForReferencePendingList(). + */ + long initialWakeupCount = ShenandoahLibrary.getReferencePendingListWakeupCount(); + + /* Throw an InterruptedException if the thread is interrupted before or after waiting. */ + if (Thread.interrupted() || !waitForPendingReferenceList(initialWakeupCount) && Thread.interrupted()) { + throw new InterruptedException(); + } + } + + private static boolean waitForPendingReferenceList(long initialWakeupCount) { + Thread currentThread = Thread.currentThread(); + int oldThreadStatus = PlatformThreads.getThreadStatus(currentThread); + PlatformThreads.setThreadStatus(currentThread, ThreadStatus.PARKED); + try { + return ShenandoahLibrary.waitForReferencePendingList(initialWakeupCount); + } finally { + PlatformThreads.setThreadStatus(currentThread, oldThreadStatus); + } + } + + @Override + public void wakeUpReferencePendingListWaiters() { + ShenandoahLibrary.wakeUpReferencePendingListWaiters(); + } + + @Override + @Uninterruptible(reason = "Prevent stack overflow exceptions and recurring callback execution.", calleeMustBe = false) + public Reference getAndClearReferencePendingList() { + Word result = ShenandoahLibrary.getAndClearReferencePendingList(); + return (Reference) result.toObject(); + } + + @Override + public boolean printLocationInfo(Log log, UnsignedWord value, boolean allowJavaHeapAccess, boolean allowUnsafeOperations) { + if (value.equal(KnownIntrinsics.heapBase())) { + log.string("is the heap base"); + return true; + } + + if (objectHeader.isEncodedObjectHeader((Word) value)) { + log.string("is the encoded object header for an object of type "); + DynamicHub hub = objectHeader.dynamicHubFromObjectHeader((Word) value); + log.string(hub.getName()); + return true; + } + + Pointer ptr = (Pointer) value; + if (printHeapLocationInfo(log, ptr)) { + if (allowJavaHeapAccess && objectHeader.pointsToObjectHeader(ptr)) { + log.indent(true); + SubstrateDiagnostics.printObjectInfo(log, ptr.toObject()); + log.redent(false); + } + return true; + } + + return printGCInternalLocationInfo(log, ptr); + } + + private static boolean printHeapLocationInfo(Log log, Pointer ptr) { + ShenandoahCommittedMemoryProvider memoryProvider = ImageSingletons.lookup(ShenandoahCommittedMemoryProvider.class); + ShenandoahRegionInfo r = UnsafeStackValue.get(ShenandoahRegionInfo.class); + for (int i = 0; i < memoryProvider.getMaxRegions(); i++) { + if (ShenandoahLibrary.getRegionInfo(i, r)) { + if (ptr.aboveOrEqual(r.bottom())) { + if (ptr.belowThan(r.top())) { + log.string("points into region ").signed(i).string(" (").string(ShenandoahRegionType.toString(r.regionType())).string(")"); + return true; + } else if (ptr.belowThan(r.end())) { + log.string("points into unused space of region ").signed(i).string(" (").string(ShenandoahRegionType.toString(r.regionType())).string(")"); + return true; + } + } + } + } + return false; + } + + @SuppressWarnings("unused") + private static boolean printGCInternalLocationInfo(Log log, Pointer ptr) { + ShenandoahInternalState state = UnsafeStackValue.get(ShenandoahInternalState.class); + fillGCInternalState(state); + + throw VMError.shouldNotReachHere("Unimplemented: check if this is a GC-internal location and print some debugging output in that case"); + } + + @Override + public void optionValueChanged(RuntimeOptionKey key) { + /* + * There is no need to inform Shenandoah about options that can only be set during isolate + * startup. + */ + if (!SubstrateUtil.HOSTED && !key.isIsolateCreationOnly()) { + assert isInImageHeap(key.getName()); + Word optionName = Word.objectToUntrackedPointer(key.getName()); + long value = convertOptionValueToLong(key); + ShenandoahLibrary.updateOptionValue(optionName, value); + } + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public long getThreadAllocatedMemory(IsolateThread thread) { + return ShenandoahLibrary.getThreadAllocatedMemory(thread); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public UnsignedWord getUsedMemoryAfterLastGC() { + return ShenandoahLibrary.getUsedMemoryAfterLastGC(); + } + + @Override + @Uninterruptible(reason = "Ensure that no GC can occur between modification of the object and this call.", callerMustBe = true) + public void dirtyAllReferencesOf(Object obj) { + if (obj == null) { + return; + } + + VMError.guarantee(obj instanceof StoredContinuation); + ShenandoahLibrary.dirtyAllReferencesOf(Word.objectToUntrackedPointer(obj)); + } + + @Override + public long getMillisSinceLastWholeHeapExamined() { + return ShenandoahLibrary.getMillisSinceLastWholeHeapExamined(); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public long getIdentityHashSalt(Object obj) { + ReplacementsUtil.staticAssert(false, "identity hash codes are never computed from addresses"); + return 0; + } + + private static long convertOptionValueToLong(RuntimeOptionKey key) { + Class valueType = key.getDescriptor().getOptionValueType(); + if (valueType == Boolean.class) { + return ((Boolean) key.getValue()) ? 1L : 0L; + } else if (valueType == Integer.class || valueType == Long.class) { + return ((Number) key.getValue()).longValue(); + } else { + throw VMError.shouldNotReachHere("Option " + key.getName() + " has an unexpected type: " + valueType); + } + } + + private static void fillGCInternalState(ShenandoahInternalState state) { + int size = SizeOf.get(ShenandoahInternalState.class); + UnmanagedMemoryUtil.fill((Pointer) state, Word.unsigned(size), (byte) 0); + ShenandoahLibrary.getGCInternalState(state); + } + + @Override + @Uninterruptible(reason = "Called during early startup.") + public boolean verifyImageHeapMapping() { + /* Read & write some data at the beginning and end of each open region. */ + for (int region = imageHeapInfo.getNumClosedRegions(); region < imageHeapInfo.getNumRegions(); region++) { + Pointer begin = imageHeapInfo.getRegionStart(region); + Pointer end = imageHeapInfo.getRegionTop(region).subtract(1); + + byte val = begin.readByte(0); + begin.writeByte(0, val); + + val = end.readByte(0); + end.writeByte(0, val); + } + return true; + } + + private static final class DumpHeapSettingsAndGCInternalState extends DiagnosticThunk { + @Override + public int maxInvocationCount() { + return 1; + } + + @Override + @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while printing diagnostics.") + public void printDiagnostics(Log log, ErrorContext context, int maxDiagnosticLevel, int invocationCount) { + log.string("Heap settings and statistics:").indent(true); + log.string("Reserved hub pointer bits: 0b").number(Heap.getHeap().getObjectHeader().getReservedHubBitsMask(), 2, false).newline(); + log.string("Region size: ").signed(ShenandoahRegionSize.getValue()).newline(); + log.string("Card table granularity: ").signed(ShenandoahConstants.cardSize()).newline(); + + ShenandoahInternalState state = UnsafeStackValue.get(ShenandoahInternalState.class); + fillGCInternalState(state); + + log.string("Full collections: ").unsigned(state.fullCollections()).newline(); + log.string("Total collections: ").unsigned(state.totalCollections()).newline(); + log.string("Card table: ").zhex(state.cardTableStart()).string(" - ").zhex(state.cardTableStart().add(state.cardTableSize()).subtract(1)).newline(); + log.indent(false); + } + } + + private static final class DumpRegionInformation extends DiagnosticThunk { + private static final int MAX_REGIONS_TO_PRINT = 128 * 1024; + + @Override + public int maxInvocationCount() { + return 1; + } + + @Override + @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while printing diagnostics.") + public void printDiagnostics(Log log, ErrorContext context, int maxDiagnosticLevel, int invocationCount) { + int maxRegions = ImageSingletons.lookup(ShenandoahCommittedMemoryProvider.class).getMaxRegions(); + ShenandoahRegionInfo regionInfo = UnsafeStackValue.get(ShenandoahRegionInfo.class); + + log.string("Heap regions:").indent(true); + int printed = 0; + for (int i = 0; i < maxRegions; i++) { + if (printed >= MAX_REGIONS_TO_PRINT) { + log.string("... (truncated)").newline(); + break; + } + + if (ShenandoahLibrary.getRegionInfo(i, regionInfo)) { + printRegion(log, i, regionInfo); + printed++; + } + } + log.indent(false); + } + + @SuppressWarnings("unused") + private static void printRegion(Log log, int regionIndex, ShenandoahRegionInfo r) { + throw VMError.shouldNotReachHere("Unimplemented: region printing"); + } + } + + private static final class DumpCurrentGCThreadName extends DiagnosticThunk { + @Override + public int maxInvocationCount() { + return 1; + } + + @Override + @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while printing diagnostics.") + public void printDiagnostics(Log log, ErrorContext context, int maxDiagnosticLevel, int invocationCount) { + if (SubstrateDiagnostics.isThreadOnlyAttachedForCrashHandler(CurrentIsolate.getCurrentThread())) { + // The failing thread is an unattached thread, so it might be a GC thread. + CCharPointer name = ShenandoahLibrary.getCurrentThreadName(); + if (name.isNonNull()) { + log.string("Internal name of crashing thread: ").string(name).newline(); + } + } + } + } +} + +@TargetClass(value = Runtime.class, onlyWith = UseShenandoahGC.class) +@SuppressWarnings({"static-method"}) +final class Target_java_lang_Runtime { + @Substitute + private long freeMemory() { + return ShenandoahLibrary.getFreeMemory(); + } + + @Substitute + private long totalMemory() { + return ShenandoahLibrary.getTotalMemory(); + } + + @Substitute + private long maxMemory() { + return ShenandoahLibrary.getMaxMemory(); + } + + @Substitute + private void gc() { + ShenandoahLibrary.collect(GCCause.JavaLangSystemGC.getId()); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahHeapWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahHeapWalker.java new file mode 100644 index 000000000000..fa66146d8c80 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahHeapWalker.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah; + +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.word.Pointer; + +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahLibrary; +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahStructs.ShenandoahRegionBoundaries; +import com.oracle.svm.core.heap.ObjectVisitor; +import com.oracle.svm.core.heap.RestrictHeapAccess; +import com.oracle.svm.core.heap.RestrictHeapAccess.Access; +import com.oracle.svm.core.hub.LayoutEncoding; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; +import com.oracle.svm.core.thread.RecurringCallbackSupport; +import com.oracle.svm.core.thread.VMOperation; + +import jdk.graal.compiler.word.Word; + +public class ShenandoahHeapWalker { + private static final OutOfMemoryError OUT_OF_MEMORY_ERROR = new OutOfMemoryError("Ran out of native memory while preparing the heap walk."); + + public static void walkCollectedHeap(ObjectVisitor visitor) { + RecurringCallbackSupport.suspendCallbackTimer("Recurring callbacks could allocate."); + try { + walkCollectedHeap0(visitor); + } finally { + RecurringCallbackSupport.resumeCallbackTimer(); + } + } + + @RestrictHeapAccess(access = Access.NO_ALLOCATION, reason = "Allocations could change the heap regions") + private static void walkCollectedHeap0(ObjectVisitor visitor) { + assert RecurringCallbackSupport.isCallbackUnsupportedOrTimerSuspended(); + VMOperation.guaranteeInProgressAtSafepoint("must only be executed at a safepoint"); + + ShenandoahCommittedMemoryProvider memoryProvider = ImageSingletons.lookup(ShenandoahCommittedMemoryProvider.class); + int maxRegions = memoryProvider.getMaxRegions(); + + ShenandoahImageHeapInfo imageHeapInfo = ShenandoahHeap.getImageHeapInfo(); + int fromRegion = imageHeapInfo.getNumRegions(); + + ShenandoahRegionBoundaries regionBoundaries = NullableNativeMemory.calloc(Word.unsigned(maxRegions).multiply(SizeOf.get(ShenandoahRegionBoundaries.class)), NmtCategory.GC); + if (regionBoundaries.isNull()) { + throw OUT_OF_MEMORY_ERROR; + } + + try { + ShenandoahLibrary.getRegionBoundaries(regionBoundaries); + for (int i = fromRegion; i < maxRegions; i++) { + ShenandoahRegionBoundaries bounds = regionBoundaries.addressOf(i); + if (bounds.bottom().equal(0)) { + continue; + } + + Word bottom = bounds.bottom(); + Word top = bounds.top(); + Pointer cur = bottom; + while (cur.belowThan(top)) { + Object o = cur.toObject(); + visitor.visitObject(o); + cur = LayoutEncoding.getObjectEndInGC(o); + } + } + } finally { + NullableNativeMemory.free(regionBoundaries); + } + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahImageHeapInfo.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahImageHeapInfo.java new file mode 100644 index 000000000000..7b214fb67378 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahImageHeapInfo.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; +import static com.oracle.svm.core.gc.shenandoah.ShenandoahOptions.ShenandoahRegionSize; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; + +import com.oracle.svm.core.BuildPhaseProvider.AfterHeapLayout; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.heap.UnknownObjectField; +import com.oracle.svm.core.heap.UnknownPrimitiveField; +import com.oracle.svm.core.snippets.KnownIntrinsics; + +import jdk.graal.compiler.word.Word; + +/** + * Stores some high-level information about the regions of the image heap. This data is passed to + * the C++ code during VM startup. + */ +public class ShenandoahImageHeapInfo { + /* + * The arrays below can be slightly larger than the total region count (elements beyond the + * total number of image heap regions don't contain valid data). + */ + @UnknownPrimitiveField(availability = AfterHeapLayout.class) int closedImageHeapRegions; + @UnknownPrimitiveField(availability = AfterHeapLayout.class) int openImageHeapRegions; + @UnknownObjectField(availability = AfterHeapLayout.class) byte[] regionTypes; + @UnknownObjectField(availability = AfterHeapLayout.class) int[] regionFreeSpaces; + @UnknownPrimitiveField(availability = AfterHeapLayout.class) int dynamicHubCount; + + @Platforms(value = Platform.HOSTED_ONLY.class) + public ShenandoahImageHeapInfo() { + } + + @SuppressWarnings("hiding") + @Platforms(value = Platform.HOSTED_ONLY.class) + public void initialize(int closedImageHeapRegions, int openImageHeapRegions, byte[] regionType, int[] regionFreeSpace, int dynamicHubCount) { + this.closedImageHeapRegions = closedImageHeapRegions; + this.openImageHeapRegions = openImageHeapRegions; + this.regionTypes = regionType; + this.regionFreeSpaces = regionFreeSpace; + this.dynamicHubCount = dynamicHubCount; + } + + @Platforms(value = Platform.HOSTED_ONLY.class) + public void writeHeapRegion(int index, ShenandoahRegionType type, int freeSpace) { + assert type != null; + assert freeSpace >= 0 : freeSpace; + regionTypes[index] = type.getTag(); + regionFreeSpaces[index] = freeSpace; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public int getNumClosedRegions() { + return closedImageHeapRegions; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public int getNumOpenRegions() { + return openImageHeapRegions; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public int getNumRegions() { + return closedImageHeapRegions + openImageHeapRegions; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public byte[] getRegionTypes() { + return regionTypes; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public int[] getRegionFreeSpaces() { + return regionFreeSpaces; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public Pointer getImageHeapStart() { + // skip the null regions before the image heap + int nullRegionsSize = Heap.getHeap().getImageHeapOffsetInAddressSpace(); + return KnownIntrinsics.heapBase().add(nullRegionsSize); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public Pointer getRegionStart(int regionIndex) { + assert regionIndex < getNumRegions(); + return getImageHeapStart().add(Word.unsigned(regionIndex).multiply(ShenandoahRegionSize.getValue())); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public Pointer getRegionTop(int regionIndex) { + assert regionIndex < getNumRegions(); + Pointer start = getRegionStart(regionIndex); + return start.add(ShenandoahRegionSize.getValue()).subtract(regionFreeSpaces[regionIndex]); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public Pointer getImageHeapEnd() { + int nullRegionsSize = Heap.getHeap().getImageHeapOffsetInAddressSpace(); + UnsignedWord imageHeapSize = Word.unsigned(ShenandoahRegionSize.getValue()).multiply(closedImageHeapRegions + openImageHeapRegions); + return KnownIntrinsics.heapBase().add(nullRegionsSize).add(imageHeapSize); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public int getDynamicHubCount() { + return dynamicHubCount; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahMemoryMXBean.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahMemoryMXBean.java new file mode 100644 index 000000000000..f1661880e79d --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahMemoryMXBean.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah; + +import java.lang.management.MemoryUsage; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahLibrary; +import com.oracle.svm.core.heap.AbstractMemoryMXBean; + +public final class ShenandoahMemoryMXBean extends AbstractMemoryMXBean { + @Platforms(Platform.HOSTED_ONLY.class) + public ShenandoahMemoryMXBean() { + } + + @Override + public MemoryUsage getHeapMemoryUsage() { + long used = ShenandoahLibrary.getUsedMemory(); + long committed = ShenandoahLibrary.getTotalMemory(); + long max = ShenandoahLibrary.getMaxMemory(); + return new MemoryUsage(UNDEFINED_MEMORY_USAGE, used, committed, max); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahObjectHeader.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahObjectHeader.java new file mode 100644 index 000000000000..1869d18e5673 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahObjectHeader.java @@ -0,0 +1,271 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; + +import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.heap.ObjectHeader; +import com.oracle.svm.core.heap.ReferenceAccess; +import com.oracle.svm.core.hub.DynamicHub; +import com.oracle.svm.core.image.ImageHeapObject; +import com.oracle.svm.core.snippets.KnownIntrinsics; +import com.oracle.svm.core.util.VMError; + +import jdk.graal.compiler.api.directives.GraalDirectives; +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.replacements.ReplacementsUtil; +import jdk.graal.compiler.word.ObjectAccess; +import jdk.graal.compiler.word.Word; +import jdk.vm.ci.code.CodeUtil; + +/** The object header consists of a 32/64 bit mark word and a 32 bit hub pointer. */ +public class ShenandoahObjectHeader extends ObjectHeader { + private final int numAlignmentBits; + private final int hubBits; + private final int numReservedHeaderBits; + private final int numReservedHubBits; + + @Platforms(Platform.HOSTED_ONLY.class) + public ShenandoahObjectHeader() { + numAlignmentBits = CodeUtil.log2(ConfigurationValues.getObjectLayout().getAlignment()); + if (useCompressedReferences()) { + /* + * Use 27 bits for the hub pointer, and 37 bits for the mark word and other VM-internal + * data. This limits the address space where all hubs must be placed to 1024 MB. + */ + hubBits = 27; + numReservedHeaderBits = 37; + numReservedHubBits = 5; + } else { + /* + * No reserved bits (64-bit mark word, 32-bit hub pointer). The hub pointer is not + * compressed, which limits the address space where all hubs must be placed to 4096 MB. + */ + hubBits = 32; + numReservedHeaderBits = 0; + numReservedHubBits = 0; + } + } + + @Fold + public static ShenandoahObjectHeader get() { + return (ShenandoahObjectHeader) ShenandoahHeap.get().getObjectHeader(); + } + + @Fold + public static int getMarkWordOffset() { + return 0; + } + + @Override + public int getReservedHubBitsMask() { + return (1 << numReservedHubBits) - 1; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private long getReservedHeaderBitsMask() { + return (1L << numReservedHeaderBits) - 1; + } + + @Override + public Word encodeAsTLABObjectHeader(DynamicHub hub) { + return encodeAsObjectHeader(hub); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private Word encodeAsObjectHeader(DynamicHub hub) { + Word heapBaseRelativeAddress = Word.objectToUntrackedPointer(hub).subtract(KnownIntrinsics.heapBase()); + assertInHubAddressSpace(heapBaseRelativeAddress); + + if (useCompressedReferences()) { + Word result = heapBaseRelativeAddress.shiftLeft(numReservedHeaderBits - numAlignmentBits); + assertReservedHeaderBitsZero(result); + return result; + } + return heapBaseRelativeAddress; + } + + @Override + public long encodeAsTLABObjectHeader(long hubOffsetFromHeapBase) { + if (useCompressedReferences()) { + return hubOffsetFromHeapBase << (numReservedHeaderBits - numAlignmentBits); + } else { + return hubOffsetFromHeapBase; + } + } + + @Override + public int constantHeaderSize() { + return useCompressedReferences() ? Long.BYTES : Integer.BYTES; + } + + @Override + public long encodeHubPointerForImageHeap(ImageHeapObject obj, long hubOffsetFromHeapBase) { + assert isInHubAddressSpace(hubOffsetFromHeapBase) : hubOffsetFromHeapBase; + + if (useCompressedReferences()) { + long result = hubOffsetFromHeapBase << (numReservedHubBits - numAlignmentBits); + assert (result & getReservedHubBitsMask()) == 0 : "all reserved bits must be zero"; + return result; + } + return hubOffsetFromHeapBase; + } + + @Override + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void verifyDynamicHubOffset(long offsetFromHeapBase) { + if (!isInHubAddressSpace(offsetFromHeapBase)) { + throw VMError.shouldNotReachHere("Hub is too far from heap base for encoding in object header"); + } + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + @Override + public Word readHeaderFromPointer(Pointer objectPointer) { + if (useCompressedReferences()) { + return objectPointer.readWord(getMarkWordOffset()); + } + return Word.unsigned(objectPointer.readInt(getHubOffset())); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + @Override + public Word readHeaderFromObject(Object o) { + if (useCompressedReferences()) { + return ObjectAccess.readWord(o, getMarkWordOffset()); + } + return Word.unsigned(ObjectAccess.readInt(o, getHubOffset())); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + @Override + public Pointer extractPotentialDynamicHubFromHeader(Word header) { + if (useCompressedReferences()) { + UnsignedWord hubPart = header.unsignedShiftRight(numReservedHeaderBits); + UnsignedWord baseRelativeBits = hubPart.shiftLeft(numAlignmentBits); + return KnownIntrinsics.heapBase().add(baseRelativeBits); + } + return header.add(KnownIntrinsics.heapBase()); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + @Override + public Word encodeAsUnmanagedObjectHeader(DynamicHub hub) { + return encodeAsObjectHeader(hub); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + @AlwaysInline(INLINE_INITIALIZE_HEADER_INIT_REASON) + @Override + protected void initializeObjectHeader(Pointer objectPointer, Word objectHeader, boolean isArrayLike, MemWriter writer) { + if (useCompressedReferences()) { + /* Write the mark word and the hub ptr as a single 64-bit value. */ + assertReservedHeaderBitsZero(objectHeader); + writer.writeWord(objectPointer, getMarkWordOffset(), objectHeader); + } else { + /* objectHeader only stores the uncompressed, 4 byte hub pointer. */ + assertInHubAddressSpace(objectHeader); + writer.writeWord(objectPointer, getMarkWordOffset(), Word.zero()); + writer.writeInt(objectPointer, getHubOffset(), (int) objectHeader.rawValue()); + } + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + @Override + public boolean hasOptionalIdentityHashField(Word header) { + if (GraalDirectives.inIntrinsic()) { + ReplacementsUtil.staticAssert(false, "all objects have the identity hash code field in the object header"); + return false; + } else { + throw VMError.shouldNotReachHereAtRuntime(); + } + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + @Override + public boolean hasIdentityHashFromAddress(Word header) { + ReplacementsUtil.staticAssert(false, "all objects have the identity hash code field in the object header"); + return false; + } + + @Uninterruptible(reason = "Prevent a GC interfering with the object's identity hash state.", callerMustBe = true) + @Override + public void setIdentityHashFromAddress(Pointer ptr, Word currentHeader) { + ReplacementsUtil.staticAssert(false, "identity hash codes are never computed from addresses"); + } + + @AlwaysInline("Otherwise, there is no guarantee that this is optimized away if assertions are disabled.") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private void assertReservedHeaderBitsZero(Word objectHeader) { + if (!GraalDirectives.inIntrinsic()) { + assert areReservedHeaderBitsZero(objectHeader) : "all reserved bits must be zero"; + } else if (ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED) { + ReplacementsUtil.dynamicAssert(areReservedHeaderBitsZero(objectHeader), "all reserved bits must be zero"); + } + } + + @AlwaysInline("Otherwise, there is no guarantee that this is optimized away if assertions are disabled.") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private void assertInHubAddressSpace(Word heapBaseRelativeAddress) { + if (!GraalDirectives.inIntrinsic()) { + assert isInHubAddressSpace(heapBaseRelativeAddress.rawValue()) : "must be in hub-specific address space"; + } else if (ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED) { + ReplacementsUtil.dynamicAssert(isInHubAddressSpace(heapBaseRelativeAddress.rawValue()), "must be in hub-specific address space"); + } + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private boolean areReservedHeaderBitsZero(Word objectHeader) { + return objectHeader.and(Word.unsigned(getReservedHeaderBitsMask())).isNull(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private boolean isInHubAddressSpace(long heapBaseRelativeAddress) { + long hubAddressSpaceSize = getHubAddressSpaceSize(); + return Long.compareUnsigned(hubAddressSpaceSize, heapBaseRelativeAddress) > 0; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private long getHubAddressSpaceSize() { + int hubAddressSpaceBits = hubBits; + if (useCompressedReferences()) { + hubAddressSpaceBits += numAlignmentBits; + } + return (1L << hubAddressSpaceBits) - 1; + } + + @Fold + static boolean useCompressedReferences() { + return ReferenceAccess.singleton().getCompressionShift() > 0; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahOptions.java new file mode 100644 index 000000000000..0c5c43fa5f3c --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahOptions.java @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah; + +import static com.oracle.svm.core.gc.shared.NativeGCOptions.K; +import static com.oracle.svm.core.gc.shared.NativeGCOptions.M; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.function.Consumer; + +import org.graalvm.nativeimage.c.type.CCharPointer; + +import com.oracle.svm.core.SubstrateGCOptions; +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.SubstrateUtil; +import com.oracle.svm.core.c.CGlobalData; +import com.oracle.svm.core.c.CGlobalDataFactory; +import com.oracle.svm.core.gc.shared.NativeGCDebugLevel; +import com.oracle.svm.core.gc.shared.NativeGCOptions; +import com.oracle.svm.core.gc.shared.NativeGCOptions.HostedArgumentsSupplier; +import com.oracle.svm.core.gc.shared.NativeGCOptions.NativeGCHostedOptionKey; +import com.oracle.svm.core.gc.shared.NativeGCOptions.NativeGCRuntimeOptionKey; +import com.oracle.svm.core.gc.shared.NativeGCOptions.RuntimeArgumentsSupplier; +import com.oracle.svm.core.option.HostedOptionKey; +import com.oracle.svm.core.option.SubstrateOptionKey; +import com.oracle.svm.core.util.UserError; + +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.options.Option; +import jdk.graal.compiler.options.OptionType; + +/** + * Contains options that are specific to Shenandoah. See {@link NativeGCOptions} for more details. + */ +public class ShenandoahOptions { + private static final String SUPPORTED_REGION_SIZES = "Supported values are 256k, 512k, 1m, 2m, 4m, 8m, 16m, or 32m"; + + @Option(help = "Specifies the debug level of the linked Shenandoah GC [product, fastdebug, or debug]", type = OptionType.Debug) // + protected static final HostedOptionKey ShenandoahDebugLevel = new ShenandoahHostedOptionKey<>("product", false); + + @Fold + public static NativeGCDebugLevel getDebugLevel() { + NativeGCDebugLevel result = NativeGCDebugLevel.fromString(ShenandoahDebugLevel.getValue()); + UserError.guarantee(result != null, "'%s' is not a valid value for the option %s.", ShenandoahDebugLevel.getValue(), ShenandoahDebugLevel.getName()); + return result; + } + + @Option(help = "Size of the Shenandoah heap regions in bytes. " + SUPPORTED_REGION_SIZES + ".", type = OptionType.User)// + public static final HostedOptionKey ShenandoahRegionSize = new ShenandoahHostedOptionKey<>(1 * M, ShenandoahOptions::validateRegionSize); + + /* Encoded option values. */ + public static final CGlobalData HOSTED_ARGUMENTS = CGlobalDataFactory.createBytes(new HostedArgumentsSupplier(getOptionFields())); + public static final CGlobalData RUNTIME_ARGUMENTS = CGlobalDataFactory.createBytes(new RuntimeArgumentsSupplier(getOptionFields())); + + public static ArrayList getOptionFields() { + Class[] optionClasses = {SubstrateGCOptions.class, SubstrateGCOptions.TlabOptions.class, NativeGCOptions.class, ShenandoahOptions.class}; + return NativeGCOptions.getOptionFields(optionClasses); + } + + private static void validateShenandoahOption(SubstrateOptionKey optionKey) { + if (optionKey.hasBeenSet() && !SubstrateOptions.useShenandoahGC()) { + throw UserError.abort("The option '%s' can only be used together with the Shenandoah garbage collector ('--gc=shenandoah').", optionKey.getName()); + } + } + + private static void validateRegionSize(HostedOptionKey optionKey) { + int value = optionKey.getValue(); + if (value % M == 0 && SubstrateUtil.isPowerOf2(value / M) && value >= 256 * K && value <= 32 * M) { + return; + } + throw UserError.invalidOptionValue(ShenandoahRegionSize, value, SUPPORTED_REGION_SIZES); + } + + private static class ShenandoahHostedOptionKey extends NativeGCHostedOptionKey { + ShenandoahHostedOptionKey(T defaultValue, Consumer> validation) { + this(defaultValue, true, validation); + } + + ShenandoahHostedOptionKey(T defaultValue, boolean passToCpp) { + this(defaultValue, passToCpp, null); + } + + ShenandoahHostedOptionKey(T defaultValue, boolean passToCpp, Consumer> validation) { + super(defaultValue, passToCpp, validation); + } + + @Override + public void validate() { + validateShenandoahOption(this); + super.validate(); + } + } + + @SuppressWarnings("unused") + private static class ShenandoahRuntimeOptionKey extends NativeGCRuntimeOptionKey { + ShenandoahRuntimeOptionKey(T defaultValue, RuntimeOptionKeyFlag... flags) { + super(defaultValue, flags); + } + + @Override + public void validate() { + validateShenandoahOption(this); + super.validate(); + } + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahPinnedObjectSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahPinnedObjectSupport.java new file mode 100644 index 000000000000..5d100903345a --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahPinnedObjectSupport.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahLibrary; +import com.oracle.svm.core.heap.AbstractPinnedObjectSupport; + +import jdk.graal.compiler.word.Word; + +public final class ShenandoahPinnedObjectSupport extends AbstractPinnedObjectSupport { + @Platforms(Platform.HOSTED_ONLY.class) + public ShenandoahPinnedObjectSupport() { + } + + @Override + @Uninterruptible(reason = "Use untracked pointers. Ensure that pinned object counts and PinnedObjects are consistent.", callerMustBe = true) + protected void pinObject(Object object) { + ShenandoahLibrary.pinObject(Word.objectToUntrackedPointer(object)); + } + + @Override + @Uninterruptible(reason = "Use untracked pointers. Ensure that pinned object counts and PinnedObjects are consistent.", callerMustBe = true) + protected void unpinObject(Object object) { + ShenandoahLibrary.unpinObject(Word.objectToUntrackedPointer(object)); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahRegionType.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahRegionType.java new file mode 100644 index 000000000000..52dfd0dd58e7 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahRegionType.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah; + +import com.oracle.svm.core.util.DuplicatedInNativeCode; + +/** + * There are two types of image heap regions, which differ in the kind of references that are + * allowed for the contained objects: + * + *
    + *
  • Closed image heap regions may only contain references to other image heap regions. The GC + * does not visit any objects in such regions. Note that parts of the closed image heap are + * read-only.
    • + *
  • Open image heap regions allow references to any other region in the image or collected Java + * heap. The GC visits open image heap regions and adjusts pointers in these regions. However, it + * doesn't move or collect any objects that are in open image heap regions.
    • + *
+ */ +@DuplicatedInNativeCode +public enum ShenandoahRegionType { + ClosedImageHeap(Flags.ClosedImageHeapBit), + ClosedImageHeapStartsHumongous(Flags.ClosedImageHeapBit | Flags.StartsHumongousBit), + ClosedImageHeapContinuesHumongous(Flags.ClosedImageHeapBit | Flags.ContinuesHumongousBit), + + OpenImageHeap(Flags.OpenImageHeapBit), + OpenImageHeapStartsHumongous(Flags.OpenImageHeapBit | Flags.StartsHumongousBit), + OpenImageHeapContinuesHumongous(Flags.OpenImageHeapBit | Flags.ContinuesHumongousBit); + + private final byte tag; + + ShenandoahRegionType(int tag) { + assert tag >= 0 && tag < Byte.MAX_VALUE : tag; + this.tag = (byte) tag; + } + + public byte getTag() { + return tag; + } + + public boolean isHumongous() { + return (tag & Flags.HumongousBits) != 0; + } + + public static boolean isHumongous(byte tag) { + return (tag & Flags.HumongousBits) != 0; + } + + public static boolean isContinuesHumongous(byte tag) { + return (tag & Flags.ContinuesHumongousBit) != 0; + } + + public boolean isClosedImageHeap() { + return (tag & Flags.ClosedImageHeapBit) != 0; + } + + public static String toString(int tag) { + if (tag == OpenImageHeap.tag) { + return "OI"; + } else if (tag == ClosedImageHeap.tag) { + return "CI"; + } else if (tag == OpenImageHeapStartsHumongous.tag) { + return "OIHS"; + } else if (tag == OpenImageHeapContinuesHumongous.tag) { + return "OIHC"; + } else if (tag == ClosedImageHeapStartsHumongous.tag) { + return "CIHS"; + } else if (tag == ClosedImageHeapContinuesHumongous.tag) { + return "CIHC"; + } else { + return "?"; + } + } + + // Constants that need to match the C++ side + private static final class Flags { + private static final byte StartsHumongousBit = 0b0001; + private static final byte ContinuesHumongousBit = 0b0010; + private static final byte HumongousBits = StartsHumongousBit | ContinuesHumongousBit; + + private static final byte ClosedImageHeapBit = 0b0100; + private static final byte OpenImageHeapBit = 0b1000; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahRelatedMXBeans.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahRelatedMXBeans.java new file mode 100644 index 000000000000..982f5d2277ad --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahRelatedMXBeans.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah; + +import java.util.Collections; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + +import com.oracle.svm.core.GCRelatedMXBeans; + +public final class ShenandoahRelatedMXBeans extends GCRelatedMXBeans { + + @Platforms(Platform.HOSTED_ONLY.class) + public ShenandoahRelatedMXBeans() { + beans.addSingleton(java.lang.management.MemoryMXBean.class, new ShenandoahMemoryMXBean()); + /* The following MX beans are not yet implemented for Shenandoah. */ + beans.addList(java.lang.management.MemoryPoolMXBean.class, Collections.emptyList()); + beans.addList(java.lang.management.BufferPoolMXBean.class, Collections.emptyList()); + beans.addList(com.sun.management.GarbageCollectorMXBean.class, Collections.emptyList()); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahRuntimeCodeInfoGCSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahRuntimeCodeInfoGCSupport.java new file mode 100644 index 000000000000..3c5e1bb205ee --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahRuntimeCodeInfoGCSupport.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah; + +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.function.CEntryPoint.Publish; +import org.graalvm.nativeimage.c.function.CEntryPointLiteral; +import org.graalvm.nativeimage.c.function.CFunctionPointer; +import org.graalvm.word.PointerBase; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.code.CodeInfo; +import com.oracle.svm.core.code.RuntimeCodeInfoMemory; +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahLibrary; +import com.oracle.svm.core.graal.RuntimeCompilation; +import com.oracle.svm.core.heap.RuntimeCodeCacheCleaner; +import com.oracle.svm.core.heap.RuntimeCodeInfoGCSupport; + +public class ShenandoahRuntimeCodeInfoGCSupport extends RuntimeCodeInfoGCSupport { + private static final RuntimeCodeCacheCleaner CODE_CACHE_CLEANER = new RuntimeCodeCacheCleaner(); + + public final CEntryPointLiteral funcCleanCodeCache; + + @Platforms(Platform.HOSTED_ONLY.class) + public ShenandoahRuntimeCodeInfoGCSupport() { + funcCleanCodeCache = RuntimeCompilation.isEnabled() ? CEntryPointLiteral.create(ShenandoahRuntimeCodeInfoGCSupport.class, "cleanCodeCache", PointerBase.class, IsolateThread.class) : null; + } + + @Override + @Uninterruptible(reason = "Called when installing code.", callerMustBe = true) + public void registerObjectFields(CodeInfo codeInfo) { + ShenandoahLibrary.registerObjectFields(codeInfo); + } + + @Override + @Uninterruptible(reason = "Called when installing code.", callerMustBe = true) + public void registerCodeConstants(CodeInfo codeInfo) { + ShenandoahLibrary.registerCodeConstants(codeInfo); + } + + @Override + @Uninterruptible(reason = "Called when installing code.", callerMustBe = true) + public void registerFrameMetadata(CodeInfo codeInfo) { + ShenandoahLibrary.registerFrameMetadata(codeInfo); + } + + @Override + @Uninterruptible(reason = "Called when installing code.", callerMustBe = true) + public void registerDeoptMetadata(CodeInfo codeInfo) { + ShenandoahLibrary.registerDeoptMetadata(codeInfo); + } + + @CEntryPoint(include = UseShenandoahGC.class, publishAs = Publish.NotPublished) + public static void cleanCodeCache(@SuppressWarnings("unused") PointerBase heapBase, @SuppressWarnings("unused") IsolateThread thread) { + RuntimeCodeInfoMemory.singleton().walkRuntimeMethodsDuringGC(CODE_CACHE_CLEANER); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahVMOperations.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahVMOperations.java new file mode 100644 index 000000000000..f00a6d5a2532 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/ShenandoahVMOperations.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah; + +import static com.oracle.svm.core.gc.shared.NativeGCVMOperationSupport.enqueue; +import static com.oracle.svm.core.thread.VMOperation.SystemEffect.SAFEPOINT; + +import org.graalvm.nativeimage.Isolate; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.function.CEntryPoint.Publish; +import org.graalvm.nativeimage.c.function.CEntryPointLiteral; +import org.graalvm.nativeimage.c.function.CFunctionPointer; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.c.function.CEntryPointOptions; +import com.oracle.svm.core.c.function.CEntryPointOptions.NoEpilogue; +import com.oracle.svm.core.gc.shared.InitializeReservedRegistersForPossiblyUnattachedThread; +import com.oracle.svm.core.gc.shared.NativeGCVMOperationSupport.NativeGCVMOperation; +import com.oracle.svm.core.gc.shared.NativeGCVMOperationSupport.NativeGCVMOperationData; +import com.oracle.svm.core.gc.shared.NativeGCVMOperationSupport.NativeGCVMOperationWrapperData; +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahLibrary; +import com.oracle.svm.core.heap.VMOperationInfo; +import com.oracle.svm.core.heap.VMOperationInfos; + +/** Shenandoah-related VM operations. */ +public class ShenandoahVMOperations { + private static final ShenandoahVMOperation OP_COLLECT_FOR_ALLOCATION = new ShenandoahVMOperation(VMOperationInfos.get(ShenandoahVMOperation.class, "Collect for allocation", SAFEPOINT), true); + private static final ShenandoahVMOperation OP_COLLECT_FULL = new ShenandoahVMOperation(VMOperationInfos.get(ShenandoahVMOperation.class, "Collect full", SAFEPOINT), true); + + public final CEntryPointLiteral funcCollectForAllocation; + public final CEntryPointLiteral funcCollectFull; + + @Platforms(Platform.HOSTED_ONLY.class) + public ShenandoahVMOperations() { + funcCollectForAllocation = CEntryPointLiteral.create(ShenandoahVMOperations.class, "collectForAllocation", + Isolate.class, IsolateThread.class, NativeGCVMOperationData.class, NativeGCVMOperationWrapperData.class); + funcCollectFull = CEntryPointLiteral.create(ShenandoahVMOperations.class, "collectFull", + Isolate.class, IsolateThread.class, NativeGCVMOperationData.class, NativeGCVMOperationWrapperData.class); + } + + @Uninterruptible(reason = "Can be called from an unattached thread.") + @CEntryPoint(include = UseShenandoahGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForPossiblyUnattachedThread.class, epilogue = NoEpilogue.class) + public static void collectForAllocation(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") IsolateThread isolateThread, NativeGCVMOperationData data, + NativeGCVMOperationWrapperData wrapperData) { + enqueue(OP_COLLECT_FOR_ALLOCATION, data, wrapperData); + } + + @Uninterruptible(reason = "Can be called from an unattached thread.") + @CEntryPoint(include = UseShenandoahGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForPossiblyUnattachedThread.class, epilogue = NoEpilogue.class) + public static void collectFull(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") IsolateThread isolateThread, NativeGCVMOperationData data, + NativeGCVMOperationWrapperData wrapperData) { + enqueue(OP_COLLECT_FULL, data, wrapperData); + } + + private static class ShenandoahVMOperation extends NativeGCVMOperation { + protected ShenandoahVMOperation(VMOperationInfo info, boolean isGC) { + super(info, isGC); + } + + @Override + public boolean executePrologue(NativeGCVMOperationData data) { + return ShenandoahLibrary.executeVMOperationPrologue(data); + } + + @Override + protected void operate0(NativeGCVMOperationData data) { + ShenandoahLibrary.executeVMOperationMain(data); + } + + @Override + public void executeEpilogue(NativeGCVMOperationData data) { + ShenandoahLibrary.executeVMOperationEpilogue(data); + } + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/UseShenandoahGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/UseShenandoahGC.java new file mode 100644 index 000000000000..074f9028e24c --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/UseShenandoahGC.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah; + +import java.util.function.BooleanSupplier; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + +import com.oracle.svm.core.SubstrateOptions; + +public class UseShenandoahGC implements BooleanSupplier { + @Platforms(Platform.HOSTED_ONLY.class) + public UseShenandoahGC() { + } + + @Override + public boolean getAsBoolean() { + return SubstrateOptions.useShenandoahGC(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/graal/ShenandoahAllocationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/graal/ShenandoahAllocationSupport.java new file mode 100644 index 000000000000..519ce6c4b52d --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/graal/ShenandoahAllocationSupport.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah.graal; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.gc.shared.graal.NativeGCAllocationSupport; +import com.oracle.svm.core.gc.shenandoah.ShenandoahConstants; +import com.oracle.svm.core.gc.shenandoah.ShenandoahHeap; +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahLibrary; +import com.oracle.svm.core.hub.DynamicHub; + +import jdk.graal.compiler.word.Word; + +public class ShenandoahAllocationSupport extends NativeGCAllocationSupport { + @Override + public Word getTLABInfo() { + return ShenandoahHeap.javaThreadTL.getAddress(); + } + + @Override + public int tlabTopOffset() { + return ShenandoahConstants.tlabTopOffset(); + } + + @Override + public int tlabEndOffset() { + return ShenandoahConstants.tlabEndOffset(); + } + + @Override + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.", callerMustBe = true, calleeMustBe = false) + protected Object allocateInstance0(DynamicHub hub) { + Word result = ShenandoahLibrary.allocateInstance(Word.objectToUntrackedPointer(hub)); + return result.toObject(); + } + + @Override + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.", callerMustBe = true, calleeMustBe = false) + protected Object allocateArray0(int length, DynamicHub hub) { + Word result = ShenandoahLibrary.allocateArray(Word.objectToUntrackedPointer(hub), length); + return result.toObject(); + } + + @Override + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.", callerMustBe = true, calleeMustBe = false) + protected Object allocateStoredContinuation0(int length, DynamicHub hub) { + Word result = ShenandoahLibrary.allocateStoredContinuation(Word.objectToUntrackedPointer(hub), length); + return result.toObject(); + } + + @Override + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.", callerMustBe = true, calleeMustBe = false) + protected Object allocatePod0(int length, DynamicHub hub) { + Word result = ShenandoahLibrary.allocatePod(Word.objectToUntrackedPointer(hub), length); + return result.toObject(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/graal/ShenandoahBarrierSetProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/graal/ShenandoahBarrierSetProvider.java new file mode 100644 index 000000000000..4c5d6e742be0 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/graal/ShenandoahBarrierSetProvider.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah.graal; + +import com.oracle.svm.core.heap.BarrierSetProvider; +import com.oracle.svm.core.heap.ReferenceInternals; + +import jdk.graal.compiler.nodes.gc.BarrierSet; +import jdk.vm.ci.meta.MetaAccessProvider; +import jdk.vm.ci.meta.ResolvedJavaField; + +public class ShenandoahBarrierSetProvider implements BarrierSetProvider { + @Override + public BarrierSet createBarrierSet(MetaAccessProvider metaAccess) { + ResolvedJavaField referentField = ReferenceInternals.getReferentField(metaAccess); + return new SubstrateShenandoahBarrierSet(metaAccess.lookupJavaType(Object[].class), referentField); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/graal/SubstrateShenandoahBarrierSet.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/graal/SubstrateShenandoahBarrierSet.java new file mode 100644 index 000000000000..a337b1508983 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/graal/SubstrateShenandoahBarrierSet.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah.graal; + +import org.graalvm.word.LocationIdentity; + +import com.oracle.svm.core.StaticFieldsSupport; + +import jdk.graal.compiler.core.common.memory.BarrierType; +import jdk.graal.compiler.core.common.type.Stamp; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.extended.RawStoreNode; +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahBarrierSet; +import jdk.graal.compiler.nodes.memory.FixedAccessNode; +import jdk.graal.compiler.nodes.spi.CoreProviders; +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.ResolvedJavaField; +import jdk.vm.ci.meta.ResolvedJavaType; + +public class SubstrateShenandoahBarrierSet extends ShenandoahBarrierSet { + public SubstrateShenandoahBarrierSet(ResolvedJavaType objectArrayType, ResolvedJavaField referentField) { + super(objectArrayType, referentField); + } + + /** + * Static fields in SVM are represented as two arrays in the native image heap: one for Object + * fields and one for all primitive fields (see {@link StaticFieldsSupport}). Therefore, we must + * emit array write barriers for static fields. + */ + @Override + public BarrierType fieldWriteBarrierType(ResolvedJavaField field, JavaKind storageKind) { + if (field.isStatic() && storageKind == JavaKind.Object) { + return arrayWriteBarrierType(storageKind); + } + return BarrierType.NONE; + } + + @Override + public BarrierType arrayWriteBarrierType(JavaKind storageKind) { + return BarrierType.NONE; + } + + @Override + public BarrierType postAllocationInitBarrier(BarrierType original) { + return BarrierType.NONE; + } + + @Override + public BarrierType readBarrierType(LocationIdentity location, ValueNode address, Stamp loadStamp) { + return BarrierType.NONE; + } + + @Override + public BarrierType writeBarrierType(RawStoreNode store) { + return BarrierType.NONE; + } + + @Override + public BarrierType fieldReadBarrierType(ResolvedJavaField field, JavaKind storageKind) { + return BarrierType.NONE; + } + + @Override + public BarrierType readWriteBarrier(ValueNode object, ValueNode value) { + return BarrierType.NONE; + } + + @Override + public void addBarriers(FixedAccessNode n, CoreProviders context) { + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/include/shenandoahGCStructs.h b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/include/shenandoahGCStructs.h new file mode 100644 index 000000000000..055c4562d793 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/include/shenandoahGCStructs.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SVM_SHENANDOAH_GC_STRUCTS_H +#define SVM_SHENANDOAH_GC_STRUCTS_H + +#include + +#ifdef __cplusplus +namespace svm_gc { +#endif + +struct ShenandoahHeapOptions { + size_t max_heap_size; + size_t heap_address_space_size; + size_t physical_memory_size; +}; + +struct ShenandoahInitState { + void* card_table_address; + int tlab_top_offset; + int tlab_end_offset; + int card_table_shift; + int log_of_heap_region_grain_bytes; + int java_thread_size; + int vm_operation_data_size; + int vm_operation_wrapper_data_size; + char dirty_card_value; +}; + +struct ShenandoahRegionBoundaries { + u_char *bottom; + u_char *top; +}; + +struct ShenandoahRegionInfo { + u_char *bottom; + u_char *top; + u_char *end; + char region_type; +}; + +struct ShenandoahInternalState { + unsigned int total_collections; + unsigned int full_collections; + + void* card_table_start; + size_t card_table_size; +}; + +#ifdef __cplusplus +} // namespace svm_gc +#endif + +#endif // SVM_SHENANDOAH_GC_STRUCTS_H diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/nativelib/ShenandoahHeaderFiles.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/nativelib/ShenandoahHeaderFiles.java new file mode 100644 index 000000000000..b31f72049b59 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/nativelib/ShenandoahHeaderFiles.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah.nativelib; + +import java.util.Collections; +import java.util.List; + +import org.graalvm.nativeimage.c.CContext; + +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.c.ProjectHeaderFile; + +/** Determines which header files are included when building a native image that uses Shenandoah. */ +public class ShenandoahHeaderFiles implements CContext.Directives { + @Override + public boolean isInConfiguration() { + return SubstrateOptions.useShenandoahGC(); + } + + @Override + public List getHeaderFiles() { + return Collections.singletonList(ProjectHeaderFile.resolve("", "include/shenandoahGCStructs.h")); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/nativelib/ShenandoahLibrary.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/nativelib/ShenandoahLibrary.java new file mode 100644 index 000000000000..d0140e6303bf --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/nativelib/ShenandoahLibrary.java @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah.nativelib; + +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.c.CContext; +import org.graalvm.nativeimage.c.function.CFunction; +import org.graalvm.nativeimage.c.function.CFunction.Transition; +import org.graalvm.nativeimage.c.function.CFunctionPointer; +import org.graalvm.nativeimage.c.type.CCharPointer; +import org.graalvm.nativeimage.c.type.CCharPointerPointer; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; + +import com.oracle.svm.core.c.function.CFunctionOptions; +import com.oracle.svm.core.code.CodeInfo; +import com.oracle.svm.core.gc.shared.NativeGCVMOperationSupport.NativeGCVMOperationData; +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahStructs.ShenandoahHeapOptions; +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahStructs.ShenandoahInitState; +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahStructs.ShenandoahInternalState; +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahStructs.ShenandoahRegionBoundaries; +import com.oracle.svm.core.gc.shenandoah.nativelib.ShenandoahStructs.ShenandoahRegionInfo; + +import jdk.graal.compiler.word.Word; + +/** + * This class contains all methods that native-image uses for calling Shenandoah C++ code. It is + * essential to use the correct kind of transition as we will either end up with deadlocks or + * incorrect behavior otherwise. In the C++ code, the expected transition for each method is + * documented in more detail. So, when changing the transition of a method, make sure to update that + * documentation as well. + * + * The method {@link #parseOptions} passes a version to Shenandoah. This is used to verify if the + * native-image really uses a compatible version. We increment this version whenever we change any + * method below. + */ +@CContext(value = ShenandoahLibraryDependencies.class) +public class ShenandoahLibrary { + // GraalVM major, GraalVM minor, interface revision + public static final int VERSION = 250101; + + @CFunction(value = "svm_gc_parse_options", transition = Transition.NO_TRANSITION) + public static native void parseOptions(int nativeImageVersion, int argc, CCharPointerPointer argv, CCharPointer imageBuildHostedArguments, CCharPointer imageBuildRuntimeArguments, + UnsignedWord maxHeapAddressSpaceSize, UnsignedWord heapBaseAlignment, UnsignedWord nullRegionSize, UnsignedWord imageHeapSize, + int compressedReferenceShift, boolean isContainerized, long containerMemoryLimitInBytes, int containerActiveProcessorCount, ShenandoahHeapOptions result); + + @CFunction(value = "svm_gc_create", transition = Transition.NO_TRANSITION) + public static native ShenandoahInitState create(IsolateThread isolateThread, Pointer heapBase, + int closedImageHeapRegions, int openImageHeapRegions, Word imageHeapRegionTypes, Word imageHeapRegionFreeSpaces, + Word dynamicHubClass, Word fillerObjectClass, Word fillerArrayClass, Word stringClass, Word systemClass, + Word staticObjectFields, Word staticPrimitiveFields, Word vmOperationThread, Word safepoint, Word runtimeCodeInfoMemory, + int referenceMapCompressedOffsetShift, Word threadLocalsReferenceMap, + Word classesAssumedReachableForCodeUnloading, boolean perfDataSupport, boolean useStringInlining, boolean closedTypeWorld, + boolean useInterfaceHashing, int interfaceHashingMaxId, int dynamicHubHashingInterfaceMask, int dynamicHubHashingShiftOffset, + Word offsets, int offsetsLength, + CFunctionPointer collectForAllocationOp, CFunctionPointer collectFullOp, + CFunctionPointer waitForVMOperationExecutionStatus, CFunctionPointer updateVMOperationExecutionStatus, CFunctionPointer isVMOperationFinished, + CFunctionPointer fetchThreadStackFrames, CFunctionPointer freeThreadStackFrames, + CFunctionPointer fetchContinuationStackFrames, CFunctionPointer freeContinuationStackFrames, + CFunctionPointer fetchCodeInfos, CFunctionPointer freeCodeInfos, CFunctionPointer cleanRuntimeCodeCache, + CFunctionPointer transitionVMToNative, CFunctionPointer fastTransitionNativeToVM, CFunctionPointer slowTransitionNativeToVM); + + @CFunction(value = "svm_gc_update_option_value", transition = Transition.NO_TRANSITION) + public static native void updateOptionValue(Word optionName, long value); + + @CFunction(value = "svm_gc_teardown", transition = Transition.NO_TRANSITION) + public static native boolean tearDown(); + + @CFunction(value = "svm_gc_attach_thread", transition = Transition.NO_TRANSITION) + public static native void attachThread(IsolateThread thread); + + @CFunction(value = "svm_gc_detach_thread", transition = Transition.NO_TRANSITION) + public static native void detachThread(IsolateThread thread); + + @CFunction(value = "svm_gc_retire_tlab", transition = Transition.NO_TRANSITION) + public static native void retireTlab(); + + @CFunction(value = "svm_gc_prepare_for_safepoint", transition = Transition.NO_TRANSITION) + public static native void prepareForSafepoint(); + + @CFunction(value = "svm_gc_end_safepoint", transition = Transition.NO_TRANSITION) + public static native void endSafepoint(); + + @CFunctionOptions(transition = CFunctionOptions.Transition.TO_VM) + @CFunction(value = "svm_gc_collect") + public static native void collect(int gcCause); + + @CFunction(value = "svm_gc_execute_vm_operation_prologue", transition = Transition.TO_NATIVE) + public static native boolean executeVMOperationPrologue(NativeGCVMOperationData data); + + @CFunction(value = "svm_gc_execute_vm_operation_main", transition = Transition.TO_NATIVE) + public static native void executeVMOperationMain(NativeGCVMOperationData data); + + @CFunction(value = "svm_gc_execute_vm_operation_epilogue", transition = Transition.TO_NATIVE) + public static native void executeVMOperationEpilogue(NativeGCVMOperationData data); + + @CFunctionOptions(transition = CFunctionOptions.Transition.TO_VM) + @CFunction(value = "svm_gc_allocate_instance") + public static native Word allocateInstance(Word hub); + + @CFunctionOptions(transition = CFunctionOptions.Transition.TO_VM) + @CFunction(value = "svm_gc_allocate_array") + public static native Word allocateArray(Word hub, int length); + + @CFunctionOptions(transition = CFunctionOptions.Transition.TO_VM) + @CFunction(value = "svm_gc_allocate_stack_chunk") + public static native Word allocateStoredContinuation(Word hub, int length); + + @CFunctionOptions(transition = CFunctionOptions.Transition.TO_VM) + @CFunction(value = "svm_gc_allocate_pod") + public static native Word allocatePod(Word hub, int length); + + @CFunction(value = "svm_gc_pin_object", transition = Transition.NO_TRANSITION) + public static native void pinObject(Word object); + + @CFunction(value = "svm_gc_unpin_object", transition = Transition.NO_TRANSITION) + public static native void unpinObject(Word object); + + @CFunction(value = "svm_gc_pre_write_barrier", transition = Transition.NO_TRANSITION) + public static native void preWriteBarrierStub(Word object); + + @CFunction(value = "svm_gc_post_write_barrier", transition = Transition.NO_TRANSITION) + public static native void postWriteBarrierStub(Word cardAddress); + + @CFunction(value = "svm_gc_dirty_all_references_of", transition = Transition.NO_TRANSITION) + public static native void dirtyAllReferencesOf(Word obj); + + @CFunction(value = "svm_gc_millis_since_last_whole_heap_examined", transition = Transition.NO_TRANSITION) + public static native long getMillisSinceLastWholeHeapExamined(); + + @CFunction(value = "svm_gc_has_reference_pending_list", transition = Transition.TO_NATIVE) + public static native boolean hasReferencePendingList(); + + @CFunctionOptions(transition = CFunctionOptions.Transition.TO_VM) + @CFunction(value = "svm_gc_get_and_clear_reference_pending_list") + public static native Word getAndClearReferencePendingList(); + + @CFunction(value = "svm_gc_get_reference_pending_list_wakeup_count", transition = Transition.NO_TRANSITION) + public static native long getReferencePendingListWakeupCount(); + + @CFunction(value = "svm_gc_wait_for_reference_pending_list", transition = Transition.TO_NATIVE) + public static native boolean waitForReferencePendingList(long initialWakeupCount); + + @CFunction(value = "svm_gc_wake_up_reference_pending_list_waiters", transition = Transition.TO_NATIVE) + public static native void wakeUpReferencePendingListWaiters(); + + @CFunction(value = "svm_gc_get_region_boundaries", transition = Transition.TO_NATIVE) + public static native void getRegionBoundaries(ShenandoahRegionBoundaries regionBoundaries); + + @CFunction(value = "svm_gc_register_object_fields", transition = Transition.NO_TRANSITION) + public static native void registerObjectFields(CodeInfo codeInfo); + + @CFunction(value = "svm_gc_register_code_constants", transition = Transition.NO_TRANSITION) + public static native void registerCodeConstants(CodeInfo codeInfo); + + @CFunction(value = "svm_gc_register_frame_metadata", transition = Transition.NO_TRANSITION) + public static native void registerFrameMetadata(CodeInfo codeInfo); + + @CFunction(value = "svm_gc_register_deopt_metadata", transition = Transition.NO_TRANSITION) + public static native void registerDeoptMetadata(CodeInfo codeInfo); + + @CFunction(value = "svm_gc_get_internal_state", transition = Transition.NO_TRANSITION) + public static native void getGCInternalState(ShenandoahInternalState result); + + @CFunction(value = "svm_gc_get_current_thread_name", transition = Transition.NO_TRANSITION) + public static native CCharPointer getCurrentThreadName(); + + @CFunction(value = "svm_gc_get_region_info", transition = Transition.NO_TRANSITION) + public static native boolean getRegionInfo(int regionIndex, ShenandoahRegionInfo result); + + @CFunction(value = "svm_gc_get_thread_allocated_memory", transition = Transition.NO_TRANSITION) + public static native long getThreadAllocatedMemory(IsolateThread isolateThread); + + @CFunctionOptions(transition = CFunctionOptions.Transition.TO_VM) + @CFunction(value = "svm_gc_get_used_memory") + public static native long getUsedMemory(); + + @CFunctionOptions(transition = CFunctionOptions.Transition.TO_VM) + @CFunction(value = "svm_gc_get_free_memory") + public static native long getFreeMemory(); + + @CFunction(value = "svm_gc_get_total_memory", transition = Transition.NO_TRANSITION) + public static native long getTotalMemory(); + + @CFunction(value = "svm_gc_get_max_memory", transition = Transition.NO_TRANSITION) + public static native long getMaxMemory(); + + @CFunction(value = "svm_gc_get_used_memory_after_last_gc", transition = Transition.NO_TRANSITION) + public static native UnsignedWord getUsedMemoryAfterLastGC(); +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/nativelib/ShenandoahLibraryDependencies.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/nativelib/ShenandoahLibraryDependencies.java new file mode 100644 index 000000000000..04a2c06f0c5e --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/nativelib/ShenandoahLibraryDependencies.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah.nativelib; + +import java.util.ArrayList; +import java.util.List; + +import org.graalvm.nativeimage.c.CContext; + +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.c.libc.LibCBase; +import com.oracle.svm.core.c.libc.MuslLibC; +import com.oracle.svm.core.gc.shenandoah.ShenandoahOptions; +import com.oracle.svm.core.heap.ReferenceAccess; + +/** + * Determines which native libraries are included when building a native image that uses Shenandoah. + */ +public class ShenandoahLibraryDependencies implements CContext.Directives { + @Override + public boolean isInConfiguration() { + return SubstrateOptions.useShenandoahGC(); + } + + @Override + public List getLibraries() { + boolean useCompressedReferences = ReferenceAccess.singleton().getCompressionShift() > 0; + String compressedReferenceSuffix = useCompressedReferences ? "-cr" : "-ur"; + String toolchainSuffix = LibCBase.targetLibCIs(MuslLibC.class) ? "-" + LibCBase.singleton().getName() : ""; + + ArrayList libraries = new ArrayList<>(); + libraries.add("shenandoahgc" + ShenandoahOptions.getDebugLevel().getLibSuffix() + toolchainSuffix + compressedReferenceSuffix); + libraries.addAll(List.of("dl", "m", "pthread")); + libraries.addAll(getCppLibs()); + + return libraries; + } + + private static List getCppLibs() { + return List.of("stdc++"); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/nativelib/ShenandoahStructs.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/nativelib/ShenandoahStructs.java new file mode 100644 index 000000000000..79ae19af34cb --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shenandoah/nativelib/ShenandoahStructs.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shenandoah.nativelib; + +import org.graalvm.nativeimage.c.CContext; +import org.graalvm.nativeimage.c.struct.CField; +import org.graalvm.nativeimage.c.struct.CStruct; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; + +import jdk.graal.compiler.word.Word; + +/** + * Defines all data structures that are imported from Shenandoah-specific header files. + */ +@CContext(ShenandoahHeaderFiles.class) +public class ShenandoahStructs { + @CStruct(addStructKeyword = true) + public interface ShenandoahHeapOptions extends PointerBase { + @CField("max_heap_size") + UnsignedWord maxHeapSize(); + + @CField("heap_address_space_size") + UnsignedWord heapAddressSpaceSize(); + + @CField("physical_memory_size") + UnsignedWord physicalMemorySize(); + } + + @CStruct(addStructKeyword = true) + public interface ShenandoahInitState extends PointerBase { + @CField("card_table_address") + Word cardTableAddress(); + + @CField("tlab_top_offset") + int tlabTopOffset(); + + @CField("tlab_end_offset") + int tlabEndOffset(); + + @CField("card_table_shift") + int cardTableShift(); + + @CField("log_of_heap_region_grain_bytes") + int logOfHeapRegionGrainBytes(); + + @CField("java_thread_size") + int javaThreadSize(); + + @CField("vm_operation_data_size") + int vmOperationDataSize(); + + @CField("vm_operation_wrapper_data_size") + int vmOperationWrapperDataSize(); + + @CField("dirty_card_value") + byte dirtyCardValue(); + } + + @CStruct(addStructKeyword = true) + public interface ShenandoahRegionBoundaries extends PointerBase { + ShenandoahRegionBoundaries addressOf(long index); + + @CField + Word bottom(); + + @CField + Word top(); + } + + @CStruct(addStructKeyword = true) + public interface ShenandoahRegionInfo extends PointerBase { + @CField + Word bottom(); + + @CField + Word top(); + + @CField + Word end(); + + @CField("region_type") + byte regionType(); + } + + @CStruct(addStructKeyword = true) + public interface ShenandoahInternalState extends PointerBase { + @CField("total_collections") + int totalCollections(); + + @CField("full_collections") + int fullCollections(); + + @CField("card_table_start") + Pointer cardTableStart(); + + @CField("card_table_size") + UnsignedWord cardTableSize(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/GC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/GC.java index f2eed9860810..4078fb70089c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/GC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/GC.java @@ -24,6 +24,9 @@ */ package com.oracle.svm.core.heap; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + public interface GC { /** Cause a collection of the Heap's choosing. */ void collect(GCCause cause); @@ -41,5 +44,6 @@ public interface GC { String getName(); /** Human-readable default heap size. */ + @Platforms(Platform.HOSTED_ONLY.class) String getDefaultMaxHeapSize(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SystemPropertiesSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SystemPropertiesSupport.java index e42f6270181f..4fd9a19f4e05 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SystemPropertiesSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SystemPropertiesSupport.java @@ -128,7 +128,7 @@ protected SystemPropertiesSupport() { initializeProperty("java.vendor", vm.vendor); initializeProperty("java.vendor.url", vm.vendorUrl); initializeProperty("java.vendor.version", vm.vendorVersion); - assert vm.info.equals(vm.info.toLowerCase(Locale.ROOT)) : "java.vm.info should not contain uppercase characters"; + assert vm.info.equals(vm.info.toLowerCase(Locale.ROOT)) : "java.vm.info should not contain uppercase characters: " + vm.info; initializeProperty("java.vm.info", vm.info); initializeProperty("java.vm.name", "Substrate VM"); initializeProperty("java.vm.vendor", vm.vendor); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/GCOptionValue.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/GCOptionValue.java index 53e17d0eabdd..f8403a7cfe6d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/GCOptionValue.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/GCOptionValue.java @@ -35,8 +35,9 @@ import jdk.graal.compiler.options.OptionsContainer; public enum GCOptionValue { - SERIAL("serial"), - EPSILON("epsilon"), + Serial("serial"), + Epsilon("epsilon"), + Shenandoah("shenandoah"), G1("G1"); private static Set supportedValues = null; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/HostedOptionKey.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/HostedOptionKey.java index b4cc5b2ea22f..7118cbefbcd8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/HostedOptionKey.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/HostedOptionKey.java @@ -40,16 +40,16 @@ * @see com.oracle.svm.core.option */ public class HostedOptionKey extends OptionKey implements SubstrateOptionKey { - private final Consumer> validation; + private final Consumer> buildTimeValidation; private OptionOrigin lastOrigin; public HostedOptionKey(T defaultValue) { this(defaultValue, null); } - public HostedOptionKey(T defaultValue, Consumer> validation) { + public HostedOptionKey(T defaultValue, Consumer> buildTimeValidation) { super(defaultValue); - this.validation = validation; + this.buildTimeValidation = buildTimeValidation; } /** @@ -103,8 +103,8 @@ public void update(EconomicMap, Object> values, Object boxedValue) @Override public void validate() { - if (validation != null) { - validation.accept(this); + if (buildTimeValidation != null) { + buildTimeValidation.accept(this); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java index d04accb06a30..c52bcce85ea8 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java @@ -115,8 +115,10 @@ public static void setDefaultIfEmpty() { CompressEncoding compressEncoding = new CompressEncoding(SubstrateOptions.SpawnIsolates.getValue() ? 1 : 0, 0); ImageSingletons.add(CompressEncoding.class, compressEncoding); - ObjectLayout objectLayout = createObjectLayout(IdentityHashMode.TYPE_SPECIFIC); - ImageSingletons.add(ObjectLayout.class, objectLayout); + if (!ImageSingletons.contains(ObjectLayout.class)) { + ObjectLayout objectLayout = createObjectLayout(IdentityHashMode.TYPE_SPECIFIC); + ImageSingletons.add(ObjectLayout.class, objectLayout); + } ImageSingletons.add(HybridLayoutSupport.class, new HybridLayoutSupport()); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/VMFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/VMFeature.java index 5f3d826e0ed1..9311098db651 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/VMFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/VMFeature.java @@ -25,6 +25,7 @@ package com.oracle.svm.hosted; import java.nio.file.Path; +import java.util.List; import java.util.Locale; import java.util.stream.Collectors; @@ -65,13 +66,13 @@ protected String determineVMInfo() { return getSelectedGCName(); } - protected static final String getSelectedGCName() { - if (SubstrateOptions.useSerialGC()) { - return "serial gc"; - } else if (SubstrateOptions.useEpsilonGC()) { - return "epsilon gc"; + protected static String getSelectedGCName() { + List values = SubstrateOptions.SupportedGCs.getValue().values(); + if (values.isEmpty()) { + return "no gc"; } else { - return "unknown gc"; + assert values.size() == 1; + return values.getFirst().toLowerCase(Locale.ROOT) + " gc"; } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ValidateGCOptionFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ValidateGCOptionFeature.java index dd3af52b8b91..882f5a2d3647 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ValidateGCOptionFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ValidateGCOptionFeature.java @@ -57,7 +57,6 @@ private static void validateGCOption() { } Set possibleValues = GCOptionValue.possibleValues(); - if (values.isEmpty()) { throw UserError.abort("Invalid option '--gc'. No GC specified. %s", getGCErrorReason(possibleValues)); } @@ -69,9 +68,10 @@ private static void validateGCOption() { } } - // Check that the specified combination is valid. - if (values.size() != 1) { - throw UserError.abort("%s is an invalid combination of GCs for option '--gc'.", StringUtil.joinSingleQuoted(values)); + // At the moment, exactly one GC must be selected at build-time. + if (values.size() > 1) { + String string = String.join(SubstrateOptions.SupportedGCs.getValue().getDelimiter(), values); + throw UserError.invalidOptionValue(SubstrateOptions.SupportedGCs, string, "Only one garbage collector can be used at a time."); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/OWNERS.toml b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/OWNERS.toml new file mode 100644 index 000000000000..1f7d0b0a8706 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/OWNERS.toml @@ -0,0 +1,7 @@ +[[rule]] +files = "*" +all = [ + "christian.haeubl@oracle.com", +] +any = [ +] diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shared/NativeGCAccessedFields.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shared/NativeGCAccessedFields.java new file mode 100644 index 000000000000..e1e6824d1607 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shared/NativeGCAccessedFields.java @@ -0,0 +1,281 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.gc.shared; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.function.BooleanSupplier; + +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.hosted.Feature.BeforeCompilationAccess; + +import com.oracle.graal.pointsto.meta.AnalysisField; +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.code.CodeInfoOffsets; +import com.oracle.svm.core.config.ObjectLayout; +import com.oracle.svm.core.jvmstat.PerfManager; +import com.oracle.svm.core.thread.ContinuationSupport; +import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.thread.VMThreads.StatusSupport; +import com.oracle.svm.core.threadlocal.FastThreadLocalBytes; +import com.oracle.svm.core.threadlocal.FastThreadLocalObject; +import com.oracle.svm.core.util.ByteArrayReader; +import com.oracle.svm.core.util.VMError; +import com.oracle.svm.hosted.FeatureImpl.BeforeAnalysisAccessImpl; +import com.oracle.svm.hosted.config.DynamicHubLayout; +import com.oracle.svm.hosted.thread.VMThreadFeature; +import com.oracle.svm.util.ClassUtil; + +import jdk.graal.compiler.core.common.util.UnsafeArrayTypeWriter; +import jdk.graal.compiler.word.Word; + +/** + * For performance reasons, the GC-related C++ code accesses certain static/thread-local/instance + * fields directly. This class computes the offsets of those fields and writes them to a byte array + * that is then passed to the C++ code upon initialization. It also ensures that the static analysis + * considers all those fields as reachable, even if they are only accessed on the C++ side. + */ +public class NativeGCAccessedFields { + public static final BooleanSupplier ALWAYS = new Always(); + public static final BooleanSupplier USE_PERF_DATA = new UsePerfData(); + public static final BooleanSupplier CLOSED_TYPE_WORLD_HUB_LAYOUT = SubstrateOptions::useClosedTypeWorldHubLayout; + public static final BooleanSupplier OPEN_TYPE_WORLD_HUB_LAYOUT = new OpenTypeWorldHubLayout(); + + /** Marks the given fields as accessed and their classes as used. */ + public static void markAsAccessed(BeforeAnalysisAccessImpl access, AccessedClass[] accessedClasses) { + for (AccessedClass accessedClass : accessedClasses) { + markAsAccessed(access, accessedClass.clazz, accessedClass.fields); + } + } + + /** Writes all offsets as integer values (4 bytes per value) into a byte array. */ + public static byte[] writeOffsets(BeforeCompilationAccess access, int markWordOffset, FastThreadLocalBytes nativeJavaThreadTL, FastThreadLocalObject podReferenceMapTL, + AccessedClass[] accessedClasses) { + UnsafeArrayTypeWriter buffer = UnsafeArrayTypeWriter.create(ByteArrayReader.supportsUnalignedMemoryAccess()); + + writeObjectLayoutOffsets(buffer, markWordOffset); + writeThreadLocalOffsets(buffer, nativeJavaThreadTL, podReferenceMapTL); + writeCodeInfoOffsets(buffer); + for (AccessedClass accessedClass : accessedClasses) { + writeFieldOffsets(access, buffer, accessedClass.clazz, accessedClass.fields); + } + + return buffer.toArray(); + } + + private static void markAsAccessed(BeforeAnalysisAccessImpl access, Class clazz, AccessedField[] accessedFields) { + access.registerAsUsed(clazz); + + Field[] fields = clazz.getDeclaredFields(); + for (AccessedField accessedField : accessedFields) { + if (accessedField.shouldInclude()) { + boolean found = false; + for (Field field : fields) { + if (accessedField.matches(field)) { + AnalysisField analysisField = access.getMetaAccess().lookupJavaField(field); + accessedField.registerAsAccessed(access, analysisField); + found = true; + break; + } + } + if (!found) { + throw VMError.shouldNotReachHere("Could not find field " + ClassUtil.getUnqualifiedName(clazz) + "." + accessedField.name); + } + } + } + } + + private static void writeObjectLayoutOffsets(UnsafeArrayTypeWriter buffer, int markWordOffset) { + ObjectLayout objectLayout = ImageSingletons.lookup(ObjectLayout.class); + assert objectLayout.getFirstFieldOffset() <= objectLayout.getArrayLengthOffset(); + int objBase = objectLayout.getFirstFieldOffset(); + int minObjectSize = objectLayout.alignUp(objBase); + + // object layout + buffer.putS4(markWordOffset); + buffer.putS4(objectLayout.getHubOffset()); + buffer.putS4(objectLayout.getAlignment()); + buffer.putS4(minObjectSize); + buffer.putS4(objBase); + buffer.putS4(objectLayout.getArrayLengthOffset()); + + // dynamic hub + buffer.putS4(SubstrateOptions.useClosedTypeWorldHubLayout() ? DynamicHubLayout.singleton().getClosedTypeWorldTypeCheckSlotsOffset() : -1); + } + + private static void writeThreadLocalOffsets(UnsafeArrayTypeWriter buffer, FastThreadLocalBytes nativeJavaThreadTL, FastThreadLocalObject podReferenceMapTL) { + VMThreadFeature vmThreadMtFeature = ImageSingletons.lookup(VMThreadFeature.class); + + buffer.putS4(vmThreadMtFeature.offsetOf(VMThreads.nextTL)); + buffer.putS4(vmThreadMtFeature.offsetOf(nativeJavaThreadTL)); + buffer.putS4(vmThreadMtFeature.offsetOf(StatusSupport.statusTL)); + buffer.putS4(vmThreadMtFeature.offsetOf(podReferenceMapTL)); + } + + private static void writeCodeInfoOffsets(UnsafeArrayTypeWriter buffer) { + buffer.putS4(CodeInfoOffsets.objectFields()); + buffer.putS4(CodeInfoOffsets.state()); + buffer.putS4(CodeInfoOffsets.gcData()); + buffer.putS4(CodeInfoOffsets.objectConstants()); + buffer.putS4(CodeInfoOffsets.deoptimizationObjectConstants()); + buffer.putS4(CodeInfoOffsets.stackReferenceMapEncoding()); + buffer.putS4(CodeInfoOffsets.codeStart()); + buffer.putS4(CodeInfoOffsets.codeConstantsReferenceMapEncoding()); + buffer.putS4(CodeInfoOffsets.codeConstantsReferenceMapIndex()); + buffer.putS4(CodeInfoOffsets.areAllObjectsInImageHeap()); + } + + private static void writeFieldOffsets(BeforeCompilationAccess access, UnsafeArrayTypeWriter buffer, Class clazz, AccessedField[] accessedFields) { + Field[] declaredFields = clazz.getDeclaredFields(); + for (AccessedField accessedField : accessedFields) { + if (accessedField.shouldInclude()) { + boolean found = false; + for (Field field : declaredFields) { + if (accessedField.matches(field)) { + buffer.putS4(access.objectFieldOffset(field)); + found = true; + break; + } + } + if (!found) { + throw VMError.shouldNotReachHere("Could not find field " + ClassUtil.getUnqualifiedName(clazz) + "." + accessedField.name); + } + } else { + // Add a dummy value so that the size of the structure is always the same. + buffer.putS4(-1); + } + } + } + + public enum FieldAccessKind { + READ, + READ_WRITE + } + + public static final class AccessedClass { + Class clazz; + AccessedField[] fields; + + public AccessedClass(Class clazz, AccessedField... fields) { + this.clazz = clazz; + this.fields = fields; + } + } + + public abstract static class AccessedField { + final String name; + final FieldAccessKind accessKind; + final BooleanSupplier include; + + AccessedField(String name, FieldAccessKind accessKind, BooleanSupplier include) { + this.name = name; + this.accessKind = accessKind; + this.include = include; + } + + public abstract boolean matches(Field field); + + public boolean shouldInclude() { + return include.getAsBoolean(); + } + + void registerAsAccessed(BeforeAnalysisAccessImpl access, AnalysisField field) { + if (accessKind == FieldAccessKind.READ) { + access.registerAsRead(field, "it is read by the GC"); + } else { + assert accessKind == FieldAccessKind.READ_WRITE; + access.registerAsAccessed(field, "it is accessed by the GC"); + } + } + } + + public static final class InstanceField extends AccessedField { + public InstanceField(String name, FieldAccessKind accessKind) { + this(name, accessKind, ALWAYS); + } + + public InstanceField(String name, FieldAccessKind accessKind, BooleanSupplier include) { + super(name, accessKind, include); + } + + @Override + public boolean matches(Field field) { + return !Modifier.isStatic(field.getModifiers()) && name.equals(field.getName()); + } + } + + public static final class StaticField extends AccessedField { + public StaticField(String name, FieldAccessKind accessKind) { + this(name, accessKind, ALWAYS); + } + + StaticField(String name, FieldAccessKind accessKind, BooleanSupplier include) { + super(name, accessKind, include); + } + + @Override + public boolean matches(Field field) { + return Modifier.isStatic(field.getModifiers()) && name.equals(field.getName()); + } + + @Override + public void registerAsAccessed(BeforeAnalysisAccessImpl access, AnalysisField field) { + /* + * Regardless of the field access kind, we always mark the field as accessed. This is + * necessary to ensure that memory gets reserved for the static field as other SVM code + * assumes that read-only static fields can always be constant-folded. + */ + access.registerAsAccessed(field, "it is accessed by the GC"); + } + } + + public static final class Always implements BooleanSupplier { + @Override + public boolean getAsBoolean() { + return true; + } + } + + public static final class UsePerfData implements BooleanSupplier { + @Override + public boolean getAsBoolean() { + return ImageSingletons.contains(PerfManager.class); + } + } + + public static final class ContinuationsSupported implements BooleanSupplier { + @Override + public boolean getAsBoolean() { + return ContinuationSupport.isSupported(); + } + } + + public static final class OpenTypeWorldHubLayout implements BooleanSupplier { + @Override + public boolean getAsBoolean() { + return !SubstrateOptions.useClosedTypeWorldHubLayout(); + } + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahAccessedFields.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahAccessedFields.java new file mode 100644 index 000000000000..c5504be38255 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahAccessedFields.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.gc.shenandoah; + +import static com.oracle.svm.hosted.gc.shared.NativeGCAccessedFields.CLOSED_TYPE_WORLD_HUB_LAYOUT; +import static com.oracle.svm.hosted.gc.shared.NativeGCAccessedFields.ContinuationsSupported; +import static com.oracle.svm.hosted.gc.shared.NativeGCAccessedFields.OPEN_TYPE_WORLD_HUB_LAYOUT; +import static com.oracle.svm.hosted.gc.shared.NativeGCAccessedFields.USE_PERF_DATA; + +import java.lang.ref.SoftReference; +import java.util.concurrent.atomic.AtomicInteger; + +import com.oracle.svm.core.code.RuntimeCodeInfoMemory; +import com.oracle.svm.core.heap.StoredContinuation; +import com.oracle.svm.core.heap.Target_java_lang_ref_Reference; +import com.oracle.svm.core.hub.DynamicHub; +import com.oracle.svm.core.jvmstat.PerfLong; +import com.oracle.svm.core.jvmstat.PerfStringVariable; +import com.oracle.svm.core.thread.PlatformThreads; +import com.oracle.svm.core.thread.Safepoint; +import com.oracle.svm.core.thread.VMOperationControl.VMOperationThread; +import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.hosted.gc.shared.NativeGCAccessedFields.AccessedClass; +import com.oracle.svm.hosted.gc.shared.NativeGCAccessedFields.FieldAccessKind; +import com.oracle.svm.hosted.gc.shared.NativeGCAccessedFields.InstanceField; +import com.oracle.svm.hosted.gc.shared.NativeGCAccessedFields.StaticField; + +public class ShenandoahAccessedFields { + static final AccessedClass[] ACCESSED_CLASSES = { + new AccessedClass(VMThreads.class, + new StaticField("head", FieldAccessKind.READ), + new StaticField("numAttachedThreads", FieldAccessKind.READ)), + new AccessedClass(PlatformThreads.class, + new StaticField("nonDaemonThreads", FieldAccessKind.READ)), + new AccessedClass(VMOperationThread.class, + new InstanceField("isolateThread", FieldAccessKind.READ)), + new AccessedClass(Safepoint.class, + new InstanceField("safepointState", FieldAccessKind.READ), + new InstanceField("safepointId", FieldAccessKind.READ)), + new AccessedClass(DynamicHub.class, + new InstanceField("name", FieldAccessKind.READ), + new InstanceField("hubType", FieldAccessKind.READ), + new InstanceField("referenceType", FieldAccessKind.READ), + new InstanceField("layoutEncoding", FieldAccessKind.READ), + new InstanceField("componentType", FieldAccessKind.READ), + new InstanceField("referenceMapCompressedOffset", FieldAccessKind.READ), + new InstanceField("typeCheckStart", FieldAccessKind.READ, CLOSED_TYPE_WORLD_HUB_LAYOUT), + new InstanceField("typeCheckRange", FieldAccessKind.READ, CLOSED_TYPE_WORLD_HUB_LAYOUT), + new InstanceField("typeCheckSlot", FieldAccessKind.READ, CLOSED_TYPE_WORLD_HUB_LAYOUT), + new InstanceField("typeID", FieldAccessKind.READ, OPEN_TYPE_WORLD_HUB_LAYOUT), + new InstanceField("typeIDDepth", FieldAccessKind.READ, OPEN_TYPE_WORLD_HUB_LAYOUT), + new InstanceField("numClassTypes", FieldAccessKind.READ, OPEN_TYPE_WORLD_HUB_LAYOUT), + new InstanceField("numIterableInterfaceTypes", FieldAccessKind.READ, OPEN_TYPE_WORLD_HUB_LAYOUT), + new InstanceField("openTypeWorldTypeCheckSlots", FieldAccessKind.READ, OPEN_TYPE_WORLD_HUB_LAYOUT), + new InstanceField("interfaceID", FieldAccessKind.READ, OPEN_TYPE_WORLD_HUB_LAYOUT), + new InstanceField("openTypeWorldInterfaceHashTable", FieldAccessKind.READ, OPEN_TYPE_WORLD_HUB_LAYOUT), + new InstanceField("openTypeWorldInterfaceHashParam", FieldAccessKind.READ, OPEN_TYPE_WORLD_HUB_LAYOUT)), + new AccessedClass(String.class, + new InstanceField("value", FieldAccessKind.READ), + new InstanceField("coder", FieldAccessKind.READ)), + new AccessedClass(AtomicInteger.class, + new InstanceField("value", FieldAccessKind.READ)), + new AccessedClass(Target_java_lang_ref_Reference.class, + new InstanceField("referent", FieldAccessKind.READ_WRITE), + new InstanceField("next", FieldAccessKind.READ_WRITE), + new InstanceField("discovered", FieldAccessKind.READ_WRITE)), + new AccessedClass(SoftReference.class, + new InstanceField("timestamp", FieldAccessKind.READ), + new StaticField("clock", FieldAccessKind.READ_WRITE)), + new AccessedClass(StoredContinuation.class, + new InstanceField("ip", FieldAccessKind.READ_WRITE, new ContinuationsSupported())), + new AccessedClass(RuntimeCodeInfoMemory.class, + new InstanceField("table", FieldAccessKind.READ)), + new AccessedClass(PerfLong.class, + new InstanceField("value", FieldAccessKind.READ_WRITE, USE_PERF_DATA)), + new AccessedClass(PerfStringVariable.class, + new InstanceField("nullTerminatedValue", FieldAccessKind.READ, USE_PERF_DATA)) + }; +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahFeature.java new file mode 100644 index 000000000000..0be85a947149 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahFeature.java @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.gc.shenandoah; + +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.hosted.Feature; +import org.graalvm.nativeimage.impl.PinnedObjectSupport; + +import com.oracle.svm.core.GCRelatedMXBeans; +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.SubstrateTargetDescription; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.config.ObjectLayout; +import com.oracle.svm.core.config.ObjectLayout.IdentityHashMode; +import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; +import com.oracle.svm.core.feature.InternalFeature; +import com.oracle.svm.core.gc.shared.graal.NativeGCAllocationSupport; +import com.oracle.svm.core.gc.shenandoah.ShenandoahCommittedMemoryProvider; +import com.oracle.svm.core.gc.shenandoah.ShenandoahHeap; +import com.oracle.svm.core.gc.shenandoah.ShenandoahImageHeapInfo; +import com.oracle.svm.core.gc.shenandoah.ShenandoahObjectHeader; +import com.oracle.svm.core.gc.shenandoah.ShenandoahOptions; +import com.oracle.svm.core.gc.shenandoah.ShenandoahPinnedObjectSupport; +import com.oracle.svm.core.gc.shenandoah.ShenandoahRelatedMXBeans; +import com.oracle.svm.core.gc.shenandoah.graal.ShenandoahAllocationSupport; +import com.oracle.svm.core.gc.shenandoah.graal.ShenandoahBarrierSetProvider; +import com.oracle.svm.core.graal.meta.RuntimeConfiguration; +import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider; +import com.oracle.svm.core.graal.snippets.GCAllocationSupport; +import com.oracle.svm.core.graal.snippets.NodeLoweringProvider; +import com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets; +import com.oracle.svm.core.heap.AllocationFeature; +import com.oracle.svm.core.heap.BarrierSetProvider; +import com.oracle.svm.core.heap.FillerArray; +import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.image.ImageHeapLayouter; +import com.oracle.svm.core.jvmstat.PerfDataFeature; +import com.oracle.svm.core.option.RuntimeOptionKey; +import com.oracle.svm.core.option.SubstrateOptionKey; +import com.oracle.svm.core.os.CommittedMemoryProvider; +import com.oracle.svm.core.util.UserError; +import com.oracle.svm.hosted.FeatureImpl.BeforeAnalysisAccessImpl; +import com.oracle.svm.hosted.FeatureImpl.BeforeCompilationAccessImpl; +import com.oracle.svm.hosted.gc.shared.NativeGCAccessedFields; +import com.oracle.svm.hosted.thread.VMThreadFeature; + +import jdk.graal.compiler.graph.Node; +import jdk.graal.compiler.options.OptionValues; +import jdk.graal.compiler.phases.util.Providers; +import jdk.vm.ci.meta.JavaKind; + +/** Shenandoah GC support. */ +@AutomaticallyRegisteredFeature +public class ShenandoahFeature implements InternalFeature { + @Override + public boolean isInConfiguration(IsInConfigurationAccess access) { + return SubstrateOptions.useShenandoahGC(); + } + + @Override + public List> getRequiredFeatures() { + return Arrays.asList(VMThreadFeature.class, PerfDataFeature.class, AllocationFeature.class); + } + + @Override + public void afterRegistration(AfterRegistrationAccess access) { + verifyOptionsAndPlatform(); + + boolean useCompressedReferences = false; + ImageSingletons.add(BarrierSetProvider.class, new ShenandoahBarrierSetProvider()); + ImageSingletons.add(ObjectLayout.class, createObjectLayout(useCompressedReferences)); + + ShenandoahCommittedMemoryProvider memoryProvider = new ShenandoahCommittedMemoryProvider(); + ImageSingletons.add(CommittedMemoryProvider.class, memoryProvider); + ImageSingletons.add(ShenandoahCommittedMemoryProvider.class, memoryProvider); + + ImageSingletons.add(GCRelatedMXBeans.class, new ShenandoahRelatedMXBeans()); + } + + @Override + public void duringSetup(DuringSetupAccess access) { + ShenandoahHeap heap = new ShenandoahHeap(); + ImageSingletons.add(Heap.class, heap); + ImageSingletons.add(ShenandoahHeap.class, heap); + ImageSingletons.add(PinnedObjectSupport.class, new ShenandoahPinnedObjectSupport()); + + ShenandoahAllocationSupport allocationSupport = new ShenandoahAllocationSupport(); + ImageSingletons.add(GCAllocationSupport.class, allocationSupport); + ImageSingletons.add(NativeGCAllocationSupport.class, allocationSupport); + } + + @Override + public void beforeAnalysis(BeforeAnalysisAccess access) { + BeforeAnalysisAccessImpl accessImpl = (BeforeAnalysisAccessImpl) access; + NativeGCAccessedFields.markAsAccessed(accessImpl, ShenandoahAccessedFields.ACCESSED_CLASSES); + + /* Shenandoah needs a custom filler array class that does not match int[].class. */ + accessImpl.registerAsUsed(FillerArray.class); + + /* Needed for the barrier set. */ + accessImpl.registerAsUsed(Object[].class); + + /* + * Ensure that SVM knows about all runtime options that Shenandoah parses on the C++ side. + */ + registerRuntimeOptionsAsRead(accessImpl); + } + + private static void registerRuntimeOptionsAsRead(BeforeAnalysisAccessImpl accessImpl) { + for (Field field : ShenandoahOptions.getOptionFields()) { + if (RuntimeOptionKey.class.isAssignableFrom(field.getType())) { + accessImpl.registerAsRead(field, "it is a GC option field"); + } + } + } + + @Override + public void afterAnalysis(AfterAnalysisAccess access) { + ImageSingletons.add(ImageHeapLayouter.class, new ShenandoahImageHeapLayouter()); + } + + @Override + public void beforeCompilation(BeforeCompilationAccess a) { + BeforeCompilationAccessImpl access = (BeforeCompilationAccessImpl) a; + ShenandoahHeap heap = ShenandoahHeap.get(); + + /* Mark the image heap info as immutable. */ + ShenandoahImageHeapInfo imageHeapInfo = ShenandoahHeap.getImageHeapInfo(); + access.registerAsImmutable(imageHeapInfo); + access.registerAsImmutable(imageHeapInfo.getRegionTypes()); + access.registerAsImmutable(imageHeapInfo.getRegionFreeSpaces()); + + /* Collect data and offsets that are needed when initializing Shenandoah. */ + byte[] fieldOffsets = NativeGCAccessedFields.writeOffsets(access, ShenandoahObjectHeader.getMarkWordOffset(), ShenandoahHeap.javaThreadTL, ShenandoahAllocationSupport.podReferenceMapTL, + ShenandoahAccessedFields.ACCESSED_CLASSES); + heap.setAccessedFieldOffsets(fieldOffsets); + access.registerAsImmutable(fieldOffsets); + } + + @Override + public void registerForeignCalls(SubstrateForeignCallsProvider foreignCalls) { + ShenandoahAllocationSupport.registerForeignCalls(foreignCalls); + } + + @Override + public void registerLowerings(RuntimeConfiguration runtimeConfig, OptionValues options, Providers providers, + Map, NodeLoweringProvider> lowerings, boolean hosted) { + SubstrateAllocationSnippets allocationSnippets = ImageSingletons.lookup(SubstrateAllocationSnippets.class); + SubstrateAllocationSnippets.Templates templates = new SubstrateAllocationSnippets.Templates(options, providers, allocationSnippets); + templates.registerLowering(lowerings); + } + + private static void verifyOptionsAndPlatform() { + UserError.guarantee(Platform.includedIn(Platform.LINUX_AMD64.class) || Platform.includedIn(Platform.LINUX_AARCH64.class), + "The Shenandoah garbage collector ('--gc=shenandoah') is currently only supported on linux/amd64 and linux/aarch64."); + + verifyOptionEnabled(SubstrateOptions.SpawnIsolates); + verifyOptionEnabled(SubstrateOptions.AllowVMInternalThreads); + verifyOptionEnabled(SubstrateOptions.ConcealedOptions.UseDedicatedVMOperationThread); + verifyOptionEnabled(SubstrateOptions.ConcealedOptions.AutomaticReferenceHandling); + verifyOptionEnabled(SubstrateOptions.UseNullRegion); + + UserError.guarantee(!SubstrateOptions.supportCompileInIsolates(), "The Shenandoah garbage collector ('--gc=shenandoah') does not support isolated compilation."); + } + + private static void verifyOptionEnabled(SubstrateOptionKey option) { + String optionMustBeEnabledFmt = "When using the Shenandoah garbage collector ('--gc=shenandoah'), please note that option '%s' must be enabled."; + UserError.guarantee(option.getValue(), optionMustBeEnabledFmt, option.getName()); + } + + /** + * Layout of instance objects: + *
    + *
  • 32/64 bit mark word/identity hashcode
    • + *
  • 32 bit hub reference
    • + *
  • instance fields (references, primitives)
    • + *
  • 32/64 bit object monitor reference (if needed)
    • + *
+ * + * Layout of array objects: + *
    + *
  • 32/64 bit mark word/identity hashcode
    • + *
  • 32 bit hub reference
    • + *
  • 32 bit array length
    • + *
  • array elements (length * elementSize)
    • + *
+ */ + private static ObjectLayout createObjectLayout(boolean useCompressedReferences) { + SubstrateTargetDescription target = ConfigurationValues.getTarget(); + int referenceSize = computeReferenceSize(target, useCompressedReferences); + int intSize = target.arch.getPlatformKind(JavaKind.Int).getSizeInBytes(); + int objectAlignment = 8; + + int markWordSize = referenceSize; + int hubSize = Integer.BYTES; + + int markWordOffset = ShenandoahObjectHeader.getMarkWordOffset(); + int headerIdentityHashOffset = markWordOffset; + int headerSize = headerIdentityHashOffset + markWordSize + hubSize + SubstrateOptions.AdditionalHeaderBytes.getValue(); + + int hubOffset = markWordOffset + markWordSize; + int firstFieldOffset = headerSize; + int arrayLengthOffset = headerSize; + int arrayBaseOffset = arrayLengthOffset + intSize; + + int identityHashNumBits = 31; + int identityHashShift = 6; + + return new ObjectLayout(target, referenceSize, objectAlignment, hubSize, hubOffset, firstFieldOffset, arrayLengthOffset, arrayBaseOffset, + headerIdentityHashOffset, IdentityHashMode.OBJECT_HEADER, identityHashNumBits, identityHashShift); + } + + private static int computeReferenceSize(SubstrateTargetDescription target, boolean useCompressedReferences) { + JavaKind referenceKind = JavaKind.Object; + if (useCompressedReferences) { + referenceKind = JavaKind.Int; + } + return target.arch.getPlatformKind(referenceKind).getSizeInBytes(); + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahImageHeapLayouter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahImageHeapLayouter.java new file mode 100644 index 000000000000..9256601b4b49 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahImageHeapLayouter.java @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.gc.shenandoah; + +import static com.oracle.svm.core.gc.shenandoah.ShenandoahOptions.ShenandoahRegionSize; + +import java.nio.ByteBuffer; +import java.util.ArrayList; + +import com.oracle.graal.pointsto.ObjectScanner.OtherReason; +import com.oracle.graal.pointsto.ObjectScanner.ScanReason; +import com.oracle.graal.pointsto.heap.ImageHeapScanner; +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.gc.shenandoah.ShenandoahHeap; +import com.oracle.svm.core.gc.shenandoah.ShenandoahImageHeapInfo; +import com.oracle.svm.core.gc.shenandoah.ShenandoahRegionType; +import com.oracle.svm.core.image.ImageHeap; +import com.oracle.svm.core.image.ImageHeapLayoutInfo; +import com.oracle.svm.core.image.ImageHeapLayouter; +import com.oracle.svm.core.image.ImageHeapObject; +import com.oracle.svm.core.image.ImageHeapPartition; +import com.oracle.svm.core.util.UnsignedUtils; +import com.oracle.svm.core.util.VMError; +import com.oracle.svm.hosted.image.NativeImageHeap; +import com.oracle.svm.util.ReflectionUtil; + +import jdk.graal.compiler.core.common.NumUtil; +import jdk.graal.compiler.word.Word; + +/** + * Layouts the heap in a way that it matches the expectations of the C++ code. Multiple image heap + * partitions can live in the same heap region. Partition alignment requirements are ensured via + * filler objects. + */ +public class ShenandoahImageHeapLayouter implements ImageHeapLayouter { + private final ShenandoahImageHeapPartition closedImageHeapReadOnly; + private final ShenandoahImageHeapPartition closedImageHeapRelocatable; + private final ShenandoahImageHeapPartition closedImageHeapWritable; + private final ShenandoahImageHeapPartition openImageHeap; + private final ShenandoahImageHeapPartition[] partitions; + + public ShenandoahImageHeapLayouter() { + assert SubstrateOptions.SpawnIsolates.getValue(); + this.closedImageHeapReadOnly = new ShenandoahImageHeapPartition("closedImageHeapReadOnly", false); + this.closedImageHeapRelocatable = new ShenandoahImageHeapPartition("closedImageHeapRelocatable", false); + this.closedImageHeapWritable = new ShenandoahImageHeapPartition("closedImageHeapWritable", true); + this.openImageHeap = new ShenandoahImageHeapPartition("openImageHeap", true); + + this.partitions = new ShenandoahImageHeapPartition[]{closedImageHeapReadOnly, closedImageHeapRelocatable, closedImageHeapWritable, openImageHeap}; + } + + @Override + public ImageHeapPartition[] getPartitions() { + return partitions; + } + + @Override + public void assignObjectToPartition(ImageHeapObject info, boolean immutable, boolean references, boolean relocatable, boolean patched) { + VMError.guarantee(!patched, "Layered native images are not supported at the moment."); + + ShenandoahImageHeapPartition partition = choosePartition(immutable, references, relocatable); + partition.add(info); + } + + private ShenandoahImageHeapPartition choosePartition(boolean immutable, boolean hasReferences, boolean hasRelocatables) { + if (immutable) { + return hasRelocatables ? closedImageHeapRelocatable : closedImageHeapReadOnly; + } else { + assert !hasRelocatables; + return hasReferences ? openImageHeap : closedImageHeapWritable; + } + } + + @Override + public ImageHeapLayoutInfo layout(ImageHeap imageHeap, int pageSize, ImageHeapLayouterCallback callback) { + int regionSize = ShenandoahRegionSize.getValue(); + int objectAlignment = ConfigurationValues.getObjectLayout().getAlignment(); + ShenandoahImageHeapObjectComparator humongousObjectsFirst = new ShenandoahImageHeapObjectComparator(regionSize, true); + ShenandoahImageHeapObjectComparator humongousObjectsLast = new ShenandoahImageHeapObjectComparator(regionSize, false); + ShenandoahImageHeapRegions regions = new ShenandoahImageHeapRegions(imageHeap); + + /* Closed image heap regions. */ + regions.setDefaultRegionType(ShenandoahRegionType.ClosedImageHeap); + + regions.allocate(closedImageHeapReadOnly, humongousObjectsFirst); + regions.endPartition(closedImageHeapReadOnly, objectAlignment); + + regions.allocate(closedImageHeapRelocatable, humongousObjectsLast); + regions.endPartition(closedImageHeapRelocatable, objectAlignment); + + regions.allocate(closedImageHeapWritable, humongousObjectsLast); + regions.endPartition(closedImageHeapWritable, regionSize); + + /* Open image heap regions. */ + regions.setDefaultRegionType(ShenandoahRegionType.OpenImageHeap); + + regions.allocate(openImageHeap, humongousObjectsFirst); + ShenandoahImageHeapInfo imageHeapInfo = initializeImageHeapInfo(imageHeap, regions); + regions.endPartition(openImageHeap, regionSize); + /* Done with the layouting, no further objects may be added to the image heap. */ + + regions.fillImageHeapInfo(imageHeapInfo); + + /* Compute the memory layout of the image heap (partitions can be empty). */ + long startOffset = ShenandoahHeap.get().getImageHeapOffsetInAddressSpace(); + long imageHeapSize = NumUtil.roundUp(regions.getSize(), SubstrateOptions.getPageSize()); + long endOffset = startOffset + imageHeapSize; + long openImageHeapBegin = getOffsetOfFirstObject(openImageHeap, endOffset); + long closedImageHeapWritableBegin = getOffsetOfFirstObject(closedImageHeapWritable, openImageHeapBegin); + long closedImageHeapRelocatableBegin = getOffsetOfFirstObject(closedImageHeapRelocatable, closedImageHeapWritableBegin); + long closedImageHeapReadOnlyBegin = getOffsetOfFirstObject(closedImageHeapReadOnly, closedImageHeapRelocatableBegin); + + assert startOffset == closedImageHeapReadOnlyBegin; + + openImageHeap.setSize(openImageHeapBegin, endOffset); + closedImageHeapWritable.setSize(closedImageHeapWritableBegin, openImageHeapBegin); + closedImageHeapRelocatable.setSize(closedImageHeapRelocatableBegin, closedImageHeapWritableBegin); + closedImageHeapReadOnly.setSize(closedImageHeapReadOnlyBegin, closedImageHeapRelocatableBegin); + + /* + * Align the writable part of the image heap to the build-time page size. As a side-effect, + * a few read-only objects may end up in the closed but writable part of the image heap. + */ + long writableBegin = UnsignedUtils.roundDown(Word.unsigned(closedImageHeapWritableBegin), Word.unsigned(pageSize)).rawValue(); + long writableEnd = endOffset; + long writableSize = writableEnd - writableBegin; + /* Layered images are not supported yet, so there is no writable-patched section. */ + long writablePatchedBegin = closedImageHeapWritableBegin; + long writablePatchedSize = 0; + + assert writableBegin % pageSize == 0; + assert openImageHeapBegin % regionSize == 0; + + return new ImageHeapLayoutInfo(startOffset, endOffset, writableBegin, writableSize, closedImageHeapRelocatableBegin, closedImageHeapRelocatable.getSize(), writablePatchedBegin, + writablePatchedSize); + } + + @Override + public void afterLayout(ImageHeap imageHeap) { + if (imageHeap instanceof NativeImageHeap nativeImageHeap) { + /* Update the arrays in the image heap info, now that the layouting is done. */ + ShenandoahImageHeapInfo imageHeapInfo = ShenandoahHeap.getImageHeapInfo(); + ImageHeapScanner heapScanner = nativeImageHeap.aUniverse.getHeapScanner(); + ScanReason reason = new OtherReason("Manual rescan triggered from " + ShenandoahImageHeapLayouter.class); + heapScanner.rescanField(imageHeapInfo, ReflectionUtil.lookupField(ShenandoahImageHeapInfo.class, "regionTypes"), reason); + heapScanner.rescanField(imageHeapInfo, ReflectionUtil.lookupField(ShenandoahImageHeapInfo.class, "regionFreeSpaces"), reason); + } + } + + private ShenandoahImageHeapInfo initializeImageHeapInfo(ImageHeap imageHeap, ShenandoahImageHeapRegions regions) { + // Below, we are adding objects to the image heap. Those objects could be placed in a new + // region, so we need one extra region. + int regionCount = regions.getCount(); + byte[] regionType = new byte[regionCount + 1]; + int[] regionFreeSpace = new int[regionCount + 1]; + + addLateToImageHeap(imageHeap, regionType, "heap metadata", openImageHeap, regions); + addLateToImageHeap(imageHeap, regionFreeSpace, "heap metadata", openImageHeap, regions); + assert regions.getCount() <= regionCount + 1; + + // After adding all the objects, we can obtain the final region count. + int closedImageHeapRegions = regions.countClosedImageHeapRegions(); + int openImageHeapRegions = regions.getCount() - closedImageHeapRegions; + + ShenandoahImageHeapInfo info = ShenandoahHeap.getImageHeapInfo(); + info.initialize(closedImageHeapRegions, openImageHeapRegions, regionType, regionFreeSpace, imageHeap.countPatchAndVerifyDynamicHubs()); + return info; + } + + private static void addLateToImageHeap(ImageHeap imageHeap, Object object, String reason, ShenandoahImageHeapPartition partition, ShenandoahImageHeapRegions regions) { + ImageHeapObject objectInfo = imageHeap.addLateToImageHeap(object, reason); + partition.add(objectInfo); + regions.allocate(objectInfo); + } + + private static long getOffsetOfFirstObject(ShenandoahImageHeapPartition partition, long defaultValue) { + ArrayList objects = partition.getObjects(); + if (objects.isEmpty()) { + return defaultValue; + } + + /* + * Any arbitrary object in the partition could have the lowest offset as we use first fit + * decreasing bin packing and place multiple partitions in one image heap region. + */ + long minOffset = Long.MAX_VALUE; + for (ImageHeapObject o : objects) { + minOffset = Math.min(minOffset, o.getOffset()); + } + return minOffset; + } + + @Override + public void writeMetadata(ByteBuffer imageHeapBytes, long imageHeapOffsetInBuffer) { + // nothing to do + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahImageHeapObjectComparator.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahImageHeapObjectComparator.java new file mode 100644 index 000000000000..e58a767aa186 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahImageHeapObjectComparator.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.gc.shenandoah; + +import java.util.Comparator; + +import com.oracle.svm.core.image.ImageHeapObject; + +public class ShenandoahImageHeapObjectComparator implements Comparator { + private final int regionSize; + private final boolean humongousObjectsFirst; + + public ShenandoahImageHeapObjectComparator(int regionSize, boolean humongousObjectsFirst) { + this.regionSize = regionSize; + this.humongousObjectsFirst = humongousObjectsFirst; + } + + @Override + public int compare(ImageHeapObject a, ImageHeapObject b) { + boolean aIsHumongous = a.getSize() > regionSize; + boolean bIsHumongous = b.getSize() > regionSize; + if (aIsHumongous != bIsHumongous) { + /* Place humongous objects at the start or at the end. */ + return aIsHumongous == humongousObjectsFirst ? -1 : 1; + } + + boolean aIsDynamicHub = a.getObjectClass() == Class.class; + boolean bIsDynamicHub = b.getObjectClass() == Class.class; + if (aIsDynamicHub != bIsDynamicHub) { + /* Place DynamicHubs before other objects, regardless of size. */ + return aIsDynamicHub ? -1 : 1; + } + + return Long.signum(b.getSize() - a.getSize()); + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahImageHeapPartition.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahImageHeapPartition.java new file mode 100644 index 000000000000..972bfbcbb421 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahImageHeapPartition.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.gc.shenandoah; + +import java.util.ArrayList; + +import com.oracle.svm.core.image.ImageHeapObject; +import com.oracle.svm.core.image.ImageHeapPartition; + +public class ShenandoahImageHeapPartition implements ImageHeapPartition { + private final String name; + private final ArrayList objects = new ArrayList<>(); + private final boolean writable; + + private long size = -1L; + + ShenandoahImageHeapPartition(String name, boolean writable) { + this.name = name; + this.writable = writable; + } + + @Override + public String getName() { + return name; + } + + @Override + public boolean isWritable() { + return writable; + } + + @Override + public long getStartOffset() { + /* All image heap objects have absolute offsets. */ + return 0; + } + + public void add(ImageHeapObject info) { + objects.add(info); + info.setHeapPartition(this); + } + + public ArrayList getObjects() { + return objects; + } + + @Override + public long getSize() { + assert size >= 0 : size; + return size; + } + + public void setSize(long startOffset, long endOffset) { + assert startOffset >= 0; + assert endOffset >= startOffset; + this.size = endOffset - startOffset; + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahImageHeapRegion.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahImageHeapRegion.java new file mode 100644 index 000000000000..0cb777257957 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahImageHeapRegion.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.gc.shenandoah; + +import static com.oracle.svm.core.gc.shenandoah.ShenandoahOptions.ShenandoahRegionSize; + +import com.oracle.svm.core.gc.shenandoah.ShenandoahRegionType; +import com.oracle.svm.core.image.ImageHeapObject; + +import jdk.graal.compiler.core.common.NumUtil; + +/** + * Holds metadata about a heap region during the image build. + */ +public class ShenandoahImageHeapRegion { + private final ShenandoahRegionType type; + private final long startOffset; + private int used; + + public ShenandoahImageHeapRegion(ShenandoahRegionType type, long startOffset) { + this.type = type; + this.startOffset = startOffset; + this.used = 0; + } + + public void allocate(ImageHeapObject info) { + /* We use absolute offsets, all partitions have an offset of 0. */ + info.setOffsetInPartition(startOffset + used); + + int regionSize = ShenandoahRegionSize.getValue(); + assert info.getSize() <= regionSize || type.isHumongous() : info; + int size = info.getSize() > regionSize ? regionSize : NumUtil.safeToInt(info.getSize()); + increaseUsed(size); + } + + public ShenandoahRegionType getType() { + return type; + } + + public void increaseUsed(int size) { + assert size > 0 && used + size <= ShenandoahRegionSize.getValue() : size; + used += size; + } + + public int getUsed() { + return used; + } + + public int getRemainingSpace() { + return ShenandoahRegionSize.getValue() - used; + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahImageHeapRegions.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahImageHeapRegions.java new file mode 100644 index 000000000000..4c933d654e1b --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shenandoah/ShenandoahImageHeapRegions.java @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.gc.shenandoah; + +import static com.oracle.svm.core.gc.shenandoah.ShenandoahOptions.ShenandoahRegionSize; + +import java.util.ArrayList; + +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.gc.shenandoah.ShenandoahHeap; +import com.oracle.svm.core.gc.shenandoah.ShenandoahImageHeapInfo; +import com.oracle.svm.core.gc.shenandoah.ShenandoahRegionType; +import com.oracle.svm.core.image.ImageHeap; +import com.oracle.svm.core.image.ImageHeapObject; + +import jdk.graal.compiler.core.common.NumUtil; + +public class ShenandoahImageHeapRegions { + private final ImageHeap imageHeap; + private final int regionSize = ShenandoahRegionSize.getValue(); + private final ArrayList regions = new ArrayList<>(); + + private int firstRegionWithFreeSpace; + private ShenandoahRegionType defaultRegionType; + private ShenandoahRegionType startsHumongousRegionType; + private ShenandoahRegionType continuesHumongousRegionType; + + public ShenandoahImageHeapRegions(ImageHeap imageHeap) { + this.imageHeap = imageHeap; + } + + public void setDefaultRegionType(ShenandoahRegionType value) { + if (value == ShenandoahRegionType.ClosedImageHeap) { + defaultRegionType = ShenandoahRegionType.ClosedImageHeap; + startsHumongousRegionType = ShenandoahRegionType.ClosedImageHeapStartsHumongous; + continuesHumongousRegionType = ShenandoahRegionType.ClosedImageHeapContinuesHumongous; + } else { + assert value == ShenandoahRegionType.OpenImageHeap; + defaultRegionType = ShenandoahRegionType.OpenImageHeap; + startsHumongousRegionType = ShenandoahRegionType.OpenImageHeapStartsHumongous; + continuesHumongousRegionType = ShenandoahRegionType.OpenImageHeapContinuesHumongous; + } + } + + public void allocate(ShenandoahImageHeapPartition partition, ShenandoahImageHeapObjectComparator comparator) { + /* Use first-fit decreasing bin packing for assigning objects to heap regions. */ + ArrayList objects = partition.getObjects(); + objects.sort(comparator); + for (ImageHeapObject info : objects) { + allocate(info); + } + } + + public void allocate(ImageHeapObject info) { + if (info.getSize() > regionSize) { + allocateHumongousObject(info); + } else { + allocateNormalObject(info); + } + } + + private void allocateHumongousObject(ImageHeapObject info) { + /* Humongous objects are always added in separate regions. */ + ShenandoahImageHeapRegion humongousStartRegion = new ShenandoahImageHeapRegion(startsHumongousRegionType, getOffsetOfNextRegion()); + humongousStartRegion.allocate(info); + regions.add(humongousStartRegion); + + long remainingObjectSize = info.getSize() - regionSize; + do { + int usedRegionSpace = remainingObjectSize > regionSize ? regionSize : NumUtil.safeToInt(remainingObjectSize); + ShenandoahImageHeapRegion continuesHumongousRegion = new ShenandoahImageHeapRegion(continuesHumongousRegionType, getOffsetOfNextRegion()); + continuesHumongousRegion.increaseUsed(usedRegionSpace); + regions.add(continuesHumongousRegion); + + remainingObjectSize -= usedRegionSpace; + } while (remainingObjectSize > 0); + } + + private void allocateNormalObject(ImageHeapObject info) { + for (int i = firstRegionWithFreeSpace; i < regions.size(); i++) { + ShenandoahImageHeapRegion region = regions.get(i); + if (!region.getType().isHumongous() && region.getRemainingSpace() >= info.getSize()) { + assert region.getType() == defaultRegionType; + region.allocate(info); + return; + } + } + + /* No existing region had sufficient free space, so start a new one. */ + ShenandoahImageHeapRegion region = new ShenandoahImageHeapRegion(defaultRegionType, getOffsetOfNextRegion()); + region.allocate(info); + regions.add(region); + } + + public void endPartition(ShenandoahImageHeapPartition partition, int alignment) { + ensureAlignment(partition, alignment); + assert firstRegionWithFreeSpace == regions.size() || firstRegionWithFreeSpace == regions.size() - 1 && regions.getLast().getUsed() % alignment == 0; + } + + private void ensureAlignment(ShenandoahImageHeapPartition partition, int alignment) { + assert alignment > 0 && alignment <= regionSize; + assert regionSize % alignment == 0 : "we assume that region starts are always aligned"; + assert alignment % ConfigurationValues.getObjectLayout().getAlignment() == 0 : "alignment must be a multiple of the object alignment"; + + ShenandoahImageHeapRegion lastRegion = regions.getLast(); + if (alignment == regionSize || lastRegion.getType().isHumongous()) { + /* Mark all regions, including the last region, as full. */ + firstRegionWithFreeSpace = regions.size(); + return; + } + + /* Check if the end of the current region is already aligned. */ + int used = lastRegion.getUsed(); + int availableBytes = lastRegion.getRemainingSpace(); + int bytesToFill = NumUtil.roundUp(used, alignment) - used; + if (bytesToFill == 0) { + /* Mark all regions, except the last region, as full. */ + firstRegionWithFreeSpace = regions.size() - 1; + return; + } + + /* Check if it makes sense to use a filler object to ensure the alignment. */ + if (bytesToFill < availableBytes) { + ImageHeapObject objectInfo = imageHeap.addFillerObject(bytesToFill); + if (objectInfo == null) { + /* The gap may be too small for the filler. Make the gap larger and try again. */ + bytesToFill += alignment; + if (bytesToFill < availableBytes) { + objectInfo = imageHeap.addFillerObject(bytesToFill); + } + } + + if (objectInfo != null) { + partition.add(objectInfo); + lastRegion.allocate(objectInfo); + assert (objectInfo.getOffset() + objectInfo.getSize()) % alignment == 0; + + /* Filler object was added - mark all regions, except the last region, as full. */ + firstRegionWithFreeSpace = regions.size() - 1; + return; + } + } + + /* + * The filler object was too large for the last region. Mark all regions, including the last + * region, as full. + */ + assert availableBytes < 2 * alignment; + firstRegionWithFreeSpace = regions.size(); + } + + private long getOffsetOfNextRegion() { + return regions.size() * ((long) regionSize) + ShenandoahHeap.get().getImageHeapOffsetInAddressSpace(); + } + + public void fillImageHeapInfo(ShenandoahImageHeapInfo info) { + for (int i = 0; i < regions.size(); i++) { + ShenandoahImageHeapRegion region = regions.get(i); + info.writeHeapRegion(i, region.getType(), region.getRemainingSpace()); + } + } + + public int getCount() { + return regions.size(); + } + + public long getSize() { + /* The last region is not necessarily full. */ + long sizeExceptLastRegion = Math.max(0, regions.size() - 1) * ((long) regionSize); + ShenandoahImageHeapRegion lastRegion = regions.getLast(); + return sizeExceptLastRegion + lastRegion.getUsed(); + } + + public int countClosedImageHeapRegions() { + int count = 0; + for (ShenandoahImageHeapRegion region : regions) { + if (region.getType().isClosedImageHeap()) { + count++; + } + } + return count; + } +} diff --git a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmLMGC.java b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmLMGC.java index d4df07c456f4..e6645a112e83 100644 --- a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmLMGC.java +++ b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmLMGC.java @@ -31,6 +31,7 @@ import java.lang.ref.Reference; import org.graalvm.collections.EconomicMap; +import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.c.function.CodePointer; @@ -226,6 +227,7 @@ public String getName() { } @Override + @Platforms(Platform.HOSTED_ONLY.class) public String getDefaultMaxHeapSize() { return "unknown"; } diff --git a/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSGC.java b/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSGC.java index d82ea71011c3..9dbc53468b66 100644 --- a/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSGC.java +++ b/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSGC.java @@ -24,6 +24,9 @@ */ package com.oracle.svm.webimage.heap; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + import com.oracle.svm.core.heap.GC; import com.oracle.svm.core.heap.GCCause; @@ -49,6 +52,7 @@ public String getName() { } @Override + @Platforms(Platform.HOSTED_ONLY.class) public String getDefaultMaxHeapSize() { return "unknown"; }