diff --git a/substratevm/mx.substratevm/mx_substratevm_namespace.py b/substratevm/mx.substratevm/mx_substratevm_namespace.py index 3a7a688fae43..3175126832dd 100644 --- a/substratevm/mx.substratevm/mx_substratevm_namespace.py +++ b/substratevm/mx.substratevm/mx_substratevm_namespace.py @@ -40,7 +40,7 @@ ignore_files = {"copy_x86.hpp", "copy_aarch64.hpp", "osThread_linux.hpp"} ignore_includes = {"CPU_HEADER(copy)", "OS_HEADER(osThread)"} -files_with_cpp_guard = {"sharedGCStructs.hpp"} +files_with_cpp_guard = {"sharedGCStructs.h", "g1GCStructs.h"} SVM_NAMESPACE = "svm_namespace" diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index b6271d04235d..1e827be91bd4 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -2103,6 +2103,7 @@ "dependency:com.oracle.svm.native.jvm.posix/*", "dependency:com.oracle.svm.native.libcontainer/*", "file:debug/include", + "file:src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/include", ], }, }, diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index dab9ee137f8a..abc534e827ec 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -153,6 +153,7 @@ public String getName() { } @Override + @Platforms(Platform.HOSTED_ONLY.class) public String getDefaultMaxHeapSize() { return String.format("%s%% of RAM", SerialAndEpsilonGCOptions.MaximumHeapSizePercent.getValue()); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 8978a570808d..dd2214436771 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -588,7 +588,7 @@ public HostedOptionKey multi @Option(help = "Please use '--gc=*' instead. Possible values are listed with '--help'.")// public static final HostedOptionKey SupportedGCs = new HostedOptionKey<>( - ReplacingLocatableMultiOptionValue.DelimitedString.buildWithCommaDelimiter(GCOptionValue.SERIAL.getValue())) { + ReplacingLocatableMultiOptionValue.DelimitedString.buildWithCommaDelimiter(GCOptionValue.Serial.getValue())) { @Override protected void onValueUpdate(EconomicMap, Object> values, ReplacingLocatableMultiOptionValue.DelimitedString oldValue, ReplacingLocatableMultiOptionValue.DelimitedString newValue) { @@ -602,17 +602,16 @@ protected void onValueUpdate(EconomicMap, Object> values, Replacing super.onValueUpdate(values, oldValue, newValue); } - }; @Fold public static boolean useSerialGC() { - return !SubstrateOptions.SupportedGCs.hasBeenSet() || SubstrateOptions.SupportedGCs.getValue().contains(GCOptionValue.SERIAL.getValue()); + return !SubstrateOptions.SupportedGCs.hasBeenSet() || SubstrateOptions.SupportedGCs.getValue().contains(GCOptionValue.Serial.getValue()); } @Fold public static boolean useEpsilonGC() { - return SubstrateOptions.SupportedGCs.getValue().contains(GCOptionValue.EPSILON.getValue()); + return SubstrateOptions.SupportedGCs.getValue().contains(GCOptionValue.Epsilon.getValue()); } @Fold @@ -629,9 +628,9 @@ public static class DeprecatedOptions { @Override protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { if (newValue) { - SubstrateOptions.SupportedGCs.update(values, ReplacingLocatableMultiOptionValue.DelimitedString.buildWithCommaDelimiter(GCOptionValue.SERIAL.getValue())); + SubstrateOptions.SupportedGCs.update(values, ReplacingLocatableMultiOptionValue.DelimitedString.buildWithCommaDelimiter(GCOptionValue.Serial.getValue())); } else if (!values.containsKey(SubstrateOptions.SupportedGCs) || - ((ReplacingLocatableMultiOptionValue.DelimitedString) values.get(SubstrateOptions.SupportedGCs)).contains(GCOptionValue.SERIAL.getValue())) { + ((ReplacingLocatableMultiOptionValue.DelimitedString) values.get(SubstrateOptions.SupportedGCs)).contains(GCOptionValue.Serial.getValue())) { SubstrateOptions.SupportedGCs.update(values, ReplacingLocatableMultiOptionValue.DelimitedString.buildWithCommaDelimiter()); } } @@ -644,8 +643,8 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o @Override protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { if (newValue) { - SubstrateOptions.SupportedGCs.update(values, ReplacingLocatableMultiOptionValue.DelimitedString.buildWithCommaDelimiter(GCOptionValue.EPSILON.getValue())); - } else if (((AccumulatingLocatableMultiOptionValue.Strings) values.get(SubstrateOptions.SupportedGCs)).contains(GCOptionValue.EPSILON.getValue())) { + SubstrateOptions.SupportedGCs.update(values, ReplacingLocatableMultiOptionValue.DelimitedString.buildWithCommaDelimiter(GCOptionValue.Epsilon.getValue())); + } else if (((AccumulatingLocatableMultiOptionValue.Strings) values.get(SubstrateOptions.SupportedGCs)).contains(GCOptionValue.Epsilon.getValue())) { SubstrateOptions.SupportedGCs.update(values, ReplacingLocatableMultiOptionValue.DelimitedString.buildWithCommaDelimiter()); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/OWNERS.toml b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/OWNERS.toml new file mode 100644 index 000000000000..1f7d0b0a8706 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/OWNERS.toml @@ -0,0 +1,7 @@ +[[rule]] +files = "*" +all = [ + "christian.haeubl@oracle.com", +] +any = [ +] diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/InitializeReservedRegistersForPossiblyUnattachedThread.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/InitializeReservedRegistersForPossiblyUnattachedThread.java new file mode 100644 index 000000000000..4df2bc074220 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/InitializeReservedRegistersForPossiblyUnattachedThread.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +import org.graalvm.nativeimage.Isolate; +import org.graalvm.nativeimage.IsolateThread; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.c.function.CEntryPointOptions.Prologue; +import com.oracle.svm.core.graal.nodes.WriteCurrentVMThreadNode; +import com.oracle.svm.core.graal.snippets.CEntryPointSnippets; + +/** + * Prologue that only initializes the base registers so that a thread can execute SVM code. + * Unattached threads need to pass null as the {@link IsolateThread}, so that this prologue behaves + * the same as {@link InitializeReservedRegistersForUnattachedThread}. + */ +public class InitializeReservedRegistersForPossiblyUnattachedThread implements Prologue { + @Uninterruptible(reason = "prologue") + @SuppressWarnings("unused") + public static void enter(Isolate heapBase, IsolateThread thread) { + CEntryPointSnippets.initBaseRegisters(heapBase); + WriteCurrentVMThreadNode.writeCurrentVMThread(thread); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/InitializeReservedRegistersForUnattachedThread.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/InitializeReservedRegistersForUnattachedThread.java new file mode 100644 index 000000000000..6e92e5e1142c --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/InitializeReservedRegistersForUnattachedThread.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +import org.graalvm.nativeimage.Isolate; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.c.function.CEntryPointOptions.Prologue; +import com.oracle.svm.core.graal.nodes.WriteCurrentVMThreadNode; +import com.oracle.svm.core.graal.snippets.CEntryPointSnippets; + +import jdk.graal.compiler.word.Word; + +/** + * Prologue that only initializes the base registers so that an unattached thread can execute SVM + * code. + */ +public final class InitializeReservedRegistersForUnattachedThread implements Prologue { + @Uninterruptible(reason = "prologue") + @SuppressWarnings("unused") + public static void enter(Isolate heapBase) { + CEntryPointSnippets.initBaseRegisters(heapBase); + WriteCurrentVMThreadNode.writeCurrentVMThread(Word.nullPointer()); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCDebugLevel.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCDebugLevel.java new file mode 100644 index 000000000000..7d5e1f51b0d5 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCDebugLevel.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +/** Defines which debug-levels are supported for GCs such as G1. */ +public enum NativeGCDebugLevel { + Product("", 0), + FastDebug("-fastdebug", 1), + Debug("-debug", 2); + + private final String libSuffix; + private final int index; + + NativeGCDebugLevel(String libSuffix, int index) { + this.libSuffix = libSuffix; + this.index = index; + } + + public String getLibSuffix() { + return libSuffix; + } + + public int getIndex() { + return index; + } + + public static NativeGCDebugLevel fromString(String value) { + return switch (value) { + case "product" -> Product; + case "fastdebug" -> FastDebug; + case "debug" -> Debug; + default -> null; + }; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCHeaderFiles.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCHeaderFiles.java new file mode 100644 index 000000000000..6bd3ae7be7bc --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCHeaderFiles.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +import java.util.Collections; +import java.util.List; + +import org.graalvm.nativeimage.c.CContext; + +import com.oracle.svm.core.c.ProjectHeaderFile; + +/** + * Determines which header files must be included when building a native-image that uses a GC such + * as G1. + */ +public class NativeGCHeaderFiles implements CContext.Directives { + @Override + public boolean isInConfiguration() { + return UseNativeGC.get(); + } + + @Override + public List getHeaderFiles() { + return Collections.singletonList(ProjectHeaderFile.resolve("", "include/sharedGCStructs.h")); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCOptions.java new file mode 100644 index 000000000000..ffd9140e7a7e --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCOptions.java @@ -0,0 +1,448 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +import static com.oracle.svm.core.option.RuntimeOptionKey.RuntimeOptionKeyFlag.IsolateCreationOnly; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.function.Consumer; +import java.util.function.Supplier; + +import org.graalvm.collections.UnmodifiableEconomicMap; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.option.HostedOptionKey; +import com.oracle.svm.core.option.HostedOptionValues; +import com.oracle.svm.core.option.RuntimeOptionKey; +import com.oracle.svm.core.option.RuntimeOptionValues; +import com.oracle.svm.core.option.SubstrateOptionKey; +import com.oracle.svm.core.util.UserError; +import com.oracle.svm.core.util.VMError; + +import jdk.graal.compiler.options.Option; +import jdk.graal.compiler.options.OptionKey; +import jdk.graal.compiler.options.OptionType; +import jdk.graal.compiler.options.OptionValues; +import jdk.vm.ci.code.CodeUtil; + +/** + * Defines options that are specific to GCs such as G1. Hosted options are properly validated at + * build-time. Runtime options, for which a default value is specified at build-time using + * {@code -R:...}, undergo only basic build-time validation. Comprehensive validation of these + * runtime options is performed by the C++ code during GC initialization. + *

+ * Please note that the default values below don't necessarily match the default values that are + * specified in files such as {@code gc_globals.hpp}. Some of these values are computed during + * startup as they depend on the used GC and operating system. + *

+ * All options that are relevant for the C++ code are serialized into byte arrays (see + * {@link HostedArgumentsSupplier} and {@link RuntimeArgumentsSupplier}). During VM startup, SVM + * passes those byte arrays to the C++ code. + *

+ * Important: After VM startup, only the C++ code knows which value each runtime option has. + * Option values are not synced back to SVM, so Java code must not access any of the runtime options + * below, as doing so may return outdated or incorrect values. + */ +public class NativeGCOptions { + public static final int K = 1024; + public static final int M = 1024 * K; + + /* gc_globals.hpp */ + + @Option(help = "Number of parallel threads that the GC will use.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ParallelGCThreads = new NativeGCRuntimeOptionKey<>(0, IsolateCreationOnly); + + @Option(help = "Dynamically choose the number of threads up to a maximum of ParallelGCThreads that the GC will use for garbage collection work.", type = OptionType.Expert)// + protected static final RuntimeOptionKey UseDynamicNumberOfGCThreads = new NativeGCRuntimeOptionKey<>(true, IsolateCreationOnly); + + @Option(help = "Size of heap (bytes) per GC thread used in calculating the number of GC threads.", type = OptionType.Expert)// + protected static final RuntimeOptionKey HeapSizePerGCThread = new NativeGCRuntimeOptionKey<>(42L * M, IsolateCreationOnly); + + @Option(help = "Number of concurrent threads that the GC will use.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ConcGCThreads = new NativeGCRuntimeOptionKey<>(0, IsolateCreationOnly); + + @Option(help = "Determines if System.gc() invokes a concurrent collection.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ExplicitGCInvokesConcurrent = new NativeGCRuntimeOptionKey<>(false, IsolateCreationOnly); + + @Option(help = "Wasted fraction of parallel allocation buffer.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ParallelGCBufferWastePct = new NativeGCRuntimeOptionKey<>(10, IsolateCreationOnly); + + @Option(help = "Target wasted space in last buffer as percent of overall allocation.", type = OptionType.Expert)// + protected static final RuntimeOptionKey TargetPLABWastePct = new NativeGCRuntimeOptionKey<>(10, IsolateCreationOnly); + + @Option(help = "Percentage (0-100) used to weight the current sample when computing exponentially decaying average for ResizePLAB.", type = OptionType.Expert)// + protected static final RuntimeOptionKey PLABWeight = new NativeGCRuntimeOptionKey<>(75, IsolateCreationOnly); + + @Option(help = "Dynamically resize (survivor space) promotion LAB's.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ResizePLAB = new NativeGCRuntimeOptionKey<>(true, IsolateCreationOnly); + + @Option(help = "Scan a subset of object array and push remainder, if array is bigger than this.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ParGCArrayScanChunk = new NativeGCRuntimeOptionKey<>(50, IsolateCreationOnly); + + @Option(help = "Force all freshly committed pages to be pre-touched.", type = OptionType.Expert)// + protected static final RuntimeOptionKey AlwaysPreTouch = new NativeGCRuntimeOptionKey<>(false, IsolateCreationOnly); + + @Option(help = "Per-thread chunk size for parallel memory pre-touch.", type = OptionType.Expert)// + protected static final RuntimeOptionKey PreTouchParallelChunkSize = new NativeGCRuntimeOptionKey<>(4L * M, IsolateCreationOnly); + + @Option(help = "Maximum size of marking stack in bytes.", type = OptionType.Expert)// + protected static final RuntimeOptionKey MarkStackSizeMax = new NativeGCRuntimeOptionKey<>(512L * M, IsolateCreationOnly); + + @Option(help = "Size of marking stack in bytes.", type = OptionType.Expert)// + protected static final RuntimeOptionKey MarkStackSize = new NativeGCRuntimeOptionKey<>(4L * M, IsolateCreationOnly); + + @Option(help = "Enable parallel reference processing whenever possible.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ParallelRefProcEnabled = new NativeGCRuntimeOptionKey<>(false, IsolateCreationOnly); + + @Option(help = "Enable balancing of reference processing queues.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ParallelRefProcBalancingEnabled = new NativeGCRuntimeOptionKey<>(true, IsolateCreationOnly); + + @Option(help = "The percent occupancy (IHOP) of the current old generation capacity above which a concurrent mark cycle will be initiated. Its value may change over time if adaptive IHOP is enabled, otherwise " + + "the value remains constant. In the latter case a value of 0 will result as frequent as possible concurrent marking cycles. A value of 100 disables concurrent marking. Fragmentation waste in the old generation " + + "is not considered free space in this calculation.", type = OptionType.Expert)// + protected static final RuntimeOptionKey InitiatingHeapOccupancyPercent = new NativeGCRuntimeOptionKey<>(45, IsolateCreationOnly); + + protected static final RuntimeOptionKey MaxRAM = SubstrateOptions.ConcealedOptions.MaxRAM; + + @Option(help = "Maximum ergonomically set heap size (in bytes); zero means use MaxRAM * MaxRAMPercentage / 100.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ErgoHeapSizeLimit = new NativeGCRuntimeOptionKey<>(0L, IsolateCreationOnly); + + @Option(help = "Maximum percentage of real memory used for maximum heap size.", type = OptionType.Expert)// + public static final RuntimeOptionKey MaxRAMPercentage = new NativeGCRuntimeOptionKey<>(25.0, IsolateCreationOnly); + + @Option(help = "Minimum percentage of real memory used for maximum heap size on systems with small physical memory size.", type = OptionType.Expert)// + protected static final RuntimeOptionKey MinRAMPercentage = new NativeGCRuntimeOptionKey<>(50.0, IsolateCreationOnly); + + @Option(help = "Percentage of real memory used for initial heap size.", type = OptionType.Expert)// + protected static final RuntimeOptionKey InitialRAMPercentage = new NativeGCRuntimeOptionKey<>(0.2, IsolateCreationOnly); + + protected static final RuntimeOptionKey ActiveProcessorCount = SubstrateOptions.ActiveProcessorCount; + + @Option(help = "Adaptive size policy maximum GC pause time goal in millisecond, or the maximum GC time per MMU time slice.", type = OptionType.Expert)// + protected static final RuntimeOptionKey MaxGCPauseMillis = new NativeGCRuntimeOptionKey<>(200L, IsolateCreationOnly); + + @Option(help = "Time slice for MMU specification.", type = OptionType.Expert)// + protected static final RuntimeOptionKey GCPauseIntervalMillis = new NativeGCRuntimeOptionKey<>(201L, IsolateCreationOnly); + + @Option(help = "Adaptive size policy application time to GC time ratio.", type = OptionType.Expert)// + protected static final RuntimeOptionKey GCTimeRatio = new NativeGCRuntimeOptionKey<>(12, IsolateCreationOnly); + + @Option(help = "How far ahead to prefetch destination area (<= 0 means off).", type = OptionType.Expert)// + protected static final RuntimeOptionKey PrefetchCopyIntervalInBytes = new NativeGCRuntimeOptionKey<>(-1L, IsolateCreationOnly); + + @Option(help = "How far ahead to prefetch scan area (<= 0 means off).", type = OptionType.Expert)// + protected static final RuntimeOptionKey PrefetchScanIntervalInBytes = new NativeGCRuntimeOptionKey<>(-1L, IsolateCreationOnly); + + @Option(help = "Verify memory system before GC.", type = OptionType.Debug)// + protected static final RuntimeOptionKey VerifyBeforeGC = new NativeGCRuntimeOptionKey<>(false, IsolateCreationOnly); + + @Option(help = "Verify memory system after GC.", type = OptionType.Debug)// + protected static final RuntimeOptionKey VerifyAfterGC = new NativeGCRuntimeOptionKey<>(false, IsolateCreationOnly); + + @Option(help = "Verify memory system during GC (between phases).", type = OptionType.Debug)// + protected static final RuntimeOptionKey VerifyDuringGC = new NativeGCRuntimeOptionKey<>(false, IsolateCreationOnly); + + @Option(help = "Initial heap size (in bytes); zero means use ergonomics.", type = OptionType.Expert)// + protected static final RuntimeOptionKey InitialHeapSize = new NativeGCRuntimeOptionKey<>(0L, IsolateCreationOnly); + + @Option(help = "Initial new generation size (in bytes).", type = OptionType.Expert)// + protected static final RuntimeOptionKey NewSize = new NativeGCRuntimeOptionKey<>(1L * M, IsolateCreationOnly); + + @Option(help = "Ratio of eden/survivor space size.", type = OptionType.Expert)// + protected static final RuntimeOptionKey SurvivorRatio = new NativeGCRuntimeOptionKey<>(8L, IsolateCreationOnly); + + @Option(help = "Ratio of old/new generation sizes.", type = OptionType.Expert)// + protected static final RuntimeOptionKey NewRatio = new NativeGCRuntimeOptionKey<>(2L, IsolateCreationOnly); + + @Option(help = "Number of times an allocation that queues behind a GC will retry before printing a warning.", type = OptionType.Expert)// + protected static final RuntimeOptionKey QueuedAllocationWarningCount = new NativeGCRuntimeOptionKey<>(0L, IsolateCreationOnly); + + @Option(help = "GC invoke count where +VerifyHeap kicks in.", type = OptionType.Debug)// + protected static final RuntimeOptionKey VerifyGCStartAt = new NativeGCRuntimeOptionKey<>(0L, IsolateCreationOnly); + + @Option(help = "Maximum value for tenuring threshold.", type = OptionType.Expert)// + protected static final RuntimeOptionKey MaxTenuringThreshold = new NativeGCRuntimeOptionKey<>(15, IsolateCreationOnly); + + @Option(help = "Desired percentage of survivor space used after scavenge.", type = OptionType.Expert)// + protected static final RuntimeOptionKey TargetSurvivorRatio = new NativeGCRuntimeOptionKey<>(50, IsolateCreationOnly); + + @Option(help = "Number of entries we will try to leave on the stack during gc.", type = OptionType.Expert)// + protected static final RuntimeOptionKey GCDrainStackTargetSize = new NativeGCRuntimeOptionKey<>(64, IsolateCreationOnly); + + @Option(help = "Card table entry size (in bytes).", type = OptionType.Expert)// + public static final HostedOptionKey GCCardSizeInBytes = new NativeGCHostedOptionKey<>(512, NativeGCOptions::validatePowerOfTwo); + + /* tlab_globals.hpp */ + @Option(help = "Zero out the newly created TLAB.", type = OptionType.Expert)// + protected static final RuntimeOptionKey ZeroTLAB = new NativeGCRuntimeOptionKey<>(false, IsolateCreationOnly); + + @Option(help = "Size of young gen promotion LAB's (in HeapWords).", type = OptionType.Expert)// + protected static final RuntimeOptionKey YoungPLABSize = new NativeGCRuntimeOptionKey<>(4096L, IsolateCreationOnly); + + @Option(help = "Size of old gen promotion LAB's (in HeapWords).", type = OptionType.Expert)// + protected static final RuntimeOptionKey OldPLABSize = new NativeGCRuntimeOptionKey<>(1024L, IsolateCreationOnly); + + @Option(help = "Allocation averaging weight.", type = OptionType.Expert)// + protected static final RuntimeOptionKey TLABAllocationWeight = new NativeGCRuntimeOptionKey<>(35L, IsolateCreationOnly); + + @Option(help = "Percentage of Eden that can be wasted (half-full TLABs at GC).", type = OptionType.Expert)// + protected static final RuntimeOptionKey TLABWasteTargetPercent = new NativeGCRuntimeOptionKey<>(1L, IsolateCreationOnly); + + @Option(help = "Maximum TLAB waste at a refill (internal fragmentation).", type = OptionType.Expert)// + protected static final RuntimeOptionKey TLABRefillWasteFraction = new NativeGCRuntimeOptionKey<>(64L, IsolateCreationOnly); + + @Option(help = "Increment allowed waste at slow allocation.", type = OptionType.Expert)// + protected static final RuntimeOptionKey TLABWasteIncrement = new NativeGCRuntimeOptionKey<>(4L, IsolateCreationOnly); + + /* globals.hpp */ + + @Option(help = "The minimum percentage of heap free after GC to avoid expansion.", type = OptionType.Expert)// + protected static final RuntimeOptionKey MinHeapFreeRatio = new NativeGCRuntimeOptionKey<>(40L, IsolateCreationOnly); + + @Option(help = "Number of milliseconds per MB of free space in the heap.", type = OptionType.Expert)// + protected static final RuntimeOptionKey SoftRefLRUPolicyMSPerMB = new NativeGCRuntimeOptionKey<>(1000L, IsolateCreationOnly); + + @Option(help = "The minimum change in heap space due to GC (in bytes).", type = OptionType.Expert)// + protected static final RuntimeOptionKey MinHeapDeltaBytes = new NativeGCRuntimeOptionKey<>(168L * K, IsolateCreationOnly); + + /* This runtime option depends on a hosted option (see special handling in svmToGC.cpp). */ + protected static final RuntimeOptionKey UsePerfData = SubstrateOptions.ConcealedOptions.UsePerfData; + + /* globals_linux.hpp */ + protected static final HostedOptionKey UseContainerSupport = SubstrateOptions.UseContainerSupport; + + @Platforms(Platform.HOSTED_ONLY.class) + public static ArrayList getOptionFields(Class[] optionClasses) { + ArrayList result = new ArrayList<>(); + for (Class clazz : optionClasses) { + for (Field field : clazz.getDeclaredFields()) { + if (Modifier.isStatic(field.getModifiers()) && OptionKey.class.isAssignableFrom(field.getType())) { + field.setAccessible(true); + result.add(field); + } + } + } + return result; + } + + private static void validatePowerOfTwo(HostedOptionKey optionKey) { + int value = optionKey.getValue(); + if (!CodeUtil.isPowerOf2(value)) { + throw UserError.invalidOptionValue(optionKey, value, "The value must be a power of two"); + } + } + + private static void validatePlatformAndGC(SubstrateOptionKey optionKey) { + if (!optionKey.hasBeenSet()) { + return; + } + + if (!Platform.includedIn(Platform.LINUX_AMD64.class) && !Platform.includedIn(Platform.LINUX_AARCH64.class)) { + throw UserError.abort("The option '%s' can only be used on linux/amd64 or linux/aarch64.", optionKey.getName()); + } else if (!SubstrateOptions.useG1GC()) { + throw UserError.abort("The option '%s' can only be used with the G1 ('--gc=G1') garbage collector.", optionKey.getName()); + } + } + + public static class NativeGCHostedOptionKey extends HostedOptionKey { + private final boolean passToCpp; + + public NativeGCHostedOptionKey(T defaultValue, Consumer> validation) { + this(defaultValue, true, validation); + } + + public NativeGCHostedOptionKey(T defaultValue, boolean passToCpp, Consumer> validation) { + super(defaultValue, validation); + this.passToCpp = passToCpp; + } + + public boolean shouldPassToCpp() { + return passToCpp; + } + + @Override + public void validate() { + validatePlatformAndGC(this); + super.validate(); + } + } + + public static class NativeGCRuntimeOptionKey extends RuntimeOptionKey { + public NativeGCRuntimeOptionKey(T defaultValue, RuntimeOptionKeyFlag... flags) { + super(defaultValue, flags); + } + + @Override + public void validate() { + validatePlatformAndGC(this); + super.validate(); + } + } + + /** Serializes GC-relevant hosted options and their build-time values into a byte array. */ + @Platforms(Platform.HOSTED_ONLY.class) + public static final class HostedArgumentsSupplier implements Supplier { + private final ArrayList optionFields; + + public HostedArgumentsSupplier(ArrayList optionFields) { + this.optionFields = optionFields; + } + + @Override + public byte[] get() { + NativeGCArgumentsBuffer buffer = new NativeGCArgumentsBuffer(); + UnmodifiableEconomicMap, Object> map = HostedOptionValues.singleton().getMap(); + for (Field field : optionFields) { + try { + Class type = field.getType(); + if (HostedOptionKey.class.isAssignableFrom(type)) { + HostedOptionKey key = (HostedOptionKey) field.get(null); + if (shouldPassToCpp(key)) { + buffer.putString(key.getName()); + buffer.putPrimitive(key.getValueOrDefault(map)); + } + } + } catch (IllegalArgumentException | IllegalAccessException e) { + throw VMError.shouldNotReachHere(e); + } + } + buffer.putEnd(); + return buffer.toArray(); + } + + private static boolean shouldPassToCpp(HostedOptionKey key) { + if (key instanceof NativeGCHostedOptionKey) { + return ((NativeGCHostedOptionKey) key).shouldPassToCpp(); + } + return true; + } + } + + /** Serializes GC-relevant runtime options and their build-time values into a byte array. */ + @Platforms(Platform.HOSTED_ONLY.class) + public static final class RuntimeArgumentsSupplier implements Supplier { + private final ArrayList optionFields; + + public RuntimeArgumentsSupplier(ArrayList optionFields) { + this.optionFields = optionFields; + } + + @Override + public byte[] get() { + NativeGCArgumentsBuffer buffer = new NativeGCArgumentsBuffer(); + OptionValues optionValues = RuntimeOptionValues.singleton(); + for (Field field : optionFields) { + try { + Class type = field.getType(); + if (RuntimeOptionKey.class.isAssignableFrom(type)) { + RuntimeOptionKey key = (RuntimeOptionKey) field.get(null); + if (key.hasBeenSet(optionValues)) { + buffer.putString(key.getName()); + buffer.putPrimitive(key.getValue(optionValues)); + } + } + } catch (IllegalArgumentException | IllegalAccessException e) { + throw VMError.shouldNotReachHere(e); + } + } + buffer.putEnd(); + return buffer.toArray(); + } + } + + /** + * Writes Strings as ISO_8859_1 with zero termination. Primitive values are encoded as 64-bit + * values (regardless of their actual size and type). + */ + @Platforms(Platform.HOSTED_ONLY.class) + private static class NativeGCArgumentsBuffer { + private ByteBuffer buffer; + + NativeGCArgumentsBuffer() { + buffer = allocateBuffer(128); + } + + public void putString(String value) { + assert value != null && !value.isEmpty() : value; + byte[] latin1String = value.getBytes(StandardCharsets.ISO_8859_1); + ensureCapacity(latin1String.length + 1); + + buffer.put(latin1String); + buffer.put((byte) 0); + } + + public void putPrimitive(Object value) { + ensureCapacity(8); + + long rawLong = switch (value) { + case Boolean _ -> ((boolean) value) ? 1L : 0L; + case Byte _ -> ((byte) value) & 0xFFL; + case Character _ -> ((char) value) & 0xFFFFL; + case Short _ -> ((short) value) & 0xFFFFL; + case Integer _ -> ((int) value) & 0xFFFFFFFFL; + case Long _ -> (long) value; + case Double _ -> Double.doubleToLongBits((double) value); + default -> throw VMError.shouldNotReachHere("Unexpected type: " + value.getClass()); + }; + + buffer.putLong(rawLong); + } + + public void putEnd() { + ensureCapacity(1); + buffer.put((byte) 0); + } + + public void ensureCapacity(int requested) { + int needed = buffer.position() + requested; + if (buffer.capacity() < needed) { + int newCapacity = buffer.capacity(); + do { + newCapacity *= 2; + } while (newCapacity < needed); + + ByteBuffer newBuffer = allocateBuffer(newCapacity); + newBuffer.put(buffer.array(), 0, buffer.position()); + buffer = newBuffer; + } + } + + public byte[] toArray() { + return Arrays.copyOf(buffer.array(), buffer.position()); + } + + private static ByteBuffer allocateBuffer(int size) { + return ByteBuffer.allocate(size).order(ByteOrder.nativeOrder()); + } + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCStackWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCStackWalker.java new file mode 100644 index 000000000000..aab39e78e427 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCStackWalker.java @@ -0,0 +1,484 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +import java.util.function.BooleanSupplier; + +import org.graalvm.nativeimage.CurrentIsolate; +import org.graalvm.nativeimage.Isolate; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.StackValue; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.function.CEntryPoint.Publish; +import org.graalvm.nativeimage.c.function.CEntryPointLiteral; +import org.graalvm.nativeimage.c.function.CFunctionPointer; +import org.graalvm.nativeimage.c.function.CodePointer; +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; + +import com.oracle.svm.core.NeverInline; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.c.NonmovableArray; +import com.oracle.svm.core.c.NonmovableArrays; +import com.oracle.svm.core.c.function.CEntryPointOptions; +import com.oracle.svm.core.code.CodeInfo; +import com.oracle.svm.core.code.CodeInfoAccess; +import com.oracle.svm.core.deopt.DeoptimizedFrame; +import com.oracle.svm.core.gc.shared.NativeGCStructs.CodeInfos; +import com.oracle.svm.core.gc.shared.NativeGCStructs.CodeInfosPerThread; +import com.oracle.svm.core.gc.shared.NativeGCStructs.StackFrame; +import com.oracle.svm.core.gc.shared.NativeGCStructs.StackFrames; +import com.oracle.svm.core.gc.shared.NativeGCStructs.StackFramesPerThread; +import com.oracle.svm.core.graal.RuntimeCompilation; +import com.oracle.svm.core.heap.StoredContinuation; +import com.oracle.svm.core.heap.StoredContinuationAccess; +import com.oracle.svm.core.heap.StoredContinuationAccess.ContinuationStackFrameVisitor; +import com.oracle.svm.core.heap.StoredContinuationAccess.ContinuationStackFrameVisitorData; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; +import com.oracle.svm.core.snippets.KnownIntrinsics; +import com.oracle.svm.core.stack.JavaStackWalker; +import com.oracle.svm.core.stack.ParameterizedStackFrameVisitor; +import com.oracle.svm.core.thread.ContinuationSupport; +import com.oracle.svm.core.thread.VMOperation; +import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.util.VMError; + +import jdk.graal.compiler.word.Word; + +/** + * Whenever GC-related C++ code needs information about the stack frames, it calls into Native Image + * and does a stack walk to collect that information. Relevant information is collected in a data + * structure that is allocated in native memory. Once the C++ code doesn't need this information + * anymore, it calls into Native Image to free the native memory. + */ +public final class NativeGCStackWalker { + private static final ThreadStackFrameCollector THREAD_STACK_FRAME_COLLECTOR = new ThreadStackFrameCollector(); + private static final ContinuationStackFrameCollector CONTINUATION_STACK_FRAME_COLLECTOR = new ContinuationStackFrameCollector(); + private static final CodeInfoCollector CODE_INFO_COLLECTOR = new CodeInfoCollector(); + + public final CEntryPointLiteral funcFetchThreadStackFrames; + public final CEntryPointLiteral funcFreeThreadStackFrames; + public final CEntryPointLiteral funcFetchContinuationStackFrames; + public final CEntryPointLiteral funcFreeContinuationStackFrames; + public final CEntryPointLiteral funcFetchCodeInfos; + public final CEntryPointLiteral funcFreeCodeInfos; + + @Platforms(Platform.HOSTED_ONLY.class) + public NativeGCStackWalker() { + funcFetchThreadStackFrames = CEntryPointLiteral.create(NativeGCStackWalker.class, "fetchThreadStackFrames", Isolate.class, IsolateThread.class); + funcFreeThreadStackFrames = CEntryPointLiteral.create(NativeGCStackWalker.class, "freeThreadStackFrames", Isolate.class, IsolateThread.class, StackFramesPerThread.class); + + if (ContinuationSupport.isSupported()) { + funcFetchContinuationStackFrames = CEntryPointLiteral.create(NativeGCStackWalker.class, "fetchContinuationStackFrames", Isolate.class, PointerBase.class, Pointer.class); + funcFreeContinuationStackFrames = CEntryPointLiteral.create(NativeGCStackWalker.class, "freeContinuationStackFrames", Isolate.class, PointerBase.class, StackFrames.class); + } else { + funcFetchContinuationStackFrames = null; + funcFreeContinuationStackFrames = null; + } + + if (RuntimeCompilation.isEnabled()) { + funcFetchCodeInfos = CEntryPointLiteral.create(NativeGCStackWalker.class, "fetchCodeInfos", Isolate.class, IsolateThread.class); + funcFreeCodeInfos = CEntryPointLiteral.create(NativeGCStackWalker.class, "freeCodeInfos", Isolate.class, IsolateThread.class, CodeInfosPerThread.class); + } else { + funcFetchCodeInfos = null; + funcFreeCodeInfos = null; + } + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForPossiblyUnattachedThread.class, epilogue = CEntryPointOptions.NoEpilogue.class) + public static StackFramesPerThread fetchThreadStackFrames(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") IsolateThread thread) { + return walkStack(THREAD_STACK_FRAME_COLLECTOR); + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForPossiblyUnattachedThread.class, epilogue = CEntryPointOptions.NoEpilogue.class) + public static void freeThreadStackFrames(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") IsolateThread thread, StackFramesPerThread stackFramesPerThread) { + for (int i = 0; i < stackFramesPerThread.count(); i++) { + NullableNativeMemory.free(stackFramesPerThread.threads().addressOf(i).read()); + } + NullableNativeMemory.free(stackFramesPerThread); + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + @CEntryPoint(include = UseNativeGCAndContinuations.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForUnattachedThread.class, epilogue = CEntryPointOptions.NoEpilogue.class) + public static StackFrames fetchContinuationStackFrames(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") PointerBase heapBase, Pointer storedContinuation) { + StoredContinuation s = (StoredContinuation) storedContinuation.toObject(); + ContinuationStackFrameCollectorData data = StackValue.get(ContinuationStackFrameCollectorData.class); + ContinuationStackFrameCollector.initialize(data); + StoredContinuationAccess.walkFrames(s, CONTINUATION_STACK_FRAME_COLLECTOR, data); + return data.getStack(); + } + + @Uninterruptible(reason = "May be called by an unattached thread (during or outside of a safepoint).") + @CEntryPoint(include = UseNativeGCAndContinuations.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForUnattachedThread.class, epilogue = CEntryPointOptions.NoEpilogue.class) + public static void freeContinuationStackFrames(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") PointerBase heapBase, StackFrames stackFrames) { + NullableNativeMemory.free(stackFrames); + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForPossiblyUnattachedThread.class, epilogue = CEntryPointOptions.NoEpilogue.class) + public static CodeInfosPerThread fetchCodeInfos(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") IsolateThread thread) { + return walkStack(CODE_INFO_COLLECTOR); + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForPossiblyUnattachedThread.class, epilogue = CEntryPointOptions.NoEpilogue.class) + public static void freeCodeInfos(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") IsolateThread thread, CodeInfosPerThread codeInfos) { + for (int i = 0; i < codeInfos.count(); i++) { + NullableNativeMemory.free(codeInfos.threads().addressOf(i).read()); + } + NullableNativeMemory.free(codeInfos); + } + + @NeverInline("Stack walking.") + @Uninterruptible(reason = "GC may only call uninterruptible code.") + private static T walkStack(Collector collector) { + VMOperation.guaranteeInProgressAtSafepoint("Doing a stack walk for every thread is only possible when we are at a safepoint."); + collector.startWalking(); + + /* Walk the current thread. */ + collector.newThread(); + JavaStackWalker.walkCurrentThread(KnownIntrinsics.readCallerStackPointer(), collector, null); + collector.finishThread(); + + /* Walk all other threads. */ + for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { + if (vmThread == CurrentIsolate.getCurrentThread()) { + /* The current thread is already walked by code above. */ + continue; + } + collector.newThread(); + JavaStackWalker.walkThread(vmThread, collector, null); + collector.finishThread(); + } + return collector.finishWalking(); + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + private static V malloc(UnsignedWord size) { + V result = NullableNativeMemory.malloc(size, NmtCategory.GC); + if (result.isNull()) { + throw VMError.shouldNotReachHere("malloc returned null."); + } + return result; + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + private static V realloc(V ptr, UnsignedWord size) { + V result = NullableNativeMemory.realloc(ptr, size, NmtCategory.GC); + if (result.isNull()) { + throw VMError.shouldNotReachHere("realloc returned null."); + } + return result; + } + + @RawStructure + private interface ContinuationStackFrameCollectorData extends ContinuationStackFrameVisitorData { + @RawField + int getCapacity(); + + @RawField + void setCapacity(int value); + + @RawField + StackFrames getStack(); + + @RawField + void setStack(StackFrames frames); + } + + private static class ContinuationStackFrameCollector extends ContinuationStackFrameVisitor { + private static final int DEFAULT_NUM_STACK_FRAMES = 10; + + @Platforms(value = Platform.HOSTED_ONLY.class) + ContinuationStackFrameCollector() { + } + + @Uninterruptible(reason = "GC may only call uninterruptible code and StoredContinuation must not move.", callerMustBe = true) + public static void initialize(ContinuationStackFrameCollectorData data) { + UnsignedWord size = computeStackFramesSize(DEFAULT_NUM_STACK_FRAMES); + StackFrames frames = malloc(size); + frames.setCount(0); + + data.setStack(frames); + data.setCapacity(DEFAULT_NUM_STACK_FRAMES); + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code and StoredContinuation must not move.", callerMustBe = true) + public void visitFrame(ContinuationStackFrameVisitorData data, Pointer sp, NonmovableArray referenceMapEncoding, long referenceMapIndex, ContinuationStackFrameVisitor visitor) { + ContinuationStackFrameCollectorData d = (ContinuationStackFrameCollectorData) data; + + long index = d.getStack().count(); + if (index == d.getCapacity()) { + int newCapacity = d.getCapacity() * 2; + StackFrames newStackFrames = realloc(d.getStack(), computeStackFramesSize(newCapacity)); + d.setStack(newStackFrames); + d.setCapacity(newCapacity); + } + + StackFrames stack = d.getStack(); + StackFrame frame = stack.frames().addressOf(index); + frame.setStackPointer(sp); + frame.setEncodedReferenceMap(NonmovableArrays.getArrayBase(referenceMapEncoding)); + frame.setReferenceMapIndex(referenceMapIndex); + stack.setCount(index + 1); + } + + @Uninterruptible(reason = "GC may only execute uninterruptible code.") + private static UnsignedWord computeStackFramesSize(int frames) { + return SizeOf.unsigned(StackFrames.class).add(SizeOf.unsigned(StackFrame.class).multiply(frames)); + } + } + + private abstract static class Collector extends ParameterizedStackFrameVisitor { + protected static final int INITIALLY_RESERVED_THREADS = 5; + protected static final int INITIALLY_RESERVED_STACK_FRAMES = 10; + + protected T threads; + protected int threadsLength; + + protected U currentStack; + protected int currentStackLength; + + @Platforms(value = Platform.HOSTED_ONLY.class) + Collector() { + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public abstract void startWalking(); + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public abstract void newThread(); + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public abstract void finishThread(); + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public T finishWalking() { + assert currentStack.isNull(); + assert currentStackLength == 0; + + T result = threads; + this.threads = Word.nullPointer(); + this.threadsLength = 0; + return result; + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + protected final boolean unknownFrame(Pointer sp, CodePointer ip, Object data) { + throw JavaStackWalker.fatalErrorUnknownFrameEncountered(sp, ip); + } + } + + private static class ThreadStackFrameCollector extends Collector { + @Platforms(value = Platform.HOSTED_ONLY.class) + ThreadStackFrameCollector() { + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public void startWalking() { + assert threads.isNull() && threadsLength == 0 && currentStack.isNull() && currentStackLength == 0; + + this.threadsLength = INITIALLY_RESERVED_THREADS; + this.threads = malloc(computeStackFramesPerThreadSize()); + this.threads.setCount(0); + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public void newThread() { + assert currentStack.isNull() && currentStackLength == 0; + + this.currentStackLength = INITIALLY_RESERVED_STACK_FRAMES; + this.currentStack = malloc(computeStackFramesSize()); + this.currentStack.setCount(0); + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public void finishThread() { + long index = this.threads.count(); + if (index == threadsLength) { + this.threadsLength *= 2; + this.threads = realloc(threads, computeStackFramesPerThreadSize()); + } + + threads.threads().addressOf(index).write(currentStack); + this.threads.setCount(index + 1); + + this.currentStack = Word.nullPointer(); + this.currentStackLength = 0; + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, Object data) { + NonmovableArray referenceMapEncoding = CodeInfoAccess.getStackReferenceMapEncoding(codeInfo); + long referenceMapIndex = CodeInfoAccess.lookupStackReferenceMapIndex(codeInfo, CodeInfoAccess.relativeIP(codeInfo, ip)); + append(sp, referenceMapEncoding, referenceMapIndex); + return true; + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame, Object data) { + /* Nothing to do - the DeoptimizedFrame is an object and therefore visible to the GC. */ + return true; + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + private void append(Pointer sp, NonmovableArray referenceMapEncoding, long referenceMapIndex) { + long index = currentStack.count(); + if (index == currentStackLength) { + currentStackLength *= 2; + currentStack = realloc(currentStack, computeStackFramesSize()); + } + + StackFrame frame = currentStack.frames().addressOf(index); + frame.setStackPointer(sp); + frame.setEncodedReferenceMap(NonmovableArrays.getArrayBase(referenceMapEncoding)); + frame.setReferenceMapIndex(referenceMapIndex); + currentStack.setCount(index + 1); + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + private UnsignedWord computeStackFramesPerThreadSize() { + return SizeOf.unsigned(StackFramesPerThread.class).add(SizeOf.unsigned(StackFrames.class).multiply(threadsLength)); + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + private UnsignedWord computeStackFramesSize() { + return SizeOf.unsigned(StackFrames.class).add(SizeOf.unsigned(StackFrame.class).multiply(currentStackLength)); + } + } + + private static class CodeInfoCollector extends Collector { + @Platforms(value = Platform.HOSTED_ONLY.class) + CodeInfoCollector() { + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public void startWalking() { + assert threads.isNull() && threadsLength == 0 && currentStack.isNull() && currentStackLength == 0; + + this.threadsLength = INITIALLY_RESERVED_THREADS; + this.threads = malloc(computeThreadsInfoSize()); + this.threads.setCount(0); + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public void newThread() { + assert currentStack.isNull() && currentStackLength == 0; + + this.currentStackLength = INITIALLY_RESERVED_STACK_FRAMES; + this.currentStack = malloc(computeStackInfoSize()); + this.currentStack.setCount(0); + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public void finishThread() { + long index = this.threads.count(); + if (index == threadsLength) { + this.threadsLength *= 2; + this.threads = realloc(threads, computeThreadsInfoSize()); + } + + threads.threads().addressOf(index).write(currentStack); + this.threads.setCount(index + 1); + + this.currentStack = Word.nullPointer(); + this.currentStackLength = 0; + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + public boolean visitRegularFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, Object data) { + if (!CodeInfoAccess.isAOTImageCode(codeInfo)) { + append(codeInfo); + } + return true; + } + + @Override + @Uninterruptible(reason = "GC may only call uninterruptible code.") + protected boolean visitDeoptimizedFrame(Pointer originalSP, CodePointer deoptStubIP, DeoptimizedFrame deoptimizedFrame, Object data) { + /* Nothing to do. */ + return true; + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + private void append(CodeInfo codeInfo) { + long index = this.currentStack.count(); + if (index == currentStackLength) { + currentStackLength *= 2; + currentStack = realloc(currentStack, computeStackInfoSize()); + } + + currentStack.codeInfos().addressOf(index).write(codeInfo); + currentStack.setCount(index + 1); + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + private UnsignedWord computeThreadsInfoSize() { + return SizeOf.unsigned(StackFramesPerThread.class).add(SizeOf.unsigned(StackFrames.class).multiply(threadsLength)); + } + + @Uninterruptible(reason = "GC may only call uninterruptible code.") + private UnsignedWord computeStackInfoSize() { + return SizeOf.unsigned(StackFrames.class).add(SizeOf.unsigned(StackFrame.class).multiply(currentStackLength)); + } + } + + private static class UseNativeGCAndContinuations implements BooleanSupplier { + @Platforms(Platform.HOSTED_ONLY.class) + UseNativeGCAndContinuations() { + } + + @Override + public boolean getAsBoolean() { + return UseNativeGC.get() && ContinuationSupport.isSupported(); + } + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCStructs.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCStructs.java new file mode 100644 index 000000000000..781ab3f92089 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCStructs.java @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +import org.graalvm.nativeimage.c.CContext; +import org.graalvm.nativeimage.c.struct.CField; +import org.graalvm.nativeimage.c.struct.CFieldAddress; +import org.graalvm.nativeimage.c.struct.CPointerTo; +import org.graalvm.nativeimage.c.struct.CStruct; +import org.graalvm.nativeimage.c.type.CCharPointer; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; + +import com.oracle.svm.core.code.CodeInfoPointer; + +/** Data structures imported from GC-related header files. */ +@CContext(NativeGCHeaderFiles.class) +public class NativeGCStructs { + /* Data structures for frames that are currently on the stack. */ + + @CStruct(addStructKeyword = true) + public interface StackFrame extends PointerBase { + StackFrame addressOf(long index); + + @CField("stack_pointer") + void setStackPointer(Pointer value); + + @CField("encoded_reference_map") + void setEncodedReferenceMap(CCharPointer value); + + @CField("reference_map_index") + void setReferenceMapIndex(long value); + } + + @CStruct(addStructKeyword = true) + public interface StackFrames extends PointerBase { + @CField + long count(); + + @CField("count") + void setCount(long value); + + @CFieldAddress + StackFrame frames(); + } + + @CStruct(addStructKeyword = true) + public interface StackFramesPerThread extends PointerBase { + @CField + long count(); + + @CField("count") + void setCount(long value); + + @CFieldAddress + StackFramesPointer threads(); + } + + @CPointerTo(StackFrames.class) + public interface StackFramesPointer extends PointerBase { + StackFramesPointer addressOf(long index); + + void write(StackFrames value); + + StackFrames read(); + } + + /* Data structures for JIT-compiled code that is currently on the stack. */ + + @CStruct(addStructKeyword = true) + public interface CodeInfos extends PointerBase { + @CField + long count(); + + @CField("count") + void setCount(long value); + + @CFieldAddress("code_infos") + CodeInfoPointer codeInfos(); + } + + @CStruct(addStructKeyword = true) + public interface CodeInfosPerThread extends PointerBase { + @CField + long count(); + + @CField("count") + void setCount(long value); + + @CFieldAddress + CodeInfosPointer threads(); + } + + @CPointerTo(CodeInfos.class) + public interface CodeInfosPointer extends PointerBase { + CodeInfosPointer addressOf(long index); + + void write(CodeInfos value); + + CodeInfos read(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCThreadTransitions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCThreadTransitions.java new file mode 100644 index 000000000000..a0192d9050f7 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCThreadTransitions.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.function.CEntryPoint.Publish; +import org.graalvm.nativeimage.c.function.CEntryPointLiteral; +import org.graalvm.nativeimage.c.function.CFunctionPointer; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.c.InitializeReservedRegistersPrologue; +import com.oracle.svm.core.c.function.CEntryPointOptions; +import com.oracle.svm.core.c.function.CEntryPointOptions.NoEpilogue; +import com.oracle.svm.core.thread.ThreadStatusTransition; + +/** + * The methods below are called if any GC-related C++ code needs to do a thread status transitions. + * This ensures that the GC doesn't need to know how thread status transitions work in detail. + */ +public class NativeGCThreadTransitions { + public final CEntryPointLiteral funcVMToNative; + public final CEntryPointLiteral funcFastNativeToVM; + public final CEntryPointLiteral funcSlowNativeToVM; + + @Platforms(Platform.HOSTED_ONLY.class) + public NativeGCThreadTransitions() { + funcVMToNative = CEntryPointLiteral.create(NativeGCThreadTransitions.class, "fromVMToNative", IsolateThread.class); + funcFastNativeToVM = CEntryPointLiteral.create(NativeGCThreadTransitions.class, "fastFromNativeToVM", IsolateThread.class); + funcSlowNativeToVM = CEntryPointLiteral.create(NativeGCThreadTransitions.class, "slowFromNativeToVM", IsolateThread.class); + } + + @Uninterruptible(reason = "Must not do a safepoint check after the thread status transition.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersPrologue.class, epilogue = NoEpilogue.class) + public static void fromVMToNative(@SuppressWarnings("unused") IsolateThread thread) { + ThreadStatusTransition.fromVMToNative(); + } + + @Uninterruptible(reason = "Must not do a safepoint check after the thread status transition.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersPrologue.class, epilogue = NoEpilogue.class) + public static boolean fastFromNativeToVM(@SuppressWarnings("unused") IsolateThread thread) { + return ThreadStatusTransition.tryFromNativeToVM(); + } + + @Uninterruptible(reason = "Must not do a safepoint check after the thread status transition.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersPrologue.class, epilogue = NoEpilogue.class) + public static void slowFromNativeToVM(@SuppressWarnings("unused") IsolateThread thread) { + ThreadStatusTransition.fromNativeToVM(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCVMOperationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCVMOperationSupport.java new file mode 100644 index 000000000000..ecbe8a53e780 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/NativeGCVMOperationSupport.java @@ -0,0 +1,342 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; +import static com.oracle.svm.core.thread.VMOperationControl.isDedicatedVMOperationThread; + +import org.graalvm.nativeimage.Isolate; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.function.CEntryPoint.Publish; +import org.graalvm.nativeimage.c.function.CEntryPointLiteral; +import org.graalvm.nativeimage.c.function.CFunctionPointer; +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.c.function.CEntryPointOptions; +import com.oracle.svm.core.c.function.CEntryPointOptions.NoEpilogue; +import com.oracle.svm.core.heap.AbstractPinnedObjectSupport; +import com.oracle.svm.core.heap.RestrictHeapAccess; +import com.oracle.svm.core.heap.RestrictHeapAccess.Access; +import com.oracle.svm.core.heap.VMOperationInfo; +import com.oracle.svm.core.heap.VMOperationInfos; +import com.oracle.svm.core.locks.VMCondition; +import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.thread.NativeVMOperation; +import com.oracle.svm.core.thread.NativeVMOperationData; +import com.oracle.svm.core.thread.RecurringCallbackSupport; +import com.oracle.svm.core.thread.ThreadStatusTransition; +import com.oracle.svm.core.thread.VMOperation; +import com.oracle.svm.core.thread.VMOperationControl; +import com.oracle.svm.core.thread.VMThreads.StatusSupport; +import com.oracle.svm.core.util.DuplicatedInNativeCode; + +/** + * GC-related VM operations can be scheduled by any isolate thread (including the VM operation + * thread), or any unattached GC-internal thread. + * + *

+ * Executing a GC-related VM operation involves a rather complex mechanism. So, avoid changes if + * possible, or be prepared to redesign the whole mechanism (changes are prone to race conditions + * and deadlocks). + * + *

+ * If an isolate thread wants to queue a VM operation while it is in C++ code (and therefore in + * {@link StatusSupport#STATUS_IN_VM}), it needs to execute the following steps: + *

    + *
  1. The queuing thread allocates all necessary data structures (e.g., + * {@link NativeGCVMOperationData}) on the C++-side in native memory.
    2. + *
  2. The queuing thread does a status transition to {@link StatusSupport#STATUS_IN_NATIVE}.
    3. + *
  3. The queuing thread calls {@link #enqueue} (via one of its callers that is annotated with + * {@link CEntryPoint}, see usages) to queue a non-blocking VM operation that wraps the GC VM + * operation (see {@link NativeGCVMOperationWrapper}). For queuing this VM operation, the thread + * needs to acquire the VM operation mutex and might get blocked. This is fine as we already did a + * transition to {@link StatusSupport#STATUS_IN_NATIVE} before.
    4. + *
  4. After queuing the VM operation, the queuing thread immediately returns to C++ code. No thread + * transition is necessary as the thread has still {@link StatusSupport#STATUS_IN_NATIVE}.
    5. + *
  5. Back in C++ code, the queuing thread waits until the VM operation thread starts executing the + * VM operation wrapper.
    6. + *
  6. The queuing thread executes the VM operation prologue which may acquire locks on the C++ + * side. Because we explicitly blocked the VM operation thread, it is guaranteed that locks (such as + * the {@code Heap_lock}) can be acquired eventually.
    7. + *
  7. The VM operation thread requests a safepoint and executes the main part of the GC-related VM + * operation while all Java threads are frozen. After the main part was executed, the safepoint is + * released.
    8. + *
  8. The queuing thread changes its thread status to {@link StatusSupport#STATUS_IN_VM} to prevent + * unexpected safepoints. This is necessary because it must be possible to transport uninitialized + * memory from the VM operation back to the queuing thread while ensuring that the GC does not see + * the uninitialized memory.
    9. + *
  9. After that, the VM operation thread may finish executing the wrapper VM operation at any + * time.
    10. + *
  10. The queuing thread executes the VM operation epilogue.
    11. + *
  11. The queuing thread waits until the execution of the VM operation wrapper finished.
    12. + *
  12. The queuing thread frees the native data structures that were allocated as it is now + * guaranteed that the VM operation thread won't access them anymore.
    13. + *
+ * + *

+ * For unattached GC threads, the mechanism is slightly simpler as the thread state transitions are + * not necessary. Besides that, pretty much the whole mechanism is the same. + * + *

+ * In Native Image, the VM operation thread is a normal Java thread. So, it can happen that for + * example a GC is needed while some non-GC-related VM operation is already in progress. Below is a + * detailed list of steps that the VM operation thread executes in that case: + *

    + *
  1. The VM operation thread is in C++ code and has {@link StatusSupport#STATUS_IN_VM} (e.g., + * currently in the allocation slowpath).
    2. + *
  2. The VM operation thread allocates all necessary data structures (e.g., + * {@link NativeGCVMOperationData}) on the C++ side in native memory.
    3. + *
  3. The VM operation thread calls one of the VM operation {@link CEntryPoint} methods (see usages + * of {@link #enqueue}).
    4. + *
  4. The VM operation thread does a transition to {@link StatusSupport#STATUS_IN_JAVA}.
    5. + *
  5. The VM operation thread executes the VM operation prologue.
    6. + *
  6. The VM operation thread initiates a safepoint if the surrounding VM operation didn't already + * start one.
    7. + *
  7. The VM operation thread executes the main part of the VM operation.
    8. + *
  8. The VM operation thread releases the safepoint if one was initiated before.
    9. + *
  9. The VM operation thread executes the VM operation epilogue.
    10. + *
  10. The VM operation thread does a transition to {@link StatusSupport#STATUS_IN_VM}.
    11. + *
  11. The VM operation thread returns to the C++ code.
    12. + *
  12. The VM operation thread frees the native data structures that were allocated before.
    13. + *
+ */ +public class NativeGCVMOperationSupport { + private static final NativeGCVMOperationWrapper VM_OPERATION_WRAPPER = new NativeGCVMOperationWrapper(); + + public final CEntryPointLiteral funcUpdateVMOperationExecutionStatus; + public final CEntryPointLiteral funcWaitForVMOperationExecutionStatus; + public final CEntryPointLiteral funcIsVMOperationFinished; + + @Platforms(Platform.HOSTED_ONLY.class) + public NativeGCVMOperationSupport() { + funcUpdateVMOperationExecutionStatus = CEntryPointLiteral.create(NativeGCVMOperationSupport.class, "updateVMOperationExecutionStatus", + Isolate.class, IsolateThread.class, NativeGCVMOperationWrapperData.class, int.class); + funcWaitForVMOperationExecutionStatus = CEntryPointLiteral.create(NativeGCVMOperationSupport.class, "waitForVMOperationExecutionStatus", + Isolate.class, IsolateThread.class, NativeGCVMOperationWrapperData.class, int.class); + funcIsVMOperationFinished = CEntryPointLiteral.create(NativeGCVMOperationSupport.class, "isVMOperationFinished", + Isolate.class, IsolateThread.class, NativeGCVMOperationWrapperData.class); + } + + @Uninterruptible(reason = "Can be called from an unattached thread.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForPossiblyUnattachedThread.class, epilogue = NoEpilogue.class) + public static void updateVMOperationExecutionStatus(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") IsolateThread isolateThread, NativeGCVMOperationWrapperData data, + int status) { + NativeGCVMOperationWrapper.updateExecutionStatus(data, status); + } + + @Uninterruptible(reason = "Can be called from an unattached thread.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForPossiblyUnattachedThread.class, epilogue = NoEpilogue.class) + public static void waitForVMOperationExecutionStatus(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") IsolateThread isolateThread, NativeGCVMOperationWrapperData data, + int minStatus) { + NativeGCVMOperationWrapper.waitForStatusUpdate(data, minStatus); + } + + @Uninterruptible(reason = "Can be called from an unattached thread.") + @CEntryPoint(include = UseNativeGC.class, publishAs = Publish.NotPublished) + @CEntryPointOptions(prologue = InitializeReservedRegistersForPossiblyUnattachedThread.class, epilogue = NoEpilogue.class) + public static boolean isVMOperationFinished(@SuppressWarnings("unused") Isolate isolate, @SuppressWarnings("unused") IsolateThread isolateThread, NativeGCVMOperationWrapperData data) { + return data.getFinished(); + } + + @Uninterruptible(reason = "Can be called from an unattached thread.") + public static void enqueue(NativeGCVMOperation op, NativeGCVMOperationData data, NativeGCVMOperationWrapperData wrapperData) { + data.setNativeVMOperation(op); + if (VMOperationControl.isDedicatedVMOperationThread()) { + /* The VM operation thread needs to execute the VM operation directly. */ + ThreadStatusTransition.fromVMToJava(false); + executeDirectly(op, data); + ThreadStatusTransition.fromJavaToVM(); + } else { + wrapperData.setNativeVMOperation(VM_OPERATION_WRAPPER); + wrapperData.setVMOperationData(data); + VM_OPERATION_WRAPPER.enqueueFromNonJavaThread(wrapperData); + } + } + + @Uninterruptible(reason = "Bridge between uninterruptible and interruptible code.", calleeMustBe = false) + private static void executeDirectly(NativeGCVMOperation op, NativeGCVMOperationData data) { + assert StatusSupport.isStatusJava() : "Thread must be in Java state to execute interruptible code"; + assert VMOperationControl.isDedicatedVMOperationThread(); + + /* Execute the prologue in the non-blocking part of the VM operation thread. */ + if (!op.executePrologue(data)) { + return; + } + + /* Queue the blocking VM operation. */ + assert op.getCausesSafepoint(); + op.enqueue(data); + + /* Execute the epilogue in the non-blocking part. */ + op.executeEpilogue(data); + } + + /** + * A non-blocking VM operation that wraps a blocking GC-related VM operation and deals with the + * necessary synchronization between SVM and C++. + * + * This has to be a {@link NativeVMOperation} as every queued instance needs a separate state + * (see {@link NativeGCVMOperationWrapperData}). + */ + @DuplicatedInNativeCode + private static class NativeGCVMOperationWrapper extends NativeVMOperation { + private static final int BLOCK_VM_THREAD = 0; + private static final int EXECUTE_PROLOGUE = BLOCK_VM_THREAD + 1; + private static final int EXECUTE_VM_OPERATION = EXECUTE_PROLOGUE + 1; + private static final int CANCELLED = EXECUTE_VM_OPERATION + 1; + private static final int ADJUST_THREAD_STATUS = CANCELLED + 1; + private static final int FINISHED = ADJUST_THREAD_STATUS + 1; + + private static final VMMutex EXECUTION_STATUS_MUTEX = new VMMutex("vmOpExecutionStatus"); + private static final VMCondition EXECUTION_STATUS_CONDITION = new VMCondition(EXECUTION_STATUS_MUTEX); + + protected NativeGCVMOperationWrapper() { + super(VMOperationInfos.get(NativeGCVMOperationWrapper.class, "Native GC VM operation wrapper", SystemEffect.NONE)); + } + + @Override + @RestrictHeapAccess(access = Access.NO_ALLOCATION, reason = "Used to implement GC functionality.") + protected void operate(NativeVMOperationData data) { + assert RecurringCallbackSupport.isCallbackUnsupportedOrTimerSuspended(); + operate0((NativeGCVMOperationWrapperData) data); + } + + private static void operate0(NativeGCVMOperationWrapperData wrapperData) { + assert wrapperData.getExecutionStatus() == BLOCK_VM_THREAD; + assert !isDedicatedVMOperationThread(wrapperData.getQueuingThread()); + assert !VMOperation.isInProgressAtSafepoint() : "GC-related VM operation that need this kind of synchronization must start their own safepoint"; + + /* Notify C++ code that the VM operation thread is in place. */ + updateExecutionStatus(wrapperData, EXECUTE_PROLOGUE); + + /* Wait until C++ code executed the prologue. */ + waitForStatusUpdate(wrapperData, EXECUTE_VM_OPERATION); + + if (wrapperData.getExecutionStatus() != CANCELLED) { + assert wrapperData.getExecutionStatus() == EXECUTE_VM_OPERATION; + + NativeGCVMOperationData opData = wrapperData.getVMOperationData(); + NativeGCVMOperation op = (NativeGCVMOperation) opData.getNativeVMOperation(); + + /* Execute the blocking part of the VM operation. */ + assert op.getCausesSafepoint(); + op.enqueue(opData); + + /* Notify C++ code that the VM operation was executed. */ + updateExecutionStatus(wrapperData, ADJUST_THREAD_STATUS); + + /* Wait until the C++ code changed the thread status to VM. */ + waitForStatusUpdate(wrapperData, FINISHED); + } + } + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.") + static void updateExecutionStatus(NativeGCVMOperationWrapperData data, int status) { + EXECUTION_STATUS_MUTEX.lockNoTransitionUnspecifiedOwner(); + try { + data.setExecutionStatus(status); + EXECUTION_STATUS_CONDITION.broadcast(); + } finally { + EXECUTION_STATUS_MUTEX.unlockNoTransitionUnspecifiedOwner(); + } + } + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.") + static void waitForStatusUpdate(NativeGCVMOperationWrapperData data, int minStatus) { + EXECUTION_STATUS_MUTEX.lockNoTransitionUnspecifiedOwner(); + try { + while (data.getExecutionStatus() < minStatus) { + EXECUTION_STATUS_CONDITION.blockNoTransitionUnspecifiedOwner(); + } + } finally { + EXECUTION_STATUS_MUTEX.unlockNoTransitionUnspecifiedOwner(); + } + } + } + + /** + * Stack allocated on the C++ side but only accessed by Native Image. We can't stack allocate + * the structure on the Native Image side because the stack is destroyed when we return to C++ + * after queuing the VM operation. + */ + @RawStructure + public interface NativeGCVMOperationWrapperData extends NativeVMOperationData { + @RawField + int getExecutionStatus(); + + @RawField + void setExecutionStatus(int value); + + @RawField + NativeGCVMOperationData getVMOperationData(); + + @RawField + void setVMOperationData(NativeGCVMOperationData value); + } + + public abstract static class NativeGCVMOperation extends NativeVMOperation { + private final boolean isGC; + + @Platforms(Platform.HOSTED_ONLY.class) + protected NativeGCVMOperation(VMOperationInfo info, boolean isGC) { + super(info); + this.isGC = isGC; + } + + @Override + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public boolean isGC() { + return isGC; + } + + public abstract boolean executePrologue(NativeGCVMOperationData data); + + @Override + protected void operate(NativeVMOperationData data) { + /* Cleanup obsolete pinned objects before calling into C++. */ + AbstractPinnedObjectSupport.singleton().removeClosedObjectsAndGetFirstOpenObject(); + operate0((NativeGCVMOperationData) data); + } + + protected abstract void operate0(NativeGCVMOperationData data); + + public abstract void executeEpilogue(NativeGCVMOperationData data); + } + + /** + * {@link NativeGCVMOperationData} structs are always allocated on the C++ side. Note that the + * C++ code uses pointer arithmetics and cast to convert this into a C++ {@code VM_Operation}. + */ + @RawStructure + public interface NativeGCVMOperationData extends NativeVMOperationData { + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/UseNativeGC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/UseNativeGC.java new file mode 100644 index 000000000000..1d61fc1f36bf --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/UseNativeGC.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared; + +import java.util.function.BooleanSupplier; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + +import com.oracle.svm.core.SubstrateOptions; + +@Platforms(Platform.HOSTED_ONLY.class) +public class UseNativeGC implements BooleanSupplier { + @Override + public boolean getAsBoolean() { + return get(); + } + + public static boolean get() { + return SubstrateOptions.useG1GC(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/graal/NativeGCAllocationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/graal/NativeGCAllocationSupport.java new file mode 100644 index 000000000000..f932c1601a8e --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/graal/NativeGCAllocationSupport.java @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.gc.shared.graal; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; +import static jdk.graal.compiler.core.common.spi.ForeignCallDescriptor.CallSideEffect.NO_SIDE_EFFECT; +import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.VERY_SLOW_PATH_PROBABILITY; +import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.probability; + +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.word.UnsignedWord; + +import com.oracle.svm.core.SubstrateGCOptions; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider; +import com.oracle.svm.core.graal.snippets.GCAllocationSupport; +import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.heap.NoAllocationVerifier; +import com.oracle.svm.core.heap.OutOfMemoryUtil; +import com.oracle.svm.core.heap.Pod; +import com.oracle.svm.core.hub.DynamicHub; +import com.oracle.svm.core.snippets.SnippetRuntime; +import com.oracle.svm.core.snippets.SnippetRuntime.SubstrateForeignCallDescriptor; +import com.oracle.svm.core.snippets.SubstrateForeignCallTarget; +import com.oracle.svm.core.stack.StackOverflowCheck; +import com.oracle.svm.core.thread.ContinuationSupport; +import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; +import com.oracle.svm.core.threadlocal.FastThreadLocalObject; + +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.core.common.spi.ForeignCallDescriptor; +import jdk.graal.compiler.word.Word; + +/** + * This class contains the {@link SubstrateForeignCallTarget}s for the allocation slow path. These + * methods are {@link Uninterruptible} to ensure that newly allocated objects are either placed in + * the young generation or that the covered cards are marked as dirty (this dirtying is done by the + * GC at a later point in time, see ReduceInitialCardMarks and DeferInitialCardMark in HotSpot). + * This allows the compiler to safely eliminate GC write barriers for initializing writes to the + * last allocated object, which reduces code size. + */ +public abstract class NativeGCAllocationSupport implements GCAllocationSupport { + private static final SubstrateForeignCallDescriptor SLOW_NEW_INSTANCE = SnippetRuntime.findForeignCall(NativeGCAllocationSupport.class, "slowPathNewInstance", NO_SIDE_EFFECT); + private static final SubstrateForeignCallDescriptor SLOW_NEW_ARRAY = SnippetRuntime.findForeignCall(NativeGCAllocationSupport.class, "slowPathNewArray", NO_SIDE_EFFECT); + private static final SubstrateForeignCallDescriptor SLOW_NEW_STORED_CONTINUATION = SnippetRuntime.findForeignCall(NativeGCAllocationSupport.class, "slowPathNewStoredContinuation", NO_SIDE_EFFECT); + private static final SubstrateForeignCallDescriptor SLOW_NEW_POD_INSTANCE = SnippetRuntime.findForeignCall(NativeGCAllocationSupport.class, "slowPathNewPodInstance", NO_SIDE_EFFECT); + private static final SubstrateForeignCallDescriptor[] UNCONDITIONAL_FOREIGN_CALLS = new SubstrateForeignCallDescriptor[]{SLOW_NEW_INSTANCE, SLOW_NEW_ARRAY}; + + /* The following thread locals may only be accessed in uninterruptible code. */ + public static final FastThreadLocalObject podReferenceMapTL = FastThreadLocalFactory.createObject(Object.class, "podReferenceMap"); + + @Fold + public static NativeGCAllocationSupport singleton() { + return ImageSingletons.lookup(NativeGCAllocationSupport.class); + } + + public static void registerForeignCalls(SubstrateForeignCallsProvider foreignCalls) { + foreignCalls.register(UNCONDITIONAL_FOREIGN_CALLS); + if (ContinuationSupport.isSupported()) { + foreignCalls.register(SLOW_NEW_STORED_CONTINUATION); + } + if (Pod.RuntimeSupport.isPresent()) { + foreignCalls.register(SLOW_NEW_POD_INSTANCE); + } + } + + @Override + public ForeignCallDescriptor getNewInstanceStub() { + return SLOW_NEW_INSTANCE; + } + + @Override + public ForeignCallDescriptor getNewArrayStub() { + return SLOW_NEW_ARRAY; + } + + @Override + public ForeignCallDescriptor getNewStoredContinuationStub() { + return SLOW_NEW_STORED_CONTINUATION; + } + + @Override + public ForeignCallDescriptor getNewPodInstanceStub() { + return SLOW_NEW_POD_INSTANCE; + } + + @Override + public boolean useTLAB() { + return SubstrateGCOptions.TlabOptions.UseTLAB.getValue(); + } + + @Override + public boolean shouldAllocateInTLAB(UnsignedWord size, boolean isArray) { + /* Always try to allocate in the TLAB (humongous objects will not fit into the TLAB). */ + return true; + } + + @SubstrateForeignCallTarget(stubCallingConvention = false) + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.") + private static Object slowPathNewInstance(Word objectHeader) { + StackOverflowCheck.singleton().makeYellowZoneAvailable(); + try { + DynamicHub hub = Heap.getHeap().getObjectHeader().dynamicHubFromObjectHeader(objectHeader); + guaranteeAllocationAllowed("allocateInstance", hub); + + Object result = NativeGCAllocationSupport.singleton().allocateInstance0(hub); + return checkForOOME(result); + } finally { + StackOverflowCheck.singleton().protectYellowZone(); + } + } + + @SubstrateForeignCallTarget(stubCallingConvention = false) + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.") + private static Object slowPathNewArray(Word objectHeader, int length) { + StackOverflowCheck.singleton().makeYellowZoneAvailable(); + try { + DynamicHub hub = Heap.getHeap().getObjectHeader().dynamicHubFromObjectHeader(objectHeader); + guaranteeAllocationAllowed("allocateArray", hub); + checkArrayLength(length); + + Object result = NativeGCAllocationSupport.singleton().allocateArray0(length, hub); + return checkForOOME(result); + } finally { + StackOverflowCheck.singleton().protectYellowZone(); + } + } + + @SubstrateForeignCallTarget(stubCallingConvention = false) + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.") + private static Object slowPathNewStoredContinuation(Word objectHeader, int length) { + StackOverflowCheck.singleton().makeYellowZoneAvailable(); + try { + DynamicHub hub = Heap.getHeap().getObjectHeader().dynamicHubFromObjectHeader(objectHeader); + guaranteeAllocationAllowed("allocateStoredContinuation", hub); + checkArrayLength(length); + + Object result = NativeGCAllocationSupport.singleton().allocateStoredContinuation0(length, hub); + return checkForOOME(result); + } finally { + StackOverflowCheck.singleton().protectYellowZone(); + } + } + + @SubstrateForeignCallTarget(stubCallingConvention = false) + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.") + private static Object slowPathNewPodInstance(Word objectHeader, int length, byte[] referenceMap) { + StackOverflowCheck.singleton().makeYellowZoneAvailable(); + try { + DynamicHub hub = Heap.getHeap().getObjectHeader().dynamicHubFromObjectHeader(objectHeader); + guaranteeAllocationAllowed("allocatePod", hub); + checkArrayLength(length); + + /* The reference map object can live in the Java heap. */ + podReferenceMapTL.set(referenceMap); + Object result = NativeGCAllocationSupport.singleton().allocatePod0(length, hub); + podReferenceMapTL.set(null); + return checkForOOME(result); + } finally { + StackOverflowCheck.singleton().protectYellowZone(); + } + } + + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.", callerMustBe = true, calleeMustBe = false) + protected abstract Object allocateInstance0(DynamicHub hub); + + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.", callerMustBe = true, calleeMustBe = false) + protected abstract Object allocateArray0(int length, DynamicHub hub); + + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.", callerMustBe = true, calleeMustBe = false) + protected abstract Object allocateStoredContinuation0(int length, DynamicHub hub); + + @Uninterruptible(reason = "The newly allocated object must be young or all its covered cards must be dirty.", callerMustBe = true, calleeMustBe = false) + protected abstract Object allocatePod0(int length, DynamicHub hub); + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static void checkArrayLength(int length) { + if (probability(VERY_SLOW_PATH_PROBABILITY, length < 0)) { + throwNegativeArraySizeExceptionInterruptibly(); + } + } + + @Uninterruptible(reason = "No need to be uninterruptible because no object was allocated.", calleeMustBe = false) + private static void throwNegativeArraySizeExceptionInterruptibly() { + throwNegativeArraySizeException(); + } + + private static void throwNegativeArraySizeException() { + throw new NegativeArraySizeException(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static void guaranteeAllocationAllowed(String callSite, DynamicHub hub) { + if (probability(VERY_SLOW_PATH_PROBABILITY, Heap.getHeap().isAllocationDisallowed())) { + throwAllocationNotAllowedInterruptibly(callSite, hub); + } + } + + @Uninterruptible(reason = "No need to be uninterruptible because it kills the process.", calleeMustBe = false) + private static void throwAllocationNotAllowedInterruptibly(String callSite, DynamicHub hub) { + throw NoAllocationVerifier.exit(callSite, DynamicHub.toClass(hub).getName()); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static Object checkForOOME(Object result) { + if (probability(VERY_SLOW_PATH_PROBABILITY, result == null)) { + throwHeapSizeExceededInterruptibly(); + } + return result; + } + + @Uninterruptible(reason = "No need to be uninterruptible because no object was allocated.", calleeMustBe = false) + private static void throwHeapSizeExceededInterruptibly() { + throw OutOfMemoryUtil.heapSizeExceeded(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/include/sharedGCStructs.h b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/include/sharedGCStructs.h new file mode 100644 index 000000000000..f94f3564c414 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/include/sharedGCStructs.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SVM_SHARED_GC_STRUCTS_HPP +#define SVM_SHARED_GC_STRUCTS_HPP + +#include + +#ifdef __cplusplus +namespace svm_gc { +#endif + +// forward declarations +typedef struct CodeInfo CodeInfo; + +// data structures for frames that are currently on the stack +struct StackFrame { + u_char *stack_pointer; + u_char *encoded_reference_map; + size_t reference_map_index; +}; + +struct StackFrames { + size_t count; + struct StackFrame frames[0]; // variable-sized array +}; + +struct StackFramesPerThread { + size_t count; + struct StackFrames *threads[0]; // variable-sized array +}; + +// data structures for JIT-compiled code that is currently on the stack +struct CodeInfos { + size_t count; + struct CodeInfo *code_infos[0]; // variable-sized array +}; + +struct CodeInfosPerThread { + size_t count; + struct CodeInfos *threads[0]; // variable-sized array +}; + +#ifdef __cplusplus +} // namespace svm_gc +#endif + +#endif // SVM_SHARED_GC_STRUCTS_HPP diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/GC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/GC.java index f2eed9860810..4078fb70089c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/GC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/GC.java @@ -24,6 +24,9 @@ */ package com.oracle.svm.core.heap; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + public interface GC { /** Cause a collection of the Heap's choosing. */ void collect(GCCause cause); @@ -41,5 +44,6 @@ public interface GC { String getName(); /** Human-readable default heap size. */ + @Platforms(Platform.HOSTED_ONLY.class) String getDefaultMaxHeapSize(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SystemPropertiesSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SystemPropertiesSupport.java index e42f6270181f..4fd9a19f4e05 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SystemPropertiesSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SystemPropertiesSupport.java @@ -128,7 +128,7 @@ protected SystemPropertiesSupport() { initializeProperty("java.vendor", vm.vendor); initializeProperty("java.vendor.url", vm.vendorUrl); initializeProperty("java.vendor.version", vm.vendorVersion); - assert vm.info.equals(vm.info.toLowerCase(Locale.ROOT)) : "java.vm.info should not contain uppercase characters"; + assert vm.info.equals(vm.info.toLowerCase(Locale.ROOT)) : "java.vm.info should not contain uppercase characters: " + vm.info; initializeProperty("java.vm.info", vm.info); initializeProperty("java.vm.name", "Substrate VM"); initializeProperty("java.vm.vendor", vm.vendor); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/GCOptionValue.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/GCOptionValue.java index 53e17d0eabdd..086e26e2c30c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/GCOptionValue.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/GCOptionValue.java @@ -35,8 +35,8 @@ import jdk.graal.compiler.options.OptionsContainer; public enum GCOptionValue { - SERIAL("serial"), - EPSILON("epsilon"), + Serial("serial"), + Epsilon("epsilon"), G1("G1"); private static Set supportedValues = null; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/HostedOptionKey.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/HostedOptionKey.java index b4cc5b2ea22f..7118cbefbcd8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/HostedOptionKey.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/HostedOptionKey.java @@ -40,16 +40,16 @@ * @see com.oracle.svm.core.option */ public class HostedOptionKey extends OptionKey implements SubstrateOptionKey { - private final Consumer> validation; + private final Consumer> buildTimeValidation; private OptionOrigin lastOrigin; public HostedOptionKey(T defaultValue) { this(defaultValue, null); } - public HostedOptionKey(T defaultValue, Consumer> validation) { + public HostedOptionKey(T defaultValue, Consumer> buildTimeValidation) { super(defaultValue); - this.validation = validation; + this.buildTimeValidation = buildTimeValidation; } /** @@ -103,8 +103,8 @@ public void update(EconomicMap, Object> values, Object boxedValue) @Override public void validate() { - if (validation != null) { - validation.accept(this); + if (buildTimeValidation != null) { + buildTimeValidation.accept(this); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java index d04accb06a30..c52bcce85ea8 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HostedConfiguration.java @@ -115,8 +115,10 @@ public static void setDefaultIfEmpty() { CompressEncoding compressEncoding = new CompressEncoding(SubstrateOptions.SpawnIsolates.getValue() ? 1 : 0, 0); ImageSingletons.add(CompressEncoding.class, compressEncoding); - ObjectLayout objectLayout = createObjectLayout(IdentityHashMode.TYPE_SPECIFIC); - ImageSingletons.add(ObjectLayout.class, objectLayout); + if (!ImageSingletons.contains(ObjectLayout.class)) { + ObjectLayout objectLayout = createObjectLayout(IdentityHashMode.TYPE_SPECIFIC); + ImageSingletons.add(ObjectLayout.class, objectLayout); + } ImageSingletons.add(HybridLayoutSupport.class, new HybridLayoutSupport()); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/VMFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/VMFeature.java index 5f3d826e0ed1..9311098db651 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/VMFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/VMFeature.java @@ -25,6 +25,7 @@ package com.oracle.svm.hosted; import java.nio.file.Path; +import java.util.List; import java.util.Locale; import java.util.stream.Collectors; @@ -65,13 +66,13 @@ protected String determineVMInfo() { return getSelectedGCName(); } - protected static final String getSelectedGCName() { - if (SubstrateOptions.useSerialGC()) { - return "serial gc"; - } else if (SubstrateOptions.useEpsilonGC()) { - return "epsilon gc"; + protected static String getSelectedGCName() { + List values = SubstrateOptions.SupportedGCs.getValue().values(); + if (values.isEmpty()) { + return "no gc"; } else { - return "unknown gc"; + assert values.size() == 1; + return values.getFirst().toLowerCase(Locale.ROOT) + " gc"; } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ValidateGCOptionFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ValidateGCOptionFeature.java index dd3af52b8b91..882f5a2d3647 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ValidateGCOptionFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ValidateGCOptionFeature.java @@ -57,7 +57,6 @@ private static void validateGCOption() { } Set possibleValues = GCOptionValue.possibleValues(); - if (values.isEmpty()) { throw UserError.abort("Invalid option '--gc'. No GC specified. %s", getGCErrorReason(possibleValues)); } @@ -69,9 +68,10 @@ private static void validateGCOption() { } } - // Check that the specified combination is valid. - if (values.size() != 1) { - throw UserError.abort("%s is an invalid combination of GCs for option '--gc'.", StringUtil.joinSingleQuoted(values)); + // At the moment, exactly one GC must be selected at build-time. + if (values.size() > 1) { + String string = String.join(SubstrateOptions.SupportedGCs.getValue().getDelimiter(), values); + throw UserError.invalidOptionValue(SubstrateOptions.SupportedGCs, string, "Only one garbage collector can be used at a time."); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/OWNERS.toml b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/OWNERS.toml new file mode 100644 index 000000000000..1f7d0b0a8706 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/OWNERS.toml @@ -0,0 +1,7 @@ +[[rule]] +files = "*" +all = [ + "christian.haeubl@oracle.com", +] +any = [ +] diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shared/NativeGCAccessedFields.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shared/NativeGCAccessedFields.java new file mode 100644 index 000000000000..e1e6824d1607 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/gc/shared/NativeGCAccessedFields.java @@ -0,0 +1,281 @@ +/* + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.gc.shared; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.function.BooleanSupplier; + +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.hosted.Feature.BeforeCompilationAccess; + +import com.oracle.graal.pointsto.meta.AnalysisField; +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.code.CodeInfoOffsets; +import com.oracle.svm.core.config.ObjectLayout; +import com.oracle.svm.core.jvmstat.PerfManager; +import com.oracle.svm.core.thread.ContinuationSupport; +import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.thread.VMThreads.StatusSupport; +import com.oracle.svm.core.threadlocal.FastThreadLocalBytes; +import com.oracle.svm.core.threadlocal.FastThreadLocalObject; +import com.oracle.svm.core.util.ByteArrayReader; +import com.oracle.svm.core.util.VMError; +import com.oracle.svm.hosted.FeatureImpl.BeforeAnalysisAccessImpl; +import com.oracle.svm.hosted.config.DynamicHubLayout; +import com.oracle.svm.hosted.thread.VMThreadFeature; +import com.oracle.svm.util.ClassUtil; + +import jdk.graal.compiler.core.common.util.UnsafeArrayTypeWriter; +import jdk.graal.compiler.word.Word; + +/** + * For performance reasons, the GC-related C++ code accesses certain static/thread-local/instance + * fields directly. This class computes the offsets of those fields and writes them to a byte array + * that is then passed to the C++ code upon initialization. It also ensures that the static analysis + * considers all those fields as reachable, even if they are only accessed on the C++ side. + */ +public class NativeGCAccessedFields { + public static final BooleanSupplier ALWAYS = new Always(); + public static final BooleanSupplier USE_PERF_DATA = new UsePerfData(); + public static final BooleanSupplier CLOSED_TYPE_WORLD_HUB_LAYOUT = SubstrateOptions::useClosedTypeWorldHubLayout; + public static final BooleanSupplier OPEN_TYPE_WORLD_HUB_LAYOUT = new OpenTypeWorldHubLayout(); + + /** Marks the given fields as accessed and their classes as used. */ + public static void markAsAccessed(BeforeAnalysisAccessImpl access, AccessedClass[] accessedClasses) { + for (AccessedClass accessedClass : accessedClasses) { + markAsAccessed(access, accessedClass.clazz, accessedClass.fields); + } + } + + /** Writes all offsets as integer values (4 bytes per value) into a byte array. */ + public static byte[] writeOffsets(BeforeCompilationAccess access, int markWordOffset, FastThreadLocalBytes nativeJavaThreadTL, FastThreadLocalObject podReferenceMapTL, + AccessedClass[] accessedClasses) { + UnsafeArrayTypeWriter buffer = UnsafeArrayTypeWriter.create(ByteArrayReader.supportsUnalignedMemoryAccess()); + + writeObjectLayoutOffsets(buffer, markWordOffset); + writeThreadLocalOffsets(buffer, nativeJavaThreadTL, podReferenceMapTL); + writeCodeInfoOffsets(buffer); + for (AccessedClass accessedClass : accessedClasses) { + writeFieldOffsets(access, buffer, accessedClass.clazz, accessedClass.fields); + } + + return buffer.toArray(); + } + + private static void markAsAccessed(BeforeAnalysisAccessImpl access, Class clazz, AccessedField[] accessedFields) { + access.registerAsUsed(clazz); + + Field[] fields = clazz.getDeclaredFields(); + for (AccessedField accessedField : accessedFields) { + if (accessedField.shouldInclude()) { + boolean found = false; + for (Field field : fields) { + if (accessedField.matches(field)) { + AnalysisField analysisField = access.getMetaAccess().lookupJavaField(field); + accessedField.registerAsAccessed(access, analysisField); + found = true; + break; + } + } + if (!found) { + throw VMError.shouldNotReachHere("Could not find field " + ClassUtil.getUnqualifiedName(clazz) + "." + accessedField.name); + } + } + } + } + + private static void writeObjectLayoutOffsets(UnsafeArrayTypeWriter buffer, int markWordOffset) { + ObjectLayout objectLayout = ImageSingletons.lookup(ObjectLayout.class); + assert objectLayout.getFirstFieldOffset() <= objectLayout.getArrayLengthOffset(); + int objBase = objectLayout.getFirstFieldOffset(); + int minObjectSize = objectLayout.alignUp(objBase); + + // object layout + buffer.putS4(markWordOffset); + buffer.putS4(objectLayout.getHubOffset()); + buffer.putS4(objectLayout.getAlignment()); + buffer.putS4(minObjectSize); + buffer.putS4(objBase); + buffer.putS4(objectLayout.getArrayLengthOffset()); + + // dynamic hub + buffer.putS4(SubstrateOptions.useClosedTypeWorldHubLayout() ? DynamicHubLayout.singleton().getClosedTypeWorldTypeCheckSlotsOffset() : -1); + } + + private static void writeThreadLocalOffsets(UnsafeArrayTypeWriter buffer, FastThreadLocalBytes nativeJavaThreadTL, FastThreadLocalObject podReferenceMapTL) { + VMThreadFeature vmThreadMtFeature = ImageSingletons.lookup(VMThreadFeature.class); + + buffer.putS4(vmThreadMtFeature.offsetOf(VMThreads.nextTL)); + buffer.putS4(vmThreadMtFeature.offsetOf(nativeJavaThreadTL)); + buffer.putS4(vmThreadMtFeature.offsetOf(StatusSupport.statusTL)); + buffer.putS4(vmThreadMtFeature.offsetOf(podReferenceMapTL)); + } + + private static void writeCodeInfoOffsets(UnsafeArrayTypeWriter buffer) { + buffer.putS4(CodeInfoOffsets.objectFields()); + buffer.putS4(CodeInfoOffsets.state()); + buffer.putS4(CodeInfoOffsets.gcData()); + buffer.putS4(CodeInfoOffsets.objectConstants()); + buffer.putS4(CodeInfoOffsets.deoptimizationObjectConstants()); + buffer.putS4(CodeInfoOffsets.stackReferenceMapEncoding()); + buffer.putS4(CodeInfoOffsets.codeStart()); + buffer.putS4(CodeInfoOffsets.codeConstantsReferenceMapEncoding()); + buffer.putS4(CodeInfoOffsets.codeConstantsReferenceMapIndex()); + buffer.putS4(CodeInfoOffsets.areAllObjectsInImageHeap()); + } + + private static void writeFieldOffsets(BeforeCompilationAccess access, UnsafeArrayTypeWriter buffer, Class clazz, AccessedField[] accessedFields) { + Field[] declaredFields = clazz.getDeclaredFields(); + for (AccessedField accessedField : accessedFields) { + if (accessedField.shouldInclude()) { + boolean found = false; + for (Field field : declaredFields) { + if (accessedField.matches(field)) { + buffer.putS4(access.objectFieldOffset(field)); + found = true; + break; + } + } + if (!found) { + throw VMError.shouldNotReachHere("Could not find field " + ClassUtil.getUnqualifiedName(clazz) + "." + accessedField.name); + } + } else { + // Add a dummy value so that the size of the structure is always the same. + buffer.putS4(-1); + } + } + } + + public enum FieldAccessKind { + READ, + READ_WRITE + } + + public static final class AccessedClass { + Class clazz; + AccessedField[] fields; + + public AccessedClass(Class clazz, AccessedField... fields) { + this.clazz = clazz; + this.fields = fields; + } + } + + public abstract static class AccessedField { + final String name; + final FieldAccessKind accessKind; + final BooleanSupplier include; + + AccessedField(String name, FieldAccessKind accessKind, BooleanSupplier include) { + this.name = name; + this.accessKind = accessKind; + this.include = include; + } + + public abstract boolean matches(Field field); + + public boolean shouldInclude() { + return include.getAsBoolean(); + } + + void registerAsAccessed(BeforeAnalysisAccessImpl access, AnalysisField field) { + if (accessKind == FieldAccessKind.READ) { + access.registerAsRead(field, "it is read by the GC"); + } else { + assert accessKind == FieldAccessKind.READ_WRITE; + access.registerAsAccessed(field, "it is accessed by the GC"); + } + } + } + + public static final class InstanceField extends AccessedField { + public InstanceField(String name, FieldAccessKind accessKind) { + this(name, accessKind, ALWAYS); + } + + public InstanceField(String name, FieldAccessKind accessKind, BooleanSupplier include) { + super(name, accessKind, include); + } + + @Override + public boolean matches(Field field) { + return !Modifier.isStatic(field.getModifiers()) && name.equals(field.getName()); + } + } + + public static final class StaticField extends AccessedField { + public StaticField(String name, FieldAccessKind accessKind) { + this(name, accessKind, ALWAYS); + } + + StaticField(String name, FieldAccessKind accessKind, BooleanSupplier include) { + super(name, accessKind, include); + } + + @Override + public boolean matches(Field field) { + return Modifier.isStatic(field.getModifiers()) && name.equals(field.getName()); + } + + @Override + public void registerAsAccessed(BeforeAnalysisAccessImpl access, AnalysisField field) { + /* + * Regardless of the field access kind, we always mark the field as accessed. This is + * necessary to ensure that memory gets reserved for the static field as other SVM code + * assumes that read-only static fields can always be constant-folded. + */ + access.registerAsAccessed(field, "it is accessed by the GC"); + } + } + + public static final class Always implements BooleanSupplier { + @Override + public boolean getAsBoolean() { + return true; + } + } + + public static final class UsePerfData implements BooleanSupplier { + @Override + public boolean getAsBoolean() { + return ImageSingletons.contains(PerfManager.class); + } + } + + public static final class ContinuationsSupported implements BooleanSupplier { + @Override + public boolean getAsBoolean() { + return ContinuationSupport.isSupported(); + } + } + + public static final class OpenTypeWorldHubLayout implements BooleanSupplier { + @Override + public boolean getAsBoolean() { + return !SubstrateOptions.useClosedTypeWorldHubLayout(); + } + } +} diff --git a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmLMGC.java b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmLMGC.java index d4df07c456f4..e6645a112e83 100644 --- a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmLMGC.java +++ b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmLMGC.java @@ -31,6 +31,7 @@ import java.lang.ref.Reference; import org.graalvm.collections.EconomicMap; +import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.c.function.CodePointer; @@ -226,6 +227,7 @@ public String getName() { } @Override + @Platforms(Platform.HOSTED_ONLY.class) public String getDefaultMaxHeapSize() { return "unknown"; } diff --git a/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSGC.java b/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSGC.java index d82ea71011c3..9dbc53468b66 100644 --- a/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSGC.java +++ b/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSGC.java @@ -24,6 +24,9 @@ */ package com.oracle.svm.webimage.heap; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + import com.oracle.svm.core.heap.GC; import com.oracle.svm.core.heap.GCCause; @@ -49,6 +52,7 @@ public String getName() { } @Override + @Platforms(Platform.HOSTED_ONLY.class) public String getDefaultMaxHeapSize() { return "unknown"; }