Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -62,15 +62,24 @@ public boolean getAsBoolean() {
}
}

@Platforms(Platform.HOSTED_ONLY.class)
public static final class SharedArenasEnabled implements BooleanSupplier {
public static boolean getValue() {
return SubstrateOptions.isForeignAPIEnabled() && SubstrateOptions.SharedArenaSupport.getValue();
private static final String VECTOR_API_SUPPORT_OPTION_NAME = SubstrateOptionsParser.commandArgument(SubstrateOptions.VectorAPISupport, "-");
private static final String SHARED_ARENA_SUPPORT_OPTION_NAME = SubstrateOptionsParser.commandArgument(SubstrateOptions.SharedArenaSupport, "-");

@Platforms(Platform.HOSTED_ONLY.class)
SharedArenasEnabled() {
}

@Override
public boolean getAsBoolean() {
return SharedArenasEnabled.getValue();
return SubstrateOptions.isSharedArenaSupportEnabled();
}

public static RuntimeException vectorAPIUnsupported() {
assert !SubstrateOptions.isSharedArenaSupportEnabled();
throw VMError.unsupportedFeature("Support for Arena.ofShared is not available if Vector API support is enabled." +
"Either disable Vector API support using " + VECTOR_API_SUPPORT_OPTION_NAME +
" or replace usages of Arena.ofShared with Arena.ofAuto and disable shared arena support using " + SHARED_ARENA_SUPPORT_OPTION_NAME + ".");
}
}

Expand All @@ -83,11 +92,11 @@ public static final class SharedArenasDisabled implements BooleanSupplier {

@Override
public boolean getAsBoolean() {
return !SharedArenasEnabled.getValue();
return !SubstrateOptions.isSharedArenaSupportEnabled();
}

public static RuntimeException fail() {
assert !SharedArenasEnabled.getValue();
assert !SubstrateOptions.isSharedArenaSupportEnabled();
throw VMError.unsupportedFeature("Support for Arena.ofShared is not active: enable with " + SHARED_ARENA_SUPPORT_OPTION_NAME);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@
import java.util.Map;
import java.util.function.BiConsumer;

import com.oracle.svm.core.util.ImageHeapMap;
import org.graalvm.collections.EconomicMap;
import org.graalvm.collections.EconomicSet;
import org.graalvm.collections.Pair;
import org.graalvm.nativeimage.ImageSingletons;
import org.graalvm.nativeimage.Platform;
Expand All @@ -54,12 +54,14 @@
import com.oracle.svm.core.SubstrateUtil;
import com.oracle.svm.core.Uninterruptible;
import com.oracle.svm.core.c.InvokeJavaFunctionPointer;
import com.oracle.svm.core.foreign.phases.SubstrateOptimizeSharedArenaAccessPhase.OptimizeSharedArenaConfig;
import com.oracle.svm.core.headers.LibC;
import com.oracle.svm.core.headers.WindowsAPIs;
import com.oracle.svm.core.image.DisallowedImageHeapObjects.DisallowedObjectReporter;
import com.oracle.svm.core.snippets.SnippetRuntime;
import com.oracle.svm.core.snippets.SubstrateForeignCallTarget;
import com.oracle.svm.core.util.BasedOnJDKFile;
import com.oracle.svm.core.util.ImageHeapMap;
import com.oracle.svm.core.util.VMError;

import jdk.graal.compiler.api.replacements.Fold;
Expand All @@ -68,8 +70,10 @@
import jdk.internal.foreign.MemorySessionImpl;
import jdk.internal.foreign.abi.CapturableState;
import jdk.internal.foreign.abi.LinkerOptions;
import jdk.vm.ci.meta.ResolvedJavaMethod;
import jdk.vm.ci.meta.ResolvedJavaType;

public class ForeignFunctionsRuntime implements ForeignSupport {
public class ForeignFunctionsRuntime implements ForeignSupport, OptimizeSharedArenaConfig {
@Fold
public static ForeignFunctionsRuntime singleton() {
return ImageSingletons.lookup(ForeignFunctionsRuntime.class);
Expand All @@ -80,6 +84,8 @@ public static ForeignFunctionsRuntime singleton() {
private final EconomicMap<NativeEntryPointInfo, FunctionPointerHolder> downcallStubs = ImageHeapMap.create("downcallStubs");
private final EconomicMap<Pair<DirectMethodHandleDesc, JavaEntryPointInfo>, FunctionPointerHolder> directUpcallStubs = ImageHeapMap.create("directUpcallStubs");
private final EconomicMap<JavaEntryPointInfo, FunctionPointerHolder> upcallStubs = ImageHeapMap.create("upcallStubs");
private final EconomicSet<ResolvedJavaType> neverAccessesSharedArenaTypes = EconomicSet.create();
private final EconomicSet<ResolvedJavaMethod> neverAccessesSharedArenaMethods = EconomicSet.create();

private final Map<Long, TrampolineSet> trampolines = new HashMap<>();
private TrampolineSet currentTrampolineSet;
Expand Down Expand Up @@ -151,6 +157,16 @@ public boolean addDirectUpcallStubPointer(DirectMethodHandleDesc desc, JavaEntry
return directUpcallStubs.putIfAbsent(key, new FunctionPointerHolder(ptr)) == null;
}

@Platforms(Platform.HOSTED_ONLY.class)
public void registerSafeArenaAccessorClass(ResolvedJavaType type) {
neverAccessesSharedArenaTypes.add(type);
}

@Platforms(Platform.HOSTED_ONLY.class)
public void registerSafeArenaAccessorMethod(ResolvedJavaMethod method) {
neverAccessesSharedArenaMethods.add(method);
}

/**
* We'd rather report the function descriptor than the native method type, but we don't have it
* available here. One could intercept this exception in
Expand Down Expand Up @@ -368,6 +384,17 @@ public static void captureCallState(int statesToCapture, CIntPointer captureBuff
@Platforms(Platform.HOSTED_ONLY.class)//
public static final SnippetRuntime.SubstrateForeignCallDescriptor CAPTURE_CALL_STATE = SnippetRuntime.findForeignCall(ForeignFunctionsRuntime.class,
"captureCallState", HAS_SIDE_EFFECT, LocationIdentity.any());

@Override
public boolean isSafeCallee(ResolvedJavaMethod method) {
if (neverAccessesSharedArenaMethods.contains(method)) {
return true;
}
if (neverAccessesSharedArenaTypes.contains(method.getDeclaringClass())) {
return true;
}
return false;
}
}

interface StubPointer extends CFunctionPointer {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import com.oracle.svm.core.nodes.foreign.ScopedMemExceptionHandlerClusterNode.ExceptionInputNode;
import com.oracle.svm.core.nodes.foreign.ScopedMemExceptionHandlerClusterNode.ExceptionPathNode;
import com.oracle.svm.core.nodes.foreign.ScopedMemExceptionHandlerClusterNode.RegularPathNode;
import com.oracle.svm.core.util.VMError;
import com.oracle.svm.util.LogUtils;

import jdk.graal.compiler.api.directives.GraalDirectives;
Expand Down Expand Up @@ -69,6 +70,24 @@ public static void checkSession(MemorySessionImpl session) {
}
}

/**
* A decorator method for {@link MemorySessionImpl#checkValidStateRaw()} which will fail if a
* shared session is passed. We use this method instead of the decorated one in runtime
* compiled @Scoped-annotated methods (and their deopt targets) to fail if shared sessions are
* used in runtime compiled methods (GR-66841).
*/
@AlwaysInline("factored out only for readability")
public static void checkValidStateRawInRuntimeCompiledCode(MemorySessionImpl session) {
/*
* A closeable session without an owner thread is a shared session (i.e. it can be closed
* concurrently).
*/
if (session.isCloseable() && session.ownerThread() == null) {
throw VMError.unsupportedFeature("Invalid session object. Using a shared session in runtime compiled methods is not supported.");
}
session.checkValidStateRaw();
}

/**
* Handles exceptions related to memory sessions within a specific arena scope.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,11 @@
import com.oracle.svm.core.util.BasedOnJDKFile;

import jdk.internal.access.foreign.MappedMemoryUtilsProxy;
import jdk.internal.foreign.AbstractMemorySegmentImpl;
import jdk.internal.foreign.MemorySessionImpl;
import jdk.internal.misc.ScopedMemoryAccess;
import jdk.internal.misc.ScopedMemoryAccess.ScopedAccessError;
import jdk.internal.vm.vector.VectorSupport;

/**
* Support for shared arenas on SVM:
Expand Down Expand Up @@ -184,6 +186,53 @@ public void forceInternal(MemorySessionImpl session, MappedMemoryUtilsProxy mapp
}
}

@SuppressWarnings("unused")
@Substitute
@TargetElement(onlyWith = SharedArenasEnabled.class)
@AlwaysInline("Safepoints must be visible in caller")
private static <V extends VectorSupport.Vector<E>, E, S extends VectorSupport.VectorSpecies<E>> V loadFromMemorySegmentScopedInternal(MemorySessionImpl session,
Class<? extends V> vmClass, Class<E> e, int length,
AbstractMemorySegmentImpl msp, long offset,
S s,
VectorSupport.LoadOperation<AbstractMemorySegmentImpl, V, S> defaultImpl) {
throw SharedArenasEnabled.vectorAPIUnsupported();
}

@SuppressWarnings("unused")
@Substitute
@TargetElement(onlyWith = SharedArenasEnabled.class)
@AlwaysInline("Safepoints must be visible in caller")
private static <V extends VectorSupport.Vector<E>, E, S extends VectorSupport.VectorSpecies<E>, M extends VectorSupport.VectorMask<E>> V loadFromMemorySegmentMaskedScopedInternal(
MemorySessionImpl session, Class<? extends V> vmClass,
Class<M> maskClass, Class<E> e, int length,
AbstractMemorySegmentImpl msp, long offset, M m,
S s, int offsetInRange,
VectorSupport.LoadVectorMaskedOperation<AbstractMemorySegmentImpl, V, S, M> defaultImpl) {
throw SharedArenasEnabled.vectorAPIUnsupported();
}

@SuppressWarnings("unused")
@Substitute
@TargetElement(onlyWith = SharedArenasEnabled.class)
@AlwaysInline("Safepoints must be visible in caller")
public static <V extends VectorSupport.Vector<E>, E> void storeIntoMemorySegment(Class<? extends V> vmClass, Class<E> e, int length,
V v,
AbstractMemorySegmentImpl msp, long offset,
VectorSupport.StoreVectorOperation<AbstractMemorySegmentImpl, V> defaultImpl) {
throw SharedArenasEnabled.vectorAPIUnsupported();
}

@SuppressWarnings("unused")
@Substitute
@TargetElement(onlyWith = SharedArenasEnabled.class)
@AlwaysInline("Safepoints must be visible in caller")
public static <V extends VectorSupport.Vector<E>, E, M extends VectorSupport.VectorMask<E>> void storeIntoMemorySegmentMasked(Class<? extends V> vmClass, Class<M> maskClass, Class<E> e,
int length, V v, M m,
AbstractMemorySegmentImpl msp, long offset,
VectorSupport.StoreVectorMaskedOperation<AbstractMemorySegmentImpl, V, M> defaultImpl) {
throw SharedArenasEnabled.vectorAPIUnsupported();
}

/**
* This method synchronizes with all other Java threads in order to be able to safely close the
* session.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package com.oracle.svm.hosted.foreign.phases;
package com.oracle.svm.core.foreign.phases;

import static jdk.graal.compiler.debug.DebugContext.VERY_DETAILED_LEVEL;

Expand All @@ -39,17 +39,13 @@
import org.graalvm.collections.Equivalence;
import org.graalvm.word.LocationIdentity;

import com.oracle.svm.core.Uninterruptible;
import com.oracle.svm.core.nodes.ClusterNode;
import com.oracle.svm.core.nodes.foreign.MemoryArenaValidInScopeNode;
import com.oracle.svm.core.nodes.foreign.ScopedMemExceptionHandlerClusterNode.ClusterBeginNode;
import com.oracle.svm.core.nodes.foreign.ScopedMemExceptionHandlerClusterNode.ExceptionInputNode;
import com.oracle.svm.core.nodes.foreign.ScopedMemExceptionHandlerClusterNode.ExceptionPathNode;
import com.oracle.svm.core.nodes.foreign.ScopedMemExceptionHandlerClusterNode.RegularPathNode;
import com.oracle.svm.core.nodes.foreign.ScopedMethodNode;
import com.oracle.svm.hosted.foreign.ForeignFunctionsFeature;
import com.oracle.svm.hosted.meta.HostedMethod;
import com.oracle.svm.hosted.meta.HostedType;

import jdk.graal.compiler.core.common.cfg.CFGLoop;
import jdk.graal.compiler.debug.Assertions;
Expand Down Expand Up @@ -346,10 +342,24 @@
*/
public class SubstrateOptimizeSharedArenaAccessPhase extends BasePhase<MidTierContext> implements RecursivePhase {

public interface OptimizeSharedArenaConfig {
/**
* Tests if calls to the given callee may remain in the critical region of
* an @Scoped-annotated method. Usually, this is the case if (1) the callee does not do any
* safepoint checks (recursively), or (2) the callee does not access native memory of a
* shared arena AND exits with an exception. The second case normally covers methods that
* are part of the path that checks the 'checkValidStateRaw' and throws an exception
* otherwise.
*/
boolean isSafeCallee(ResolvedJavaMethod method);
}

final CanonicalizerPhase canonicalizer;
final OptimizeSharedArenaConfig config;

public SubstrateOptimizeSharedArenaAccessPhase(CanonicalizerPhase canonicalizer) {
public SubstrateOptimizeSharedArenaAccessPhase(CanonicalizerPhase canonicalizer, OptimizeSharedArenaConfig config) {
this.canonicalizer = canonicalizer;
this.config = config;
}

@Override
Expand Down Expand Up @@ -638,13 +648,20 @@ protected void run(StructuredGraph graph, MidTierContext context) {
cleanupClusterNodes(graph, context, insertSessionChecks(graph, context));
}

private static EconomicSet<DominatedCall> insertSessionChecks(StructuredGraph graph, MidTierContext context) {
private EconomicSet<DominatedCall> insertSessionChecks(StructuredGraph graph, MidTierContext context) {
if (graph.getNodes().filter(ScopedMethodNode.class).count() == 0) {
/*
* We are, for whatever reason, compiling the exception handler template method, we do
* not verify no calls and we do not duplicate any session checks inside.
*/
return null;
}
ControlFlowGraph cfg = ControlFlowGraph.newBuilder(graph).modifiableBlocks(true).connectBlocks(true).computeFrequency(true).computeLoops(true).computeDominators(true)
.computePostdominators(true)
.build();
// Compute the graph with all the necessary data about scoped memory accesses.
EconomicSet<DominatedCall> calls = EconomicSet.create();
EconomicMap<Node, List<ScopedAccess>> sugaredGraph = enumerateScopedAccesses(cfg, context, calls);
EconomicMap<Node, List<ScopedAccess>> sugaredGraph = enumerateScopedAccesses(config, cfg, context, calls);
if (sugaredGraph != null) {
ReentrantBlockIterator.apply(new MinimalSessionChecks(graph, sugaredGraph, cfg, calls), cfg.getStartBlock());
}
Expand Down Expand Up @@ -848,7 +865,8 @@ record DominatedCall(MemoryArenaValidInScopeNode defNode, Invoke invoke) {

}

private static EconomicMap<Node, List<ScopedAccess>> enumerateScopedAccesses(ControlFlowGraph cfg, MidTierContext context, EconomicSet<DominatedCall> dominatedCalls) {
private static EconomicMap<Node, List<ScopedAccess>> enumerateScopedAccesses(OptimizeSharedArenaConfig config, ControlFlowGraph cfg, MidTierContext context,
EconomicSet<DominatedCall> dominatedCalls) {
EconomicMap<Node, List<ScopedAccess>> nodeAccesses = EconomicMap.create();
final ResolvedJavaType memorySessionType = context.getMetaAccess().lookupJavaType(MemorySessionImpl.class);
assert memorySessionType != null;
Expand Down Expand Up @@ -909,7 +927,7 @@ public void run() {

private void processNode(FixedNode f) {
if (!scopes.isEmpty() && f instanceof Invoke i) {
if (i.getTargetMethod() != null && calleeMightUseArena(i.getTargetMethod())) {
if (i.getTargetMethod() != null && !config.isSafeCallee(i.getTargetMethod())) {
if (!defs.isEmpty()) {
dominatedCalls.add(new DominatedCall(defs.peek().defNode, i));
}
Expand Down Expand Up @@ -974,21 +992,6 @@ private void cacheSafepointPosition(ReachingDefScope existingDef, ValueNode scop
existingAccesses.add(new ScopedSafepoint(scopeAssociatedVal, existingDef.defNode));
}

/**
* Special methods known to never access a memory arena. Normally part of the path that
* checks the `checkValidStateRaw` and throws an exception otherwise.
*/
private boolean calleeMightUseArena(ResolvedJavaMethod targetMethod) {
if (Uninterruptible.Utils.isUninterruptible(targetMethod)) {
// Uninterruptible can never safepoint
return false;
}
if (ForeignFunctionsFeature.singleton().getNeverAccessesSharedArena().contains(((HostedType) targetMethod.getDeclaringClass()).getWrapped())) {
return false;
}
return !ForeignFunctionsFeature.singleton().getNeverAccessesSharedArenaMethods().contains(((HostedMethod) targetMethod).getWrapped());
}

private static boolean visitInputsUntil(Node key, Node start) {
NodeStack toProcess = new NodeStack();
toProcess.push(start);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@
import com.oracle.svm.core.c.libc.LibCBase;
import com.oracle.svm.core.c.libc.MuslLibC;
import com.oracle.svm.core.config.ConfigurationValues;
import com.oracle.svm.core.graal.RuntimeCompilation;
import com.oracle.svm.core.heap.ReferenceHandler;
import com.oracle.svm.core.jdk.VectorAPIEnabled;
import com.oracle.svm.core.option.APIOption;
Expand Down Expand Up @@ -1478,16 +1477,24 @@ public static boolean isForeignAPIEnabled() {
@Option(help = "Enable support for Arena.ofShared ", type = Expert)//
public static final HostedOptionKey<Boolean> SharedArenaSupport = new HostedOptionKey<>(false, key -> {
if (key.getValue()) {
// GR-65268: Shared arenas cannot be used together with runtime compilations
UserError.guarantee(!RuntimeCompilation.isEnabled(), "Arena.ofShared is not supported with runtime compilations. " +
"Replace usages of Arena.ofShared with Arena.ofAuto and disable shared arena support.");
UserError.guarantee(isForeignAPIEnabled(), "Support for Arena.ofShared is only available with foreign API support. " +
"Enable foreign API support with %s",
SubstrateOptionsParser.commandArgument(ForeignAPISupport, "+"));

// GR-65162: Shared arenas cannot be used together with Vector API support
UserError.guarantee(!VectorAPIEnabled.getValue(), "Support for Arena.ofShared is not available with Vector API support. " +
"Either disable Vector API support using %s or replace usages of Arena.ofShared with Arena.ofAuto and disable shared arena support.",
SubstrateOptionsParser.commandArgument(VectorAPISupport, "-"));
}
});

@Fold
public static boolean isSharedArenaSupportEnabled() {
// GR-65162: Shared arenas cannot be used together with Vector API support
return isForeignAPIEnabled() && SubstrateOptions.SharedArenaSupport.getValue() &&
(SubstrateOptions.SharedArenaSupport.hasBeenSet() || !VectorAPIEnabled.getValue());
}

@Option(help = "Assume new types cannot be added after analysis", type = OptionType.Expert) //
public static final HostedOptionKey<Boolean> ClosedTypeWorld = new HostedOptionKey<>(true) {
@Override
Expand Down
Loading