Validate reachable object before propagating it.

Author: cstancu

sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/hosted/Feature.java

@@ -194,8 +194,9 @@ interface DuringSetupAccess extends FeatureAccess {
 
         /**
          * Register a callback that is executed when an object of type {@code clazz}, or any of its
-         * subtypes, is marked as reachable during heap scanning. The callback may be executed for
-         * the same object by multiple worker threads concurrently.
+         * subtypes, is marked as reachable during heap scanning. The callback is executed before
+         * the object is added to the shadow heap. The callback may be executed for the same object
+         * by multiple worker threads concurrently.
          *
          * @since 24.2
          */

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/api/HostVM.java

@@ -130,6 +130,14 @@ public void notifyClassReachabilityListener(AnalysisUniverse universe, DuringAna
         }
     }
 
+    /**
+     * Run validation checks for reachable objects before registering them in the shadow heap.
+     *
+     * @param obj the object to validate
+     */
+    public void validateReachableObject(Object obj) {
+    }
+
     /**
      * Register newly created type.
      * 

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java

@@ -587,15 +587,25 @@ private ImageHeapConstant markReachable(ImageHeapConstant imageHeapConstant, Sca
     }
 
     protected void onObjectReachable(ImageHeapConstant imageHeapConstant, ScanReason reason, Consumer<ScanReason> onAnalysisModified) {
-        AnalysisType objectType = metaAccess.lookupJavaType(imageHeapConstant);
-        imageHeap.addReachableObject(objectType, imageHeapConstant);
 
-        AnalysisType type = imageHeapConstant.getType();
-        Object object = bb.getSnippetReflectionProvider().asObject(Object.class, imageHeapConstant);
-        /* Simulated constants don't have a backing object and don't need to be processed. */
-        if (object != null) {
+        AnalysisType objectType = imageHeapConstant.getType();
+        if (imageHeapConstant.isBackedByHostedObject()) {
+            /* Simulated constants don't have a backing object and don't need to be processed. */
             try {
-                type.notifyObjectReachable(universe.getConcurrentAnalysisAccess(), object, reason);
+                Object object = bb.getSnippetReflectionProvider().asObject(Object.class, imageHeapConstant);
+                /*
+                 * Before adding the object to ImageHeap.reachableObjects, where it could be read by
+                 * other threads, run validation checks, e.g., verify that the object's type can be
+                 * initialized at build time. Also run the validation before exposing the object to
+                 * other reachability hooks to avoid propagating an invalid object.
+                 */
+                hostVM.validateReachableObject(object);
+                /*
+                 * Note that reachability hooks can also reject objects based on specific validation
+                 * conditions, e.g., a started Thread should never be added to the image heap, but
+                 * the structure of the object is valid, as ensured by the validity check above.
+                 */
+                objectType.notifyObjectReachable(universe.getConcurrentAnalysisAccess(), object, reason);
             } catch (UnsupportedFeatureException e) {
                 /* Enhance the unsupported feature message with the object trace and rethrow. */
                 StringBuilder backtrace = new StringBuilder();
@@ -604,6 +614,8 @@ protected void onObjectReachable(ImageHeapConstant imageHeapConstant, ScanReason
             }
         }
 
+        imageHeap.addReachableObject(objectType, imageHeapConstant);
+
         markTypeInstantiated(objectType, reason);
         if (imageHeapConstant instanceof ImageHeapObjectArray imageHeapArray) {
             AnalysisType arrayType = imageHeapArray.getType();

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/FeatureImpl.java

@@ -55,6 +55,7 @@
 
 import com.oracle.graal.pointsto.BigBang;
 import com.oracle.graal.pointsto.ObjectScanner;
+import com.oracle.graal.pointsto.constraints.UnsupportedFeatureException;
 import com.oracle.graal.pointsto.heap.ImageHeapConstant;
 import com.oracle.graal.pointsto.heap.ImageHeapScanner;
 import com.oracle.graal.pointsto.infrastructure.SubstitutionProcessor;
@@ -309,7 +310,10 @@ public void registerObjectToConstantReplacer(Function<Object, ImageHeapConstant>
 
         /**
          * Register a callback that is executed when an object of the specified type or any of its
-         * subtypes is marked as reachable.
+         * subtypes is marked as reachable. The callback is executed before the object is added to
+         * the shadow heap. A callback may throw an {@link UnsupportedFeatureException} to reject an
+         * object based on specific validation rules. This will stop the image build and report how
+         * the object was reached.
          *
          * @since 24.0
          */

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java

@@ -45,6 +45,7 @@
 import java.util.function.BiPredicate;
 import java.util.function.Function;
 
+import com.oracle.svm.hosted.classinitialization.ClassInitializationFeature;
 import org.graalvm.nativeimage.AnnotationAccess;
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.Platform;
@@ -360,6 +361,11 @@ public boolean isRelocatedPointer(JavaConstant constant) {
         return constant instanceof RelocatableConstant;
     }
 
+    @Override
+    public void validateReachableObject(Object obj) {
+        ImageSingletons.lookup(ClassInitializationFeature.class).checkImageHeapInstance(obj);
+    }
+
     @Override
     public void registerType(AnalysisType analysisType) {
         DynamicHub hub = createHub(analysisType);

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/classinitialization/ClassInitializationFeature.java

@@ -38,7 +38,6 @@
 
 import org.graalvm.nativeimage.impl.clinit.ClassInitializationTracking;
 
-import com.oracle.graal.pointsto.ObjectScanner;
 import com.oracle.graal.pointsto.constraints.UnsupportedFeatureException;
 import com.oracle.graal.pointsto.meta.AnalysisMethod;
 import com.oracle.graal.pointsto.meta.AnalysisType;
@@ -142,11 +141,10 @@ private static void initializeNativeImagePackagesAtBuildTime(ClassInitialization
     public void duringSetup(DuringSetupAccess a) {
         FeatureImpl.DuringSetupAccessImpl access = (FeatureImpl.DuringSetupAccessImpl) a;
         classInitializationSupport = access.getHostVM().getClassInitializationSupport();
-        access.registerObjectReachableCallback(Object.class, this::checkImageHeapInstance);
     }
 
     @SuppressWarnings("unused")
-    private void checkImageHeapInstance(DuringAnalysisAccess access, Object obj, ObjectScanner.ScanReason reason) {
+    public void checkImageHeapInstance(Object obj) {
         /*
          * Note that initializeAtBuildTime also memoizes the class as InitKind.BUILD_TIME, which
          * means that the user cannot later manually register it as RERUN or RUN_TIME.