Support dynamic class loading

Support dynamic class loading for native image

Author: ziyilin

substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/BreakpointInterceptor.java

@@ -27,6 +27,7 @@
 import static com.oracle.svm.core.util.VMError.guarantee;
 import static com.oracle.svm.jni.JNIObjectHandles.nullHandle;
 import static com.oracle.svm.jvmtiagentbase.Support.callObjectMethod;
+import static com.oracle.svm.jvmtiagentbase.Support.callObjectMethodL;
 import static com.oracle.svm.jvmtiagentbase.Support.check;
 import static com.oracle.svm.jvmtiagentbase.Support.checkJni;
 import static com.oracle.svm.jvmtiagentbase.Support.checkNoException;
@@ -50,6 +51,7 @@
 import static com.oracle.svm.jvmtiagentbase.jvmti.JvmtiEvent.JVMTI_EVENT_BREAKPOINT;
 import static com.oracle.svm.jvmtiagentbase.jvmti.JvmtiEvent.JVMTI_EVENT_CLASS_PREPARE;
 import static com.oracle.svm.jvmtiagentbase.jvmti.JvmtiEvent.JVMTI_EVENT_NATIVE_METHOD_BIND;
+import static com.oracle.svm.jvmtiagentbase.jvmti.JvmtiEvent.JVMTI_EVENT_CLASS_FILE_LOAD_HOOK;
 import static org.graalvm.word.WordFactory.nullPointer;
 
 import java.nio.ByteBuffer;
@@ -63,6 +65,7 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.locks.ReentrantLock;
 
+import com.oracle.svm.core.util.JavaClassUtil;
 import org.graalvm.compiler.core.common.NumUtil;
 import org.graalvm.nativeimage.StackValue;
 import org.graalvm.nativeimage.UnmanagedMemory;
@@ -126,7 +129,7 @@ final class BreakpointInterceptor {
     private static NativeImageAgent agent;
 
     private static Map<Long, Breakpoint> installedBreakpoints;
-
+    private static List<String> unsupportedExceptions = new ArrayList<>();
     /**
      * A map from {@link JNIMethodId} to entry point addresses for bound Java {@code native}
      * methods, NOT considering our intercepting functions, i.e., these are the original entry
@@ -1085,6 +1088,51 @@ private static void installBreakpointIfClassLoader(JNIEnvironment jni, JNIObject
         }
     }
 
+    @CEntryPoint
+    @CEntryPointOptions(prologue = AgentIsolate.Prologue.class)
+    private static void onClassFileLoadHook(@SuppressWarnings("unused") JvmtiEnv jvmti, JNIEnvironment jni,
+                                            @SuppressWarnings("unused") JNIObjectHandle classBeingRedefined, JNIObjectHandle loader, CCharPointer name, JNIObjectHandle protectionDomain, int classDataLen,
+                                            CCharPointer classData, @SuppressWarnings("unused") CIntPointer newClassDataLen, @SuppressWarnings("unused") CCharPointerPointer newClassData) {
+        boolean nameIsNull = name.isNull();
+        if (isDynamicallyGenerated(jni, loader, nameIsNull, nameIsNull ? "" : fromCString(name))) {
+            byte[] contents = new byte[classDataLen];
+            CTypeConversion.asByteBuffer(classData, classDataLen).get(contents);
+            String definedClassName = nameIsNull ? JavaClassUtil.getClassName(contents) : fromCString(name);
+            ClassLoaderDefineClassSupport.trace(traceWriter, contents, definedClassName, true);
+        }
+    }
+
+    private static boolean isDynamicallyGenerated(JNIEnvironment jni, JNIObjectHandle classLoader, boolean inputNameIsNull, String definedClassName) {
+        boolean isDynamicallyGenerated;
+        // 1. Classloader is null, it's a system class.
+        // The class is not dynamically generated.
+        if(classLoader.equal(nullHandle())){
+            isDynamicallyGenerated = false;
+        } else {
+            // 2. Don't have a name for class before defining.
+            // The class is dynamically generated.
+            if (inputNameIsNull) {
+                isDynamicallyGenerated = true;
+            } else {
+                // 3. A dynamically defined class always return null
+                // when call java.lang.ClassLoader.getResource(classname)
+                // This is the accurate but slow way.
+                String asResourceName = definedClassName.replace('.', '/') + ".class";
+                try (CCharPointerHolder resourceNameHolder = toCString(asResourceName);) {
+                    JNIObjectHandle resourceNameJString = jniFunctions().getNewStringUTF().invoke(jni, resourceNameHolder.get());
+                    if (agent.handles() == null) {
+                        // agent's handles is created at onVMStart.
+                        isDynamicallyGenerated = false;
+                    } else {
+                        JNIObjectHandle returnValue = callObjectMethodL(jni, classLoader, agent.handles().javaLangClassLoaderGetResource, resourceNameJString);
+                        isDynamicallyGenerated = returnValue.equal(nullHandle());
+                    }
+                }
+            }
+        }
+        return isDynamicallyGenerated;
+    }
+
     private static final CEntryPointLiteral<CFunctionPointer> onBreakpointLiteral = CEntryPointLiteral.create(BreakpointInterceptor.class, "onBreakpoint",
                     JvmtiEnv.class, JNIEnvironment.class, JNIObjectHandle.class, JNIMethodId.class, long.class);
 
@@ -1094,6 +1142,9 @@ private static void installBreakpointIfClassLoader(JNIEnvironment jni, JNIObject
     private static final CEntryPointLiteral<CFunctionPointer> onClassPrepareLiteral = CEntryPointLiteral.create(BreakpointInterceptor.class, "onClassPrepare",
                     JvmtiEnv.class, JNIEnvironment.class, JNIObjectHandle.class, JNIObjectHandle.class);
 
+    private static final CEntryPointLiteral<CFunctionPointer> onClassFileLoadHookLiteral = CEntryPointLiteral.create(BreakpointInterceptor.class, "onClassFileLoadHook",
+            JvmtiEnv.class, JNIEnvironment.class, JNIObjectHandle.class, JNIObjectHandle.class, CCharPointer.class, JNIObjectHandle.class, int.class, CCharPointer.class, CIntPointer.class, CCharPointerPointer.class);
+
     public static void onLoad(JvmtiEnv jvmti, JvmtiEventCallbacks callbacks, TraceWriter writer, NativeImageAgent nativeImageTracingAgent,
                     boolean exptlClassLoaderSupport) {
 
@@ -1106,6 +1157,7 @@ public static void onLoad(JvmtiEnv jvmti, JvmtiEventCallbacks callbacks, TraceWr
         capabilities.setCanGenerateBreakpointEvents(1);
         capabilities.setCanAccessLocalVariables(1);
         capabilities.setCanGenerateNativeMethodBindEvents(1);
+        capabilities.setCanGenerateAllClassHookEvents(1);
         if (exptlClassLoaderSupport) {
             capabilities.setCanGetBytecodes(1);
             capabilities.setCanGetConstantPool(1);
@@ -1125,6 +1177,9 @@ public static void onLoad(JvmtiEnv jvmti, JvmtiEventCallbacks callbacks, TraceWr
 
         BreakpointInterceptor.boundNativeMethods = new HashMap<>();
         Support.check(jvmti.getFunctions().SetEventNotificationMode().invoke(jvmti, JvmtiEventMode.JVMTI_ENABLE, JVMTI_EVENT_NATIVE_METHOD_BIND, nullHandle()));
+
+        callbacks.setClassFileLoadHook(onClassFileLoadHookLiteral.getFunctionPointer());
+        Support.check(jvmti.getFunctions().SetEventNotificationMode().invoke(jvmti, JvmtiEventMode.JVMTI_ENABLE, JVMTI_EVENT_CLASS_FILE_LOAD_HOOK, nullHandle()));
     }
 
     public static void onVMInit(JvmtiEnv jvmti, JNIEnvironment jni) {
@@ -1269,6 +1324,19 @@ private static void bindNativeBreakpoint(JNIEnvironment jni, NativeBreakpoint bp
         }
     }
 
+    public static void reportExceptions() {
+        if (!unsupportedExceptions.isEmpty()) {
+            System.err.println(unsupportedExceptions.size() + " unsupported features are detected ");
+            StringBuilder errorMsg = new StringBuilder();
+            for (int i = 0; i < unsupportedExceptions.size(); i++) {
+                errorMsg.append(unsupportedExceptions.get(i)).append("\n");
+            }
+            throw new UnsupportedOperationException(errorMsg.toString());
+        } else {
+            unsupportedExceptions = null;
+        }
+    }
+
     public static void onUnload() {
         installedBreakpoints = null;
         nativeBreakpoints = null;

substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/ClassLoaderDefineClassSupport.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2021, Alibaba Group Holding Limited. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.agent;
+
+import com.oracle.svm.core.util.JavaClassUtil;
+import com.oracle.svm.jni.nativeapi.JNIEnvironment;
+
+import java.security.NoSuchAlgorithmException;
+
+import static com.oracle.svm.jvmtiagentbase.Support.getMethodFullNameAtFrame;
+
+/**
+ * Support dynamic class loading that is implemented by java.lang.ClassLoader.defineClass.
+ */
+public class ClassLoaderDefineClassSupport {
+
+    private static String calculateGeneratedClassSHA(byte[] values) {
+        String generatedClassHashCode;
+        try {
+            generatedClassHashCode = JavaClassUtil.getSHAWithoutSourceFileInfo(values);
+        } catch (NoSuchAlgorithmException e) {
+            generatedClassHashCode = null;
+        }
+        return generatedClassHashCode;
+    }
+
+    public static void trace(TraceWriter traceWriter, byte[] classContents, String generatedClassName, Object result) {
+        assert classContents != null;
+        if (generatedClassName != null && result != null) {
+            // Trace dynamically generated class in config file
+            traceWriter.traceCall("classDefiner", "onClassFileLoadHook", null, null, null, result, generatedClassName.replace('/', '.'), calculateGeneratedClassSHA(classContents), classContents);
+        }
+    }
+
+    public static StringBuilder getStackTrace(JNIEnvironment jni) {
+        StringBuilder trace = new StringBuilder();
+        int i = 0;
+        int maxDepth = 20;
+        while (i < maxDepth) {
+            String methodName = getMethodFullNameAtFrame(jni, i++);
+            if (methodName == null) {
+                break;
+            }
+            trace.append("    ").append(methodName).append("\n");
+        }
+        if (i >= maxDepth) {
+            trace.append("    ").append("...").append("\n");
+        }
+        return trace;
+    }
+}

substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/NativeImageAgent.java

@@ -46,6 +46,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import org.graalvm.nativeimage.ProcessProperties;
 import org.graalvm.nativeimage.hosted.Feature;
@@ -216,7 +217,8 @@ protected int onLoadCallback(JNIJavaVM vm, JvmtiEnv jvmti, JvmtiEventCallbacks c
                 // They should use the same filter sets, however.
                 AccessAdvisor advisor = createAccessAdvisor(builtinHeuristicFilter, callerFilter, accessFilter);
                 TraceProcessor processor = new TraceProcessor(advisor, mergeConfigs.loadJniConfig(handler), mergeConfigs.loadReflectConfig(handler),
-                                mergeConfigs.loadProxyConfig(handler), mergeConfigs.loadResourceConfig(handler), mergeConfigs.loadSerializationConfig(handler));
+                                mergeConfigs.loadProxyConfig(handler), mergeConfigs.loadResourceConfig(handler), mergeConfigs.loadSerializationConfig(handler),
+                                mergeConfigs.loadDynamicClassesConfig(handler));
                 traceWriter = new TraceProcessorWriterAdapter(processor);
             } catch (Throwable t) {
                 return error(2, t.toString());
@@ -425,6 +427,7 @@ private void writeConfigurationFiles() {
             allConfigFiles.put(ConfigurationFiles.DYNAMIC_PROXY_NAME, p.getProxyConfiguration());
             allConfigFiles.put(ConfigurationFiles.RESOURCES_NAME, p.getResourceConfiguration());
             allConfigFiles.put(ConfigurationFiles.SERIALIZATION_NAME, p.getSerializationConfiguration());
+            allConfigFiles.put(ConfigurationFiles.DYNAMIC_CLASSES_NAME, p.getDynamicClassesConfiguration());
 
             for (Map.Entry<String, JsonPrintable> configFile : allConfigFiles.entrySet()) {
                 Path tempPath = tempDirectory.resolve(configFile.getKey());
@@ -439,6 +442,18 @@ private void writeConfigurationFiles() {
                 tryAtomicMove(source, target);
             }
 
+            Path dumpedClassSrc = tempDirectory.resolve(ConfigurationFiles.DUMP_CLASSES_DIR);
+            Path dumpedClassTarget = configOutputDirPath.resolve(ConfigurationFiles.DUMP_CLASSES_DIR);
+            // Move the entire directory if the target directory does not exist
+            if (Files.notExists(dumpedClassTarget)) {
+                tryAtomicMove(dumpedClassSrc, dumpedClassTarget);
+            }else {
+                // Move each file inside the source directory if the target directory exists
+                for (Path filePath : Files.list(dumpedClassSrc).collect(Collectors.toList())) {
+                    tryAtomicMove(filePath, dumpedClassTarget.resolve(filePath.getFileName()));
+                }
+            }
+
             compulsoryDelete(tempDirectory);
         } catch (IOException e) {
             warnUpToLimit(currentFailuresWritingConfigs++, MAX_WARNINGS_FOR_WRITING_CONFIGS_FAILURES, "Error when writing configuration files: " + e.toString());
@@ -509,7 +524,7 @@ protected int onUnloadCallback(JNIJavaVM vm) {
          * (unless another JVM is launched in this process).
          */
         // cleanupOnUnload(vm);
-
+        BreakpointInterceptor.reportExceptions();
         /*
          * The epilogue of this method does not tear down our VM: we don't seem to observe all
          * threads that end and therefore can't detach them, so we would wait forever for them.

substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/NativeImageAgentJNIHandleSet.java

@@ -39,17 +39,15 @@ public class NativeImageAgentJNIHandleSet extends JNIHandleSet {
     final JNIObjectHandle javaLangClass;
     final JNIMethodId javaLangClassForName3;
     final JNIMethodId javaUtilEnumerationNextElement;
-    final JNIMethodId javaLangClassGetDeclaredMethod;
-    final JNIMethodId javaLangClassGetDeclaredConstructor;
-    final JNIMethodId javaLangClassGetDeclaredField;
-    final JNIMethodId javaLangClassGetName;
-
     final JNIMethodId javaLangReflectMemberGetName;
     final JNIMethodId javaLangReflectMemberGetDeclaringClass;
-
     final JNIMethodId javaUtilEnumerationHasMoreElements;
-
+    final JNIMethodId javaLangClassLoaderGetResource;
     final JNIObjectHandle javaLangClassLoader;
+    final JNIMethodId javaLangClassGetDeclaredMethod;
+    final JNIMethodId javaLangClassGetDeclaredConstructor;
+    final JNIMethodId javaLangClassGetDeclaredField;
+    final JNIMethodId javaLangClassGetName;
 
     final JNIMethodId javaLangObjectGetClass;
 
@@ -78,6 +76,7 @@ public class NativeImageAgentJNIHandleSet extends JNIHandleSet {
         javaLangClassGetDeclaredField = getMethodId(env, javaLangClass, "getDeclaredField", "(Ljava/lang/String;)Ljava/lang/reflect/Field;", false);
         javaLangClassGetName = getMethodId(env, javaLangClass, "getName", "()Ljava/lang/String;", false);
 
+        javaLangClassLoaderGetResource = getMethodId(env, findClass(env, "java/lang/ClassLoader"), "getResource", "(Ljava/lang/String;)Ljava/net/URL;", false);
         JNIObjectHandle javaLangReflectMember = findClass(env, "java/lang/reflect/Member");
         javaLangReflectMemberGetName = getMethodId(env, javaLangReflectMember, "getName", "()Ljava/lang/String;", false);
         javaLangReflectMemberGetDeclaringClass = getMethodId(env, javaLangReflectMember, "getDeclaringClass", "()Ljava/lang/Class;", false);

substratevm/src/com.oracle.svm.configure/src/com/oracle/svm/configure/ConfigurationTool.java

@@ -195,6 +195,12 @@ private static void generate(Iterator<String> argsIter, boolean acceptTraceFileA
                     set.getSerializationConfigPaths().add(requirePathUri(current, value));
                     break;
 
+                case "--dynamic-classes-input":
+                    set = inputSet; // fall through
+                case "--dynamic-classes-output":
+                    set.getDynamicClassesConfigPaths().add(requirePathUri(current, value));
+                    break;
+
                 case "--trace-input":
                     traceInputs.add(requirePathUri(current, value));
                     break;
@@ -256,7 +262,8 @@ private static void generate(Iterator<String> argsIter, boolean acceptTraceFileA
         try {
             p = new TraceProcessor(advisor, inputSet.loadJniConfig(ConfigurationSet.FAIL_ON_EXCEPTION), inputSet.loadReflectConfig(ConfigurationSet.FAIL_ON_EXCEPTION),
                             inputSet.loadProxyConfig(ConfigurationSet.FAIL_ON_EXCEPTION), inputSet.loadResourceConfig(ConfigurationSet.FAIL_ON_EXCEPTION),
-                            inputSet.loadSerializationConfig(ConfigurationSet.FAIL_ON_EXCEPTION));
+                            inputSet.loadSerializationConfig(ConfigurationSet.FAIL_ON_EXCEPTION),
+                            inputSet.loadDynamicClassesConfig(ConfigurationSet.FAIL_ON_EXCEPTION));
         } catch (IOException e) {
             throw e;
         } catch (Throwable t) {
@@ -299,6 +306,11 @@ private static void generate(Iterator<String> argsIter, boolean acceptTraceFileA
                 p.getSerializationConfiguration().printJson(writer);
             }
         }
+        for (URI uri : outputSet.getDynamicClassesConfigPaths()) {
+            try (JsonWriter writer = new JsonWriter(Paths.get(uri))) {
+                p.getDynamicClassesConfiguration().printJson(writer);
+            }
+        }
     }
 
     private static void generateFilterRules(Iterator<String> argsIter) throws IOException {