diff --git a/core/src/main/java/org/apache/spark/classloader/GreedyUrlClassLoader.java b/core/src/main/java/org/apache/spark/classloader/GreedyUrlClassLoader.java new file mode 100644 index 0000000000000..3195a7d695565 --- /dev/null +++ b/core/src/main/java/org/apache/spark/classloader/GreedyUrlClassLoader.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.classloader; + +import java.net.URL; +import java.net.URLClassLoader; +import java.net.URLStreamHandlerFactory; + +/** + *
+ * Instead of the usual delegate-first strategy employed by all the built-in classloaders, this one + * calls findClass first (after checking the cache), then delegates. The net effect is to allow + * the classes that this classloader knows how to find and load itself (the URLs that are + * registered with it) to shadow class defs of the same name from parent classloaders. + *
+ *+ * This shouldn't cause JVM class loader violations, because the world is still internally + * consistent from the perspective of classes loaded by this CL. + *
+ */ +public class GreedyUrlClassLoader extends URLClassLoader { + /* + * It would be great to make this a Scala Trait, but I don't know of a way to write a static + * initializer into a trait. It's a class loader; timing matters. + */ + static { + //This spooky Java magic declares that we're smart enough to avoid class loading deadlocks. + //Requires 1.7 + registerAsParallelCapable(); + } + + public GreedyUrlClassLoader(URL[] urls, ClassLoader parent) { + super(urls, parent); + } + + public GreedyUrlClassLoader(URL[] urls, ClassLoader parent, + URLStreamHandlerFactory factory) { + super(urls, parent, factory); + } + + @Override + protected Class> loadClass(String name, boolean resolve) throws ClassNotFoundException { + synchronized (getClassLoadingLock(name)) { + // First, check if the class has already been loaded + Class c = findLoadedClass(name); + if (c == null) { + try { + c = findClass(name); + } catch (ClassNotFoundException ignored) { + } + + if (c == null) { + // Couldn't load it ourselves; delegate to the parent + c = getParent().loadClass(name); + } + } + if (resolve) { + resolveClass(c); + } + return c; + } + } +} diff --git a/core/src/main/scala/org/apache/spark/executor/ExecutorURLClassLoader.scala b/core/src/main/scala/org/apache/spark/executor/ExecutorURLClassLoader.scala index 218ed7b5d2d39..baa0807b6c97e 100644 --- a/core/src/main/scala/org/apache/spark/executor/ExecutorURLClassLoader.scala +++ b/core/src/main/scala/org/apache/spark/executor/ExecutorURLClassLoader.scala @@ -19,56 +19,21 @@ package org.apache.spark.executor import java.net.{URLClassLoader, URL} -import org.apache.spark.util.ParentClassLoader +import org.apache.spark.classloader.GreedyUrlClassLoader /** * The addURL method in URLClassLoader is protected. We subclass it to make this accessible. * We also make changes so user classes can come before the default classes. */ - private[spark] trait MutableURLClassLoader extends ClassLoader { - def addURL(url: URL) + def addURL(url: URL) //XXX: this is sketchy and dangerous. ClassLoader instances aren't designed to be mutable. def getURLs: Array[URL] } private[spark] class ChildExecutorURLClassLoader(urls: Array[URL], parent: ClassLoader) - extends MutableURLClassLoader { - - private object userClassLoader extends URLClassLoader(urls, null){ - override def addURL(url: URL) { - super.addURL(url) - } - override def findClass(name: String): Class[_] = { - super.findClass(name) - } - } - - private val parentClassLoader = new ParentClassLoader(parent) - - override def findClass(name: String): Class[_] = { - try { - userClassLoader.findClass(name) - } catch { - case e: ClassNotFoundException => { - parentClassLoader.loadClass(name) - } - } - } - - def addURL(url: URL) { - userClassLoader.addURL(url) - } - - def getURLs() = { - userClassLoader.getURLs() - } + extends GreedyUrlClassLoader(urls, parent) with MutableURLClassLoader { } private[spark] class ExecutorURLClassLoader(urls: Array[URL], parent: ClassLoader) extends URLClassLoader(urls, parent) with MutableURLClassLoader { - - override def addURL(url: URL) { - super.addURL(url) - } } - diff --git a/pom.xml b/pom.xml index b993391b15042..236d5fc06d94e 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@