1- package org . apache . spark . sql
1+ package frameless
22
3- import org .apache .spark .sql .catalyst .ScalaReflection .{
4- cleanUpReflectionObjects ,
5- getClassFromType ,
6- localTypeOf
7- }
8- import org .apache .spark .sql .types .{
9- BinaryType ,
10- BooleanType ,
11- ByteType ,
12- CalendarIntervalType ,
13- DataType ,
14- Decimal ,
15- DecimalType ,
16- DoubleType ,
17- FloatType ,
18- IntegerType ,
19- LongType ,
20- NullType ,
21- ObjectType ,
22- ShortType
23- }
3+ import org .apache .spark .sql .types ._
244import org .apache .spark .unsafe .types .CalendarInterval
255
266/**
@@ -45,6 +25,59 @@ package object reflection {
4525
4626 import universe ._
4727
28+ // Since we are creating a runtime mirror using the class loader of current thread,
29+ // we need to use def at here. So, every time we call mirror, it is using the
30+ // class loader of the current thread.
31+ def mirror : universe.Mirror = {
32+ universe.runtimeMirror(Thread .currentThread().getContextClassLoader)
33+ }
34+
35+ /**
36+ * Any codes calling `scala.reflect.api.Types.TypeApi.<:<` should be wrapped by this method to
37+ * clean up the Scala reflection garbage automatically. Otherwise, it will leak some objects to
38+ * `scala.reflect.runtime.JavaUniverse.undoLog`.
39+ *
40+ * @see https://github.com/scala/bug/issues/8302
41+ */
42+ def cleanUpReflectionObjects [T ](func : => T ): T = {
43+ universe.asInstanceOf [scala.reflect.runtime.JavaUniverse ].undoLog.undo(func)
44+ }
45+
46+ /**
47+ * Return the Scala Type for `T` in the current classloader mirror.
48+ *
49+ * Use this method instead of the convenience method `universe.typeOf`, which
50+ * assumes that all types can be found in the classloader that loaded scala-reflect classes.
51+ * That's not necessarily the case when running using Eclipse launchers or even
52+ * Sbt console or test (without `fork := true`).
53+ *
54+ * @see SPARK-5281
55+ */
56+ def localTypeOf [T : TypeTag ]: `Type` = {
57+ val tag = implicitly[TypeTag [T ]]
58+ tag.in(mirror).tpe.dealias
59+ }
60+
61+ /*
62+ * Retrieves the runtime class corresponding to the provided type.
63+ */
64+ def getClassFromType (tpe : Type ): Class [_] =
65+ mirror.runtimeClass(erasure(tpe).dealias.typeSymbol.asClass)
66+
67+ private def erasure (tpe : Type ): Type = {
68+ // For user-defined AnyVal classes, we should not erasure it. Otherwise, it will
69+ // resolve to underlying type which wrapped by this class, e.g erasure
70+ // `case class Foo(i: Int) extends AnyVal` will return type `Int` instead of `Foo`.
71+ // But, for other types, we do need to erasure it. For example, we need to erasure
72+ // `scala.Any` to `java.lang.Object` in order to load it from Java ClassLoader.
73+ // Please see SPARK-17368 & SPARK-31190 for more details.
74+ if (isSubtype(tpe, localTypeOf[AnyVal ]) && ! tpe.toString.startsWith(" scala"
75+ tpe
76+ } else {
77+ tpe.erasure
78+ }
79+ }
80+
4881 /**
4982 * Returns the Spark SQL DataType for a given scala type. Where this is not an exact mapping
5083 * to a native type, an ObjectType is returned. Special handling is also used for Arrays including
@@ -62,7 +95,7 @@ package object reflection {
6295 *
6396 * See https://github.com/scala/bug/issues/10766
6497 */
65- private [sql] def isSubtype (tpe1 : `Type`, tpe2 : `Type`): Boolean = {
98+ private def isSubtype (tpe1 : `Type`, tpe2 : `Type`): Boolean = {
6699 ScalaSubtypeLock .synchronized {
67100 tpe1 <:< tpe2
68101 }
0 commit comments