forked from feathr-ai/feathr
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* untested prototype of UDF plugin API * demo of how to enable coercion between different kinds of FeatureValue classes in MVEL expressions * add plugin APIs (untested) * added docs, cleaned up APIs, still working on testing * update how MVEL is invoked, to conditionally apply plugin to convert to FeatureValue. also added tests. * small doc fix * attempted bugfix regarding not holding on to the adapted anchorExtractor object Co-authored-by: David Stein <dstein@dstein-mn1.linkedin.biz>
- Loading branch information
Showing
20 changed files
with
666 additions
and
14 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
23 changes: 23 additions & 0 deletions
23
src/main/scala/com/linkedin/feathr/offline/client/plugins/FeathrUdfPluginContext.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| package com.linkedin.feathr.offline.client.plugins | ||
|
|
||
| import scala.collection.mutable | ||
|
|
||
| /** | ||
| * A shared registry for loading [[UdfAdaptor]]s, which basically can tell Feathr's runtime how to support different | ||
| * kinds of "external" UDFs not natively known to Feathr, but which have similar behavior to Feathr's. | ||
| * | ||
| * All "external" UDF classes are required to have a public default zero-arg constructor. | ||
| */ | ||
| object FeathrUdfPluginContext { | ||
| val registeredUdfAdaptors = mutable.Buffer[UdfAdaptor[_]]() | ||
|
|
||
| def registerUdfAdaptor(adaptor: UdfAdaptor[_]): Unit = { | ||
| this.synchronized { | ||
| registeredUdfAdaptors += adaptor | ||
| } | ||
| } | ||
|
|
||
| def getRegisteredUdfAdaptor(clazz: Class[_]): Option[UdfAdaptor[_]] = { | ||
| registeredUdfAdaptors.find(_.canAdapt(clazz)) | ||
| } | ||
| } |
53 changes: 53 additions & 0 deletions
53
src/main/scala/com/linkedin/feathr/offline/client/plugins/UdfAdaptor.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| package com.linkedin.feathr.offline.client.plugins | ||
|
|
||
| import com.linkedin.feathr.common.{AnchorExtractor, FeatureDerivationFunction} | ||
| import com.linkedin.feathr.sparkcommon.{SimpleAnchorExtractorSpark, SourceKeyExtractor} | ||
|
|
||
| /** | ||
| * Tells Feathr how to use UDFs that are defined in "external" non-Feathr UDF classes (i.e. that don't extend from | ||
| * Feathr's AnchorExtractor or other UDF traits). An adaptor must match the external UDF class to a specific kind of | ||
| * Feathr UDF – see child traits below for the various options. | ||
| * | ||
| * All "external" UDF classes are required to have a public default zero-arg constructor. | ||
| * | ||
| * @tparam T the internal Feathr UDF class whose behavior the external UDF can be translated to | ||
| */ | ||
| sealed trait UdfAdaptor[T] extends Serializable { | ||
| /** | ||
| * Indicates whether this adaptor can be applied to an object of the provided class. | ||
| * | ||
| * Implementations should usually look like <pre>classOf[UdfTraitThatIsNotPartOfFeathr].isAssignableFrom(clazz)</pre> | ||
| * | ||
| * @param clazz some external UDF type | ||
| * @return true if this adaptor can "adapt" the given class type; false otherwise | ||
| */ | ||
| def canAdapt(clazz: Class[_]): Boolean | ||
|
|
||
| /** | ||
| * Returns an instance of a Feathr UDF, that follows the behavior of some external UDF instance, e.g. via delegation. | ||
| * | ||
| * @param externalUdf instance of the "external" UDF | ||
| * @return the Feathr UDF | ||
| */ | ||
| def adaptUdf(externalUdf: AnyRef): T | ||
| } | ||
|
|
||
| /** | ||
| * An adaptor that can "tell Feathr how to use" a UDF type that can act in place of [[AnchorExtractor]] | ||
| */ | ||
| trait AnchorExtractorAdaptor extends UdfAdaptor[AnchorExtractor[_]] | ||
|
|
||
| /** | ||
| * An adaptor that can "tell Feathr how to use" a UDF type that can act in place of [[SimpleAnchorExtractorSpark]] | ||
| */ | ||
| trait SimpleAnchorExtractorSparkAdaptor extends UdfAdaptor[SimpleAnchorExtractorSpark] | ||
|
|
||
| /** | ||
| * An adaptor that can "tell Feathr how to use" a UDF type that can act in place of [[FeatureDerivationFunction]] | ||
| */ | ||
| trait FeatureDerivationFunctionAdaptor extends UdfAdaptor[FeatureDerivationFunction] | ||
|
|
||
| /** | ||
| * An adaptor that can "tell Feathr how to use" a UDF type that can act in place of [[SourceKeyExtractor]] | ||
| */ | ||
| trait SourceKeyExtractorAdaptor extends UdfAdaptor[SourceKeyExtractor] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
79 changes: 79 additions & 0 deletions
79
src/main/scala/com/linkedin/feathr/offline/mvel/plugins/FeathrMvelPluginContext.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| package com.linkedin.feathr.offline.mvel.plugins; | ||
|
|
||
| import com.linkedin.feathr.common.FeatureValue; | ||
| import com.linkedin.feathr.common.InternalApi; | ||
| import org.mvel2.ConversionHandler; | ||
| import org.mvel2.DataConversion; | ||
|
|
||
| import java.util.concurrent.ConcurrentHashMap; | ||
| import java.util.concurrent.ConcurrentMap; | ||
|
|
||
|
|
||
| /** | ||
| * A plugin that allows an advanced user to add additional capabilities or behaviors to Feathr's MVEL runtime. | ||
| * | ||
| * NOTE: This class is intended for advanced users only, and specifically as a "migration aid" for migrating from | ||
| * some previous versions of Feathr whose FeatureValue representations had a different class name, while preserving | ||
| * compatibility with feature definitions written against those older versions of Feathr. | ||
| */ | ||
| public class FeathrMvelPluginContext { | ||
| // TODO: Does this need to be "translated" into a different pattern whereby we track the CLASSNAME of the type adaptors | ||
| // instead of the instance, such that the class mappings can be broadcasted via Spark and then reinitialized on | ||
| // executor hosts? | ||
| private static final ConcurrentMap<Class<?>, FeatureValueTypeAdaptor<?>> TYPE_ADAPTORS; | ||
|
|
||
| static { | ||
| TYPE_ADAPTORS = new ConcurrentHashMap<>(); | ||
| DataConversion.addConversionHandler(FeatureValue.class, new FeathrFeatureValueConversionHandler()); | ||
| } | ||
|
|
||
| /** | ||
| * Add a type adaptor to Feathr's MVEL runtime, that will enable Feathr's expressions to support some alternative | ||
| * class representation of {@link FeatureValue} via coercion. | ||
| * @param clazz the class of the "other" alternative representation of feature value | ||
| * @param typeAdaptor the type adaptor that can convert between the "other" representation and {@link FeatureValue} | ||
| * @param <T> type parameter for the "other" feature value class | ||
| */ | ||
| @SuppressWarnings("unchecked") | ||
| public static <T> void addFeatureTypeAdaptor(Class<T> clazz, FeatureValueTypeAdaptor<T> typeAdaptor) { | ||
| // TODO: MAKE SURE clazz IS NOT ONE OF THE CLASSES ALREADY COVERED IN org.mvel2.DataConversion.CONVERTERS! | ||
| // IF WE OVERRIDE ANY OF THOSE, IT MIGHT CAUSE MVEL TO BEHAVE IN STRANGE AND UNEXPECTED WAYS! | ||
| TYPE_ADAPTORS.put(clazz, typeAdaptor); | ||
| DataConversion.addConversionHandler(clazz, new ExternalFeatureValueConversionHandler(typeAdaptor)); | ||
| } | ||
|
|
||
| static class FeathrFeatureValueConversionHandler implements ConversionHandler { | ||
| @Override | ||
| @SuppressWarnings("unchecked") | ||
| public Object convertFrom(Object in) { | ||
| FeatureValueTypeAdaptor<Object> adaptor = (FeatureValueTypeAdaptor<Object>) TYPE_ADAPTORS.get(in.getClass()); | ||
| if (adaptor == null) { | ||
| throw new IllegalArgumentException("Can't convert to Feathr FeatureValue from " + in); | ||
| } | ||
| return adaptor.toFeathrFeatureValue(in); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean canConvertFrom(Class cls) { | ||
| return TYPE_ADAPTORS.containsKey(cls); | ||
| } | ||
| } | ||
|
|
||
| static class ExternalFeatureValueConversionHandler implements ConversionHandler { | ||
| private final FeatureValueTypeAdaptor<?> _adaptor; | ||
|
|
||
| public ExternalFeatureValueConversionHandler(FeatureValueTypeAdaptor<?> adaptor) { | ||
| _adaptor = adaptor; | ||
| } | ||
|
|
||
| @Override | ||
| public Object convertFrom(Object in) { | ||
| return _adaptor.fromFeathrFeatureValue((FeatureValue) in); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean canConvertFrom(Class cls) { | ||
| return FeatureValue.class.equals(cls); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.