fix for handling missing feature data (#1163)

Co-authored-by: Anirudh Agarwal <aniagarwal@linkedin.biz>
feathr-ai · May 9, 2023 · 7d91781 · 7d91781
1 parent 44e4c7f
commit 7d91781
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 3 deletions.
diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/AclCheckUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/AclCheckUtils.scala
@@ -68,12 +68,12 @@ private[offline] object AclCheckUtils {
     } yield featureAnchorWithSource.source.path
 
     val shouldSkipFeature = FeathrUtils.getFeathrJobParam(ss.sparkContext.getConf, FeathrUtils.SKIP_MISSING_FEATURE).toBoolean
-    val shouldAddDefaultCol = FeathrUtils.getFeathrJobParam(ss.sparkContext.getConf, FeathrUtils.SKIP_MISSING_FEATURE).toBoolean
+    val shouldAddDefaultCol = FeathrUtils.getFeathrJobParam(ss.sparkContext.getConf, FeathrUtils.ADD_DEFAULT_COL_FOR_MISSING_DATA).toBoolean
     val invalidPaths = AclCheckUtils.checkReadAuthorization(conf, allRequiredPaths.distinct)
     if (invalidPaths.isEmpty) {
       (Success(()), invalidPaths.map(_._2))
     } else {
-      if (!shouldSkipFeature || !shouldAddDefaultCol) {
+      if (!shouldSkipFeature && !shouldAddDefaultCol) {
         (Failure(
           new RuntimeException(
             "Can not verify read authorization on the following paths. This can be due to" +

diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/SlidingWindowAggIntegTest.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/SlidingWindowAggIntegTest.scala
@@ -820,6 +820,7 @@ class SlidingWindowAggIntegTest extends FeathrIntegTest {
     setFeathrJobParam(SKIP_MISSING_FEATURE, "false")
   }
 
+  @Test
   def testSWAWithMissingFeatureDataFlag(): Unit = {
     setFeathrJobParam(FeathrUtils.ADD_DEFAULT_COL_FOR_MISSING_DATA, "true")
     val joinConfigAsString =

diff --git a/gradle.properties b/gradle.properties
@@ -1,3 +1,3 @@
-version=1.0.3-rc3
+version=1.0.4-rc1
 SONATYPE_AUTOMATIC_RELEASE=true
 POM_ARTIFACT_ID=feathr_2.12