#1461 DataType now validated

AbsaOSS · Jul 29, 2020 · 7a7a6d9 · 7a7a6d9
1 parent b049bd3
commit 7a7a6d9
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 12 deletions.
diff --git a/...la/za/co/absa/enceladus/conformance/interpreter/exceptions/InvalidDataTypeException.scala b/...la/za/co/absa/enceladus/conformance/interpreter/exceptions/InvalidDataTypeException.scala
@@ -0,0 +1,7 @@
+package za.co.absa.enceladus.conformance.interpreter.exceptions
+
+import org.apache.spark.sql.types.DataType
+
+case class InvalidDataTypeException(input: String, dataType: DataType) extends Exception(
+  s"Data type ${dataType.typeName} is not valid or not supported"
+)
diff --git a/...n/scala/za/co/absa/enceladus/conformance/interpreter/rules/FillNullsRuleInterpreter.scala b/...n/scala/za/co/absa/enceladus/conformance/interpreter/rules/FillNullsRuleInterpreter.scala
@@ -15,14 +15,15 @@
 
 package za.co.absa.enceladus.conformance.interpreter.rules
 
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.types._
 import org.apache.spark.sql.{Column, Dataset, Row, SparkSession}
+import za.co.absa.enceladus.conformance.config.ConformanceConfig
 import za.co.absa.enceladus.conformance.interpreter.{ExplosionState, RuleValidators}
 import za.co.absa.enceladus.dao.MenasDAO
 import za.co.absa.enceladus.model.conformanceRule.{ConformanceRule, FillNullsConformanceRule}
-import za.co.absa.spark.hats.Extensions._
-import org.apache.spark.sql.functions._
-import za.co.absa.enceladus.conformance.config.ConformanceConfig
 import za.co.absa.enceladus.utils.schema.SchemaUtils
+import za.co.absa.spark.hats.Extensions._
 
 import scala.util.{Failure, Success}
 
@@ -44,15 +45,14 @@ case class FillNullsRuleInterpreter(rule: FillNullsConformanceRule) extends Rule
       rule.outputColumn
     )
 
-    val dataType = SchemaUtils.getFieldType(rule.inputColumn, df.schema).get
+    val dataType: DataType = SchemaUtils.getFieldType(rule.inputColumn, df.schema).get
     val default: Column = simpleLiteralCast(rule.value, dataType) match {
       case Success(value) => value
-      case Failure(_) =>
-        log.warn(
+      case Failure(exception) =>
+        throw new ValidationException(
           s"""Unable to cast literal ${rule.value} to $dataType
-             |for FillNulls conformance rule number ${rule.order}.
-             |Using string as a fallback.""".stripMargin.replaceAll("[\\r\\n]", ""))
-        lit(rule.value)
+             |for FillNulls conformance rule number ${rule.order}.""".stripMargin.replaceAll("[\\r\\n]", ""),
+          cause = exception)
     }
 
     if (rule.outputColumn.contains('.')) {

diff --git a/...s/src/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/RuleInterpreter.scala b/...s/src/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/RuleInterpreter.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.{Column, Dataset, Row, SparkSession}
 import org.slf4j.{Logger, LoggerFactory}
 import za.co.absa.enceladus.conformance.config.ConformanceConfig
 import za.co.absa.enceladus.conformance.interpreter.ExplosionState
+import za.co.absa.enceladus.conformance.interpreter.exceptions.InvalidDataTypeException
 import za.co.absa.enceladus.dao.MenasDAO
 import za.co.absa.enceladus.model.conformanceRule.ConformanceRule
 import za.co.absa.enceladus.utils.transformations.ArrayTransformations
@@ -32,7 +33,6 @@ import scala.util.Try
 
 trait RuleInterpreter {
 
-
   /**
     * Returns the conformance rule the interpreter is intended to interpret.
     * The return value is optional since some interpreters are generated during conformance rules processing optimization
@@ -109,8 +109,9 @@ trait RuleInterpreter {
           lit(Timestamp.valueOf(input))
         case _: DateType =>
           lit(Date.valueOf(input))
-        case _ =>
+        case _: StringType =>
           lit(input)
+        case _ => throw InvalidDataTypeException(input, dataType)
       }
     })
   }

diff --git a/...c/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/ValidationException.scala b/...c/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/ValidationException.scala
@@ -15,4 +15,7 @@
 
 package za.co.absa.enceladus.conformance.interpreter.rules
 
-class ValidationException(val message: String, val techDetails: String = "") extends Exception(message)
+class ValidationException(val message: String,
+                          val techDetails: String = "",
+                          cause: Throwable = None.orNull) extends Exception(message, cause)
+