#1371 Implemented feedback

AbsaOSS · Jul 27, 2020 · daa4617 · daa4617
1 parent 5054da2
commit daa4617
Show file tree

Hide file tree

Showing 5 changed files with 13 additions and 12 deletions.
diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/common/CommonJobExecution.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/common/CommonJobExecution.scala
@@ -161,16 +161,14 @@ trait CommonJobExecution {
     }
   }
 
-  protected def getDefaultPathConfig[T](cmd: JobConfigParser[T], dataset: Dataset, reportVersion: Int): PathConfig = {
+  protected def getPathConfig[T](cmd: JobConfigParser[T], dataset: Dataset, reportVersion: Int): PathConfig = {
     PathConfig(
       rawPath = buildRawPath(cmd, dataset, reportVersion),
       publishPath = buildPublishPath(cmd, dataset, reportVersion),
       standardizationPath = getStandardizationPath(cmd, reportVersion)
     )
   }
 
-  protected def getPathConfig[T](cmd: JobConfigParser[T], dataset: Dataset, reportVersion: Int): PathConfig
-
   private def buildPublishPath[T](cmd: JobConfigParser[T], ds: Dataset, reportVersion: Int): String = {
     val infoDateCol: String = InfoDateColumn
     val infoVersionCol: String = InfoVersionColumn

diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/ConformanceExecution.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/ConformanceExecution.scala
@@ -82,9 +82,8 @@ trait ConformanceExecution extends CommonJobExecution {
   }
 
   override def getPathConfig[T](cmd: JobConfigParser[T], dataset: Dataset, reportVersion: Int): PathConfig = {
-    val pathOverride = cmd.asInstanceOf[ConformanceConfig].publishPathOverride
-    val initialConfig = getDefaultPathConfig(cmd, dataset, reportVersion)
-    pathOverride match {
+    val initialConfig = super.getPathConfig(cmd, dataset, reportVersion)
+    cmd.asInstanceOf[ConformanceConfig].publishPathOverride match {
       case None => initialConfig
       case Some(providedRawPath) => initialConfig.copy(publishPath = providedRawPath)
     }

diff --git a/...in/scala/za/co/absa/enceladus/conformance/interpreter/rules/CoalesceRuleInterpreter.scala b/...in/scala/za/co/absa/enceladus/conformance/interpreter/rules/CoalesceRuleInterpreter.scala
@@ -33,7 +33,9 @@ case class CoalesceRuleInterpreter(rule: CoalesceConformanceRule) extends RuleIn
   override def conformanceRule: Option[ConformanceRule] = Some(rule)
 
   def conform(df: Dataset[Row])
-             (implicit spark: SparkSession, explosionState: ExplosionState, dao: MenasDAO,
+             (implicit spark: SparkSession,
+              explosionState: ExplosionState,
+              dao: MenasDAO,
               progArgs: InterpreterContextArgs): Dataset[Row] = {
     // Validate the rule parameters
     RuleValidators.validateSameParent(progArgs.datasetName, ruleName, rule.inputColumns :+ rule.outputColumn: _*)

diff --git a/...k-jobs/src/main/scala/za/co/absa/enceladus/standardization/StandardizationExecution.scala b/...k-jobs/src/main/scala/za/co/absa/enceladus/standardization/StandardizationExecution.scala
@@ -87,9 +87,8 @@ trait StandardizationExecution extends CommonJobExecution {
   }
 
   override def getPathConfig[T](cmd: JobConfigParser[T], dataset: Dataset, reportVersion: Int): PathConfig = {
-    val pathOverride = cmd.asInstanceOf[StandardizationConfig].rawPathOverride
-    val initialConfig = getDefaultPathConfig(cmd, dataset, reportVersion)
-    pathOverride match {
+    val initialConfig = super.getPathConfig(cmd, dataset, reportVersion)
+    cmd.asInstanceOf[StandardizationConfig].rawPathOverride match {
       case None => initialConfig
       case Some(providedRawPath) => initialConfig.copy(rawPath = providedRawPath)
     }

diff --git a/...o/absa/enceladus/standardization_conformance/StandardizationAndConformanceExecution.scala b/...o/absa/enceladus/standardization_conformance/StandardizationAndConformanceExecution.scala
@@ -15,20 +15,23 @@
 
 package za.co.absa.enceladus.standardization_conformance
 
+import za.co.absa.enceladus.common.CommonJobExecution
 import za.co.absa.enceladus.common.config.{JobConfigParser, PathConfig}
 import za.co.absa.enceladus.conformance.ConformanceExecution
 import za.co.absa.enceladus.model.Dataset
 import za.co.absa.enceladus.standardization.StandardizationExecution
 import za.co.absa.enceladus.standardization_conformance.config.StandardizationConformanceConfig
 import za.co.absa.enceladus.utils.fs.FileSystemVersionUtils
 
-trait StandardizationAndConformanceExecution extends StandardizationExecution with ConformanceExecution {
+trait StandardizationAndConformanceExecution extends StandardizationExecution
+  with ConformanceExecution
+  with CommonJobExecution{
 
   override def getPathConfig[T](cmd: JobConfigParser[T], dataset: Dataset, reportVersion: Int): PathConfig = {
+    val defaultConfig = super[CommonJobExecution].getPathConfig(cmd, dataset, reportVersion)
     val jobCmd = cmd.asInstanceOf[StandardizationConformanceConfig]
     val rawPathOverride = jobCmd.rawPathOverride
     val publishPathOverride = jobCmd.publishPathOverride
-    val defaultConfig = getDefaultPathConfig(cmd, dataset, reportVersion)
     defaultConfig.copy(rawPath = rawPathOverride.getOrElse(defaultConfig.rawPath),
       publishPath = publishPathOverride.getOrElse(defaultConfig.publishPath))
   }