diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/common/CommonJobExecution.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/common/CommonJobExecution.scala index 329ad1637..8dd4cdfb6 100644 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/common/CommonJobExecution.scala +++ b/spark-jobs/src/main/scala/za/co/absa/enceladus/common/CommonJobExecution.scala @@ -161,7 +161,7 @@ trait CommonJobExecution { } } - protected def getDefaultPathConfig[T](cmd: JobConfigParser[T], dataset: Dataset, reportVersion: Int): PathConfig = { + protected def getPathConfig[T](cmd: JobConfigParser[T], dataset: Dataset, reportVersion: Int): PathConfig = { PathConfig( rawPath = buildRawPath(cmd, dataset, reportVersion), publishPath = buildPublishPath(cmd, dataset, reportVersion), @@ -169,8 +169,6 @@ trait CommonJobExecution { ) } - protected def getPathConfig[T](cmd: JobConfigParser[T], dataset: Dataset, reportVersion: Int): PathConfig - private def buildPublishPath[T](cmd: JobConfigParser[T], ds: Dataset, reportVersion: Int): String = { val infoDateCol: String = InfoDateColumn val infoVersionCol: String = InfoVersionColumn diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/ConformanceExecution.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/ConformanceExecution.scala index 6fa08acad..f335f39b1 100644 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/ConformanceExecution.scala +++ b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/ConformanceExecution.scala @@ -82,9 +82,8 @@ trait ConformanceExecution extends CommonJobExecution { } override def getPathConfig[T](cmd: JobConfigParser[T], dataset: Dataset, reportVersion: Int): PathConfig = { - val pathOverride = cmd.asInstanceOf[ConformanceConfig].publishPathOverride - val initialConfig = getDefaultPathConfig(cmd, dataset, reportVersion) - pathOverride match { + val initialConfig = super.getPathConfig(cmd, dataset, reportVersion) + cmd.asInstanceOf[ConformanceConfig].publishPathOverride match { case None => initialConfig case Some(providedRawPath) => initialConfig.copy(publishPath = providedRawPath) } diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/CoalesceRuleInterpreter.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/CoalesceRuleInterpreter.scala index 3bafba81a..cccd395d3 100644 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/CoalesceRuleInterpreter.scala +++ b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/interpreter/rules/CoalesceRuleInterpreter.scala @@ -33,7 +33,9 @@ case class CoalesceRuleInterpreter(rule: CoalesceConformanceRule) extends RuleIn override def conformanceRule: Option[ConformanceRule] = Some(rule) def conform(df: Dataset[Row]) - (implicit spark: SparkSession, explosionState: ExplosionState, dao: MenasDAO, + (implicit spark: SparkSession, + explosionState: ExplosionState, + dao: MenasDAO, progArgs: InterpreterContextArgs): Dataset[Row] = { // Validate the rule parameters RuleValidators.validateSameParent(progArgs.datasetName, ruleName, rule.inputColumns :+ rule.outputColumn: _*) diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/standardization/StandardizationExecution.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/standardization/StandardizationExecution.scala index 52c3ca9fb..ac8bb3368 100644 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/standardization/StandardizationExecution.scala +++ b/spark-jobs/src/main/scala/za/co/absa/enceladus/standardization/StandardizationExecution.scala @@ -87,9 +87,8 @@ trait StandardizationExecution extends CommonJobExecution { } override def getPathConfig[T](cmd: JobConfigParser[T], dataset: Dataset, reportVersion: Int): PathConfig = { - val pathOverride = cmd.asInstanceOf[StandardizationConfig].rawPathOverride - val initialConfig = getDefaultPathConfig(cmd, dataset, reportVersion) - pathOverride match { + val initialConfig = super.getPathConfig(cmd, dataset, reportVersion) + cmd.asInstanceOf[StandardizationConfig].rawPathOverride match { case None => initialConfig case Some(providedRawPath) => initialConfig.copy(rawPath = providedRawPath) } diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/standardization_conformance/StandardizationAndConformanceExecution.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/standardization_conformance/StandardizationAndConformanceExecution.scala index 4cda253c9..7208bbcb7 100644 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/standardization_conformance/StandardizationAndConformanceExecution.scala +++ b/spark-jobs/src/main/scala/za/co/absa/enceladus/standardization_conformance/StandardizationAndConformanceExecution.scala @@ -15,6 +15,7 @@ package za.co.absa.enceladus.standardization_conformance +import za.co.absa.enceladus.common.CommonJobExecution import za.co.absa.enceladus.common.config.{JobConfigParser, PathConfig} import za.co.absa.enceladus.conformance.ConformanceExecution import za.co.absa.enceladus.model.Dataset @@ -22,13 +23,15 @@ import za.co.absa.enceladus.standardization.StandardizationExecution import za.co.absa.enceladus.standardization_conformance.config.StandardizationConformanceConfig import za.co.absa.enceladus.utils.fs.FileSystemVersionUtils -trait StandardizationAndConformanceExecution extends StandardizationExecution with ConformanceExecution { +trait StandardizationAndConformanceExecution extends StandardizationExecution + with ConformanceExecution + with CommonJobExecution{ override def getPathConfig[T](cmd: JobConfigParser[T], dataset: Dataset, reportVersion: Int): PathConfig = { + val defaultConfig = super[CommonJobExecution].getPathConfig(cmd, dataset, reportVersion) val jobCmd = cmd.asInstanceOf[StandardizationConformanceConfig] val rawPathOverride = jobCmd.rawPathOverride val publishPathOverride = jobCmd.publishPathOverride - val defaultConfig = getDefaultPathConfig(cmd, dataset, reportVersion) defaultConfig.copy(rawPath = rawPathOverride.getOrElse(defaultConfig.rawPath), publishPath = publishPathOverride.getOrElse(defaultConfig.publishPath)) }