From 85570d75010aa7c733ccfd03992849811bb47179 Mon Sep 17 00:00:00 2001 From: Daniel Kavan Date: Mon, 17 Aug 2020 14:53:51 +0200 Subject: [PATCH] PR updates --- .../enceladus/conformance/ConformanceExecution.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/ConformanceExecution.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/ConformanceExecution.scala index f0c953f04..d0177fdc8 100644 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/ConformanceExecution.scala +++ b/spark-jobs/src/main/scala/za/co/absa/enceladus/conformance/ConformanceExecution.scala @@ -150,7 +150,7 @@ trait ConformanceExecution extends CommonJobExecution { .withColumnIfDoesNotExist(InfoVersionColumn, lit(preparationResult.reportVersion)) // TODO fix for s3 [ref issue #1416] - val recordCount = 100 + val recordCount = -1 // val recordCount = result.lastCheckpointRowCount match { // case None => withPartCols.count // case Some(p) => p @@ -179,9 +179,10 @@ trait ConformanceExecution extends CommonJobExecution { //withPartCols.writeInfoFile(preparationResult.pathCfg.publishPath) //writePerformanceMetrics(preparationResult.performance, cmd) - if (conformanceReader.isAutocleanStdFolderEnabled()) { - fsUtils.deleteDirectoryRecursively(preparationResult.pathCfg.standardizationPath) - } + // TODO fix for s3 [ref issue #1416] +// if (conformanceReader.isAutocleanStdFolderEnabled()) { +// fsUtils.deleteDirectoryRecursively(preparationResult.pathCfg.standardizationPath) +// } log.info(s"$sourceId finished successfully") } }