diff --git a/README.md b/README.md index 2c271bd6f..791fcef0f 100644 --- a/README.md +++ b/README.md @@ -556,6 +556,10 @@ is determined by the pipeline configuration. # Specifies the maximum number of records to fetch. Good for testing purposes. #limit.records = 100 + + # Optionally, you can specify a class for a custom SQL generator for your RDMS engine. + # The class whould extend 'za.co.absa.pramen.api.sql.SqlGenerator' + #sql.generator.class = "com.example.MySqlGenerator" } ``` diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/model/QuotingPolicy.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/QuotingPolicy.scala similarity index 96% rename from pramen/core/src/main/scala/za/co/absa/pramen/core/reader/model/QuotingPolicy.scala rename to pramen/api/src/main/scala/za/co/absa/pramen/api/sql/QuotingPolicy.scala index 2fb74f531..3830409b6 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/model/QuotingPolicy.scala +++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/QuotingPolicy.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package za.co.absa.pramen.core.reader.model +package za.co.absa.pramen.api.sql sealed trait QuotingPolicy { def name: String diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlColumnType.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlColumnType.scala similarity index 97% rename from pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlColumnType.scala rename to pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlColumnType.scala index 6a1f1108d..b89d73f3d 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlColumnType.scala +++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlColumnType.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package za.co.absa.pramen.core.sql +package za.co.absa.pramen.api.sql sealed trait SqlColumnType diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlConfig.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlConfig.scala similarity index 77% rename from pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlConfig.scala rename to pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlConfig.scala index 5ad5f925a..eba62e797 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlConfig.scala +++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlConfig.scala @@ -14,13 +14,15 @@ * limitations under the License. */ -package za.co.absa.pramen.core.sql +package za.co.absa.pramen.api.sql -import za.co.absa.pramen.core.reader.model.QuotingPolicy +import com.typesafe.config.Config case class SqlConfig( infoDateColumn: String, infoDateType: SqlColumnType, dateFormatApp: String, - identifierQuotingPolicy: QuotingPolicy + identifierQuotingPolicy: QuotingPolicy, + sqlGeneratorClass: Option[String], + extraConfig: Config ) diff --git a/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlGenerator.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlGenerator.scala new file mode 100644 index 000000000..827d4e5e9 --- /dev/null +++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlGenerator.scala @@ -0,0 +1,81 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.pramen.api.sql + +import java.sql.Connection +import java.time.LocalDate + +trait SqlGenerator { + /** + * Returns wrapped query that can be passed as .option("dtable", here) to the Spark JDBC reader. + * For example, given "SELECT * FROM abc", returns "(SELECT * FROM abc) tbl" + */ + def getDtable(sql: String): String + + /** + * Generates a query that returns the record count of a table that does not have the information date field. + */ + def getCountQuery(tableName: String): String + + /** + * Generates a query that returns the record count of a table for the given period when the table does have the information date field. + */ + def getCountQuery(tableName: String, infoDateBegin: LocalDate, infoDateEnd: LocalDate): String + + /** + * Generates a query that returns data of a table that does not have the information date field. + */ + def getDataQuery(tableName: String, columns: Seq[String], limit: Option[Int]): String + + /** + * Generates a query that returns the data of a table for the given period when the table does have the information date field. + */ + def getDataQuery(tableName: String, infoDateBegin: LocalDate, infoDateEnd: LocalDate, columns: Seq[String], limit: Option[Int]): String + + /** + * Returns WHERE condition for table that has the information date field given the time period. + */ + def getWhere(dateBegin: LocalDate, dateEnd: LocalDate): String + + /** Returns the date literal for the dialect of the SQL. */ + def getDateLiteral(date: LocalDate): String + + /** + * Validates and escapes an identifier (table or column name) if needed. + * Escaping happens according to the quoting policy. + */ + def escape(identifier: String): String + + /** + * Quotes an identifier name with characters specific to SQL dialects. + * If the identifier is already wrapped, it won't be double wrapped. + * It supports partially quoted identifiers. E.g. '"my_catalog".my table' will be quoted as '"my_catalog"."my table"'. + */ + def quote(identifier: String): String + + /** + * Returns true if the SQL generator can only work if it has an active connection. + * This can be for database engines that does not support "SELECT * FROM table" and require explicit list of columns. + * The connection can be used to query the list of columns first. + */ + def requiresConnection: Boolean = false + + /** + * Sets the connection for the the SQL generator can only work if it has an active connection. + */ + def setConnection(connection: Connection): Unit = {} +} diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorBase.scala b/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlGeneratorBase.scala similarity index 94% rename from pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorBase.scala rename to pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlGeneratorBase.scala index 78e1cda9a..a928fe45a 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorBase.scala +++ b/pramen/api/src/main/scala/za/co/absa/pramen/api/sql/SqlGeneratorBase.scala @@ -14,9 +14,7 @@ * limitations under the License. */ -package za.co.absa.pramen.core.sql - -import za.co.absa.pramen.core.reader.model.QuotingPolicy +package za.co.absa.pramen.api.sql import scala.collection.mutable.ListBuffer @@ -51,8 +49,7 @@ abstract class SqlGeneratorBase(sqlConfig: SqlConfig) extends SqlGenerator { splitComplexIdentifier(identifier).map(quoteSingleIdentifier).mkString(".") } - /** This validates and escapes an identifier (table or column name) if needed. Escaping does not happen always to maintain backwards compatibility. */ - override final def escape(identifier: String): String = { + override final def escape(identifier: String): String = { if (needsEscaping(sqlConfig.identifierQuotingPolicy, identifier)) { quote(identifier) } else { diff --git a/pramen/api/src/test/scala/za/co/absa/pramen/api/sql/QuotingPolicySuite.scala b/pramen/api/src/test/scala/za/co/absa/pramen/api/sql/QuotingPolicySuite.scala new file mode 100644 index 000000000..5cd6d3f68 --- /dev/null +++ b/pramen/api/src/test/scala/za/co/absa/pramen/api/sql/QuotingPolicySuite.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.pramen.api.sql + +import org.scalatest.wordspec.AnyWordSpec + +class QuotingPolicySuite extends AnyWordSpec { + "QuotingPolicy.fromString" should { + "return Never for 'never'" in { + assert(QuotingPolicy.fromString("never") == QuotingPolicy.Never) + assert(QuotingPolicy.fromString("never").name == "never") + } + + "return Always for 'always'" in { + assert(QuotingPolicy.fromString("always") == QuotingPolicy.Always) + assert(QuotingPolicy.fromString("always").name == "always") + } + + "return Auto for 'auto'" in { + assert(QuotingPolicy.fromString("auto") == QuotingPolicy.Auto) + assert(QuotingPolicy.fromString("auto").name == "auto") + } + + "throw an exception for an unknown quoting policy" in { + assertThrows[IllegalArgumentException] { + QuotingPolicy.fromString("unknown") + } + } + } +} diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderJdbc.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderJdbc.scala index 9b3f3ea98..6fafe0de2 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderJdbc.scala +++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/TableReaderJdbc.scala @@ -19,10 +19,11 @@ package za.co.absa.pramen.core.reader import com.typesafe.config.Config import org.apache.spark.sql.{DataFrame, SparkSession} import org.slf4j.LoggerFactory +import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig} import za.co.absa.pramen.api.{Query, TableReader} import za.co.absa.pramen.core.config.Keys import za.co.absa.pramen.core.reader.model.TableReaderJdbcConfig -import za.co.absa.pramen.core.sql.{SqlColumnType, SqlConfig, SqlGenerator} +import za.co.absa.pramen.core.sql.SqlGeneratorLoader import za.co.absa.pramen.core.utils.JdbcNativeUtils.JDBC_WORDS_TO_REDACT import za.co.absa.pramen.core.utils.{ConfigUtils, JdbcSparkUtils, TimeUtils} @@ -46,9 +47,7 @@ class TableReaderJdbc(jdbcReaderConfig: TableReaderJdbcConfig, logConfiguration() private[core] lazy val sqlGen = { - val gen = SqlGenerator.fromDriverName(jdbcReaderConfig.jdbcConfig.driver, - getSqlConfig, - ConfigUtils.getExtraConfig(conf, "sql")) + val gen = SqlGeneratorLoader.getSqlGenerator(jdbcReaderConfig.jdbcConfig.driver, getSqlConfig) if (gen.requiresConnection) { val (connection, url) = jdbcUrlSelector.getWorkingConnection(jdbcRetries) @@ -211,7 +210,9 @@ class TableReaderJdbc(jdbcReaderConfig: TableReaderJdbcConfig, SqlConfig(jdbcReaderConfig.infoDateColumn, infoDateType, jdbcReaderConfig.infoDateFormat, - jdbcReaderConfig.identifierQuotingPolicy) + jdbcReaderConfig.identifierQuotingPolicy, + jdbcReaderConfig.sqlGeneratorClass, + ConfigUtils.getExtraConfig(conf, "sql")) case None => throw new IllegalArgumentException(s"Unknown info date type specified (${jdbcReaderConfig.infoDateType}). " + s"It should be one of: date, string, number") } diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/model/TableReaderJdbcConfig.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/model/TableReaderJdbcConfig.scala index 90b4e1f45..529b9311c 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/model/TableReaderJdbcConfig.scala +++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/reader/model/TableReaderJdbcConfig.scala @@ -18,6 +18,7 @@ package za.co.absa.pramen.core.reader.model import com.typesafe.config.Config import org.slf4j.LoggerFactory +import za.co.absa.pramen.api.sql.QuotingPolicy import za.co.absa.pramen.core.utils.ConfigUtils case class TableReaderJdbcConfig( @@ -32,7 +33,8 @@ case class TableReaderJdbcConfig( correctDecimalsFixPrecision: Boolean = false, enableSchemaMetadata: Boolean = false, useJdbcNative: Boolean = false, - identifierQuotingPolicy: QuotingPolicy = QuotingPolicy.Auto + identifierQuotingPolicy: QuotingPolicy = QuotingPolicy.Auto, + sqlGeneratorClass: Option[String] = None ) object TableReaderJdbcConfig { @@ -51,6 +53,7 @@ object TableReaderJdbcConfig { val ENABLE_SCHEMA_METADATA_KEY = "enable.schema.metadata" val USE_JDBC_NATIVE = "use.jdbc.native" val IDENTIFIER_QUOTING_POLICY = "identifier.quoting.policy" + val SQL_GENERATOR_CLASS_KEY = "sql.generator.class" def load(conf: Config, parent: String = ""): TableReaderJdbcConfig = { ConfigUtils.validatePathsExistence(conf, parent, HAS_INFO_DATE :: Nil) @@ -87,7 +90,8 @@ object TableReaderJdbcConfig { correctDecimalsFixPrecision = ConfigUtils.getOptionBoolean(conf, CORRECT_DECIMALS_FIX_PRECISION).getOrElse(false), enableSchemaMetadata = ConfigUtils.getOptionBoolean(conf, ENABLE_SCHEMA_METADATA_KEY).getOrElse(false), useJdbcNative = ConfigUtils.getOptionBoolean(conf, USE_JDBC_NATIVE).getOrElse(false), - identifierQuotingPolicy = identifierQuotingPolicy + identifierQuotingPolicy = identifierQuotingPolicy, + sqlGeneratorClass = ConfigUtils.getOptionString(conf, SQL_GENERATOR_CLASS_KEY) ) } diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorDb2.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorDb2.scala index 66b36759d..0ce3beb30 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorDb2.scala +++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorDb2.scala @@ -16,6 +16,8 @@ package za.co.absa.pramen.core.sql +import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase} + import java.time.LocalDate import java.time.format.DateTimeFormatter diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorDenodo.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorDenodo.scala index 4ee429c6c..651845d28 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorDenodo.scala +++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorDenodo.scala @@ -16,6 +16,8 @@ package za.co.absa.pramen.core.sql +import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase} + import java.time.LocalDate import java.time.format.DateTimeFormatter diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorGeneric.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorGeneric.scala index 8eb7d0bcf..7f102bd42 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorGeneric.scala +++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorGeneric.scala @@ -16,6 +16,8 @@ package za.co.absa.pramen.core.sql +import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase} + import java.time.LocalDate import java.time.format.DateTimeFormatter diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorHive.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorHive.scala index 8a5ee54b4..d54d2ff6d 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorHive.scala +++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorHive.scala @@ -18,6 +18,7 @@ package za.co.absa.pramen.core.sql import org.apache.spark.sql.jdbc.JdbcDialects import org.slf4j.LoggerFactory +import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase} import za.co.absa.pramen.core.sql.impl.HiveDialect import java.time.LocalDate diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorHsqlDb.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorHsqlDb.scala index 8adea0c72..4da4f2fd6 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorHsqlDb.scala +++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorHsqlDb.scala @@ -16,6 +16,8 @@ package za.co.absa.pramen.core.sql +import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase} + import java.time.LocalDate import java.time.format.DateTimeFormatter diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGenerator.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorLoader.scala similarity index 59% rename from pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGenerator.scala rename to pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorLoader.scala index 8cd87622b..5b306a38b 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGenerator.scala +++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorLoader.scala @@ -16,50 +16,31 @@ package za.co.absa.pramen.core.sql -import com.typesafe.config.Config import org.slf4j.LoggerFactory +import za.co.absa.pramen.api.sql.{SqlConfig, SqlGenerator} -import java.sql.Connection -import java.time.LocalDate - -trait SqlGenerator { - def getDtable(sql: String): String - - def getCountQuery(tableName: String): String - - def getCountQuery(tableName: String, infoDateBegin: LocalDate, infoDateEnd: LocalDate): String - - def getDataQuery(tableName: String, columns: Seq[String], limit: Option[Int]): String - - def getDataQuery(tableName: String, infoDateBegin: LocalDate, infoDateEnd: LocalDate, columns: Seq[String], limit: Option[Int]): String - - def getWhere(dateBegin: LocalDate, dateEnd: LocalDate): String - - /** Returns the date literal for the dialect of the SQL. */ - def getDateLiteral(date: LocalDate): String - - /** - * Quotes an identifier, if needed according to the generator configuration. - */ - def escape(identifier: String): String +object SqlGeneratorLoader { + private val log = LoggerFactory.getLogger(this.getClass) /** - * Always quotes an identifier name with characters specific to SQL dialects. - * If the identifier is already wrapped, it won't be double wrapped. - * It supports partially quoted identifiers. E.g. '"my_catalog".my table' will be quoted as '"my_catalog"."my table"'. + * Loads an SQL generator, If SQL configuration contains a generator class name, it will be loaded. + * If not, the generator will be selected based on the driver name based on the internal mapping. + * @param driver The driver class. + * @param sqlConfig The SQL configuration. + * @return The SQL generator. */ - def quote(identifier: String): String - - def requiresConnection: Boolean = false - - def setConnection(connection: Connection): Unit = {} -} + def getSqlGenerator(driver: String, sqlConfig: SqlConfig): SqlGenerator = { + val sqlGenerator = sqlConfig.sqlGeneratorClass match { + case Some(clazz) => fromClass(clazz, sqlConfig) + case None => fromDriverName(driver, sqlConfig) + } -object SqlGenerator { - private val log = LoggerFactory.getLogger(this.getClass) + log.info(s"Using SQL generator: ${sqlGenerator.getClass.getCanonicalName}") + sqlGenerator + } - def fromDriverName(driver: String, sqlConfig: SqlConfig, extraConfig: Config): SqlGenerator = { - val sqlGenerator = driver match { + private def fromDriverName(driver: String, sqlConfig: SqlConfig): SqlGenerator = { + driver match { case "org.postgresql.Driver" => new SqlGeneratorPostgreSQL(sqlConfig) case "oracle.jdbc.OracleDriver" => new SqlGeneratorOracle(sqlConfig) case "net.sourceforge.jtds.jdbc.Driver" => new SqlGeneratorMicrosoft(sqlConfig) @@ -75,7 +56,12 @@ object SqlGenerator { log.warn(s"Unsupported JDBC driver: '$d'. Trying to use a generic SQL generator.") new SqlGeneratorGeneric(sqlConfig) } - log.info(s"Using SQL generator: ${sqlGenerator.getClass.getCanonicalName}") - sqlGenerator + } + + private def fromClass(clazz: String, sqlConfig: SqlConfig): SqlGenerator = { + Class.forName(clazz) + .getConstructor(classOf[SqlConfig]) + .newInstance(sqlConfig) + .asInstanceOf[SqlGenerator] } } diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorMicrosoft.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorMicrosoft.scala index db8011221..cb6bca98b 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorMicrosoft.scala +++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorMicrosoft.scala @@ -16,6 +16,8 @@ package za.co.absa.pramen.core.sql +import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase} + import java.time.LocalDate import java.time.format.DateTimeFormatter diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorOracle.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorOracle.scala index e78887a15..c2a7bdc44 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorOracle.scala +++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorOracle.scala @@ -16,6 +16,8 @@ package za.co.absa.pramen.core.sql +import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase} + import java.time.LocalDate import java.time.format.DateTimeFormatter diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorPostgreSQL.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorPostgreSQL.scala index 4f3dafc90..95901c05b 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorPostgreSQL.scala +++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorPostgreSQL.scala @@ -16,6 +16,8 @@ package za.co.absa.pramen.core.sql +import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase} + import java.time.LocalDate import java.time.format.DateTimeFormatter diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorSas.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorSas.scala index 0c757bee8..966040f37 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorSas.scala +++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorSas.scala @@ -18,6 +18,7 @@ package za.co.absa.pramen.core.sql import org.apache.spark.sql.jdbc.JdbcDialects import org.slf4j.LoggerFactory +import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase} import za.co.absa.pramen.core.sql.impl.SasDialect import java.sql.{Connection, ResultSet} diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/utils/CsvUtils.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/utils/CsvUtils.scala index 6763d7148..c16842c3c 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/utils/CsvUtils.scala +++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/utils/CsvUtils.scala @@ -37,7 +37,6 @@ object CsvUtils { obj.getClass.getDeclaredFields .filterNot(_.getName.contains('$')) .map(field => { - println(field.getName) field.setAccessible(true) field.get(obj).toString.replace(separator, ' ') }).mkString("", s"$separator", "\n") diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/DummySqlConfigFactory.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/DummySqlConfigFactory.scala index b0dfd4122..42d8bfdf0 100644 --- a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/DummySqlConfigFactory.scala +++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/DummySqlConfigFactory.scala @@ -16,17 +16,20 @@ package za.co.absa.pramen.core.mocks -import za.co.absa.pramen.core.reader.model.QuotingPolicy -import za.co.absa.pramen.core.sql.{SqlColumnType, SqlConfig} +import com.typesafe.config.ConfigFactory +import za.co.absa.pramen.api.sql.{QuotingPolicy, SqlColumnType, SqlConfig} object DummySqlConfigFactory { def getDummyConfig(infoDateColumn: String = "col", infoDateType: SqlColumnType = SqlColumnType.DATE, dateFormatApp: String = "yyyy-MM-dd", - identifierQuotingPolicy: QuotingPolicy = QuotingPolicy.Auto + identifierQuotingPolicy: QuotingPolicy = QuotingPolicy.Auto, + sqlGeneratorClass: Option[String] = None ): SqlConfig = SqlConfig( infoDateColumn = infoDateColumn, infoDateType = infoDateType, dateFormatApp = dateFormatApp, - identifierQuotingPolicy = identifierQuotingPolicy) + identifierQuotingPolicy = identifierQuotingPolicy, + sqlGeneratorClass = sqlGeneratorClass, + ConfigFactory.empty()) } diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/SqlGeneratorDummy.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/SqlGeneratorDummy.scala new file mode 100644 index 000000000..4a5f71991 --- /dev/null +++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/mocks/SqlGeneratorDummy.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.pramen.core.mocks + +import za.co.absa.pramen.api.sql.{SqlConfig, SqlGenerator} + +import java.time.LocalDate + +class SqlGeneratorDummy(sqlConfig: SqlConfig) extends SqlGenerator { + def getSqlConfig: SqlConfig = sqlConfig + + override def getDtable(sql: String): String = null + + override def getCountQuery(tableName: String): String = null + + override def getCountQuery(tableName: String, infoDateBegin: LocalDate, infoDateEnd: LocalDate): String = null + + override def getDataQuery(tableName: String, columns: Seq[String], limit: Option[Int]): String = null + + override def getDataQuery(tableName: String, infoDateBegin: LocalDate, infoDateEnd: LocalDate, columns: Seq[String], limit: Option[Int]): String = null + + override def getWhere(dateBegin: LocalDate, dateEnd: LocalDate): String = null + + override def getDateLiteral(date: LocalDate): String = null + + override def escape(identifier: String): String = null + + override def quote(identifier: String): String = null +} diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/QuotingPolicySuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/QuotingPolicySuite.scala deleted file mode 100644 index 1601d3dc6..000000000 --- a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/QuotingPolicySuite.scala +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2022 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.pramen.core.tests.reader - -import org.scalatest.wordspec.AnyWordSpec -import za.co.absa.pramen.core.reader.model._ - -class QuotingPolicySuite extends AnyWordSpec { - "QuotingPolicy.fromString" should { - "return Never for 'never'" in { - assert(QuotingPolicy.fromString("never") == QuotingPolicy.Never) - assert(QuotingPolicy.fromString("never").name == "never") - } - - "return Always for 'always'" in { - assert(QuotingPolicy.fromString("always") == QuotingPolicy.Always) - assert(QuotingPolicy.fromString("always").name == "always") - } - - "return Auto for 'auto'" in { - assert(QuotingPolicy.fromString("auto") == QuotingPolicy.Auto) - assert(QuotingPolicy.fromString("auto").name == "auto") - } - - "throw an exception for an unknown quoting policy" in { - assertThrows[IllegalArgumentException] { - QuotingPolicy.fromString("unknown") - } - } - } -} diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/TableReaderJdbcSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/TableReaderJdbcSuite.scala index 76f20e2c5..f4a82d48c 100644 --- a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/TableReaderJdbcSuite.scala +++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/reader/TableReaderJdbcSuite.scala @@ -21,9 +21,11 @@ import org.mockito.Mockito.{mock, when => whenMock} import org.scalatest.BeforeAndAfterAll import org.scalatest.wordspec.AnyWordSpec import za.co.absa.pramen.api.Query +import za.co.absa.pramen.api.sql.QuotingPolicy import za.co.absa.pramen.core.base.SparkTestBase import za.co.absa.pramen.core.fixtures.RelationalDbFixture -import za.co.absa.pramen.core.reader.model.{QuotingPolicy, TableReaderJdbcConfig} +import za.co.absa.pramen.core.mocks.SqlGeneratorDummy +import za.co.absa.pramen.core.reader.model.TableReaderJdbcConfig import za.co.absa.pramen.core.reader.{JdbcUrlSelector, TableReaderJdbc} import za.co.absa.pramen.core.samples.RdbExampleTable import za.co.absa.pramen.core.sql.SqlGeneratorHsqlDb @@ -75,6 +77,7 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark | information.date.column = "INFO_DATE" | information.date.type = "date" | information.date.app.format = "YYYY-MM-dd" + | sql.generator.class = "za.co.absa.pramen.core.mocks.SqlGeneratorDummy" |} |reader_minimal { | jdbc { @@ -103,6 +106,8 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark assert(jdbc.infoDateColumn == "INFO_DATE") assert(jdbc.infoDateType == "number") assert(jdbc.infoDateFormat == "yyyy-MM-DD") + assert(jdbc.identifierQuotingPolicy == QuotingPolicy.Never) + assert(jdbc.sqlGeneratorClass.isEmpty) assert(!jdbc.hasInfoDate) assert(!jdbc.saveTimestampsAsDates) } @@ -120,6 +125,8 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark assert(jdbc.infoDateColumn == "INFO_DATE") assert(jdbc.infoDateType == "date") assert(jdbc.infoDateFormat == "YYYY-MM-dd") + assert(jdbc.identifierQuotingPolicy == QuotingPolicy.Auto) + assert(jdbc.sqlGeneratorClass.contains("za.co.absa.pramen.core.mocks.SqlGeneratorDummy")) assert(jdbc.hasInfoDate) assert(!jdbc.saveTimestampsAsDates) } @@ -139,14 +146,22 @@ class TableReaderJdbcSuite extends AnyWordSpec with BeforeAndAfterAll with Spark assert(jdbc.infoDateFormat == "yyyy-MM-dd") assert(jdbc.hasInfoDate) assert(!jdbc.saveTimestampsAsDates) + assert(jdbc.identifierQuotingPolicy == QuotingPolicy.Auto) + assert(jdbc.sqlGeneratorClass.isEmpty) } - "ensure sql query generator is properly selected" in { + "ensure sql query generator is properly selected 1" in { val reader = TableReaderJdbc(conf.getConfig("reader"), "reader") assert(reader.sqlGen.isInstanceOf[SqlGeneratorHsqlDb]) } + "ensure sql query generator is properly selected 2" in { + val reader = TableReaderJdbc(conf.getConfig("reader_legacy"), "reader_legacy") + + assert(reader.sqlGen.isInstanceOf[SqlGeneratorDummy]) + } + "ensure jdbc config properties are passed correctly" in { val testConfig = conf .withValue("reader.save.timestamps.as.dates", ConfigValueFactory.fromAnyRef(true)) diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlColumnTypeSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlColumnTypeSuite.scala index a0572ee66..c689fa684 100644 --- a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlColumnTypeSuite.scala +++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlColumnTypeSuite.scala @@ -19,7 +19,7 @@ package za.co.absa.pramen.core.tests.sql import org.scalatest.wordspec.AnyWordSpec class SqlColumnTypeSuite extends AnyWordSpec { - import za.co.absa.pramen.core.sql.SqlColumnType._ + import za.co.absa.pramen.api.sql.SqlColumnType._ "fromString()" should { "return corresponding type" in { diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorLoaderSuite.scala similarity index 90% rename from pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorSuite.scala rename to pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorLoaderSuite.scala index 34d09c5d0..f20712d7b 100644 --- a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorSuite.scala +++ b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorLoaderSuite.scala @@ -16,19 +16,18 @@ package za.co.absa.pramen.core.tests.sql -import com.typesafe.config.ConfigFactory import org.scalatest.wordspec.AnyWordSpec +import za.co.absa.pramen.api.sql.{QuotingPolicy, SqlColumnType, SqlGeneratorBase} import za.co.absa.pramen.core.fixtures.RelationalDbFixture -import za.co.absa.pramen.core.mocks.DummySqlConfigFactory -import za.co.absa.pramen.core.reader.model.QuotingPolicy +import za.co.absa.pramen.core.mocks.{DummySqlConfigFactory, SqlGeneratorDummy} import za.co.absa.pramen.core.samples.RdbExampleTable import za.co.absa.pramen.core.sql._ import java.time.LocalDate -class SqlGeneratorSuite extends AnyWordSpec with RelationalDbFixture { +class SqlGeneratorLoaderSuite extends AnyWordSpec with RelationalDbFixture { - import za.co.absa.pramen.core.sql.SqlGenerator._ + import za.co.absa.pramen.core.sql.SqlGeneratorLoader._ private val sqlConfigDate = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATE, infoDateColumn = "D") private val sqlConfigEscape = DummySqlConfigFactory.getDummyConfig(infoDateColumn = "Info date", identifierQuotingPolicy = QuotingPolicy.Always) @@ -40,8 +39,6 @@ class SqlGeneratorSuite extends AnyWordSpec with RelationalDbFixture { private val date1 = LocalDate.of(2020, 8, 17) private val date2 = LocalDate.of(2020, 8, 30) - private val config = ConfigFactory.empty() - override protected def beforeAll(): Unit = { super.beforeAll() RdbExampleTable.Company.initTable(getConnection) @@ -52,27 +49,35 @@ class SqlGeneratorSuite extends AnyWordSpec with RelationalDbFixture { super.afterAll() } - "fromDriverName" should { + "loadSqlGenerator" should { "return an Oracle SQL generator" in { - assert(fromDriverName("oracle.jdbc.OracleDriver", sqlConfigDate, config).isInstanceOf[SqlGeneratorOracle]) + assert(getSqlGenerator("oracle.jdbc.OracleDriver", sqlConfigDate).isInstanceOf[SqlGeneratorOracle]) } "return a Microsoft SQL generator" in { - assert(fromDriverName("net.sourceforge.jtds.jdbc.Driver", sqlConfigDate, config).isInstanceOf[SqlGeneratorMicrosoft]) - assert(fromDriverName("com.microsoft.sqlserver.jdbc.SQLServerDriver", sqlConfigDate, config).isInstanceOf[SqlGeneratorMicrosoft]) + assert(getSqlGenerator("net.sourceforge.jtds.jdbc.Driver", sqlConfigDate).isInstanceOf[SqlGeneratorMicrosoft]) + assert(getSqlGenerator("com.microsoft.sqlserver.jdbc.SQLServerDriver", sqlConfigDate).isInstanceOf[SqlGeneratorMicrosoft]) } "return a generic SQL generator" in { - assert(fromDriverName("unknown", sqlConfigDate, config).isInstanceOf[SqlGeneratorGeneric]) + assert(getSqlGenerator("unknown", sqlConfigDate).isInstanceOf[SqlGeneratorGeneric]) + } + + "return from a class" in { + val sqlConfig = sqlConfigDate.copy(sqlGeneratorClass = Some("za.co.absa.pramen.core.mocks.SqlGeneratorDummy")) + val generator = getSqlGenerator("unknown", sqlConfig) + + assert(generator.isInstanceOf[SqlGeneratorDummy]) + assert(generator.asInstanceOf[SqlGeneratorDummy].getSqlConfig == sqlConfig) } } "Oracle SQL generator" should { - val gen = fromDriverName("oracle.jdbc.OracleDriver", sqlConfigDate, config) - val genStr = fromDriverName("oracle.jdbc.OracleDriver", sqlConfigString, config) - val genNum = fromDriverName("oracle.jdbc.OracleDriver", sqlConfigNumber, config) - val genDateTime = fromDriverName("oracle.jdbc.OracleDriver", sqlConfigDateTime, config) - val genEscaped = fromDriverName("oracle.jdbc.OracleDriver", sqlConfigEscape, config) + val gen = getSqlGenerator("oracle.jdbc.OracleDriver", sqlConfigDate) + val genStr = getSqlGenerator("oracle.jdbc.OracleDriver", sqlConfigString) + val genNum = getSqlGenerator("oracle.jdbc.OracleDriver", sqlConfigNumber) + val genDateTime = getSqlGenerator("oracle.jdbc.OracleDriver", sqlConfigDateTime) + val genEscaped = getSqlGenerator("oracle.jdbc.OracleDriver", sqlConfigEscape) "generate count queries without date ranges" in { assert(gen.getCountQuery("A") == "SELECT COUNT(*) FROM A") @@ -177,13 +182,13 @@ class SqlGeneratorSuite extends AnyWordSpec with RelationalDbFixture { } "Microsoft SQL generator" should { - val genDate = fromDriverName("net.sourceforge.jtds.jdbc.Driver", sqlConfigDate, config) - val genDateTime = fromDriverName("net.sourceforge.jtds.jdbc.Driver", sqlConfigDateTime, config) - val genStr = fromDriverName("net.sourceforge.jtds.jdbc.Driver", sqlConfigString, config) - val genStr2 = fromDriverName("net.sourceforge.jtds.jdbc.Driver", DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.STRING, dateFormatApp = "yyyyMMdd", infoDateColumn = "D"), config) - val genNum = fromDriverName("net.sourceforge.jtds.jdbc.Driver", sqlConfigNumber, config) - val genEscaped = fromDriverName("net.sourceforge.jtds.jdbc.Driver", sqlConfigEscape, config) - val genEscaped2 = fromDriverName("net.sourceforge.jtds.jdbc.Driver", DummySqlConfigFactory.getDummyConfig(infoDateColumn = "[Info date]", identifierQuotingPolicy = QuotingPolicy.Auto), config) + val genDate = getSqlGenerator("net.sourceforge.jtds.jdbc.Driver", sqlConfigDate) + val genDateTime = getSqlGenerator("net.sourceforge.jtds.jdbc.Driver", sqlConfigDateTime) + val genStr = getSqlGenerator("net.sourceforge.jtds.jdbc.Driver", sqlConfigString) + val genStr2 = getSqlGenerator("net.sourceforge.jtds.jdbc.Driver", DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.STRING, dateFormatApp = "yyyyMMdd", infoDateColumn = "D")) + val genNum = getSqlGenerator("net.sourceforge.jtds.jdbc.Driver", sqlConfigNumber) + val genEscaped = getSqlGenerator("net.sourceforge.jtds.jdbc.Driver", sqlConfigEscape) + val genEscaped2 = getSqlGenerator("net.sourceforge.jtds.jdbc.Driver", DummySqlConfigFactory.getDummyConfig(infoDateColumn = "[Info date]", identifierQuotingPolicy = QuotingPolicy.Auto)) "generate count queries without date ranges" in { assert(genDate.getCountQuery("A") == "SELECT COUNT(*) AS CNT FROM A WITH (NOLOCK)") @@ -368,12 +373,12 @@ class SqlGeneratorSuite extends AnyWordSpec with RelationalDbFixture { } "Denodo SQL generator" should { - val gen = fromDriverName("com.denodo.vdp.jdbc.Driver", sqlConfigDate, config) - val genStr = fromDriverName("com.denodo.vdp.jdbc.Driver", sqlConfigString, config) - val genNum = fromDriverName("com.denodo.vdp.jdbc.Driver", sqlConfigNumber, config) - val genDateTime = fromDriverName("com.denodo.vdp.jdbc.Driver", sqlConfigDateTime, config) - val genEscaped = fromDriverName("com.denodo.vdp.jdbc.Driver", sqlConfigEscape, config) - val genEscapedAuto = fromDriverName("com.denodo.vdp.jdbc.Driver", sqlConfigEscape.copy(identifierQuotingPolicy = QuotingPolicy.Auto), config) + val gen = getSqlGenerator("com.denodo.vdp.jdbc.Driver", sqlConfigDate) + val genStr = getSqlGenerator("com.denodo.vdp.jdbc.Driver", sqlConfigString) + val genNum = getSqlGenerator("com.denodo.vdp.jdbc.Driver", sqlConfigNumber) + val genDateTime = getSqlGenerator("com.denodo.vdp.jdbc.Driver", sqlConfigDateTime) + val genEscaped = getSqlGenerator("com.denodo.vdp.jdbc.Driver", sqlConfigEscape) + val genEscapedAuto = getSqlGenerator("com.denodo.vdp.jdbc.Driver", sqlConfigEscape.copy(identifierQuotingPolicy = QuotingPolicy.Auto)) "generate count queries without date ranges" in { assert(gen.getCountQuery("A") == "SELECT COUNT(*) FROM A") @@ -577,11 +582,11 @@ class SqlGeneratorSuite extends AnyWordSpec with RelationalDbFixture { "SAS SQL generator" should { val connection = getConnection - val gen = fromDriverName("com.sas.rio.MVADriver", sqlConfigDate, config) - val genStr = fromDriverName("com.sas.rio.MVADriver", sqlConfigString, config) - val genNum = fromDriverName("com.sas.rio.MVADriver", sqlConfigNumber, config) - val genDateTime = fromDriverName("com.sas.rio.MVADriver", sqlConfigDateTime, config) - val genEscaped = fromDriverName("com.sas.rio.MVADriver", sqlConfigEscape, config) + val gen = getSqlGenerator("com.sas.rio.MVADriver", sqlConfigDate) + val genStr = getSqlGenerator("com.sas.rio.MVADriver", sqlConfigString) + val genNum = getSqlGenerator("com.sas.rio.MVADriver", sqlConfigNumber) + val genDateTime = getSqlGenerator("com.sas.rio.MVADriver", sqlConfigDateTime) + val genEscaped = getSqlGenerator("com.sas.rio.MVADriver", sqlConfigEscape) gen.setConnection(connection) genStr.setConnection(connection) @@ -681,12 +686,12 @@ class SqlGeneratorSuite extends AnyWordSpec with RelationalDbFixture { } "Hive SQL generator" should { - val gen = fromDriverName("com.cloudera.hive.jdbc41.HS2Driver", sqlConfigDate, config) - val genStr = fromDriverName("com.cloudera.hive.jdbc41.HS2Driver", sqlConfigString, config) - val genNum = fromDriverName("com.cloudera.hive.jdbc41.HS2Driver", sqlConfigNumber, config) - val genDateTime = fromDriverName("com.cloudera.hive.jdbc41.HS2Driver", sqlConfigDateTime, config) - val genEscaped = fromDriverName("com.cloudera.hive.jdbc41.HS2Driver", sqlConfigEscape, config) - val genEscaped2 = fromDriverName("com.cloudera.hive.jdbc41.HS2Driver", DummySqlConfigFactory.getDummyConfig(infoDateColumn = "`Info date`", identifierQuotingPolicy = QuotingPolicy.Auto), config) + val gen = getSqlGenerator("com.cloudera.hive.jdbc41.HS2Driver", sqlConfigDate) + val genStr = getSqlGenerator("com.cloudera.hive.jdbc41.HS2Driver", sqlConfigString) + val genNum = getSqlGenerator("com.cloudera.hive.jdbc41.HS2Driver", sqlConfigNumber) + val genDateTime = getSqlGenerator("com.cloudera.hive.jdbc41.HS2Driver", sqlConfigDateTime) + val genEscaped = getSqlGenerator("com.cloudera.hive.jdbc41.HS2Driver", sqlConfigEscape) + val genEscaped2 = getSqlGenerator("com.cloudera.hive.jdbc41.HS2Driver", DummySqlConfigFactory.getDummyConfig(infoDateColumn = "`Info date`", identifierQuotingPolicy = QuotingPolicy.Auto)) "generate count queries without date ranges" in { assert(gen.getCountQuery("A") == "SELECT COUNT(*) FROM A") @@ -805,11 +810,11 @@ class SqlGeneratorSuite extends AnyWordSpec with RelationalDbFixture { } "PostgreSQL SQL generator" should { - val genDate = fromDriverName("org.postgresql.Driver", sqlConfigDate, config) - val genDateTime = fromDriverName("org.postgresql.Driver", sqlConfigDateTime, config) - val genStr = fromDriverName("org.postgresql.Driver", sqlConfigString, config) - val genNum = fromDriverName("org.postgresql.Driver", sqlConfigNumber, config) - val genEscaped = fromDriverName("org.postgresql.Driver", sqlConfigEscape, config) + val genDate = getSqlGenerator("org.postgresql.Driver", sqlConfigDate) + val genDateTime = getSqlGenerator("org.postgresql.Driver", sqlConfigDateTime) + val genStr = getSqlGenerator("org.postgresql.Driver", sqlConfigString) + val genNum = getSqlGenerator("org.postgresql.Driver", sqlConfigNumber) + val genEscaped = getSqlGenerator("org.postgresql.Driver", sqlConfigEscape) "generate count queries without date ranges" in { assert(genDate.getCountQuery("A") == "SELECT COUNT(*) FROM A") @@ -921,11 +926,11 @@ class SqlGeneratorSuite extends AnyWordSpec with RelationalDbFixture { } "DB2 SQL generator" should { - val genDate = fromDriverName("com.ibm.db2.jcc.DB2Driver", sqlConfigDate, config) - val genDateTime = fromDriverName("com.ibm.db2.jcc.DB2Driver", sqlConfigDateTime, config) - val genStr = fromDriverName("com.ibm.db2.jcc.DB2Driver", sqlConfigString, config) - val genNum = fromDriverName("com.ibm.db2.jcc.DB2Driver", sqlConfigNumber, config) - val genEscaped = fromDriverName("com.ibm.db2.jcc.DB2Driver", sqlConfigEscape, config) + val genDate = getSqlGenerator("com.ibm.db2.jcc.DB2Driver", sqlConfigDate) + val genDateTime = getSqlGenerator("com.ibm.db2.jcc.DB2Driver", sqlConfigDateTime) + val genStr = getSqlGenerator("com.ibm.db2.jcc.DB2Driver", sqlConfigString) + val genNum = getSqlGenerator("com.ibm.db2.jcc.DB2Driver", sqlConfigNumber) + val genEscaped = getSqlGenerator("com.ibm.db2.jcc.DB2Driver", sqlConfigEscape) "generate count queries without date ranges" in { assert(genDate.getCountQuery("A") == "SELECT COUNT(*) AS CNT FROM A") @@ -1037,11 +1042,11 @@ class SqlGeneratorSuite extends AnyWordSpec with RelationalDbFixture { } "HSQL generator" should { - val genDate = fromDriverName("org.hsqldb.jdbc.JDBCDriver", sqlConfigDate, config) - val genDateTime = fromDriverName("org.hsqldb.jdbc.JDBCDriver", sqlConfigDateTime, config) - val genStr = fromDriverName("org.hsqldb.jdbc.JDBCDriver", sqlConfigString, config) - val genNum = fromDriverName("org.hsqldb.jdbc.JDBCDriver", sqlConfigNumber, config) - val genEscaped = fromDriverName("org.hsqldb.jdbc.JDBCDriver", sqlConfigEscape, config) + val genDate = getSqlGenerator("org.hsqldb.jdbc.JDBCDriver", sqlConfigDate) + val genDateTime = getSqlGenerator("org.hsqldb.jdbc.JDBCDriver", sqlConfigDateTime) + val genStr = getSqlGenerator("org.hsqldb.jdbc.JDBCDriver", sqlConfigString) + val genNum = getSqlGenerator("org.hsqldb.jdbc.JDBCDriver", sqlConfigNumber) + val genEscaped = getSqlGenerator("org.hsqldb.jdbc.JDBCDriver", sqlConfigEscape) "generate count queries without date ranges" in { assert(genDate.getCountQuery("A") == "SELECT COUNT(*) AS CNT FROM A") @@ -1165,11 +1170,11 @@ class SqlGeneratorSuite extends AnyWordSpec with RelationalDbFixture { } "Generic SQL generator" should { - val genDate = fromDriverName("generic", sqlConfigDate, config) - val genDateTime = fromDriverName("generic", sqlConfigDateTime, config) - val genStr = fromDriverName("generic", sqlConfigString, config) - val genNum = fromDriverName("generic", sqlConfigNumber, config) - val genEscaped = fromDriverName("generic", sqlConfigEscape, config) + val genDate = getSqlGenerator("generic", sqlConfigDate) + val genDateTime = getSqlGenerator("generic", sqlConfigDateTime) + val genStr = getSqlGenerator("generic", sqlConfigString) + val genNum = getSqlGenerator("generic", sqlConfigNumber) + val genEscaped = getSqlGenerator("generic", sqlConfigEscape) "generate count queries without date ranges" in { assert(genDate.getCountQuery("A") == "SELECT COUNT(*) AS CNT FROM A") @@ -1280,7 +1285,7 @@ class SqlGeneratorSuite extends AnyWordSpec with RelationalDbFixture { "escape should not escape if turned off by config" in { val sql = DummySqlConfigFactory.getDummyConfig(infoDateType = SqlColumnType.DATE, infoDateColumn = "D", identifierQuotingPolicy = QuotingPolicy.Never) - val genDate = fromDriverName("generic", sql, config) + val genDate = getSqlGenerator("generic", sql) val actual = genDate.asInstanceOf[SqlGeneratorBase].escape("System User.\"Table Name\"")