diff --git a/README.md b/README.md index 8a7cd70e..ecd74bc8 100644 --- a/README.md +++ b/README.md @@ -299,6 +299,19 @@ It may be useful to have some GRANT commands or similar run here when l table, the changes will be reverted and the backup table restored if post actions fail.

+ + extracopyoptions + No + No default + +

A list extra options to append to the Redshift COPY command when loading data, e.g. TRUNCATECOLUMNS +or MAXERROR n (see the Redshift docs +for other options).

+ +

Note that since these options are appended to the end of the COPY command, only options that make sense +at the end of the command can be used, but that should cover most possible use cases.

+ + ## Additional configuration options diff --git a/src/main/scala/com/databricks/spark/redshift/Parameters.scala b/src/main/scala/com/databricks/spark/redshift/Parameters.scala index 78be9840..e7e92c70 100644 --- a/src/main/scala/com/databricks/spark/redshift/Parameters.scala +++ b/src/main/scala/com/databricks/spark/redshift/Parameters.scala @@ -178,6 +178,12 @@ private[redshift] object Parameters { */ def useStagingTable: Boolean = parameters("usestagingtable").toBoolean + /** + * Extra options to pass to the Redshift COPY command (e.g. "MAXERROR 100"). + * + */ + def extraCopyOptions: String = parameters.get("extracopyoptions").getOrElse("") + /** * List of semi-colon separated SQL statements to run after successful write operations. * This can be useful for running GRANT operations to make your new tables readable to other diff --git a/src/main/scala/com/databricks/spark/redshift/RedshiftWriter.scala b/src/main/scala/com/databricks/spark/redshift/RedshiftWriter.scala index 00c68e87..1d86afab 100644 --- a/src/main/scala/com/databricks/spark/redshift/RedshiftWriter.scala +++ b/src/main/scala/com/databricks/spark/redshift/RedshiftWriter.scala @@ -98,7 +98,7 @@ private[redshift] class RedshiftWriter( val credsString: String = AWSCredentialsUtils.getRedshiftCredentialsString(creds) val fixedUrl = Utils.fixS3Url(manifestUrl) s"COPY ${params.table.get} FROM '$fixedUrl' CREDENTIALS '$credsString' FORMAT AS " + - "AVRO 'auto' DATEFORMAT 'YYYY-MM-DD HH:MI:SS' manifest" + "AVRO 'auto' DATEFORMAT 'YYYY-MM-DD HH:MI:SS' manifest ${params.extraCopyOptions}" } /**