Skip to content

Commit

Permalink
Update spark-defaults.conf
Browse files Browse the repository at this point in the history
  • Loading branch information
ilias1111 committed Nov 11, 2024
1 parent 64424ee commit dc5c492
Showing 1 changed file with 18 additions and 0 deletions.
18 changes: 18 additions & 0 deletions .github/workflows/spark_deployment/spark-defaults.conf
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,24 @@ spark.sql.catalog.glue.io-impl org.apache.iceberg.aws.s3.S3FileI
spark.sql.defaultCatalog glue
spark.sql.catalog.glue.database dbt-spark-iceberg

# Table capabilities and operation settings
spark.sql.catalog.glue.table-default.format-version 2
spark.sql.catalog.glue.table-default.write.update.mode merge-on-read
spark.sql.catalog.glue.table-default.write.delete.mode merge-on-read
spark.sql.catalog.glue.table-default.write.operation.mode batch
spark.sql.catalog.glue.table-default.write.merge.mode merge-on-read
spark.sql.catalog.glue.table-default.write.distribution-mode none
spark.sql.catalog.glue.table-default.write.data.path s3a://dbt-spark-iceberg/github-integration-testing
spark.sql.catalog.glue.table-default.write.metadata.path s3a://dbt-spark-iceberg/github-integration-testing/metadata
spark.sql.catalog.glue.table-default.write.metadata.previous-versions-max 10
spark.sql.catalog.glue.table-default.write.format.default iceberg
spark.sql.catalog.glue.table-default.engine.hive.enabled true
spark.sql.table.is.transactional true
spark.sql.iceberg.check-nullability false
spark.sql.iceberg.vectorization.enabled true
spark.sql.iceberg.allow-append-in-batch-mode true
spark.sql.iceberg.handle-timestamp-without-timezone true

# Performance Tuning
spark.master local[3]
spark.driver.memory 10g
Expand Down

0 comments on commit dc5c492

Please sign in to comment.