Enable Iceberg in Spark CI (#174)

Closes #42.
microsoft · Nov 6, 2023 · 9395582 · 9395582
1 parent 3d68aee
commit 9395582
Show file tree

Hide file tree

Showing 7 changed files with 17 additions and 2 deletions.
diff --git a/.github/scripts/spark/start_thrift_server.sh b/.github/scripts/spark/start_thrift_server.sh
@@ -11,7 +11,7 @@ if [[ $1 == "delta" ]]; then
 elif [[ $1 == "hudi" ]]; then
   PARAMS="--conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.hudi.catalog.HoodieCatalog --conf spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension"
 elif [[ $1 == "iceberg" ]]; then
-  PARAMS="--conf spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkSessionCatalog --conf spark.sql.catalog.spark_catalog.type=hive --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions"
+  PARAMS="--conf spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkSessionCatalog --conf spark.sql.catalog.spark_catalog.type=hadoop --conf spark.sql.catalog.spark_catalog.warehouse=/warehouse --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions"
 else
   echo "Unknown LST: $1"
   exit 1

diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven.yaml
@@ -72,7 +72,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        lst: [ delta, hudi ]
+        lst: [ delta, hudi, iceberg ]
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4

diff --git a/src/test/resources/config/spark/w_all_tpcds-iceberg.yaml b/src/test/resources/config/spark/w_all_tpcds-iceberg.yaml
@@ -19,6 +19,9 @@ phases:
   sessions:
   - tasks:
     - template_id: build
+      replace_regex:
+      - pattern: '(?i)options\((.|\n)*?\)'
+        replacement: ''
 - id: single_user
   sessions:
   - tasks:

diff --git a/src/test/resources/config/spark/w_all_tpcds_single_session-iceberg.yaml b/src/test/resources/config/spark/w_all_tpcds_single_session-iceberg.yaml
@@ -10,6 +10,9 @@ phases:
     - template_id: setup_data_maintenance
     - template_id: init
     - template_id: build
+      replace_regex:
+        - pattern: '(?i)options\((.|\n)*?\)'
+          replacement: ''
     - template_id: single_user
     - template_id: data_maintenance_iceberg
     - template_id: optimize_iceberg
diff --git a/src/test/resources/config/spark/w_all_tpcds_single_session_jdbc-iceberg.yaml b/src/test/resources/config/spark/w_all_tpcds_single_session_jdbc-iceberg.yaml
@@ -10,6 +10,9 @@ phases:
     - template_id: setup_data_maintenance
     - template_id: init
     - template_id: build
+      replace_regex:
+        - pattern: '(?i)options\((.|\n)*?\)'
+          replacement: ''
     - template_id: single_user
     - template_id: data_maintenance_iceberg
     - template_id: data_maintenance_dependent

diff --git a/src/test/resources/config/spark/w_all_tpch-iceberg.yaml b/src/test/resources/config/spark/w_all_tpch-iceberg.yaml
@@ -19,6 +19,9 @@ phases:
   sessions:
   - tasks:
     - template_id: build
+      replace_regex:
+        - pattern: '(?i)options\((.|\n)*?\)'
+          replacement: ''
 - id: single_user
   sessions:
   - tasks:

diff --git a/src/test/resources/config/spark/w_multi_connection-iceberg.yaml b/src/test/resources/config/spark/w_multi_connection-iceberg.yaml
@@ -19,6 +19,9 @@ phases:
   sessions:
   - tasks:
     - template_id: build
+      replace_regex:
+        - pattern: '(?i)options\((.|\n)*?\)'
+          replacement: ''
 - id: multi_single_user
   sessions:
   - tasks: