Updated the circle ci to use spark3

dbt-labs · Oct 25, 2021 · 35f5fc7 · 35f5fc7
1 parent 84d8547
commit 35f5fc7
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 22 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -11,11 +11,14 @@ jobs:
       - run: tox -e flake8,unit
 
   integration-spark-thrift:
+    parameters:
+      spark-version:
+        type: string
     environment:
       DBT_INVOCATION_ENV: circle
     docker:
       - image: fishtownanalytics/test-container:10
-      - image: godatadriven/spark:2
+      - image: godatadriven/spark:<<parameters.spark-version>>
         environment:
           WAIT_FOR: localhost:5432
         command: >
@@ -26,7 +29,7 @@ jobs:
           --conf spark.hadoop.javax.jdo.option.ConnectionPassword=dbt
           --conf spark.hadoop.javax.jdo.option.ConnectionDriverName=org.postgresql.Driver
           --conf spark.serializer=org.apache.spark.serializer.KryoSerializer
-          --conf spark.jars.packages=org.apache.hudi:hudi-spark-bundle_2.11:0.9.0,org.apache.spark:spark-avro_2.11:2.4.4
+          --conf spark.jars.packages=org.apache.hudi:hudi-spark3-bundle_2.12:0.9.0
           --conf spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension
           --conf spark.driver.userClassPathFirst=true
           --conf spark.hadoop.datanucleus.autoCreateTables=true
@@ -70,7 +73,7 @@ jobs:
       - store_artifacts:
           path: ./logs
 
-  integration-spark-databricks-odbc-cluster: &databricks-odbc
+  integration-spark-databricks-odbc:
     environment:
       DBT_INVOCATION_ENV: circle
       ODBC_DRIVER: Simba # TODO: move env var to Docker image
@@ -84,18 +87,7 @@ jobs:
       - checkout
       - run:
           name: Run integration tests
-          command: tox -e integration-spark-databricks-odbc-cluster
-          no_output_timeout: 1h
-      - store_artifacts:
-          path: ./logs
-
-  integration-spark-databricks-odbc-endpoint:
-    <<: *databricks-odbc
-    steps:
-      - checkout
-      - run:
-          name: Run integration tests
-          command: tox -e integration-spark-databricks-odbc-sql-endpoint
+          command: tox -e integration-spark-databricks-odbc-cluster,integration-spark-databricks-odbc-sql-endpoint
           no_output_timeout: 1h
       - store_artifacts:
           path: ./logs
@@ -106,14 +98,14 @@ workflows:
     jobs:
       - unit
       - integration-spark-thrift:
+          matrix:
+            parameters:
+              spark-version: ["3.0.0"]
           requires:
             - unit
       - integration-spark-databricks-http:
           requires:
-            - integration-spark-thrift
-      - integration-spark-databricks-odbc-cluster:
-          requires:
-            - integration-spark-thrift
-      - integration-spark-databricks-odbc-endpoint:
+            - unit
+      - integration-spark-databricks-odbc:
           requires:
-            - integration-spark-thrift
+            - unit
diff --git a/docker/hive-site.xml b/docker/hive-site.xml
@@ -43,4 +43,9 @@
         <name>hive.metastore.schema.verification</name>
         <value>false</value>
     </property>
+
+    <property>
+        <name>datanucleus.schema.autoCreateAll</name>
+        <value>true</value>
+    </property>
 </configuration>
diff --git a/test/integration/spark-thrift-hudi.dbtspec b/test/integration/spark-thrift-hudi.dbtspec
@@ -18,7 +18,7 @@ projects:
         dbt_test_project:
           file_format: hudi
   - overrides: snapshot_strategy_timestamp
-    dbt_project_yml: *file_format_delta
+    dbt_project_yml: *file_format_hudi
 sequences:
   test_dbt_empty: empty
   test_dbt_base: base