Update the build and test infra for connectors

## Description
- Updated build.sbt to add new subprojects for connectors.
  - Updated the unidoc settings to make sure it ignores all the new kernel and connectors code.
  - Moved unidoc generation testing into an explicit separate step (explicitly call `sbt test unidoc` from run-tests.py) instead of implicitly via sbt task dependencies.
   - Disabled code coverage because it was having problems with scala-2.13 builds and we were not really using it anyways.
- Added a new Github action for running all connector tests by copying the existing action from the connectors dir.
- Renamed Spark test Github action file from tests.yaml to spark_tests.yaml
- Increased SBT's Java heap size with .sbtoptions

New and existing Github actions

Closes delta-io#1845

GitOrigin-RevId: d195e7123e37aae67d117d434139f2f28e3c6f2b

.github/workflows/connectors_test.yaml

@@ -0,0 +1,45 @@
+name: "Delta Connectors Tests"
+on: [push, pull_request]
+jobs:
+  build:
+    name: "Run tests"
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        # These Scala versions must match those in the build.sbt
+        scala: [2.13.5, 2.12.15]
+    steps:
+      - uses: actions/checkout@v2
+      - name: install java
+        uses: actions/setup-java@v2
+        with:
+          distribution: 'zulu'
+          java-version: '8'
+      - name: Cache Scala, SBT
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.sbt
+            ~/.ivy2
+            ~/.cache/coursier
+            ~/.m2
+          key: build-cache-3-with-scala_${{ matrix.scala }}
+      - name: Run Scala Style tests on test sources (Scala 2.12 only)
+        run: build/sbt "++ ${{ matrix.scala }}" testScalastyle
+        if: startsWith(matrix.scala, '2.12.')
+      - name: Run sqlDeltaImport tests (Scala 2.12 and 2.13 only)
+        run: build/sbt "++ ${{ matrix.scala }}" sqlDeltaImport/test
+        if: ${{ !startsWith(matrix.scala, '2.11.') }}
+        # These tests are not working yet
+        # - name: Run Delta Standalone Compatibility tests (Scala 2.12 only)
+        #  run: build/sbt "++ ${{ matrix.scala }}" compatibility/test
+        #  if: startsWith(matrix.scala, '2.12.')
+      - name: Run Delta Standalone tests
+        run: build/sbt "++ ${{ matrix.scala }}" standalone/test testStandaloneCosmetic/test standaloneParquet/test testParquetUtilsWithStandaloneCosmetic/test
+      - name: Run Hive 3 tests
+        run: build/sbt "++ ${{ matrix.scala }}" hiveMR/test hiveTez/test
+      - name: Run Hive 2 tests
+        run: build/sbt "++ ${{ matrix.scala }}" hive2MR/test hive2Tez/test
+      - name: Run Flink tests (Scala 2.12 only)
+        run: build/sbt -mem 3000 "++ ${{ matrix.scala }}" flink/test
+        if: ${{ startsWith(matrix.scala, '2.12.') }}

.github/workflows/test.yaml → .github/workflows/spark_test.yaml

@@ -5,6 +5,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
+        # These Scala versions must match those in the build.sbt
         scala: [2.12.15, 2.13.5]
     env:
       SCALA_VERSION: ${{ matrix.scala }}
@@ -16,6 +17,7 @@ jobs:
           PATTERNS: |
             **
             !kernel/**
+            !connectors/**
       - name: install java
         uses: actions/setup-java@v3
         with:
@@ -58,6 +60,6 @@ jobs:
         if: steps.git-diff.outputs.diff
       - name: Run Scala/Java and Python tests
         run: |
-          pipenv run python run-tests.py --group spark --coverage
+          pipenv run python run-tests.py --group spark
           cd examples/scala && build/sbt "++ $SCALA_VERSION compile"
         if: steps.git-diff.outputs.diff

.sbtopts

@@ -0,0 +1 @@
+-J-Xmx4G