feat(table/scanner): Initial pass for planning a scan and returning the files to use

.github/workflows/go-ci.yml

@@ -20,7 +20,7 @@ name: Go
 on:
   push:
     branches:
-      - 'master'
+      - 'main'
     tags:
       - 'v**'          
   pull_request:    
@@ -39,7 +39,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [ '1.21' ]
+        go: [ '1.21', '1.22' ]
         os: [ 'ubuntu-latest', 'windows-latest', 'macos-latest' ]
     steps:
     - uses: actions/checkout@v3

.github/workflows/go-integration.yml

@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: "Go Integration"
+
+on:
+  push:
+    branches:
+      - 'main'
+    tags:
+      - 'v**'
+  pull_request:
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+permissions:
+  contents: read
+
+jobs:
+  integration-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - name: Install Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.22
+          cache: true
+          cache-dependency-path: go.sum
+
+      - name: Start docker
+        run: |
+          docker compose -f dev/docker-compose.yml up -d
+          sleep 10
+      - name: Provision Tables
+        run: |
+          docker compose -f dev/docker-compose.yml exec -T spark-iceberg ipython ./provision.py
+          sleep 10
+
+      - name: Get minio container IP
+        run: |
+          echo "AWS_S3_ENDPOINT=http://$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' minio):9000" >> $GITHUB_ENV
+
+      - name: Run integration tests
+        env:
+          AWS_S3_ENDPOINT: "${{ env.AWS_S3_ENDPOINT }}"
+          AWS_REGION: "us-east-1"
+        run: |          
+          go test -tags integration -v -run="^TestScanner" ./table
+
+      - name: Show debug logs
+        if: ${{ failure() }}
+        run: docker compose -f dev/docker-compose.yml logs

dev/Dockerfile

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM tabulario/spark-iceberg
+
+RUN pip3 install -q ipython
+RUN pip3 install pyiceberg[s3fs,hive]
+RUN pip3 install pyarrow
+
+COPY provision.py .
+
+ENTRYPOINT ["./entrypoint.sh"]
+CMD ["notebook"]

dev/docker-compose.yml

@@ -18,11 +18,12 @@ version: "3"
 
 services:
   spark-iceberg:
-    image: tabulario/spark-iceberg
+    image: pyiceberg-spark
     container_name: spark-iceberg    
+    build: .
     networks:
       iceberg_net:
-    depends_on:
+    depends_on:      
       - rest
       - minio
     volumes:
@@ -85,5 +86,6 @@ services:
       /usr/bin/mc policy set public minio/warehouse;
       tail -f /dev/null
       "
+
 networks:
-  iceberg_net:
+  iceberg_net: