Reactivate the integration tests for remote file access (GCS, S3) #331
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Dev Docker | |
| env: | |
| DEFAULT_KHIOPS_REVISION: 10.2.3 | |
| DEFAULT_IMAGE_INCREMENT: 0 | |
| DEFAULT_SERVER_REVISION: main | |
| DEFAULT_PYTHON_VERSIONS: 3.8 3.9 3.10 3.11 3.12 3.13 | |
| DEFAULT_KHIOPS_GCS_DRIVER_REVISION: 0.0.10 | |
| DEFAULT_KHIOPS_S3_DRIVER_REVISION: 0.0.12 | |
| on: | |
| pull_request: | |
| paths: [packaging/docker/khiopspydev/Dockerfile.*, .github/workflows/dev-docker.yml] | |
| workflow_dispatch: | |
| inputs: | |
| khiops-revision: | |
| type: string | |
| default: 10.2.3 | |
| description: Khiops Revision | |
| image-increment: | |
| type: number | |
| default: 0 | |
| description: Image Tag Increment | |
| push: | |
| type: boolean | |
| default: false | |
| description: Push to GH Registry | |
| set-latest: | |
| type: boolean | |
| default: false | |
| description: Set as 'latest' | |
| python-versions: | |
| type: string | |
| default: 3.8 3.9 3.10 3.11 3.12 3.13 | |
| description: Python versions to support | |
| server-revision: | |
| type: string | |
| default: main | |
| description: Khiops Server Revision | |
| khiops-gcs-driver-revision: | |
| type: string | |
| default: 0.0.10 | |
| description: Version of the driver used for remote files on Google Cloud Storage (GCS) | |
| khiops-s3-driver-revision: | |
| type: string | |
| default: 0.0.12 | |
| description: Version of the driver used for remote files AWS compatible storages (S3) | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-push: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| khiopsdev-os: [ubuntu22.04, rocky8, rocky9] | |
| permissions: | |
| packages: write # to write in the Github package registry | |
| steps: | |
| - name: Set input parameters as env or output | |
| run: | | |
| set -x | |
| echo "KHIOPS_REVISION=${{ inputs.khiops-revision || env.DEFAULT_KHIOPS_REVISION }}" >> "$GITHUB_ENV" | |
| echo "IMAGE_INCREMENT=${{ inputs.image-increment || env.DEFAULT_IMAGE_INCREMENT }}" >> "$GITHUB_ENV" | |
| echo "KHIOPSDEV_OS_CODENAME=$(echo '${{ matrix.khiopsdev-os }}' | tr -d '0-9.')" >> "$GITHUB_ENV" | |
| echo "SERVER_REVISION=${{ inputs.server-revision || env.DEFAULT_SERVER_REVISION }}" >> "$GITHUB_ENV" | |
| echo "IMAGE_URL=ghcr.io/khiopsml/khiops-python/khiopspydev-${{ matrix.khiopsdev-os }}" >> "$GITHUB_ENV" | |
| echo "KHIOPS_GCS_DRIVER_REVISION=${{ inputs.khiops-gcs-driver-revision || env.DEFAULT_KHIOPS_GCS_DRIVER_REVISION }}" >> "$GITHUB_ENV" | |
| echo "KHIOPS_S3_DRIVER_REVISION=${{ inputs.khiops-s3-driver-revision || env.DEFAULT_KHIOPS_S3_DRIVER_REVISION }}" >> "$GITHUB_ENV" | |
| - name: Checkout khiops-python sources | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| id: docker-build | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Github Packages | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up the Docker Image Tags | |
| run: | | |
| set -x | |
| # Set latest only if requested and on "dev" or "main" branch | |
| if [[ "${{ inputs.set-latest }}" == "true" && | |
| ("$GITHUB_REF_NAME" == "dev" || "$GITHUB_REF_NAME" == "main") ]] | |
| then | |
| DOCKER_IMAGE_TAGS="$IMAGE_URL:latest,$IMAGE_URL:$KHIOPS_REVISION.$IMAGE_INCREMENT" | |
| else | |
| DOCKER_IMAGE_TAGS="$IMAGE_URL:$KHIOPS_REVISION.$IMAGE_INCREMENT" | |
| fi | |
| echo "DOCKER_IMAGE_TAGS=$DOCKER_IMAGE_TAGS" >> "$GITHUB_ENV" | |
| - name: Build image and push it to GitHub Container Registry | |
| uses: docker/build-push-action@v5 | |
| with: | |
| # Special hostname used by the integration tests for remote file access | |
| # added using inputs because /etc/hosts is read-only for alternate builders (buildx via moby buildkit) | |
| add-hosts: s3-bucket.localhost:127.0.0.1 | |
| context: ./packaging/docker/khiopspydev/ | |
| file: ./packaging/docker/khiopspydev/Dockerfile.${{ env.KHIOPSDEV_OS_CODENAME }} | |
| build-args: | | |
| "KHIOPS_REVISION=${{ env.KHIOPS_REVISION }}" | |
| "KHIOPSDEV_OS=${{ matrix.khiopsdev-os }}" | |
| "SERVER_REVISION=${{ env.SERVER_REVISION }}" | |
| "PYTHON_VERSIONS=${{ inputs.python-versions || env.DEFAULT_PYTHON_VERSIONS }}" | |
| "KHIOPS_GCS_DRIVER_REVISION=${{ env.KHIOPS_GCS_DRIVER_REVISION }}" | |
| "KHIOPS_S3_DRIVER_REVISION=${{ env.KHIOPS_S3_DRIVER_REVISION }}" | |
| tags: ${{ env.DOCKER_IMAGE_TAGS }} | |
| # Push only on manual request | |
| push: ${{ inputs.push || false }} | |
| - name: Display the image digest | |
| run: echo ${{ steps.docker-build.outputs.digest }} |