Reactivate the integration tests for remote file access (GCS, S3)

- 2 fake servers (fakes3 and fake-gcs-server) are used locally instead of a real S3 and GCS servers.
- The fake servers are pre-provisioned with sample files to be ready before the tests run
- During the construction of the khiopsdev docker image we need to install the s3/gcs drivers required by khiops

.github/workflows/dev-docker.yml

@@ -75,6 +75,9 @@ jobs:
       - name: Build image and push it to GitHub Container Registry
         uses: docker/build-push-action@v5
         with:
+          # Special hostname used by the integration tests for remote file access
+          # added using inputs because /etc/hosts is read-only for alternate builders (buildx via moby buildkit)
+          add-hosts: s3-bucket.localhost:127.0.0.1
           context: ./packaging/docker/khiopspydev/
           file: ./packaging/docker/khiopspydev/Dockerfile.${{ env.KHIOPSDEV_OS_CODENAME }}
           build-args: |

.github/workflows/unit-tests.yml → .github/workflows/unit-and-integration-tests.yml

@@ -1,5 +1,5 @@
 ---
-name: Unit Tests
+name: Unit & Integration Tests
 env:
   DEFAULT_SAMPLES_REVISION: main
 on:
@@ -199,3 +199,83 @@ jobs:
           # modules are currently not initializing the shell anyway
           if [ -n "$MODULESHOME" ]; then module unload mpi; fi
           python -m unittest -v tests.test_khiops_integrations.KhiopsRunnerEnvironmentTests.test_runner_has_mpiexec_on_linux
+  test-access-to-remote-files:
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/khiopsml/khiops-python/khiopspydev-ubuntu22.04:10.2.2.s3-gcs-remote-files
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    permissions:
+      id-token: write
+      contents: read
+      checks: write
+      packages: read
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          # Get Git tags so that versioneer can function correctly
+          # See issue https://github.com/actions/checkout/issues/701
+          fetch-depth: 0
+      - name: Install khiops-python dev dependencies
+        run: |
+          # The following git command is required, 
+          # as the Git repository is in a directory the current user does not own,
+          # Python versioneer fails to compute the current version correctly otherwise
+          git config --global --add safe.directory $(realpath .)
+          python setup.py egg_info
+          pip install `grep -v "^\[" khiops.egg-info/requires.txt`
+          rm -rf khiops.egg-info
+      - name: Setup and Install Test Requirements
+        run: |
+          pip install -r test-requirements.txt
+          pip install unittest-xml-reporting
+      - name: Launch the tests
+        env:
+          # for the tests with GCS
+          GCS_BUCKET_NAME: gcs-bucket
+          STORAGE_EMULATOR_HOST: http://localhost:4443
+          GCS_DRIVER_LOGLEVEL: debug  # for diagnosis
+          # for the tests with S3
+          S3_BUCKET_NAME: s3-bucket
+          AWS_SECRET_ACCESS_KEY: XXX
+          AWS_ENDPOINT_URL: http://localhost:4569
+          AWS_ACCESS_KEY_ID: XXX
+          # common vars
+          no_proxy: localhost
+          # Force > 2 CPU cores to launch mpiexec
+          KHIOPS_PROC_NUMBER: 4
+          # Oversubscribe for MPI 4.x
+          rmaps_base_oversubscribe: true
+          # Oversubscribe for MPI > 4.x
+          OMPI_MCA_rmaps_base_oversubscribe: true
+          PRTE_MCA_rmaps_default_mapping_policy: :oversubscribe
+        run: |-
+          /scripts/run_fake_remote_file_servers.sh .  # launch the servers in the background
+          python -m xmlrunner tests.test_remote_access.KhiopsGCSRemoteFileTests -o reports/
+          python -m xmlrunner tests.test_remote_access.KhiopsS3RemoteFileTests -o reports/
+      - name: Display Integration Test Reports on remote files
+        uses: dorny/test-reporter@v1
+        with:
+          name: Integration Tests on remote files
+          path: >-
+            reports/TEST-tests.test_remote_access.*.xml
+          reporter: java-junit
+          path-replace-backslashes: 'true'  # Necessary for windows paths
+      - name: Upload Test Reports as Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration-test-reports-on-remote-files
+          path: |-
+            reports/TEST-tests.test_remote_access.*.xml
+            test/resources/tmp/khiops-cicd/*/*
+            tests/resources/scenario_generation/*/ref/*._kh
+            tests/resources/scenario_generation/*/output/*._kh
+            tests/resources/*/output_reports/*.txt
+            tests/resources/*/ref_reports/*.txt
+            tests/resources/dictionary/ref_kdic/*.kdic
+            tests/resources/dictionary/output_kdic/*.kdic
+            tests/resources/dictionary/copy_output_kdic/*.kdic
+            tests/resources/general_options/general_options/*/*._kh
+          retention-days: 7

packaging/docker/khiopspydev/Dockerfile.ubuntu

@@ -10,15 +10,14 @@ ARG KHIOPS_REVISION
 RUN true \
   # Install git (for khiops-python version calculation) and pip \
   && apt-get -y update \
-  && apt-get -y --no-install-recommends install git python3-pip zip pandoc wget \
+  && apt-get -y --no-install-recommends install git python3-pip zip pandoc wget ruby-dev \
   # Get Linux distribution codename \
   && if [ -f /etc/os-release ]; then . /etc/os-release; fi \
   # Obtain the Khiops native package \
   && KHIOPS_PKG_FILE=$KHIOPS_REVISION/khiops-core-openmpi_$KHIOPS_REVISION-1-$VERSION_CODENAME.amd64.deb \
   && wget -O KHIOPS_CORE.deb "https://github.com/KhiopsML/khiops/releases/download/${KHIOPS_PKG_FILE}" \
-  # Install the Khiops native package \
-  && dpkg -i --force-all KHIOPS_CORE.deb \
-  && apt-get -f -y install \
+  # Install the Khiops native package : do not break immediately if dpkg fails because apt will install the missing dependencies \
+  && (dpkg -i --force-all KHIOPS_CORE.deb || true) && apt-get -f -y install \
   && rm -f KHIOPS_CORE.deb \
   # Set python to python3 \
   && update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
@@ -48,9 +47,35 @@ done'
 
 RUN mkdir -p /scripts
 COPY ./run_service.sh /scripts/run_service.sh
-RUN chmod +x /scripts/run_service.sh && \
+COPY ./run_fake_remote_file_servers.sh /scripts/run_fake_remote_file_servers.sh
+RUN chmod +x /scripts/*.sh && \
   useradd -rm -d /home/ubuntu -s /bin/bash -g root -u 1000 ubuntu
 
+# new gcs & s3 drivers (written in C++)
+RUN true \
+    && wget -O khiops-gcs.deb https://github.com/sgouache/khiopsdriver-gcs/releases/download/0.0.10/khiops-gcs_0.0.10.deb \
+    && wget -O khiops-s3.deb https://github.com/sgouache/khiopsdriver-s3/releases/download/0.0.12/khiops-s3_0.0.12.deb \
+    && (dpkg -i --force-all khiops-gcs.deb khiops-s3.deb || true) && apt-get -f -y install \
+    && rm -f khiops-gcs.deb khiops-s3.deb \
+    && true
+
 FROM ghcr.io/khiopsml/khiops-server:${SERVER_REVISION} AS server
+FROM fsouza/fake-gcs-server:1.49 AS gcs-server
+
 FROM khiopsdev AS base
 COPY --from=server /service /usr/bin/service
+
+# GCS fake file server (only in the ubuntu container)
+COPY --from=gcs-server /bin/fake-gcs-server /bin/fake-gcs-server
+EXPOSE 4443
+
+# S3 fake file server (only in the ubuntu container)
+# Do not use the latest fakes3 version because starting from 1.3.1 a licence is required
+RUN gem install fakes3:0.2.5 sorted_set
+# Avoid resolving a fake s3-bucket.localhost hostname
+# Alternate builders (buildx via moby buildkit) mount /etc/hosts read-only, the following command will fail
+# echo "127.0.0.1 s3-bucket.localhost" >> /etc/hosts
+# You will have to add the `add-hosts` input instead (https://github.com/docker/build-push-action/#inputs)
+
+EXPOSE 4569
+

packaging/docker/khiopspydev/run_fake_remote_file_servers.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# We cannot start the fake remote files servers until the source code is fetched
+# because we expose pre-provisioned files to be read
+
+ROOT_FOLDER=${1:-.} # defaults to current folder
+
+# File server for GCS (runs in background)
+# WARNING : there are 3 major features actived by the options ...
+# -data : exposes pre-provisioned files to be read remotely : the direct child folders will be the bucket names
+# -filesystem-root : let upload and read new files remotely at the same location as the source
+# -public-host : must expose localhost (https://github.com/fsouza/fake-gcs-server/issues/201)
+nohup /bin/fake-gcs-server \
+  -data "${ROOT_FOLDER}"/tests/resources/remote-access \
+  -filesystem-root "${ROOT_FOLDER}"/tests/resources/remote-access \
+  -scheme http \
+  -public-host localhost &
+
+# File server for S3 (runs in background)
+# WARNING :
+# -r : exposes pre-provisioned files : the direct child folders will be the bucket names
+#      these files were uploaded once because fake-s3 creates metadata
+nohup /usr/local/bin/fakes3 \
+  -r "${ROOT_FOLDER}"/tests/resources/remote-access \
+  -p 4569 &

run_fake_remote_file_servers.sh

@@ -0,0 +1 @@
+packaging/docker/khiopspydev/run_fake_remote_file_servers.sh

tests/resources/remote-access/gcs-bucket/khiops-cicd/samples/Iris/Iris.kdic

@@ -0,0 +1,9 @@
+
+Dictionary	Iris
+{
+	Numerical	SepalLength	;	
+	Numerical	SepalWidth	;	
+	Numerical	PetalLength	;	
+	Numerical	PetalWidth	;	
+	Categorical	Class	;	
+};

tests/resources/remote-access/gcs-bucket/khiops-cicd/samples/Iris/Iris.txt

@@ -0,0 +1,151 @@
+SepalLength	SepalWidth	PetalLength	PetalWidth	Class
+5.1	3.5	1.4	0.2	Iris-setosa
+4.9	3.0	1.4	0.2	Iris-setosa
+4.7	3.2	1.3	0.2	Iris-setosa
+4.6	3.1	1.5	0.2	Iris-setosa
+5.0	3.6	1.4	0.2	Iris-setosa
+5.4	3.9	1.7	0.4	Iris-setosa
+4.6	3.4	1.4	0.3	Iris-setosa
+5.0	3.4	1.5	0.2	Iris-setosa
+4.4	2.9	1.4	0.2	Iris-setosa
+4.9	3.1	1.5	0.1	Iris-setosa
+5.4	3.7	1.5	0.2	Iris-setosa
+4.8	3.4	1.6	0.2	Iris-setosa
+4.8	3.0	1.4	0.1	Iris-setosa
+4.3	3.0	1.1	0.1	Iris-setosa
+5.8	4.0	1.2	0.2	Iris-setosa
+5.7	4.4	1.5	0.4	Iris-setosa
+5.4	3.9	1.3	0.4	Iris-setosa
+5.1	3.5	1.4	0.3	Iris-setosa
+5.7	3.8	1.7	0.3	Iris-setosa
+5.1	3.8	1.5	0.3	Iris-setosa
+5.4	3.4	1.7	0.2	Iris-setosa
+5.1	3.7	1.5	0.4	Iris-setosa
+4.6	3.6	1.0	0.2	Iris-setosa
+5.1	3.3	1.7	0.5	Iris-setosa
+4.8	3.4	1.9	0.2	Iris-setosa
+5.0	3.0	1.6	0.2	Iris-setosa
+5.0	3.4	1.6	0.4	Iris-setosa
+5.2	3.5	1.5	0.2	Iris-setosa
+5.2	3.4	1.4	0.2	Iris-setosa
+4.7	3.2	1.6	0.2	Iris-setosa
+4.8	3.1	1.6	0.2	Iris-setosa
+5.4	3.4	1.5	0.4	Iris-setosa
+5.2	4.1	1.5	0.1	Iris-setosa
+5.5	4.2	1.4	0.2	Iris-setosa
+4.9	3.1	1.5	0.1	Iris-setosa
+5.0	3.2	1.2	0.2	Iris-setosa
+5.5	3.5	1.3	0.2	Iris-setosa
+4.9	3.1	1.5	0.1	Iris-setosa
+4.4	3.0	1.3	0.2	Iris-setosa
+5.1	3.4	1.5	0.2	Iris-setosa
+5.0	3.5	1.3	0.3	Iris-setosa
+4.5	2.3	1.3	0.3	Iris-setosa
+4.4	3.2	1.3	0.2	Iris-setosa
+5.0	3.5	1.6	0.6	Iris-setosa
+5.1	3.8	1.9	0.4	Iris-setosa
+4.8	3.0	1.4	0.3	Iris-setosa
+5.1	3.8	1.6	0.2	Iris-setosa
+4.6	3.2	1.4	0.2	Iris-setosa
+5.3	3.7	1.5	0.2	Iris-setosa
+5.0	3.3	1.4	0.2	Iris-setosa
+7.0	3.2	4.7	1.4	Iris-versicolor
+6.4	3.2	4.5	1.5	Iris-versicolor
+6.9	3.1	4.9	1.5	Iris-versicolor
+5.5	2.3	4.0	1.3	Iris-versicolor
+6.5	2.8	4.6	1.5	Iris-versicolor
+5.7	2.8	4.5	1.3	Iris-versicolor
+6.3	3.3	4.7	1.6	Iris-versicolor
+4.9	2.4	3.3	1.0	Iris-versicolor
+6.6	2.9	4.6	1.3	Iris-versicolor
+5.2	2.7	3.9	1.4	Iris-versicolor
+5.0	2.0	3.5	1.0	Iris-versicolor
+5.9	3.0	4.2	1.5	Iris-versicolor
+6.0	2.2	4.0	1.0	Iris-versicolor
+6.1	2.9	4.7	1.4	Iris-versicolor
+5.6	2.9	3.6	1.3	Iris-versicolor
+6.7	3.1	4.4	1.4	Iris-versicolor
+5.6	3.0	4.5	1.5	Iris-versicolor
+5.8	2.7	4.1	1.0	Iris-versicolor
+6.2	2.2	4.5	1.5	Iris-versicolor
+5.6	2.5	3.9	1.1	Iris-versicolor
+5.9	3.2	4.8	1.8	Iris-versicolor
+6.1	2.8	4.0	1.3	Iris-versicolor
+6.3	2.5	4.9	1.5	Iris-versicolor
+6.1	2.8	4.7	1.2	Iris-versicolor
+6.4	2.9	4.3	1.3	Iris-versicolor
+6.6	3.0	4.4	1.4	Iris-versicolor
+6.8	2.8	4.8	1.4	Iris-versicolor
+6.7	3.0	5.0	1.7	Iris-versicolor
+6.0	2.9	4.5	1.5	Iris-versicolor
+5.7	2.6	3.5	1.0	Iris-versicolor
+5.5	2.4	3.8	1.1	Iris-versicolor
+5.5	2.4	3.7	1.0	Iris-versicolor
+5.8	2.7	3.9	1.2	Iris-versicolor
+6.0	2.7	5.1	1.6	Iris-versicolor
+5.4	3.0	4.5	1.5	Iris-versicolor
+6.0	3.4	4.5	1.6	Iris-versicolor
+6.7	3.1	4.7	1.5	Iris-versicolor
+6.3	2.3	4.4	1.3	Iris-versicolor
+5.6	3.0	4.1	1.3	Iris-versicolor
+5.5	2.5	4.0	1.3	Iris-versicolor
+5.5	2.6	4.4	1.2	Iris-versicolor
+6.1	3.0	4.6	1.4	Iris-versicolor
+5.8	2.6	4.0	1.2	Iris-versicolor
+5.0	2.3	3.3	1.0	Iris-versicolor
+5.6	2.7	4.2	1.3	Iris-versicolor
+5.7	3.0	4.2	1.2	Iris-versicolor
+5.7	2.9	4.2	1.3	Iris-versicolor
+6.2	2.9	4.3	1.3	Iris-versicolor
+5.1	2.5	3.0	1.1	Iris-versicolor
+5.7	2.8	4.1	1.3	Iris-versicolor
+6.3	3.3	6.0	2.5	Iris-virginica
+5.8	2.7	5.1	1.9	Iris-virginica
+7.1	3.0	5.9	2.1	Iris-virginica
+6.3	2.9	5.6	1.8	Iris-virginica
+6.5	3.0	5.8	2.2	Iris-virginica
+7.6	3.0	6.6	2.1	Iris-virginica
+4.9	2.5	4.5	1.7	Iris-virginica
+7.3	2.9	6.3	1.8	Iris-virginica
+6.7	2.5	5.8	1.8	Iris-virginica
+7.2	3.6	6.1	2.5	Iris-virginica
+6.5	3.2	5.1	2.0	Iris-virginica
+6.4	2.7	5.3	1.9	Iris-virginica
+6.8	3.0	5.5	2.1	Iris-virginica
+5.7	2.5	5.0	2.0	Iris-virginica
+5.8	2.8	5.1	2.4	Iris-virginica
+6.4	3.2	5.3	2.3	Iris-virginica
+6.5	3.0	5.5	1.8	Iris-virginica
+7.7	3.8	6.7	2.2	Iris-virginica
+7.7	2.6	6.9	2.3	Iris-virginica
+6.0	2.2	5.0	1.5	Iris-virginica
+6.9	3.2	5.7	2.3	Iris-virginica
+5.6	2.8	4.9	2.0	Iris-virginica
+7.7	2.8	6.7	2.0	Iris-virginica
+6.3	2.7	4.9	1.8	Iris-virginica
+6.7	3.3	5.7	2.1	Iris-virginica
+7.2	3.2	6.0	1.8	Iris-virginica
+6.2	2.8	4.8	1.8	Iris-virginica
+6.1	3.0	4.9	1.8	Iris-virginica
+6.4	2.8	5.6	2.1	Iris-virginica
+7.2	3.0	5.8	1.6	Iris-virginica
+7.4	2.8	6.1	1.9	Iris-virginica
+7.9	3.8	6.4	2.0	Iris-virginica
+6.4	2.8	5.6	2.2	Iris-virginica
+6.3	2.8	5.1	1.5	Iris-virginica
+6.1	2.6	5.6	1.4	Iris-virginica
+7.7	3.0	6.1	2.3	Iris-virginica
+6.3	3.4	5.6	2.4	Iris-virginica
+6.4	3.1	5.5	1.8	Iris-virginica
+6.0	3.0	4.8	1.8	Iris-virginica
+6.9	3.1	5.4	2.1	Iris-virginica
+6.7	3.1	5.6	2.4	Iris-virginica
+6.9	3.1	5.1	2.3	Iris-virginica
+5.8	2.7	5.1	1.9	Iris-virginica
+6.8	3.2	5.9	2.3	Iris-virginica
+6.7	3.3	5.7	2.5	Iris-virginica
+6.7	3.0	5.2	2.3	Iris-virginica
+6.3	2.5	5.0	1.9	Iris-virginica
+6.5	3.0	5.2	2.0	Iris-virginica
+6.2	3.4	5.4	2.3	Iris-virginica
+5.9	3.0	5.1	1.8	Iris-virginica

tests/resources/remote-access/gcs-bucket/khiops-cicd/samples/SpliceJunction/SpliceJunction.kdic

@@ -0,0 +1,14 @@
+
+Root	Dictionary	SpliceJunction(SampleId)
+{
+	Categorical	SampleId	;	
+	Categorical	Class	;	
+	Table(SpliceJunctionDNA)	DNA	;
+};
+
+Dictionary	SpliceJunctionDNA(SampleId)
+{
+	Categorical	SampleId	;	
+	Numerical	Pos	;	
+	Categorical	Char	;	
+};