312 accept types in internal csv reading function #792
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Unit Tests | |
| env: | |
| DEFAULT_SAMPLES_REVISION: 10.2.4 | |
| DEFAULT_KHIOPS_DESKTOP_REVISION: 10.2.3 | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| samples-revision: | |
| default: 10.2.4 | |
| description: Git Tag/Branch/Commit for the khiops-samples Repo | |
| image-tag: | |
| default: latest | |
| description: Development Docker Image Tag | |
| khiops-desktop-revision: | |
| default: 10.2.3 | |
| description: Khiops Windows Desktop Application Version | |
| run-long-tests: | |
| type: boolean | |
| required: false | |
| default: false | |
| description: Execute long tests | |
| pull_request: | |
| paths: | |
| - khiops/**.py | |
| - pykhiops/**.py # TODO: Remove for Khiops 11 | |
| - tests/**.py | |
| - tests/resources/** | |
| - '!tests/resources/**.md' | |
| - .github/workflows/unit-tests.yml | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| run: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] | |
| container: | |
| # 'latest' default image tag cannot be set as an environment variable, | |
| # because the `env` context is only accessible at the step level; | |
| # hence, it is hard-coded | |
| image: |- | |
| ghcr.io/khiopsml/khiops-python/khiopspydev-ubuntu22.04:${{ inputs.image-tag || 'latest' }} | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| checks: write | |
| packages: read | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| with: | |
| # Get Git tags so that versioneer can function correctly | |
| # See issue https://github.com/actions/checkout/issues/701 | |
| fetch-depth: 0 | |
| # We move SAMPLES_REVISION to the environment so that we can use | |
| # them in both push and workflow_dispatch events | |
| - name: Set SAMPLES_REVISION on 'pull_request' events | |
| if: github.event_name == 'pull_request' | |
| run: echo "SAMPLES_REVISION=${DEFAULT_SAMPLES_REVISION}" >> "$GITHUB_ENV" | |
| - name: Set SAMPLES_REVISION on 'workflow_dispatch' events | |
| if: github.event_name == 'workflow_dispatch' | |
| run: echo "SAMPLES_REVISION=${{ inputs.samples-revision }}" >> "$GITHUB_ENV" | |
| - name: Checkout Khiops samples | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: khiopsml/khiops-samples | |
| ref: ${{ env.SAMPLES_REVISION }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| path: khiops-samples | |
| - name: Setup and Install Test Requirements | |
| if: success() || failure() | |
| run: | | |
| CONDA="/root/miniforge3/bin/conda" | |
| # Native Khiops-based Conda environment, and | |
| # `khiops-core`-based Conda environment | |
| CONDA_ENVS="py${{ matrix.python-version }} py${{ matrix.python-version }}_conda" | |
| for CONDA_ENV in $CONDA_ENVS | |
| do | |
| mkdir -p -m u+rwx reports/"$CONDA_ENV" | |
| # install within the conda environments without activating them | |
| $CONDA install -y -n "$CONDA_ENV" unittest-xml-reporting | |
| $CONDA install -y -n "$CONDA_ENV" --file test-requirements.txt | |
| done | |
| - name: Install khiops-python dependencies | |
| if: success() || failure() | |
| run: | | |
| # The following git command is required, | |
| # as the Git repository is in a directory the current user does not own, | |
| # Python versioneer fails to compute the current version correctly otherwise | |
| git config --global --add safe.directory $(realpath .) | |
| CONDA="/root/miniforge3/bin/conda" | |
| # Native Khiops-based Conda environment, and | |
| # `khiops-core`-based Conda environment | |
| CONDA_ENVS="py${{ matrix.python-version }} py${{ matrix.python-version }}_conda" | |
| for CONDA_ENV in $CONDA_ENVS | |
| do | |
| # Since Python 3.13, setuptools is not installed automatically anymore | |
| $CONDA install -y -n "$CONDA_ENV" setuptools | |
| $CONDA run --no-capture-output -n "$CONDA_ENV" python setup.py egg_info | |
| $CONDA install -y -n "$CONDA_ENV" `grep -v "^\[" khiops.egg-info/requires.txt` | |
| rm -rf khiops.egg-info | |
| done | |
| - name: Prepare Unit Tests Environment | |
| if: github.ref != 'dev' && github.ref != 'main' && ! inputs.run-long-tests | |
| run: echo "UNITTEST_ONLY_SHORT_TESTS=true" >> "$GITHUB_ENV" | |
| - name: Run Unit Tests | |
| env: | |
| KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples | |
| KHIOPS_DOCKER_RUNNER_URL: https://localhost:11000 | |
| KHIOPS_DOCKER_RUNNER_SHARED_DIR: /tmp/sandbox | |
| KHIOPS_RUNNER_SERVICE_PATH: /scripts/run_service.sh | |
| # Force > 2 CPU cores to launch mpiexec | |
| KHIOPS_PROC_NUMBER: 4 | |
| # Oversubscribe for MPI 4.x | |
| rmaps_base_oversubscribe: true | |
| # Oversubscribe for MPI > 4.x | |
| OMPI_MCA_rmaps_base_oversubscribe: true | |
| run: | | |
| # This is needed so that the Git tag is parsed and the khiops-python | |
| # version is retrieved | |
| git config --global --add safe.directory $(realpath .) | |
| CONDA="/root/miniforge3/bin/conda" | |
| # Native Khiops-based Conda environment, and | |
| # `khiops-core`-based Conda environment | |
| CONDA_ENVS="py${{ matrix.python-version }} py${{ matrix.python-version }}_conda" | |
| for CONDA_ENV in $CONDA_ENVS | |
| do | |
| $CONDA run --no-capture-output -n "$CONDA_ENV" coverage run -m xmlrunner -o "reports/$CONDA_ENV" -v | |
| $CONDA run --no-capture-output -n "$CONDA_ENV" coverage report -m | |
| $CONDA run --no-capture-output -n "$CONDA_ENV" coverage xml -o "reports/$CONDA_ENV/py-coverage.xml" | |
| done | |
| - name: Display Unit Test Reports | |
| uses: dorny/test-reporter@v1 | |
| with: | |
| name: Unit Tests ${{ matrix.python-version }} | |
| path: >- | |
| reports/py${{ matrix.python-version }}/TEST-tests.*.*.xml, | |
| reports/py${{ matrix.python-version }}_conda/TEST-tests.*.*.xml | |
| reporter: java-junit | |
| path-replace-backslashes: 'true' # Necessary for windows paths | |
| - name: Upload Test Reports as Artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-reports-${{ matrix.python-version }} | |
| path: |- | |
| reports/py${{ matrix.python-version }}/TEST-tests.*.*.xml | |
| reports/py${{ matrix.python-version }}/py-coverage.xml | |
| reports/py${{ matrix.python-version }}_conda/TEST-tests.*.*.xml | |
| reports/py${{ matrix.python-version }}_conda/py-coverage.xml | |
| tests/resources/scenario_generation/*/ref/*._kh | |
| tests/resources/scenario_generation/*/output/*._kh | |
| tests/resources/*/output_reports/*.txt | |
| tests/resources/*/ref_reports/*.txt | |
| tests/resources/dictionary/ref_kdic/*.kdic | |
| tests/resources/dictionary/output_kdic/*.kdic | |
| tests/resources/dictionary/copy_output_kdic/*.kdic | |
| tests/resources/general_options/general_options/*/*._kh | |
| retention-days: 7 | |
| check-khiops-integration-on-windows: | |
| runs-on: windows-2019 | |
| steps: | |
| - name: Download the Khiops Desktop NSIS Installer | |
| shell: pwsh | |
| run: | | |
| $KHIOPS_DESKTOP_REVISION = '${{ inputs.khiops-desktop-revision || env.DEFAULT_KHIOPS_DESKTOP_REVISION }}' | |
| $KHIOPS_DOWNLOAD_URL = "https://github.com/KhiopsML/khiops/releases/download/${KHIOPS_DESKTOP_REVISION}/khiops-${KHIOPS_DESKTOP_REVISION}-setup.exe" | |
| Invoke-WebRequest "${KHIOPS_DOWNLOAD_URL}" ` | |
| -OutFile .\khiops-setup.exe ` | |
| -UseBasicParsing | |
| Unblock-File .\khiops-setup.exe | |
| - name: Install the Khiops Desktop Application | |
| shell: pwsh | |
| run: | | |
| # Execute the installer | |
| $ErrorActionPreference = 'Stop' | |
| $ProgressPreference = 'SilentlyContinue' | |
| Start-Process ` | |
| -FilePath .\khiops-setup.exe ` | |
| -ArgumentList '/S' ` | |
| -Wait | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| with: | |
| # Get Git tags so that versioneer can function correctly | |
| # See issue https://github.com/actions/checkout/issues/701 | |
| fetch-depth: 0 | |
| - name: Checkout Khiops samples | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: khiopsml/khiops-samples | |
| ref: ${{ env.SAMPLES_REVISION }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| path: khiops-samples | |
| - name: Setup Python | |
| id: setup-python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install khiops-python dev dependencies | |
| shell: pwsh | |
| run: | | |
| # The following git command is required, | |
| # as the Git repository is in a directory the current user does not own, | |
| # Python versioneer fails to compute the current version correctly otherwise | |
| git config --global --add safe.directory $(Resolve-Path '.' | % {$_.toString()}) | |
| python setup.py egg_info | |
| # Install the Pyrhon requirements | |
| Get-Content .\khiops.egg-info\requires.txt ` | |
| | Select-String -Pattern '^\[' -NotMatch ` | |
| | Select-String -Pattern '^$' -NotMatch ` | |
| | % {$_.Line} ` | |
| | ForEach-Object {python -m pip install $_.toString()} | |
| Remove-Item -r -force khiops.egg-info | |
| - name: Setup and Install Test Requirements | |
| run: python -m pip install -r test-requirements.txt | |
| - name: Test Khiops Integration | |
| env: | |
| # Force > 2 CPU cores to launch MPI | |
| KHIOPS_PROC_NUMBER: 4 | |
| KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples | |
| shell: pwsh | |
| run: | | |
| $ErrorActionPreference = 'Stop' | |
| # Refresh environment variables by using the Chocolatey tool | |
| # Otherwise, the env vars set in the registry by the Khiops installer do not get updated | |
| # See also https://github.com/actions/runner-images/discussions/6065#discussioncomment-3517318 | |
| Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 | |
| refreshenv | |
| # Invoke the Python version set-up by the `setup-python` step / action | |
| # Otherwise invoking `python` directly invokes the system-cached Python on the runner | |
| # This is a side-effect of refreshing the environment | |
| $Python = "${{ steps.setup-python.outputs.python-path }}" | |
| # Print status | |
| Invoke-Expression -Command "$Python -c 'import khiops.core as kh; kh.get_runner().print_status()'" | |
| # Run integration tests on Windows | |
| Invoke-Expression -Command "$Python -m unittest -v tests.test_khiops_integrations" | |
| # Execute Khiops sample (train and deploy model) | |
| Invoke-Expression -Command "$Python -m khiops.samples.samples -i deploy_model -e" | |
| # Execute Khiops Coclustering sample (train and deploy model) | |
| Invoke-Expression -Command "$Python -m khiops.samples.samples -i deploy_coclustering -e" | |
| check-khiops-integration-on-linux: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| container: [ubuntu22.04, rocky8, rocky9] | |
| runs-on: ubuntu-latest | |
| container: | |
| # 'latest' default image tag cannot be set as an environment variable, | |
| # because the `env` context is only accessible at the step level; | |
| # hence, it is hard-coded | |
| image: |- | |
| ghcr.io/khiopsml/khiops-python/khiopspydev-${{ matrix.container }}:${{ inputs.image-tag || 'latest' }} | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| checks: write | |
| packages: read | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| with: | |
| # Get Git tags so that versioneer can function correctly | |
| # See issue https://github.com/actions/checkout/issues/701 | |
| fetch-depth: 0 | |
| - name: Checkout Khiops samples | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: khiopsml/khiops-samples | |
| ref: ${{ env.SAMPLES_REVISION }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| path: khiops-samples | |
| - name: Install khiops-python dev dependencies | |
| run: | | |
| # The following git command is required, | |
| # as the Git repository is in a directory the current user does not own, | |
| # Python versioneer fails to compute the current version correctly otherwise | |
| git config --global --add safe.directory $(realpath .) | |
| python setup.py egg_info | |
| pip install `grep -v "^\[" khiops.egg-info/requires.txt` | |
| rm -rf khiops.egg-info | |
| - name: Setup and Install Test Requirements | |
| run: | | |
| pip install -r test-requirements.txt | |
| - name: Test Khiops Integration | |
| env: | |
| # Force > 2 CPU cores to launch mpiexec | |
| KHIOPS_PROC_NUMBER: 4 | |
| KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples | |
| # Oversubscribe for MPI 4.x | |
| rmaps_base_oversubscribe: true | |
| # Oversubscribe for MPI > 4.x | |
| OMPI_MCA_rmaps_base_oversubscribe: true | |
| run: |- | |
| # Make sure MPI support is not loaded through env modules | |
| # Note: As Docker container's shell is non-interactive, environment | |
| # modules are currently not initializing the shell anyway | |
| if [ -n "$MODULESHOME" ]; then module unload mpi; fi | |
| # Print status | |
| python -c "import khiops.core as kh; kh.get_runner().print_status()" | |
| # Run integration tests on Ubuntu and Rocky | |
| python -m unittest -v tests.test_khiops_integrations | |
| # Execute Khiops sample (train and deploy model), which uses MPI | |
| python -m khiops.samples.samples -i deploy_model -e |