This repository has been archived by the owner on Aug 25, 2024. It is now read-only.
docs: discussions: Alice Engineering Comms: 2024-06-27T06:02:18+00:00 #5218
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| permissions: | |
| contents: read | |
| on: | |
| workflow_dispatch: null | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - 'docs/arch/**' | |
| pull_request: | |
| paths-ignore: | |
| - 'docs/arch/**' | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| attestations: write | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - "3.12" | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get pip cache | |
| id: pip-cache | |
| run: | | |
| python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" | |
| - name: pip cache | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.cfg') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dev dependencies | |
| run: | | |
| pip install -U pip setuptools wheel build | |
| pip install -U sbom4python | |
| pip install -U https://github.com/scitt-community/scitt-api-emulator/archive/e89a60584fa717382f279ae24b8a1a93d458bb4d.zip | |
| pip install -e .[dev] | |
| python -m pip freeze | |
| - name: Build | |
| run: | | |
| python -m build . | |
| - name: Generate SBOM | |
| id: generate-sbom | |
| uses: anthonyharrison/sbom4python@5b458354df89357bf0253e62ea4567b1807120e2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| module-name: dffml | |
| output-directory: sbom | |
| - name: Get built filenames | |
| id: filename | |
| run: | | |
| echo "tar=$(cd dist/ && echo *.tar.gz)" >> $GITHUB_OUTPUT | |
| echo "whl=$(cd dist/ && echo *.tar.gz)" >> $GITHUB_OUTPUT | |
| - name: Attest Build Provenance for tar | |
| uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 | |
| with: | |
| subject-path: "dist/${{ steps.filename.outputs.tar }}" | |
| - name: Attest Build Provenance for whl | |
| uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 | |
| with: | |
| subject-path: "dist/${{ steps.filename.outputs.whl }}" | |
| - uses: openvex/generate-vex@159b7ee4845fb48f1991395ce8501d6263407360 | |
| name: Run vexctl | |
| id: vexctl | |
| with: | |
| product: pkg:github/${{ github.repository }}@${{ github.sha }} | |
| - name: Output vexctl to file | |
| shell: cp -v {0} vex.json | |
| run: | | |
| ${{ steps.vexctl.outputs.openvex }} | |
| - name: Submit OpenVEX to Transparency Service | |
| uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 | |
| with: | |
| subject-path: vex.json | |
| lint: | |
| if: false | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 40 | |
| matrix: | |
| check: [changelog, whitespace, commit, lines] | |
| python-version: [3.7] | |
| node-version: [12.x] | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| - name: Checkout full upstream repo | |
| run: | | |
| git remote set-url origin https://github.com/intel/dffml | |
| git fetch --prune --unshallow | |
| git fetch --depth=1 origin +refs/tags/*:refs/tags/* | |
| git config --global user.email "dffml@intel.com" | |
| git config --global user.name "DFFML CI/CD" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Use Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Get pip cache | |
| id: pip-cache | |
| run: | | |
| python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" | |
| - name: pip cache | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: | | |
| set -x | |
| ./.ci/deps.sh ${{ matrix.check }} | |
| - name: Run check | |
| run: | | |
| export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} | |
| SSH_DFFML_GH_PAGES=${{ secrets.SSH_DFFML_GH_PAGES }} ./.ci/run.sh ${{ matrix.check }} | |
| container: | |
| if: false | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| - name: Install dependencies | |
| run: | | |
| set -x | |
| curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - | |
| sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | |
| sudo apt-get update && sudo apt-get install -y docker-ce docker-ce-cli containerd.io | |
| - name: Run check | |
| run: | | |
| ./.ci/run.sh container | |
| test: | |
| if: false | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 100 | |
| matrix: | |
| plugin: | |
| - model/daal4py | |
| - model/tensorflow | |
| - model/scratch | |
| - model/scikit | |
| - model/vowpalWabbit | |
| - model/autosklearn | |
| - model/xgboost | |
| - operations/binsec | |
| - operations/data | |
| - operations/deploy | |
| - operations/image | |
| - operations/nlp | |
| - source/mysql | |
| - feature/git | |
| - feature/auth | |
| - service/http | |
| - configloader/yaml | |
| - configloader/image | |
| - source/mongodb | |
| - entities/alice | |
| python-version: | |
| - "3.12" | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| - name: Checkout full upstream repo | |
| run: | | |
| git remote set-url origin https://github.com/intel/dffml | |
| git fetch --prune --unshallow | |
| git fetch --depth=1 origin +refs/tags/*:refs/tags/* | |
| git config --global user.email "dffml@intel.com" | |
| git config --global user.name "DFFML CI/CD" | |
| - name: Remove unused software | |
| run: | | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get pip cache | |
| id: pip-cache | |
| run: | | |
| python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" | |
| - name: pip cache | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: shouldi test binary cache | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 | |
| with: | |
| path: examples/shouldi/tests/downloads | |
| key: ${{ hashFiles('examples/shouldi/tests/binaries.py') }} | |
| restore-keys: | | |
| ${{ runner.os }}-shouldi-test-binaries- | |
| - name: Install dependencies | |
| run: | | |
| set -x | |
| ./.ci/deps.sh ${{ matrix.plugin }} | |
| - name: Test | |
| run: | | |
| set -x | |
| export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} | |
| export PYTHON=python${{ matrix.python-version }} | |
| export PLUGIN=${{ matrix.plugin }} | |
| export LOGGING=debug | |
| export PATH="${HOME}/.local/bin:${PATH}" | |
| export PYPI_TOKENS=$(mktemp) | |
| cat > ${PYPI_TOKENS} <<EOF | |
| examples/shouldi=${{ secrets.PYPI_SHOULDI }} | |
| model/daal4py=${{ secrets.PYPI_MODEL_DAAL4PY }} | |
| model/tensorflow=${{ secrets.PYPI_MODEL_TENSORFLOW }} | |
| model/tensorflow_hub=${{ secrets.PYPI_MODEL_TENSORFLOW_HUB }} | |
| model/scratch=${{ secrets.PYPI_MODEL_SCRATCH }} | |
| model/scikit=${{ secrets.PYPI_MODEL_SCIKIT }} | |
| model/spacy=${{ secrets.PYPI_MODEL_SPACY }} | |
| model/vowpalWabbit=${{ secrets.PYPI_MODEL_VOWPALWABBIT }} | |
| model/autosklearn=${{ secrets.PYPI_MODEL_AUTOSKLEARN }} | |
| model/pytorch=${{ secrets.PYPI_MODEL_PYTORCH }} | |
| model/xgboost=${{ secrets.PYPI_MODEL_XGBOOST }} | |
| source/mysql=${{ secrets.PYPI_SOURCE_MYSQL }} | |
| source/mongodb=${{ secrets.PYPI_SOURCE_MONGODB }} | |
| feature/git=${{ secrets.PYPI_FEATURE_GIT }} | |
| feature/auth=${{ secrets.PYPI_FEATURE_AUTH }} | |
| operations/binsec=${{ secrets.PYPI_OPERATIONS_BINSEC }} | |
| operations/data=${{ secrets.PYPI_OPERATIONS_DATA }} | |
| operations/deploy=${{ secrets.PYPI_OPERATIONS_DEPLOY }} | |
| operations/image=${{ secrets.PYPI_OPERATIONS_IMAGE }} | |
| operations/nlp=${{ secrets.PYPI_OPERATIONS_NLP }} | |
| service/http=${{ secrets.PYPI_SERVICE_HTTP }} | |
| configloader/yaml=${{ secrets.PYPI_CONFIG_YAML }} | |
| configloader/image=${{ secrets.PYPI_CONFIG_IMAGE }} | |
| EOF | |
| export TWINE_USERNAME=__token__ | |
| if [ "x${PLUGIN}" = "x." ]; then | |
| export TWINE_PASSWORD=${{ secrets.PYPI_DFFML }} | |
| else | |
| export TWINE_PASSWORD=$(grep "${PLUGIN}=" "${PYPI_TOKENS}" | sed 's/^[^=]*=//g') | |
| fi | |
| rm ${PYPI_TOKENS} | |
| ./.ci/run.sh "${PLUGIN}" | |
| if [ "x${PLUGIN}" = "x." ]; then | |
| CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} codecov | |
| fi | |
| tutorials: | |
| runs-on: ubuntu-latest | |
| if: false | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 100 | |
| matrix: | |
| python-version: [3.7] | |
| docs: | |
| - docs/cli.rst | |
| - docs/examples/dataflows.rst | |
| - docs/examples/integration.rst | |
| - docs/examples/or_covid_data_by_county.rst | |
| - docs/examples/shouldi.rst | |
| - docs/examples/innersource/swportal.rst | |
| - docs/examples/innersource/microservice.rst | |
| - docs/examples/innersource/kubernetes.rst | |
| - docs/examples/icecream_sales.rst | |
| - docs/examples/data_cleanup/data_cleanup.rst | |
| - docs/examples/data_cleanup/data_cleanup_classfication.rst | |
| - docs/installation.rst | |
| - docs/troubleshooting.rst | |
| - docs/tutorials/accuracy/mse.rst | |
| - docs/tutorials/dataflows/io.rst | |
| - docs/tutorials/dataflows/nlp.rst | |
| - docs/tutorials/models/iris.rst | |
| - docs/tutorials/models/package.rst | |
| - docs/tutorials/models/docs.rst | |
| - docs/tutorials/models/slr.rst | |
| - docs/tutorials/sources/complex.rst | |
| - docs/tutorials/sources/file.rst | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| - name: Checkout full upstream repo | |
| run: | | |
| git remote set-url origin https://github.com/intel/dffml | |
| git fetch --prune --unshallow | |
| git fetch --depth=1 origin +refs/tags/*:refs/tags/* | |
| git config --global user.email "dffml@intel.com" | |
| git config --global user.name "DFFML CI/CD" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get pip cache | |
| id: pip-cache | |
| run: | | |
| python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" | |
| - name: pip cache | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: | | |
| set -x | |
| ./.ci/deps.sh . | |
| - name: Test | |
| run: | | |
| set -x | |
| export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} | |
| export PYTHON=python${{ matrix.python-version }} | |
| export PLUGIN=${{ matrix.plugin }} | |
| export LOGGING=debug | |
| export PATH="${HOME}/.local/bin:${PATH}" | |
| ./.ci/run.sh consoletest ${{ matrix.docs }} | |
| macos: | |
| runs-on: macos-latest | |
| if: false | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [3.7] | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| - name: Checkout full upstream repo | |
| run: | | |
| git remote set-url origin https://github.com/intel/dffml | |
| git fetch --prune --unshallow | |
| git config --global user.email "dffml@intel.com" | |
| git config --global user.name "DFFML CI/CD" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get pip cache | |
| id: pip-cache | |
| run: | | |
| python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" | |
| - name: pip cache | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Setup DFFML | |
| run: | | |
| pip install -U pip setuptools wheel | |
| pip install -e .[dev] | |
| dffml service dev install -skip model/daal4py | |
| # XGBoost requires libomp on OSX | |
| brew install libomp | |
| - name: Test | |
| run: | | |
| python -m unittest discover -v | |
| python -m pip freeze |