Skip to content

[ONNX] Added QuickGelu from com.microsoft domain #8762

[ONNX] Added QuickGelu from com.microsoft domain

[ONNX] Added QuickGelu from com.microsoft domain #8762

name: Windows (VS 2019, Python 3.11, Release)
on:
workflow_dispatch:
pull_request:
merge_group:
push:
branches:
- master
- 'releases/**'
concurrency:
# github.ref is not unique in post-commit
group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-windows
cancel-in-progress: true
env:
TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }}
PIP_CACHE_PATH: /mount/caches/pip/win
PYTHON_VERSION: '3.11'
permissions: read-all
jobs:
Smart_CI:
runs-on: ubuntu-latest
outputs:
affected_components: "${{ steps.smart_ci.outputs.affected_components }}"
skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}"
target_branch: ${{ steps.set_target_branch.outputs.target_branch }}
steps:
- name: checkout action
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
sparse-checkout: .github/actions/smart-ci
- name: Get affected components
id: smart_ci
uses: ./.github/actions/smart-ci
with:
repository: ${{ github.repository }}
pr: ${{ github.event.number }}
commit_sha: ${{ github.sha }}
ref_name: ${{ github.ref_name }}
component_pattern: "category: (.*)"
repo_token: ${{ secrets.GITHUB_TOKEN }}
skip_when_only_listed_labels_set: 'docs'
skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg,*/layer_tests_summary/*,*/conformance/*'
- name: Get target branch
id: set_target_branch
run: |
echo "target_branch=${TARGET_BRANCH#refs/heads/}" >> $GITHUB_OUTPUT
Build:
needs: [ Smart_CI ]
if: "!needs.smart_ci.outputs.skip_workflow"
uses: ./.github/workflows/job_build_windows.yml
with:
runner: 'aks-win-16-cores-32gb'
affected-components: ${{ needs.smart_ci.outputs.affected_components }}
build-type: 'Release'
target-branch: ${{ needs.smart_ci.outputs.target_branch }}
cmake-options: |-
-G "Ninja Multi-Config" `
-DENABLE_PYTHON=ON `
-DENABLE_CPPLINT=OFF `
-DBUILD_SHARED_LIBS=ON `
-DENABLE_TESTS=ON `
-DCMAKE_COMPILE_WARNING_AS_ERROR=ON `
-DENABLE_STRICT_DEPENDENCIES=OFF `
-DCMAKE_DISABLE_FIND_PACKAGE_PkgConfig=ON `
-DCUSTOM_OPERATIONS="calculate_grid;complex_mul;fft;grid_sample;sparse_conv;sparse_conv_transpose" `
-DOPENVINO_EXTRA_MODULES="${env:OPENVINO_CONTRIB_REPO }}/modules/custom_operations;${env:OPENVINO_CONTRIB_REPO}/modules/java_api"
Samples:
needs: [ Build, Smart_CI ]
if: fromJSON(needs.smart_ci.outputs.affected_components).samples
timeout-minutes: 20
defaults:
run:
shell: pwsh
runs-on: aks-win-4-cores-8gb
env:
OPENVINO_REPO: "${{ github.workspace }}\\openvino"
INSTALL_DIR: "${{ github.workspace }}\\install"
INSTALL_TEST_DIR: "${{ github.workspace }}\\install\\tests"
INSTALL_WHEELS_DIR: "${{ github.workspace }}\\install\\wheels"
SAMPLES_INSTALL_DIR: "${{ github.workspace }}\\install\\samples"
BUILD_DIR: "${{ github.workspace }}\\build"
steps:
- name: Download OpenVINO artifacts (tarballs)
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
pattern: openvino_[package|tests]*
path: ${{ env.INSTALL_DIR }}
merge-multiple: true
- name: Download OpenVINO artifacts (wheels)
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
pattern: openvino_[wheels]*
path: ${{ env.INSTALL_WHEELS_DIR }}
merge-multiple: true
- name: Extract OpenVINO packages
run: |
Expand-Archive openvino_package.zip -DestinationPath .
Expand-Archive openvino_tests.zip -DestinationPath .
working-directory: ${{ env.INSTALL_DIR }}
- name: Fetch setup_python action
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
sparse-checkout: |
.github/actions/setup_python/action.yml
sparse-checkout-cone-mode: false
path: 'openvino'
- name: Setup Python ${{ env.PYTHON_VERSION }}
uses: ./openvino/.github/actions/setup_python
with:
version: ${{ env.PYTHON_VERSION }}
should-setup-pip-paths: 'false'
self-hosted-runner: 'true'
# Test with the short names of the arguments
- name: Build cpp samples
run: |
& ${{ env.SAMPLES_INSTALL_DIR }}/cpp/build_samples.ps1 -i ${{ env.INSTALL_DIR }} -b ${{ env.BUILD_DIR }}/cpp_samples
env:
CMAKE_COMPILE_WARNING_AS_ERROR: 'ON'
# Test with the full names of the arguments
- name: Build c samples
run: |
& ${{ env.SAMPLES_INSTALL_DIR }}/c/build_samples.ps1 -InstallDirectory ${{ env.INSTALL_DIR }} -BuildDirectory ${{ env.BUILD_DIR }}/c_samples
- name: Samples tests
run: |
# Install Python benchmark_app by installing openvino-*.whl
$ovCoreWheelPath=Get-ChildItem -Path ./wheels -Filter openvino-*.whl | % { $_.FullName }
python3 -m pip install --ignore-installed PyYAML -r ./tests/smoke_tests/requirements.txt "$ovCoreWheelPath"
. "./setupvars.ps1"
$Env:PYTHONCOERCECLOCALE="warn"
python3 -bb -W error -X dev -X warn_default_encoding -m pytest ./tests/smoke_tests --numprocesses auto
env:
IE_APP_PATH: ${{ env.INSTALL_DIR }}/samples_bin
IE_APP_PYTHON_PATH: ${{ env.INSTALL_DIR }}/samples/python
SHARE: ${{ env.INSTALL_TEST_DIR }}/smoke_tests/samples_smoke_tests_data
WORKSPACE: ${{ env.INSTALL_DIR }}
working-directory: ${{ env.INSTALL_DIR }}
# Test .bat scripts for samples building
- name: Build cpp samples (bat)
run: |
& ${{ env.SAMPLES_INSTALL_DIR }}/cpp/build_samples_msvc.bat -i ${{ env.INSTALL_DIR }}/samples_bat -b ${{ env.BUILD_DIR }}/cpp_samples_bat
env:
CMAKE_COMPILE_WARNING_AS_ERROR: 'ON'
- name: Build c samples (bat)
run: |
& ${{ env.SAMPLES_INSTALL_DIR }}/c/build_samples_msvc.bat -i ${{ env.INSTALL_DIR }}/samples_bat -b ${{ env.BUILD_DIR }}/c_samples_bat
JS_API:
name: JS API
needs: [ Build, Smart_CI ]
defaults:
run:
shell: pwsh
runs-on: 'aks-win-4-cores-8gb'
env:
OPENVINO_JS_DIR: "${{ github.workspace }}\\openvino\\src\\bindings\\js"
OPENVINO_JS_LIBS_DIR: "${{ github.workspace }}\\openvino\\src\\bindings\\js\\node\\bin"
NODE_VERSION: 21
if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API
steps:
- name: Fetch OpenVINO JS sources
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
sparse-checkout: |
src/bindings/js
path: 'openvino'
- name: Download OpenVINO js package
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: openvino_js_package
path: ${{ env.OPENVINO_JS_LIBS_DIR }}
- name: Setup Node ${{ env.NODE_VERSION }}
uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0
with:
node-version: ${{ env.NODE_VERSION }}
- name: Configure OpenVINO JS API
working-directory: ${{ env.OPENVINO_JS_DIR }}/node
run: npm i
- name: Test OpenVINO JS API
working-directory: ${{ env.OPENVINO_JS_DIR }}/node
run: npm test
- name: Test OpenVINO JS API (cmd)
shell: cmd
working-directory: ${{ env.OPENVINO_JS_DIR }}/node
run: call npm test
- name: Add msbuild to PATH
uses: microsoft/setup-msbuild@v2
- name: E2E of openvino-node package
working-directory: ${{ env.OPENVINO_JS_DIR }}/node
run: npm run test:e2e
- name: E2E of openvino-node package (cmd)
shell: cmd
working-directory: ${{ env.OPENVINO_JS_DIR }}/node
run: call npm run test:e2e
- name: Create package dir
working-directory: ${{ env.OPENVINO_JS_DIR }}
run: mkdir project-uses-openvino-node
- name: Test installation of openvino-node package
working-directory: ${{ env.OPENVINO_JS_DIR }}/project-uses-openvino-node
run: |
npm i openvino-node
node -e "const { addon: ov } = require('openvino-node'); console.log(ov);"
Openvino_tokenizers:
name: OpenVINO tokenizers extension
needs: [ Build, Smart_CI ]
uses: ./.github/workflows/job_tokenizers.yml
with:
runner: 'aks-win-4-cores-8gb'
shell: pwsh
affected-components: ${{ needs.smart_ci.outputs.affected_components }}
python-version: '3.11'
if: fromJSON(needs.smart_ci.outputs.affected_components).TOKENIZERS
Python_Unit_Tests:
name: Python unit tests
needs: [ Build, Smart_CI ]
timeout-minutes: 75
defaults:
run:
shell: pwsh
runs-on: aks-win-8-cores-16gb
env:
OPENVINO_REPO: "${{ github.workspace }}\\openvino"
INSTALL_DIR: "${{ github.workspace }}\\install"
INSTALL_TEST_DIR: "${{ github.workspace }}\\install\\tests"
INSTALL_WHEELS_DIR: "${{ github.workspace }}\\install\\wheels"
LAYER_TESTS_INSTALL_DIR: "${{ github.workspace }}\\install\\tests\\layer_tests"
PYTHON_STATIC_ARGS: -m "not dynamic_library and not template_plugin"
steps:
- name: Download OpenVINO artifacts (tarballs)
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
pattern: openvino_[tests]*
path: ${{ env.INSTALL_DIR }}
merge-multiple: true
- name: Download OpenVINO artifacts (wheels)
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
pattern: openvino_[wheels]*
path: ${{ env.INSTALL_WHEELS_DIR }}
merge-multiple: true
- name: Extract OpenVINO packages
run: |
Expand-Archive openvino_tests.zip -DestinationPath .
working-directory: ${{ env.INSTALL_DIR }}
- name: Fetch setup_python action
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
sparse-checkout: |
.github/actions/setup_python/action.yml
sparse-checkout-cone-mode: false
path: 'openvino'
- name: Setup Python ${{ env.PYTHON_VERSION }}
uses: ./openvino/.github/actions/setup_python
with:
version: ${{ env.PYTHON_VERSION }}
pip-cache-path: ${{ env.PIP_CACHE_PATH }}
should-setup-pip-paths: 'false'
self-hosted-runner: 'true'
- name: Install OpenVINO Python wheels
run: |
# Find and install the core OV wheel
$ovCoreWheelPath=Get-ChildItem -Path . -Filter openvino-*.whl | % { $_.FullName }
python3 -m pip install "$ovCoreWheelPath"
# Find and install the dev OV wheel
$ovDevWheelPath=Get-ChildItem -Path . -Filter openvino_dev*.whl | % { $_.FullName }
python3 -m pip install "$ovDevWheelPath[caffe,kaldi,onnx,tensorflow2,pytorch]"
working-directory: ${{ env.INSTALL_WHEELS_DIR }}
- name: Install Python API tests dependencies
run: |
# To enable pytest parallel features
python3 -m pip install pytest-xdist[psutil]
# For torchvision to OpenVINO preprocessing converter
python3 -m pip install -r ${{ env.INSTALL_TEST_DIR }}/python/preprocess/torchvision/requirements.txt
# TODO: replace with Python API tests requirements
python3 -m pip install -r ${{ env.INSTALL_TEST_DIR }}/mo/requirements_dev.txt
# For getting rid of SSL issues during model downloading for unit tests
python3 -m pip install certifi
- name: Set SSL_CERT_FILE for model downloading for unit tests
run: echo SSL_CERT_FILE=$(python3 -m certifi) >> $env:GITHUB_ENV
- name: Python API Tests
#if: fromJSON(needs.smart_ci.outputs.affected_components).Python_API.test # Ticket: 127101
shell: cmd
run: |
set PYTHONPATH=${{ env.LAYER_TESTS_INSTALL_DIR }};%PYTHONPATH%
python3 -m pytest -sv ${{ env.INSTALL_TEST_DIR }}/pyopenvino ${{ env.PYTHON_STATIC_ARGS }} --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-Pyngraph.xml --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_utils/test_utils.py
- name: Model Optimizer UT
if: fromJSON(needs.smart_ci.outputs.affected_components).MO.test
shell: cmd
run: |
python3 -m pytest -s ${{ env.INSTALL_TEST_DIR }}/mo/unit_tests --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-ModelOptimizer.xml
- name: Install Python Layer tests dependencies
run: |
# layer test requirements
python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt
- name: ONNX Layer Tests
if: fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE.test
shell: cmd
run: |
:: requires 'unit_tests' from 'tools/mo'
set PYTHONPATH=${{ env.INSTALL_TEST_DIR }}\mo;%PYTHONPATH%
python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/onnx_tests -n logical -m "not launch_only_if_manually_specified and precommit" --junitxml=${INSTALL_TEST_DIR}/TEST-onnx.xml
env:
TEST_DEVICE: CPU
TEST_PRECISION: FP16
- name: TensorFlow Lite Layer Tests - TFL FE
if: fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test
shell: cmd
run: |
python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow_lite_tests/ -n logical --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tfl_fe.xml
env:
TEST_DEVICE: CPU
TEST_PRECISION: FP16
- name: Python ONNX operators tests
if: fromJSON(needs.smart_ci.outputs.affected_components).Python_API.test ||
fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE.test
shell: cmd
run: |
:: Skip test_onnx/test_zoo_models and test_onnx/test_backend due to long execution time - ONNX Model Zoo tests are run separately
python3 -m pytest ${{ env.INSTALL_TEST_DIR }}/onnx -k "not cuda" ^
--junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-onnx_frontend.xml ^
--ignore=${{ env.INSTALL_TEST_DIR }}/onnx/test_python/test_zoo_models.py
- name: MO Python API Tests
if: fromJSON(needs.smart_ci.outputs.affected_components).MO.test
shell: cmd
run: |
:: Used for 'test_utils' installed in '<test_package>\python\openvino\test_utils'
set PYTHONPATH=${{ env.INSTALL_TEST_DIR }}\python\openvino\test_utils;${{ env.INSTALL_TEST_DIR }}\python;%PYTHONPATH%
python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/mo_python_api_tests --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-test_mo_convert.xml
env:
TEST_DEVICE: CPU
TEST_PRECISION: FP16
- name: OVC Python API Tests
if: fromJSON(needs.smart_ci.outputs.affected_components).MO.test
shell: cmd
run: |
:: Used for 'test_utils' installed in '<test_package>\python\openvino\test_utils'
set PYTHONPATH=${{ env.INSTALL_TEST_DIR }}\python\openvino\test_utils;${{ env.INSTALL_TEST_DIR }}\python;%PYTHONPATH%
:: Skip test ticket: 126319
python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/ovc_python_api_tests -k "not test_ovc_tool_non_existng_output_dir" --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-test_ovc_convert.xml
env:
TEST_DEVICE: CPU
TEST_PRECISION: FP16
- name: Python Frontend tests
if: fromJSON(needs.smart_ci.outputs.affected_components).PyTorch_FE.test ||
fromJSON(needs.smart_ci.outputs.affected_components).PDPD_FE.test
run: |
python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/py_frontend_tests --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-test_py_fontend.xml
- name: OVC unit tests
if: fromJSON(needs.smart_ci.outputs.affected_components).MO.test
shell: cmd
run: python3 -m pytest -s ${{ env.INSTALL_TEST_DIR }}/ovc/unit_tests --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-OpenVinoConversion.xml
- name: Upload Test Results
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
if: ${{ !cancelled() }}
with:
name: test-results-python
path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml
if-no-files-found: 'error'
TensorFlow_Layer_Tests:
name: TensorFlow Layer Tests
needs: [ Build, Smart_CI, Openvino_tokenizers ]
uses: ./.github/workflows/job_tensorflow_layer_tests.yml
with:
runner: 'aks-win-8-cores-16gb'
affected-components: ${{ needs.smart_ci.outputs.affected_components }}
python-version: '3.11'
Pytorch_Layer_Tests:
name: Pytorch Layer Tests
needs: [ Build, Smart_CI ]
uses: ./.github/workflows/job_pytorch_layer_tests.yml
with:
runner: 'aks-win-8-cores-16gb'
affected-components: ${{ needs.smart_ci.outputs.affected_components }}
python-version: '3.11'
CXX_Unit_Tests:
name: C++ unit tests
needs: [ Build, Smart_CI ]
uses: ./.github/workflows/job_cxx_unit_tests.yml
with:
runner: 'aks-win-4-cores-8gb'
affected-components: ${{ needs.smart_ci.outputs.affected_components }}
os: 'windows_2019'
build-type: 'Release'
timeout-minutes: 50
CPU_Functional_Tests:
name: CPU functional tests
needs: [ Build, Smart_CI ]
timeout-minutes: 70
defaults:
run:
shell: pwsh
runs-on: aks-win-8-cores-16gb
env:
OPENVINO_REPO: "${{ github.workspace }}\\openvino"
INSTALL_DIR: "${{ github.workspace }}\\install"
INSTALL_TEST_DIR: "${{ github.workspace }}\\install\\tests"
PARALLEL_TEST_SCRIPT: "${{ github.workspace }}\\install\\tests\\functional_test_utils\\layer_tests_summary\\run_parallel.py"
PARALLEL_TEST_CACHE: "${{ github.workspace }}\\install\\tests\\test_cache.lst"
if: fromJSON(needs.smart_ci.outputs.affected_components).CPU.test
steps:
- name: Download OpenVINO package
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: openvino_package
path: ${{ env.INSTALL_DIR }}
- name: Download OpenVINO tests package
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: openvino_tests
path: ${{ env.INSTALL_TEST_DIR }}
- name: Extract OpenVINO packages
run: |
pushd ${{ env.INSTALL_DIR }}
Expand-Archive openvino_package.zip -DestinationPath "${{ env.INSTALL_DIR }}"
popd
pushd ${{ env.INSTALL_TEST_DIR }}
Expand-Archive openvino_tests.zip -DestinationPath "${{ env.INSTALL_DIR }}"
popd
- name: Fetch setup_python action
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
sparse-checkout: |
.github/actions/setup_python/action.yml
sparse-checkout-cone-mode: false
path: 'openvino'
- name: Setup Python ${{ env.PYTHON_VERSION }}
uses: ./openvino/.github/actions/setup_python
with:
version: ${{ env.PYTHON_VERSION }}
should-setup-pip-paths: 'false'
self-hosted-runner: 'true'
- name: Install python dependencies
run: python3 -m pip install -r ${{ github.workspace }}\install\tests\functional_test_utils\layer_tests_summary\requirements.txt
- name: Restore tests execution time
uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
with:
path: ${{ env.PARALLEL_TEST_CACHE }}
key: ${{ runner.os }}-tests-functional-cpu-stamp-${{ github.sha }}
restore-keys: |
${{ runner.os }}-tests-functional-cpu-stamp
- name: Intel CPU plugin func tests (parallel)
run: |
. "${{ env.INSTALL_DIR }}/setupvars.ps1"
python3 ${{ env.PARALLEL_TEST_SCRIPT }} -e ${{ env.INSTALL_TEST_DIR }}/ov_cpu_func_tests.exe -c ${{ env.PARALLEL_TEST_CACHE }} -w ${{ env.INSTALL_TEST_DIR }} -s suite -rf 0 -- --gtest_filter=*smoke*
timeout-minutes: 60
- name: Save tests execution time
uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
if: github.ref_name == 'master'
with:
path: ${{ env.PARALLEL_TEST_CACHE }}
key: ${{ runner.os }}-tests-functional-cpu-stamp-${{ github.sha }}
- name: Upload Test Results
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
if: ${{ !cancelled() }}
with:
name: test-results-functional-cpu
path: |
${{ env.INSTALL_TEST_DIR }}/temp/*.log
${{ env.INSTALL_TEST_DIR }}/logs/*.log
${{ env.INSTALL_TEST_DIR }}/logs/failed/*.log
${{ env.INSTALL_TEST_DIR }}/logs/crashed/*.log
${{ env.INSTALL_TEST_DIR }}/logs/hanged/*.log
${{ env.INSTALL_TEST_DIR }}/logs/interapted/*.log
${{ env.INSTALL_TEST_DIR }}/logs/hash_table.csv
${{ env.PARALLEL_TEST_CACHE }}
if-no-files-found: 'error'
Overall_Status:
name: ci/gha_overall_status_windows
needs: [ Smart_CI, Build, Samples, CXX_Unit_Tests, Python_Unit_Tests, CPU_Functional_Tests, Openvino_tokenizers, TensorFlow_Layer_Tests, Pytorch_Layer_Tests ]
if: ${{ always() }}
runs-on: ubuntu-latest
steps:
- name: Check status of all jobs
if: >-
${{
contains(needs.*.result, 'failure') ||
contains(needs.*.result, 'cancelled')
}}
run: exit 1